mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-31 00:17:47 +00:00
Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Ben Duncan
|
cf4a6abafb | ||
|
Ben Duncan
|
06f93802b8 | ||
|
Josh Siviour
|
0fc93e975a | ||
|
Josh Siviour
|
d066786faa |
@@ -102,9 +102,9 @@ jobs:
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
|
||||
- name: Build microVM artifacts (lb-agent + kernel + initramfs + qboot + SQS store image)
|
||||
- name: Build microVM artifacts (lb-agent + kernel + initramfs + qboot + SQS store and worker images)
|
||||
if: steps.version.outputs.exists != 'true'
|
||||
run: cd spinifex && make build-microvm-image build-sqs-store-microvm-image
|
||||
run: cd spinifex && make build-microvm-image build-sqs-store-microvm-image build-sqs-worker-microvm-image
|
||||
|
||||
- name: Build amd64 Tarball
|
||||
if: steps.version.outputs.exists != 'true'
|
||||
|
||||
@@ -57,8 +57,8 @@ jobs:
|
||||
go-version-file: spinifex/go.mod
|
||||
cache: false
|
||||
|
||||
- name: Build microVM artifacts (lb-agent + kernel + initramfs + qboot + SQS store image)
|
||||
run: cd spinifex && make build-microvm-image build-sqs-store-microvm-image
|
||||
- name: Build microVM artifacts (lb-agent + kernel + initramfs + qboot + SQS store and worker images)
|
||||
run: cd spinifex && make build-microvm-image build-sqs-store-microvm-image build-sqs-worker-microvm-image
|
||||
|
||||
- name: Build amd64 Tarball
|
||||
env:
|
||||
|
||||
@@ -24,6 +24,9 @@ build/microvm/qboot.LICENSE
|
||||
build/microvm/sqs-store/vmlinuz
|
||||
build/microvm/sqs-store/initramfs.cpio.gz
|
||||
build/microvm/sqs-store/.stage.*/
|
||||
build/microvm/sqs-worker/vmlinuz
|
||||
build/microvm/sqs-worker/initramfs.cpio.gz
|
||||
build/microvm/sqs-worker/.stage.*/
|
||||
build/qboot/
|
||||
!spinifex/runtime/roles/spinifexui/frontend/dist/
|
||||
tmp/
|
||||
|
||||
@@ -157,6 +157,21 @@ bin/nats-server: go.mod go.sum
|
||||
|
||||
build-sqs-store-microvm-image: $(SQS_STORE_MICROVM_ARTIFACTS) ## Build the SQS store microVM kernel and initramfs
|
||||
|
||||
# The shared SQS worker microVM image: the store image's /init and kernel with
|
||||
# the worker binary instead of nats-server. Not installed or released yet.
|
||||
SQS_WORKER_MICROVM_OUT_DIR := $(MICROVM_OUT_DIR)/sqs-worker
|
||||
SQS_WORKER_MICROVM_ARTIFACTS := $(SQS_WORKER_MICROVM_OUT_DIR)/vmlinuz $(SQS_WORKER_MICROVM_OUT_DIR)/initramfs.cpio.gz
|
||||
SQS_WORKER_MICROVM_INPUTS := scripts/build-sqs-store-microvm-image.sh scripts/lib/sqs-store-microvm-publish.sh $(SQS_STORE_MICROVM_OUT_DIR)/init.sh $(SQS_WORKER_MICROVM_OUT_DIR)/inittab $(addprefix $(SQS_WORKER_MICROVM_OUT_DIR)/,sqs-worker-lib.sh sqs-worker-prepare sqs-worker-ready sqs-worker-stop) bin/sqs-worker
|
||||
|
||||
$(SQS_WORKER_MICROVM_ARTIFACTS) &: $(SQS_WORKER_MICROVM_INPUTS)
|
||||
SQS_MICROVM_PROFILE=worker ./scripts/build-sqs-store-microvm-image.sh
|
||||
|
||||
bin/sqs-worker: go.mod go.sum $(shell find cmd/sqs-worker spinifex/agents/sqs -name '*.go' ! -name '*_test.go')
|
||||
@mkdir -p bin
|
||||
GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags '-s -w' -o $@ ./cmd/sqs-worker
|
||||
|
||||
build-sqs-worker-microvm-image: $(SQS_WORKER_MICROVM_ARTIFACTS) ## Build the shared SQS worker microVM kernel and initramfs
|
||||
|
||||
# qboot is the microvm firmware. The source tarball ships with each release
|
||||
# because qboot is GPL-2.0.
|
||||
QBOOT_COMMIT := $(shell sed -n 's/^QBOOT_COMMIT="\(.*\)"$$/\1/p' scripts/build-qboot.sh)
|
||||
@@ -174,6 +189,12 @@ install-sqs-store-microvm: $(SQS_STORE_MICROVM_ARTIFACTS) ## Install the SQS sto
|
||||
sudo install -m 0644 $(SQS_STORE_MICROVM_OUT_DIR)/vmlinuz /usr/share/spinifex/microvm/sqs-store/vmlinuz
|
||||
sudo install -m 0644 $(SQS_STORE_MICROVM_OUT_DIR)/initramfs.cpio.gz /usr/share/spinifex/microvm/sqs-store/initramfs.cpio.gz
|
||||
|
||||
install-sqs-worker-microvm: $(SQS_WORKER_MICROVM_ARTIFACTS) ## Install the SQS worker microVM image to /usr/share/spinifex/microvm/sqs-worker/
|
||||
sudo install -d /usr/share/spinifex/microvm/sqs-worker
|
||||
sudo rm -f /usr/share/spinifex/microvm/sqs-worker/initramfs.cpio.gz
|
||||
sudo install -m 0644 $(SQS_WORKER_MICROVM_OUT_DIR)/vmlinuz /usr/share/spinifex/microvm/sqs-worker/vmlinuz
|
||||
sudo install -m 0644 $(SQS_WORKER_MICROVM_OUT_DIR)/initramfs.cpio.gz /usr/share/spinifex/microvm/sqs-worker/initramfs.cpio.gz
|
||||
|
||||
install-microvm: $(MICROVM_ARTIFACTS) $(QBOOT_ARTIFACTS) ## Install microVM artifacts to /usr/share/spinifex/microvm/
|
||||
sudo install -d /usr/share/spinifex/microvm
|
||||
sudo install -m 0644 $(MICROVM_OUT_DIR)/vmlinuz /usr/share/spinifex/microvm/vmlinuz
|
||||
@@ -475,6 +496,7 @@ distro-amd64:
|
||||
echo "[distro-amd64] run 'make build-microvm-image' before 'make distro-amd64'"; \
|
||||
fi
|
||||
./scripts/lib/stage-sqs-store-microvm.sh $(SQS_STORE_MICROVM_OUT_DIR) dist/amd64
|
||||
./scripts/lib/stage-sqs-store-microvm.sh $(SQS_WORKER_MICROVM_OUT_DIR) dist/amd64 sqs-worker
|
||||
tar -czf dist/spinifex-$(VERSION)-linux-amd64.tar.gz -C dist/amd64 .
|
||||
sha256sum dist/spinifex-$(VERSION)-linux-amd64.tar.gz > dist/spinifex-$(VERSION)-linux-amd64.tar.gz.sha256
|
||||
|
||||
@@ -493,7 +515,7 @@ distro-arm64:
|
||||
distro-clean:
|
||||
rm -rf dist/
|
||||
|
||||
.PHONY: test-package-check build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image build-sqs-store-microvm-image install-microvm install-sqs-store-microvm go_build preflight test test-cover test-race diff-coverage bench fuzz test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle \
|
||||
.PHONY: test-package-check build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image build-sqs-store-microvm-image build-sqs-worker-microvm-image install-microvm install-sqs-store-microvm install-sqs-worker-microvm go_build preflight test test-cover test-race diff-coverage bench fuzz test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle \
|
||||
deploy reinstall clean \
|
||||
install-system install-go install-aws quickinstall \
|
||||
lint fix govulncheck nilaway \
|
||||
|
||||
@@ -59,9 +59,6 @@ var allowlist = []allowance{
|
||||
{From: "domains/ec2/awsapi/instance", To: "runtime/compute/cache", Debt: "domains/ec2/awsapi/instance (dependency)", Reason: "node liveness and VM state; needs an EC2-owned projection"},
|
||||
{From: "domains/ec2/awsapi/instance", To: "runtime/compute/vm", Debt: "domains/ec2/awsapi/instance (dependency)", Reason: "node liveness and VM state; needs an EC2-owned projection"},
|
||||
|
||||
{From: "domains/ec2/awsapi/spotinstance", To: "domains/admission/quota", Debt: "domains/ec2/awsapi/spotinstance (dependency)", Reason: "RequestSpotInstances.go uses IAMService and quota.Service"},
|
||||
{From: "domains/ec2/awsapi/spotinstance", To: "domains/iam", Debt: "domains/ec2/awsapi/spotinstance (dependency)", Reason: "RequestSpotInstances.go uses IAMService and quota.Service"},
|
||||
|
||||
{From: "domains/ec2/eip", To: "domains/network/topology", Debt: "EC2 → network implementation", Reason: "EC2 reads network implementation; clears with the EC2-owned projection"},
|
||||
|
||||
{From: "domains/ec2/guestmetadata", To: "domains/dns", Debt: "Guest metadata: the cross-domain and runtime imports above", Reason: "guest-metadata relocation residual; needs capabilities, projections or contracts"},
|
||||
@@ -103,25 +100,7 @@ var allowlist = []allowance{
|
||||
{From: "domains/ochre/vector", To: "domains/iam", Debt: "Cross-domain", Reason: "consumes another domain's implementation rather than a capability"},
|
||||
|
||||
{From: "gateway", To: "domains/ec2/awsapi", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
|
||||
{From: "gateway", To: "domains/ec2/awsapi/account", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
|
||||
{From: "gateway", To: "domains/ec2/awsapi/capacityreservation", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
|
||||
{From: "gateway", To: "domains/ec2/awsapi/eigw", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
|
||||
{From: "gateway", To: "domains/ec2/awsapi/eip", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
|
||||
{From: "gateway", To: "domains/ec2/awsapi/idem", Debt: "gateway/ec2.go (registration; ec2_idempotency.go)", Reason: "dispatch-time idempotency wrapper; moves with the EC2 dispatch map"},
|
||||
{From: "gateway", To: "domains/ec2/awsapi/igw", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
|
||||
{From: "gateway", To: "domains/ec2/awsapi/image", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
|
||||
{From: "gateway", To: "domains/ec2/awsapi/instance", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
|
||||
{From: "gateway", To: "domains/ec2/awsapi/key", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
|
||||
{From: "gateway", To: "domains/ec2/awsapi/launchtemplate", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
|
||||
{From: "gateway", To: "domains/ec2/awsapi/natgw", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
|
||||
{From: "gateway", To: "domains/ec2/awsapi/placementgroup", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
|
||||
{From: "gateway", To: "domains/ec2/awsapi/routetable", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
|
||||
{From: "gateway", To: "domains/ec2/awsapi/snapshot", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
|
||||
{From: "gateway", To: "domains/ec2/awsapi/spotinstance", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
|
||||
{From: "gateway", To: "domains/ec2/awsapi/tags", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
|
||||
{From: "gateway", To: "domains/ec2/awsapi/volume", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
|
||||
{From: "gateway", To: "domains/ec2/awsapi/vpc", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
|
||||
{From: "gateway", To: "domains/ec2/awsapi/zone", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
|
||||
|
||||
{From: "domains/rds/awsapi", To: "domains/ec2/awsapi/instance", Debt: "Instance adapter consumers outside EC2", Reason: "enginecatalog.go calls the instance adapter; needs an EC2 capability"},
|
||||
|
||||
|
||||
@@ -161,7 +161,8 @@ sqs_run() {
|
||||
return 2
|
||||
fi
|
||||
sqs_log "run: starting pid=$$"
|
||||
exec "$NATS_BIN" -c "$SQS_RUN/nats.conf"
|
||||
# ADR-0011:S11 bounds the store's Go heap; the guest must be larger.
|
||||
GOMEMLIMIT=512MiB exec "$NATS_BIN" -c "$SQS_RUN/nats.conf"
|
||||
}
|
||||
|
||||
# sqs_stop: first shutdown action. Block any respawn, SIGTERM the recorded run
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
# The power button loads here, not in /init: busybox init has its SIGUSR2
|
||||
# (poweroff) handler installed before sysinit runs, so a press can never reach
|
||||
# the pre-exec shell PID 1. A press before this line is lost; the host re-sends.
|
||||
::sysinit:/sbin/modprobe tiny_power_button power_signal=12
|
||||
::sysinit:/usr/local/lib/sqs-worker/sqs-worker-prepare
|
||||
::respawn:/usr/local/lib/sqs-worker/sqs-worker-ready
|
||||
::ctrlaltdel:/sbin/reboot
|
||||
::shutdown:/usr/local/lib/sqs-worker/sqs-worker-stop
|
||||
@@ -0,0 +1,115 @@
|
||||
# sqs-worker-lib.sh — shared by the SQS worker guest's readiness and stop
|
||||
# steps. Sourced, never executed. Every path and command is overridable so the
|
||||
# host test can drive the same code against test-owned processes and stub tools.
|
||||
#
|
||||
# store-ready exists only while the running watcher's last check passed. A
|
||||
# check proves the store accepted the worker key, not the store server's
|
||||
# identity, and a store failure is noticed up to one interval plus call time late.
|
||||
|
||||
SQSW_RUN=${SQSW_RUN:-/run/sqs-worker}
|
||||
SQSW_DATA=${SQSW_DATA:-/data}
|
||||
SQSW_CONSOLE=${SQSW_CONSOLE:-/dev/console}
|
||||
SQSW_MOUNTS=${SQSW_MOUNTS:-/proc/mounts}
|
||||
SQSW_BIN=${SQSW_BIN:-/usr/local/bin/sqs-worker}
|
||||
SQSW_STOP_BUDGET_TENTHS=${SQSW_STOP_BUDGET_TENTHS:-100}
|
||||
SQSW_RESPAWN_GAP=${SQSW_RESPAWN_GAP:-5}
|
||||
SQSW_PARK_INTERVAL=${SQSW_PARK_INTERVAL:-30}
|
||||
UMOUNT=${UMOUNT:-umount}
|
||||
|
||||
sqsw_log() { echo "[sqs-worker] $*" >> "$SQSW_CONSOLE"; }
|
||||
|
||||
# sqsw_alive <pid>: running or stopped, not exited (a zombie counts as exited).
|
||||
sqsw_alive() { [ -r "/proc/$1/status" ] && ! grep -q '^State:[[:space:]]*Z' "/proc/$1/status" 2>/dev/null; }
|
||||
|
||||
# sqsw_starttime <pid>: kernel start time (stat field 22). It survives exec, and
|
||||
# pairing it with the pid reduces PID-reuse risk; it is not an absolute guarantee.
|
||||
sqsw_starttime() { sed 's/.*) //' "/proc/$1/stat" 2>/dev/null | cut -d' ' -f20; }
|
||||
|
||||
# sqsw_same <pid> <starttime>: the recorded process is still alive.
|
||||
sqsw_same() { [ -n "$2" ] && sqsw_alive "$1" && [ "$(sqsw_starttime "$1")" = "$2" ]; }
|
||||
|
||||
# sqsw_run: the respawned readiness body. A predecessor that was killed or
|
||||
# crashed could not remove its marker, so every start removes it first. Its
|
||||
# identity is recorded before the stopping check, as the store's run step does.
|
||||
sqsw_run() {
|
||||
mkdir -p "$SQSW_RUN"
|
||||
echo "$$ $(sqsw_starttime $$)" > "$SQSW_RUN/ready.pid"
|
||||
if [ -e "$SQSW_RUN/store-ready" ]; then
|
||||
rm -f "$SQSW_RUN/store-ready"
|
||||
sqsw_log "ready: removed a store-ready marker left by an earlier watcher"
|
||||
fi
|
||||
# Returning here would make init respawn and log at once, forever. Park
|
||||
# instead, logging once: shutdown stops this recorded process, and a key
|
||||
# that appears later lets the watcher start.
|
||||
if [ -e "$SQSW_RUN/stopping" ] || [ ! -s "$SQSW_RUN/public-key" ]; then
|
||||
if [ -e "$SQSW_RUN/stopping" ]; then
|
||||
sqsw_log "ready: shutdown in progress, not starting"
|
||||
else
|
||||
sqsw_log "ready: no worker key; readiness not started, waiting"
|
||||
fi
|
||||
while [ -e "$SQSW_RUN/stopping" ] || [ ! -s "$SQSW_RUN/public-key" ]; do
|
||||
sleep "$SQSW_PARK_INTERVAL"
|
||||
done
|
||||
fi
|
||||
# Init respawns at once; a watcher that keeps exiting restarts at most
|
||||
# once per gap instead of in a tight loop.
|
||||
now=$(date +%s)
|
||||
last=$(cat "$SQSW_RUN/ready.last" 2>/dev/null || echo 0)
|
||||
wait=$((last + SQSW_RESPAWN_GAP - now))
|
||||
if [ "$wait" -gt 0 ] && [ "$wait" -le "$SQSW_RESPAWN_GAP" ]; then sleep "$wait"; fi
|
||||
date +%s > "$SQSW_RUN/ready.last"
|
||||
sqsw_log "ready: starting watcher pid=$$"
|
||||
exec "$SQSW_BIN" ready >> "$SQSW_CONSOLE" 2>&1
|
||||
}
|
||||
|
||||
# sqsw_stop: block any respawn, drop the marker, SIGTERM the recorded watcher,
|
||||
# SIGKILL only after the budget, and unmount the worker disk only once the
|
||||
# watcher is confirmed gone. A survivor keeps the disk mounted.
|
||||
sqsw_stop() {
|
||||
mkdir -p "$SQSW_RUN"
|
||||
: > "$SQSW_RUN/stopping"
|
||||
rm -f "$SQSW_RUN/store-ready"
|
||||
rec=$(cat "$SQSW_RUN/ready.pid" 2>/dev/null || true)
|
||||
pid=${rec%% *} st=${rec#* }
|
||||
[ "$st" != "$rec" ] || st=""
|
||||
if [ -n "$pid" ] && sqsw_same "$pid" "$st"; then
|
||||
kill -TERM "$pid" 2>/dev/null || true
|
||||
sqsw_log "stop: SIGTERM sent to readiness watcher pid=$pid"
|
||||
i=0
|
||||
while sqsw_same "$pid" "$st" && [ "$i" -lt "$SQSW_STOP_BUDGET_TENTHS" ]; do sleep 0.1; i=$((i + 1)); done
|
||||
if sqsw_same "$pid" "$st"; then
|
||||
sqsw_log "stop: readiness watcher pid=$pid still running after $((SQSW_STOP_BUDGET_TENTHS / 10))s, SIGKILL"
|
||||
kill -KILL "$pid" 2>/dev/null || true
|
||||
j=0
|
||||
while sqsw_same "$pid" "$st" && [ "$j" -lt 30 ]; do sleep 0.1; j=$((j + 1)); done
|
||||
fi
|
||||
if sqsw_same "$pid" "$st"; then
|
||||
rm -f "$SQSW_RUN/store-ready"
|
||||
sqsw_log "stop: readiness watcher pid=$pid STILL ALIVE, leaving $SQSW_DATA mounted"
|
||||
return 1
|
||||
fi
|
||||
sqsw_log "stop: readiness watcher pid=$pid exited"
|
||||
else
|
||||
sqsw_log "stop: no running readiness watcher"
|
||||
fi
|
||||
rm -f "$SQSW_RUN/store-ready" "$SQSW_RUN/ready.pid"
|
||||
sync
|
||||
if grep -q " $SQSW_DATA " "$SQSW_MOUNTS" 2>/dev/null; then
|
||||
rc=0
|
||||
"$UMOUNT" "$SQSW_DATA" || rc=$?
|
||||
sqsw_log "stop: umount $SQSW_DATA rc=$rc"
|
||||
fi
|
||||
}
|
||||
|
||||
# sqsw_shutdown: the only shutdown action. The generic read-only remount runs
|
||||
# only after the watcher stopped; a survivor keeps every mount as it is.
|
||||
sqsw_shutdown() {
|
||||
if ! sqsw_stop; then
|
||||
sync
|
||||
sqsw_log "shutdown: readiness watcher survived, skipping umount -a -r"
|
||||
return 1
|
||||
fi
|
||||
rc=0
|
||||
"$UMOUNT" -a -r || rc=$?
|
||||
echo "[init] shutdown: umount -a -r rc=$rc" >> "$SQSW_CONSOLE"
|
||||
}
|
||||
Executable
+8
@@ -0,0 +1,8 @@
|
||||
#!/bin/sh
|
||||
# sysinit: load the disk drivers, then select, check and mount the worker disk
|
||||
# and load or create the worker's store key on it. Any refusal is logged; the
|
||||
# key is never read or created on a disk that is not proven mounted.
|
||||
modprobe virtio_blk 2>/dev/null || true
|
||||
modprobe ext4 2>/dev/null || true
|
||||
/usr/local/bin/sqs-worker boot >> /dev/console 2>&1 || echo "[sqs-worker] boot refused" >> /dev/console
|
||||
exit 0
|
||||
Executable
+6
@@ -0,0 +1,6 @@
|
||||
#!/bin/sh
|
||||
# Respawned by init. A watcher that must not run parks here instead of exiting,
|
||||
# so init does not respawn it in a tight loop.
|
||||
. /usr/local/lib/sqs-worker/sqs-worker-lib.sh
|
||||
sqsw_run
|
||||
exec sleep 2147483647
|
||||
Executable
+5
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
# The only shutdown action: stop the readiness watcher and unmount the worker
|
||||
# disk, then run the generic read-only remount only if the watcher is gone.
|
||||
. /usr/local/lib/sqs-worker/sqs-worker-lib.sh
|
||||
sqsw_shutdown
|
||||
@@ -0,0 +1,120 @@
|
||||
// sqs-worker runs inside the shared SQS worker microVM.
|
||||
//
|
||||
// Usage:
|
||||
//
|
||||
// sqs-worker boot
|
||||
// sqs-worker ready
|
||||
//
|
||||
// boot selects and mounts the worker's data disk named by fw_cfg, asks the
|
||||
// gateway for its current first-store decision with its instance-role
|
||||
// credentials, then loads or creates the store key on that disk and
|
||||
// publishes only the public key. ready then keeps checking the bound store's
|
||||
// JetStream with that key until it is stopped, keeping a readiness marker only
|
||||
// while the check passes; it proves the store accepted the worker key, not
|
||||
// the store server's identity.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/internal/gwsign"
|
||||
"github.com/mulgadc/spinifex/internal/sqsworkergw"
|
||||
"github.com/mulgadc/spinifex/spinifex/agents/sqs/workerboot"
|
||||
"github.com/mulgadc/spinifex/spinifex/agents/sqs/workerkey"
|
||||
)
|
||||
|
||||
// The cluster CA the daemon supplies; the only root the client trusts.
|
||||
const caPath = "/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/ca-cert/raw"
|
||||
|
||||
// newClaimer signs with the IMDS instance-role credentials only; there is
|
||||
// no static-key fallback.
|
||||
func newClaimer() workerboot.NewClaimer {
|
||||
return func(region, gatewayURL string) (workerkey.Claimer, error) {
|
||||
return newClient(region, gatewayURL)
|
||||
}
|
||||
}
|
||||
|
||||
func newClient(region, gatewayURL string) (*sqsworkergw.Client, error) {
|
||||
ca, err := os.ReadFile(caPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read cluster CA: %w", err)
|
||||
}
|
||||
return sqsworkergw.New(gatewayURL, region, ca, gwsign.NewInstanceRole())
|
||||
}
|
||||
|
||||
type describer struct{ c *sqsworkergw.Client }
|
||||
|
||||
func (d describer) DescribeStore(ctx context.Context) (workerboot.Store, error) {
|
||||
st, err := d.c.DescribeStore(ctx)
|
||||
return workerboot.Store(st), err
|
||||
}
|
||||
|
||||
func main() {
|
||||
if len(os.Args) != 2 || (os.Args[1] != "boot" && os.Args[1] != "ready") {
|
||||
fmt.Fprintln(os.Stderr, "usage: sqs-worker boot|ready")
|
||||
os.Exit(2)
|
||||
}
|
||||
logf := func(msg string) { fmt.Fprintf(os.Stderr, "[sqs-worker] %s\n", msg) }
|
||||
run := boot
|
||||
if os.Args[1] == "ready" {
|
||||
run = ready
|
||||
}
|
||||
if err := run(logf); err != nil {
|
||||
logf("FATAL: " + err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func config(logf func(string)) workerboot.Config {
|
||||
return workerboot.Config{
|
||||
FwCfgDir: "/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/sqs-worker",
|
||||
SysBlock: "/sys/block",
|
||||
DevDir: "/dev",
|
||||
MountInfo: "/proc/self/mountinfo",
|
||||
DataDir: "/data",
|
||||
RunDir: "/run/sqs-worker",
|
||||
Disk: workerboot.SystemDisk{E2fsck: "/sbin/e2fsck"},
|
||||
IsBlockDevice: workerboot.IsBlockDevice,
|
||||
NewClaimer: newClaimer(),
|
||||
Attempts: 60,
|
||||
Wait: 5 * time.Second,
|
||||
Log: logf,
|
||||
}
|
||||
}
|
||||
|
||||
func boot(logf func(string)) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
|
||||
defer cancel()
|
||||
_, err := workerboot.Boot(ctx, config(logf))
|
||||
return err
|
||||
}
|
||||
|
||||
// ready runs until init stops it; the marker is removed on the way out.
|
||||
func ready(logf func(string)) error {
|
||||
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT)
|
||||
defer cancel()
|
||||
err := workerboot.Watch(ctx, workerboot.ReadyConfig{
|
||||
Config: config(logf),
|
||||
NewDescriber: func(region, gatewayURL string) (workerboot.StoreDescriber, error) {
|
||||
c, err := newClient(region, gatewayURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return describer{c}, nil
|
||||
},
|
||||
Checker: workerboot.NATSChecker{},
|
||||
CallTimeout: 30 * time.Second,
|
||||
Refresh: 30 * time.Second,
|
||||
Refused: 5 * time.Minute,
|
||||
})
|
||||
if errors.Is(err, context.Canceled) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -192,6 +192,21 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
|
||||
|---------|-------|-------------|
|
||||
| `spx admin eks restore-snapshot` | `--cluster` (required), `--snapshot` (optional, defaults to the latest snapshot in predastore), `--account` (optional, defaults to the bootstrap account) | Single-CP total-loss DR path (fail-safe): validates the snapshot exists in predastore BEFORE any mutation (a typo'd/missing key hard-fails, never resets into an empty datastore) → launches a fresh control-plane VM as a cluster-init seed (replaying the persisted create-time launch template) → sets a required-snapshot `RecoveryDirective` (`cluster-reset`) so the boot-time recovery agent aborts rather than resets-into-empty if it cannot fetch the snapshot → persists the replacement in cluster meta BEFORE re-pointing the NLB (so an NLB failure is convergeable by the reconciler, returned as a provisional status, not a hard error) → re-points the cluster NLB's apiserver and konnectivity target groups from the old CP's ENI to the new one → fences the old CP with retries, failing loudly if it cannot be confirmed terminated (split-brain guard). Any failure before the meta commit unwinds the fresh CP (terminate + clear directive) so a re-run does not stack a second resetting control plane. The returned status is provisional — success means the sequence completed, not that etcd is restored and serving; verify cluster health. HA clusters (a spread with a potentially surviving quorum) are rejected — recover those via quorum reformation instead. |
|
||||
|
||||
### SQS System Instances
|
||||
|
||||
These commands start the first SQS system instances on one node. They do not serve the SQS API, and a bound VM is not a ready service.
|
||||
|
||||
Both send a signed `spinifex` admin action (`EnsureSQSWorker`, `EnsureSQSStore`) to the AWS gateway using the standard AWS credential chain, for example `AWS_PROFILE=spinifex`. The caller must be in the super-admin account (`000000000001`) and hold `spinifex:<Action>`. The VMs, volumes, private VPC and ENIs belong to the system account (`000000000000`). `--endpoint`, `--region` and `--ca-bundle` default to this node's gateway, region and `config/ca.pem`; off a node, `--endpoint` and `--region` are required. Each call waits up to 11 minutes.
|
||||
|
||||
| Command | Flags | Description |
|
||||
|---------|-------|-------------|
|
||||
| `spx admin sqs ensure-worker` | `--node` (required), `--instance-type` (required), `--endpoint`, `--region`, `--ca-bundle` | Launches the Region's one SQS worker VM on `--node`, or reports the VM already bound there: node, region, instance, instance type and generation. The worker is pinned to its first node; another node is refused with `InvalidParameterValue`, and there is no takeover. It gets a private system VPC with one /24 subnet and no internet gateway, NAT or Elastic IP, and an encrypted 1 GiB system-account data volume on which the guest generates its own store key. The platform records only the public key. A different `--instance-type` is accepted only while no volume, network interface or launch step has been attempted; once any has, it is `IncorrectState`, whether or not the step completed. |
|
||||
| `spx admin sqs ensure-store` | `--node` (required, the worker's node), `--instance-type` (required, more than 512 MiB memory), `--volume-size-gib` (required, 1–16384), `--endpoint`, `--region`, `--ca-bundle` | Launches the Region's SQS store VM on the worker's node with an encrypted system-account data volume, or reports the VM already bound there: node, region, instance, instance type and address. Needs the worker bound with its public key recorded, else `IncorrectState`. The store accepts only that key. A different size, type or key than the first call is `IncorrectState`. |
|
||||
|
||||
`ServiceUnavailable` never proves that no SQS resources exist, because an earlier attempt may already have created some. Its message says what it does establish. When no node serves the subject, this request was not delivered. When a volume create was not sent, that particular create did not happen. When the node cannot boot the guest yet or its launcher lacks a capability, this request stopped before allocating or recording anything. When the gateway times out waiting for the node's reply, nothing is known: the launch may still be running, so never assume it created nothing. In every case, call the command again with the same arguments. A retry first checks the durable record of the attempt and continues only when that is safe; it can proceed to the first launch after a create that was proven unsent, and otherwise reports the recorded outcome as bound or `IncorrectState`, or returns `ServiceUnavailable` again. `IncorrectState` from a volume, network interface or launch step means that step was recorded as sent but its outcome was never observed. It is never resent, the message names the volume when its ID is known, and no tool reconciles it yet: recovery is manual.
|
||||
|
||||
`bound` says only that the VM was launched for the current record. The worker decides store readiness itself, by checking that the store is bound to its own public key and that a NATS connection with that key succeeds; nothing reports that readiness to the operator. The worker-to-store connection does not authenticate the store server. Real VPC routing, IMDS and gateway TLS from the guest, and worker-to-store traffic on a real interface have not been tested.
|
||||
|
||||
## Private Admin API
|
||||
|
||||
Not an AWS API: a Spinifex-internal surface on the AWS gateway, JSON in and JSON out, one method per path segment at `POST /admin/<Method>`. It exists so a self-service signup form, a provisioning system or a load-test harness can create and remove tenant accounts without an operator shell on a node.
|
||||
|
||||
@@ -360,6 +360,91 @@ Evidence is local only: unit tests, real QEMU boots against disposable scratch d
|
||||
Viperblock, volume encryption, the EC2 volume record path, multiple nodes and host reachability of the guest address are not tested.
|
||||
Credential issuance and rotation, store placement and NATS readiness checks do not exist, and there is no failover: a lost node leaves its store unavailable.
|
||||
|
||||
### SQS worker guest (operator-launched)
|
||||
|
||||
The shared SQS worker guest holds the SQS store's client credential, so that no platform component does (ADR-0011:S2).
|
||||
Only an operator activates it, with the super-admin `EnsureSQSWorker` action described below; nothing starts it at daemon startup, and no worker service runs after boot.
|
||||
|
||||
`make build-sqs-worker-microvm-image` builds it with the store image's script and network `/init` under `SQS_MICROVM_PROFILE=worker`; it carries the static `sqs-worker` binary and the ext4 tools, no `nats-server`, and an empty `/data` mountpoint.
|
||||
Releases build it beside the store image and ship it as `microvm-sqs-worker`, which `setup.sh` installs to `/usr/share/spinifex/microvm/sqs-worker` with the store image's rules: a release without it removes an earlier one, and an incomplete one is refused.
|
||||
At boot `sqs-worker boot` reads three firmware config values under `opt/spinifex/sqs-worker/`: the worker disk's serial, its Region and the gateway URL; none is a first-store decision, because a static value survives a guest reset.
|
||||
A missing, empty or malformed value refuses before any disk is touched.
|
||||
|
||||
It selects exactly one disk with that serial, runs `e2fsck -p`, mounts it at `/data` and then confirms from `/proc/self/mountinfo` that this disk, as a whole ext4 filesystem, is what is mounted there; an empty mountpoint, a foreign mount or a failed check or mount refuses, and the guest never formats.
|
||||
Only then does it ask the gateway for the current decision with `ClaimFirstStore`, signed by `internal/sqsworkergw` with credentials from the instance metadata service only (`gwsign.NewInstanceRole`, no environment, profile or static-key fallback), over TLS 1.3 that trusts only the cluster CA from `opt/spinifex/ca-cert` and never follows a redirect.
|
||||
Under that decision `workerkey.Acquire` creates the key on `/data/sqs-worker-key`, owner-only and synced, records only its public key and then uses it, or loads the recorded key; a missing or different key once one is recorded refuses and is never replaced.
|
||||
The VM can start before its launch binds the instance, so its first calls are legitimately `AccessDenied`; that denial creates nothing and is asked again, as are a lost reply and a `ServiceUnavailable` or `Throttling` refusal, all within one bound (60 attempts 5 seconds apart), after which a worker that was never bound, or was replaced, refuses the boot.
|
||||
A signature or other refusal, or an answer that is neither "create" nor one valid public key, refuses the boot at once.
|
||||
|
||||
`domains/sqs/firststore` is the SQS domain's first-store decision: one record per Region in the `spinifex-sqs-first-store` KV bucket moves from `claimed` to `key-recorded` to `launch-intent` under compare-and-swap, bound to the worker's instance ID, claim ID and generation.
|
||||
A claim authorizes key creation only while no public key is recorded; once one is, every claim returns only that key, and a store launch intent needs it recorded first.
|
||||
An older generation, a different identity at the same generation or an unreadable record authorizes nothing, and the record holds no private key material.
|
||||
Reads are stream message gets answered by the stream leader, never direct gets a lagging replica can answer, and every authorizing answer is also confirmed by a compare-and-swap at the revision it read, so a stale or unavailable read authorizes nothing.
|
||||
The authenticated caller that will carry this decision must derive the worker's instance ID and generation from authoritative instance state; a generation supplied in a request cannot itself authorize a takeover.
|
||||
|
||||
`domains/sqs/workerident` is that authoritative instance state: one record per Region in the `spinifex-sqs-worker-identity` KV bucket holds a monotonic generation, claim ID, pinned node and encrypted data volume, and moves from `launching` to `bound` to `stopped`, or from `launching` to `failed`, under compare-and-swap.
|
||||
A launch intent is refused while the previous attempt is `launching` or `bound`, including one whose outcome is unknown, so only one caller ever holds an intent to start a VM on the pinned volume.
|
||||
A replacement takes the next generation only after `FailLaunch` or `RecordStopped` records that nothing holds the volume, and each request names the generation it supersedes, so a replayed request cannot take a generation it never observed; there is no automatic takeover.
|
||||
The launch path then raises the first-store fence to the successor's generation with `firststore.Fence` before `BeginLaunch`, which refuses a replacement until a leader read shows that fence; the fence only rises, and `firststore` refuses every call from a generation below it.
|
||||
Only the current claim ID is recorded, so claim IDs are minted fresh per attempt and a VM launch must be idempotent on generation and claim together.
|
||||
Binding writes the instance's index entry first and then binds the record to that instance.
|
||||
The index is only a lookup: `Authorize` accepts an instance only while the Region's record is `bound` to exactly that instance, generation and claim, confirmed by a compare-and-swap at the leader-read revision, so an old, orphaned, rewritten or unbound entry authorizes nothing.
|
||||
A same-instance reboot keeps its instance ID and identity; a replacement takes a new generation, and the first cut refuses another node or data volume and an unencrypted volume, although the encryption flag is the caller's claim and the launch wiring must verify it from the volume's metadata.
|
||||
Both buckets are opened through `OpenDurableKV`, which refuses an existing bucket whose effective stream configuration could expire, evict, lose, lag or refuse records: non-limits retention, an age, per-message or delete-marker TTL, memory storage, asynchronous flush, a size, count or message-size cap, a discard policy other than discard-new, unacknowledged writes, a sealed stream, a mirror or sources, or fewer replicas than the cluster needs.
|
||||
`workerkey.Acquire` drives that decision: it syncs a new key to disk, records its public key and only then returns it, so a crash before recording reuses the key already on disk.
|
||||
|
||||
`domains/sqs/awsapi/worker` carries the decision to the worker as `ClaimFirstStore` and `RecordWorkerPublicKey`, served under the internal `spinifex-sqs` signing service, which no AWS SDK signs for; the public AmazonSQS action inventory is untouched.
|
||||
The gateway admits only an assumed-role session of `sqsWorkerRole` in the system account, then the role's policy, and only then resolves the session name, the instance ID STS set, through `workerident.Authorize`; the request carries no identity, and any parameter beyond the action's own is refused.
|
||||
The first-store identity is the current record's instance, claim and generation, and a worker bound to another Region is refused.
|
||||
A request that passed `Authorize` just before a confirmed stop and replacement still carries the old generation; the fence was raised before the successor began, so `firststore` refuses it whether or not the successor has claimed yet.
|
||||
The adapter never logs a caller-chosen parameter name, because one could carry a secret.
|
||||
`RecordWorkerPublicKey` accepts only a valid public user key, so a seed is refused without being recorded, logged or echoed.
|
||||
The gateway binds both buckets per request and never creates them, so it provisions nothing and answers `ServiceUnavailable` until a worker launch has created them; the worker launch below ensures the role.
|
||||
The private key stays on the worker disk; the guest logs and publishes only the public key to `/run/sqs-worker/public-key`.
|
||||
|
||||
`domains/sqs/workerlaunch` realizes the worker on the node it runs on, its only node: a bound record is returned as is; otherwise it takes the intent, then on the first launch marks the intent's one volume create, prepares the volume and records its ID, reads the volume's own record and refuses it unless encrypted, ensures the role, finds or creates the network for the claim and reads the recorded store key, and marks the intent dispatched before launching.
|
||||
A dispatched launch is never sent again: a lost reply or error leaves the intent `launching` and dispatched, and every later call returns `ErrOutcomeUnknown` until the outcome is reconciled, since the launch path is not idempotent.
|
||||
A failure before dispatch leaves an undispatched intent that the next call resumes with the same claim; a confirmed stop or failure is replaced only after `firststore.Fence` excludes the old generation, reusing the pinned volume.
|
||||
The launch passes no store decision or key: the guest asks the gateway at boot.
|
||||
The intent exists before any volume, so concurrent first launches allocate nothing for the losers. A Prepare proven to have sent no create withdraws the mark and returns `ServiceUnavailable`, or `UnsupportedOperation` when the cluster storage key is not configured, and a retry may prepare the volume; a create that may have applied is never sent again, and every later call returns `IncorrectState` until it is reconciled. A volume the call knows of but cannot use is reported with its ID in a `VolumeError` and left in place, never deleted. An intent that has sent no create may be re-typed; one that has keeps its type until replaced. The worker role is created with EC2-only trust, and an existing role is checked for that trust before any policy is granted; the instance profile must carry exactly the worker role, and one holding any other role is refused unchanged.
|
||||
The daemon's `LaunchSQSWorkerInstance` boots the `sqs-worker` image through the same direct-boot path as the store, with the data volume attached at boot and kept on terminate, a required instance profile, and a required mgmt NIC; it re-reads the volume's own record and refuses one that is not available and encrypted, before anything is allocated.
|
||||
Its firmware config adds the gateway URL on this node's mgmt bridge and the cluster CA from the daemon's own trust store; the caller cannot supply either, and no value written is secret.
|
||||
A relaunch rebuilds that config from the VM record on the node that launched it and is refused anywhere else, and membership takeover already skips SQS-managed instances.
|
||||
`sqsWorkerRole` is ensured in the system account granting exactly `spinifex-sqs:ClaimFirstStore` and `spinifex-sqs:RecordWorkerPublicKey`, from the `domains/sqs/workerrole` contract the gateway gate also reads; an existing role whose trust admits more than the EC2 service is refused.
|
||||
|
||||
Evidence is local only: unit tests with a fake firmware config, sysfs and mount table, tests running the boot through the real client against the real `spinifex-sqs` registration and records, and QEMU boots of the built image against disposable scratch disks and a host-side test gateway and metadata service, showing the first boot creates the key, reboots reuse it, a key lost after recording is never replaced, and the seed reaches no log or request.
|
||||
The worker's network is a private SQS system VPC per Region, built by `systemvpc.EnsurePrivate` from the `[sqs]` supernet (default `10.240.0.0/14`): one VPC and one /24 subnet, with no internet gateway, NAT gateway, EIP, route table or daemon-to-store node port. Each launch claim gets one ENI in that subnet, in the VPC default security group, found again by its claim on a retry; two ENIs for one claim are refused. The launch intent is marked before the one create is sent, so a create whose ENI is not yet visible is never sent again: the launch fails closed with `IncorrectState` until the ENI appears or the outcome is reconciled. Worker-to-store TCP and the guest's route to the metadata service are not yet proven by any traffic test.
|
||||
The super-admin `EnsureSQSWorker` action on the internal `spinifex` operator service takes a `NodeId`, which must be a node of this cluster and a single subject token, and an `InstanceType`, and sends a request to `sqs.worker.Ensure.v1.<node>` with no queue group, so only that node answers and the worker stays pinned.
|
||||
That node's daemon re-checks the node the request names and the instance type, and refuses as `ServiceUnavailable` before allocating a volume or network or recording an intent unless the worker image's kernel and initramfs are non-empty files, its mgmt bridge exists and the cluster CA is readable; a call after the image is installed then launches.
|
||||
It then runs its one composed launcher for its own Region, one launch at a time, in the daemon's own context with a 30-minute bound, so a caller's timeout does not cancel it.
|
||||
A reply lost to that timeout is reported as `ServiceUnavailable`; calling again reports the bound worker, or `IncorrectState` for a dispatched launch whose outcome is unknown, which is never resent; a Region pinned to another node is refused as `InvalidParameterValue`.
|
||||
The reply names the bound VM's node, Region, instance ID and generation; bound is not ready, and it says nothing about the guest's boot, its key or the store.
|
||||
Stop/detach evidence for `RecordStopped` is not wired, the guest's metadata route and gateway TLS on a real ENI and worker-to-store TCP are unproven, and Viperblock's at-rest key is held by the platform.
|
||||
|
||||
The worker identity record holds the instance type of its generation; a request for another type is refused as `IncorrectState`, before dispatch and after binding, the reply names the recorded type, and only a replacement after a confirmed stop may change it.
|
||||
Operators call both actions with `spx admin sqs ensure-worker --node --instance-type` and `spx admin sqs ensure-store --node --instance-type --volume-size-gib`, signed for the `spinifex` service with the standard AWS credential chain like the other `spx admin` gateway commands.
|
||||
Each command refuses missing inputs before sending anything and reports only the bound VM; a bound VM is not a ready worker or store.
|
||||
The super-admin `EnsureSQSStore` action takes a `NodeId`, an `InstanceType` and a `VolumeSizeGiB` and sends a request to `sqs.store.Ensure.v1.<node>` with no queue group; only the node the Region's worker is bound to can launch the store.
|
||||
That node refuses an instance type whose memory does not exceed 512 MiB, the store's `GOMEMLIMIT` under ADR-0011:S11; this is a floor, not a validated size, and no type is defaulted.
|
||||
It refuses before recording or allocating anything unless the store image and the cluster storage key are present, the worker is bound on this node, and that worker's identity has recorded its public key in the first-store decision, which it then advances to a launch intent.
|
||||
The store's own launch record in `spinifex-sqs-store-launch` holds only that public key, the claim, the node and the operator's instance type and size, and marks each step before its uncertain create: the volume, the ENI in the worker's private subnet, and the launch.
|
||||
A later request for a different instance type or size is refused as `IncorrectState`, before dispatch and after binding, and the reply names the type the store was launched with.
|
||||
A volume preparation that failed before sending any create, such as a missing `mkfs.ext4` or a scratch or format failure, withdraws its mark and returns `ServiceUnavailable`, so a retry may try again once the host is fixed.
|
||||
Any other marked step whose result is not visible is reported as `IncorrectState` and never resent; a volume that is not encrypted by its own record is refused as `UnsupportedOperation` with its ID kept, and the daemon re-checks the volume's encryption and the guest's size at launch.
|
||||
The daemon never receives the worker's seed; the store's NATS server accepts only that public key, and the reply's bound VM and address are not readiness, which only the worker can check.
|
||||
|
||||
After boot, the worker's long-running `sqs-worker ready` step mounts and verifies its disk again and calls the internal `spinifex-sqs` `DescribeStore` action, which admits only the current worker and returns the bound store's address and recorded public key, or pending.
|
||||
It loads its key only if it matches that public key, then connects to the store's NATS on port 4222 with it and asks JetStream for account info, each pass bounded to 30 seconds.
|
||||
The step runs until shutdown with no overall deadline, so a store the operator launches at any time is picked up: it asks again every 5 seconds while the store is pending, not answering or the reply was lost, and re-checks a ready store every 30 seconds.
|
||||
A pass writes only the address and public key to `/run/sqs-worker/store-ready`; any failed pass removes that file, and so does the step's own exit.
|
||||
A start that finds shutdown in progress or no worker key logs once and waits in place rather than exiting, so init does not respawn it in a loop; shutdown stops it like a running watcher.
|
||||
Init respawns the step, at most once every 5 seconds, and every start removes a marker left by a watcher that was killed or crashed before the new watcher runs.
|
||||
A ready store that fails is noticed up to one re-check interval plus one bounded call later, so the marker can be up to about a minute stale.
|
||||
Shutdown blocks further respawns, removes the marker, and stops the watcher by its recorded PID and start time, escalating to SIGKILL after a budget; it unmounts the worker disk only once the watcher is confirmed gone, and a survivor leaves every mount in place.
|
||||
A mismatched key, a store that refuses the key, a key missing from the disk or an unclassified gateway refusal is asked again only every 5 minutes, so it never spins and still recovers if the operator fixes it; a missing key is never replaced.
|
||||
The NKey authenticates the worker to the store, but nothing authenticates the store to the worker: there is no TLS, so a pass means only that something at that address accepted the key.
|
||||
This is store readiness for the worker alone; there is no SQS API, and no production sizing or support is claimed.
|
||||
|
||||
## Configuration
|
||||
|
||||
Cluster configuration (`spinifex/bootstrap/config/config.go`):
|
||||
|
||||
@@ -365,6 +365,7 @@ while the branch is in flight; they do not describe the umbrella repository's
|
||||
| `spinifex#1348` | `spinifex/gateway/ecs.go` (`ECS_Request`) and `spinifex/gateway/eks.go` (`EKS_Request`, the EKS route table) | `spinifex/domains/ecs/awsapi` and `spinifex/domains/eks/awsapi` (`NewRegistration`, `Deps`, `OperationInventory`), composed by `spinifex/runtime/roles/awsgw` | ECS (AWS JSON 1.1) and EKS (REST JSON) register through the ADR-0002 seam with the JSON error envelopes they had, both declared as AWS JSON 1.1 (see the envelope divergence below). The check order and every response are kept, pinned by the pipeline characterisation tests committed first. EKS and ECS resolve roles through `foundation/aws/passrole` `Check` with `domains/iam` `RoleResolver`: a malformed or foreign role is refused first, a missing role is authorized as written before it is reported, and a resolver outage is returned unchanged. Role handling now follows the AWS capture of 2026-10-30. ECS RegisterTaskDefinition refuses a foreign role with `ClientException` "Role is not valid" before PassRole and accepts a nonexistent, invented-path or non-ARN role from a caller who can pass roles. ECS RunTask refuses a foreign `taskRoleArn` or `executionRoleArn` override the same way and answers a nonexistent or invented-path one with "ECS was unable to assume the role". EKS CreateCluster and CreateNodegroup answer a non-ARN or foreign role with `AccessDeniedException` "Cross-account pass role is not allowed.". EKS public routes now report every parameter fault as `InvalidParameterException`, the code the EKS model declares, while the internal control-plane routes keep `InvalidParameterValue` because the guest agents match on it. Two deliberate EKS divergences remain. First, CreateCluster still refuses a nonexistent or invented-path role with `InvalidParameterException` "role ... does not exist", where AWS accepts it and the cluster later ends FAILED. Our EKS only stores, validates and echoes the cluster role and never assumes it, so accepting would leave an ACTIVE cluster bound to a role that does not exist. Second, CreateNodegroup keeps its existing refusal of a nonexistent role because AWS was not captured for it. A third divergence is the error envelope. EKS is restJson1 but registers `ErrorEnvelopeJSON`, so errors carry `application/x-amz-json-1.1` and a `__type` body where AWS sends `application/json` and a `message` body. The `X-Amzn-Errortype` header is correct and SDKs read the code from it, so impact is low. Switching to `ErrorEnvelopeRESTJSON` also changes the throttle code and the default internal-error code (`ServiceException` where the EKS model declares `ServerException`), so it is left for its own change. Verified live on env14: the ECS and EKS role cases above, and a valid CreateCluster reaching ACTIVE with the control-plane agent publishing through the gateway. The EKS route table and its tests move with the registration, and the gateway keeps no ECS or EKS fallback. The `cmd/aws-model-coverage` named exception gains an `ecs` and an `eks` row, and the two `gateway` allowlist rows are removed. |
|
||||
| `spinifex#1350` | `spinifex/gateway` (`ecr.go`, `ecrauth_middleware.go`, the `/v2/token` exchange, `ecr_principal.go` with `ResolveECRPrincipal`, the per-operation `/v2` authorization, the registry host matcher, `ctxECRPrincipal`) | `spinifex/domains/ecr/registry` (`Bridge`, `NewBridge`, `BridgeConfig`, and its own `Principal`, `Rehydrator`, `Authorizer`, `Admission` types with `ErrInvalidCredential` and `ErrUnavailable`), `spinifex/ingress/http` (`HostMatcher`, `TargetAccount`, `RequestClientIP`; names no service), `spinifex/ingress/aws/credential` (`Rehydrator`, `Principal.CallerARN`), and under `spinifex/runtime/roles/awsgw`: `credverify` (`Rehydrate`, sharing the SigV4 key and session lookups), `policyauth`, `accountadmission`, `registryadapt` | ADR-0001 amendment 2026-10-29 and ADR-0005:S2: the OCI `/v2` bridge and its token auth move out of the gateway, and ECR resolves its principal through the credential capability instead of reading `IAMService` and `STSService` directly. ADR-0002:S5 keeps `/v2` outside the seam, so the gateway keeps only a `RegistryHandler` mount that answers 501 when unset. `domains/ecr/registry` imports no ingress package (`TestADR0001_Layering` unchanged): the principal, rehydrator and authorizer are registry-local types and `registryadapt` maps `credential.Failure` codes onto `ErrInvalidCredential` (401) and `ErrUnavailable` (503). No widening of `dispatch.Actor` or `dispatch.Invocation`. The SigV4 path and the bridge share one policy evaluator (`policyauth`) and one account-admission check. Deliberate change: the open-registry mode for a nil token verifier is deleted and `NewBridge` refuses a nil Verifier, Issuer, Rehydrator, Authorizer or Admission, so tests that want an unauthenticated registry mount `registry.Handler` directly. The pre-move pipeline characterisation (status codes, error bodies, the `WWW-Authenticate` challenge, referrers, token exchange, revoked and deleted principals) passes unchanged apart from setup, and the retired `ResolveECRPrincipal` parity rows live on as `credverify` `Rehydrate` rows and `Bridge` claim-check rows. Live on env14 (DHCP external IP): crane, skopeo and docker push and pull (`tests/e2e/ecr`), a cosign sign and verify in the legacy tag mode and in OCI 1.1 referrers mode with the signature returned by `GET /v2/{repo}/referrers/{digest}`, `/v2/token` refresh from fresh logins, and a deactivated access key answering 401 on `/v2/token`, manifest GET, crane pull and crane push, then 200 again once reactivated. The ECR signing-key deadlock bead (`mulga-6wqce`) is untouched. |
|
||||
| `spinifex#1351` | `spinifex/gateway/{bedrock,bedrockruntime,bedrockagent,bedrockagentruntime}.go` (`Bedrock_Request`, `BedrockRuntime_Request`, `BedrockAgent_Request`, `BedrockAgentRuntime_Request`, the four route tables), `spinifex/domains/ochre/authz.go`, and the 13 `Bedrock*` fields of `GatewayConfig` | `spinifex/domains/ochre/awsapi` (`NewRegistration`, `Deps`, `Quota`, `OperationInventory`, `ServiceName`), composed by `spinifex/runtime/roles/awsgw` | The four Bedrock services register through the ADR-0002 seam as one registration, per ADR-0002:S1 (one per signing service) and the ADR-0001 amendment of 2026-10-06: the service is `bedrock` and `bedrock-runtime`, `bedrock-agent` and `bedrock-agent-runtime` are aliases. A request signed with a sub-service scope is dispatched to that sub-service whatever its path, so a path owned by another sub-service answers `InvalidActionException` as before; only the shared `bedrock` scope is split by path. The pipeline characterisation tests committed first (status, error type and envelope per service, metering order, nil-capability answers) pass unchanged apart from setup. The 13 gateway fields become `awsapi.Deps`: `Credentials`, `Endpoints`, `Access`, `Recorder`, `Logging`, `Provisioned`, `Guardrails`, `Embedder`, `KB`, `DataSources`, `Vector` and `Quota`, with the documented nil fallbacks (no grants denies every model, a nil store errors rather than panics); `BedrockAccessAdmin` had no reader outside one test and is gone, and `Deps.NATS` is dropped because the nil-NATS answer is given by the gateway-wide gate before dispatch. `Logging`, `Provisioned`, `Guardrails`, `KB` and `DataSources` are the concrete exported store types, a temporary edge until Ochre exposes capability interfaces for them. The wire-typed action functions stay in the `domains/ochre` root and the SDK-typed adapters move; `authz.go` moves with them, which removes the `domains/ochre` to `ingress/aws/bodyscope` allowlist row, the only allowlist change. The legacy switch cases, the `supportedServices` and `jsonErrorService` entries and the path-based sub-service rewrite in `GetService` are removed, so `ec2` is the only service left on the legacy table; the rewrite's rows are pinned in `awsapi` as `resolveSubservice` and dispatch tests. The registration declares the AWS JSON 1.1 envelope it had and `cmd/aws-model-coverage` is not wired, so no coverage allowlist row is added. Recorded divergences, not changed by this move: Bedrock is restJson1 in AWS but answers with the JSON 1.1 envelope (`application/x-amz-json-1.1`, `__type` body), and CreateGuardrail answers 200 where AWS answers 202; both are tracked by `mulga-xug8b`. No change to `dispatch.Actor` or `dispatch.Invocation`. |
|
||||
| `spinifex#1354` | `spinifex/gateway/ec2.go` (the EC2 action table, `ec2Handler`, `ec2Actions`, the dispatch pipeline), `ec2_idempotency.go`, `ec2_validation.go`, and the EC2 token-bucket fields of `GatewayConfig` | `spinifex/domains/ec2/awsapi` (`Handler`, `NewHandler`, `Deps`, `Quota`, `Dispatch`, `ActionNames`, `InputHasField`), composed by `spinifex/gateway/ec2.go` | Slice 9a of the EC2 move: the capabilities first, with no behaviour change. The EC2 action table, idempotency and validation move to `domains/ec2/awsapi` behind `Deps{NATS, IAM, Quota, ActiveNodes, ExpectedNodes, NodeIDs, AZ, Endpoint, InstanceStatus}`; the gateway keeps the composition and `EC2_Request`, still on the legacy switch until slice 9b registers EC2 through the ADR-0002 seam. `Quota` is declared in `awsapi` because `domains/admission/quota` imports the EC2 action packages and cannot be imported back, and the gateway wraps `quota.Service` to supply the NATS instance-type resolver its retype gate takes. The instance and spot-instance adapters take the two-method `InstanceProfiles` interface (`ResolveInstanceProfile`, `CanonicalResourceARN`) instead of `iamdomain.IAMService`, which pays off the `spotinstance` to `domains/admission/quota` and `domains/iam` allowlist rows. Allowlist changes are 20 deletions and no additions: the 18 `gateway` to `domains/ec2/awsapi/*` rows and those 2. The pipeline characterisation tests committed first (error envelopes, check order, gateway-local actions, idempotent replay, PassRole, ImportKeyPair padding, launch-template expansion before authorization, RunInstances instance-profile refusal order) pass unchanged. Package-local tests drive every routed action through `Dispatch` against a stub daemon. No change to `dispatch.Actor` or `dispatch.Invocation`. Live on env14: DescribeRegions, DescribeAvailabilityZones and DescribeAccountAttributes, an idempotent CreateVolume replay and its parameter mismatch, the DryRun and MaxResults errors, a no-policy user refused with UnauthorizedOperation (including on the gateway-local DescribeRegions), an assumed-role session allowed and refused per its policy, and one RunInstances to running then terminated. |
|
||||
| `spinifex#1334` | `spinifex/gateway/oidc_discovery.go` (`eksAccountKV`, the discovery-document and JWKS read and construction) | `spinifex/domains/eks/cluster` (`OIDCPublisher`, `OIDCDiscoveryDocument`), composed by `spinifex/runtime/roles/awsgw` | The EKS domain now reads its own account bucket and builds the per-cluster discovery document and JWKS responses (ADR-0005:S4), and the gateway only maps the unauthenticated `/oidc/eks/...` routes onto an `OIDCPublisher` interface it is handed. Responses are byte-identical, pinned by `oidc_discovery_current_behaviour_test.go`, whose assertions are unchanged. The gateway no longer imports `domains/eks/cluster` in production code. `STS` keeps its own `ClusterJWKSResolver` (`OIDCJWKSReader`), untouched. Recorded debt, not repaired here: a stored JWKS is served verbatim with 200 even when it is not valid JSON. |
|
||||
The EC2 contract row is intentionally different from the directory moves: it creates
|
||||
a compatibility boundary. `ec2.cmd.*` retains its deployed one-token NATS
|
||||
@@ -471,12 +472,11 @@ These residuals were recorded by individual slices and must be resolved once the
|
||||
- `domains/eks/cluster` (potential security risk): `ClusterMeta.ControlPlaneTemplate` (`domains/eks/cluster/service_impl.go`) clears the template's credentials but keeps `OIDCPrivateKeyPEM` and the k3s `JoinToken`, so both are persisted in the `eks-account-{acct}` cluster record, which the gateway also reads. Whether that record is encrypted at rest is unconfirmed; if it is not, a reader of the bucket can forge service-account tokens for the cluster's OIDC issuer and join nodes to it. This predates the refactor and is not fixed in it. Removal condition: the key and token are held by a dedicated secret store (a future KMS or Secrets Manager capability is the likely owner) and the cluster record keeps only a reference, with a test proving the persisted record contains neither secret.
|
||||
- `operator/cli` (dependency): the command tree still imports domain and service implementations directly (`handlers/{iam,eks}`, `domains/admission/quota`, `domains/ochre`, `vpcd`, the Viperblock adapter and runtime roles). The ADR's `cmd/spinifex` current-to-target map row requires command packages to reach domains through public capabilities; splitting supported use cases into `operator/*` packages is later work.
|
||||
- `providers/ebs/viperblock` → `runtime/compute/nbd` and `runtime/host/process` (dependency, moved unchanged from `services/viperblockd`): the adapter formats NBD endpoints and manages nbdkit processes through runtime packages. ADR-0001 does not list providers → runtime as forbidden and the layering test does not enforce it (same category as `providers/ebs/conformance` → `runtime/compute/nbd`), but a provider should not depend on node runtime. Clears when NBD endpoint naming has a provider-facing home (a candidate for `contracts/ebs/v1`) and process supervision is injected by the role. The adapter also imports `providers/objectstore`, a provider-to-provider edge that is allowed.
|
||||
- `gateway/ec2.go` (registration): the EC2 dispatch map stays a `GatewayConfig` method in `gateway` and imports every `domains/ec2/awsapi/*` package; `gateway/ec2_idempotency.go` also imports `domains/ec2/awsapi/idem` (`KVBucket`, `TTL`, `TokenAndParams`) for the dispatch-time idempotency wrapper. ADR-0001 makes the domain own its action registration; moving the map waits on the generic ingress registration seam and must not be done as a package move.
|
||||
- `gateway/ec2.go` (registration): the action table, idempotency and validation moved to `domains/ec2/awsapi` behind `Handler` and `Deps` (slice 9a, `spinifex#1354`), but the gateway still composes the handler and serves it from the legacy switch, so `gateway` keeps importing `domains/ec2/awsapi` and `domains/ec2/awsapi/instance`. Registering EC2 through the ADR-0002 seam and emptying the legacy switch is slice 9b.
|
||||
- `gateway` (service registration): the root `gateway` dispatch for IAM, STS, EKS, RDS and ECS imports `domains/{iam,sts,eks,rds,ecs}/awsapi`, which only AWSGW may wire. It clears when each service registers through the ADR-0002 ingress registration seam; as with EC2, moving the dispatch must not be done as a package move.
|
||||
- `operator/accountlifecycle` (dependency): its EC2 reapers call `domains/ec2/awsapi/*` action adapters directly, as they did under `gateway/ec2`. The amended ADR-0001 `accountteardown` row keeps replacing these with domain-owned cleanup capabilities as deferred debt; it was not a relocation prerequisite. It also holds a read-only consumer of the EC2 platform-inventory contract (`contracts/ec2/v1`, `ec2.v1.PlatformInventory`), served by `domains/ec2/accountinventory`; only the explicit dry run (`Engine.DryRun`) calls it, to log a shadow comparison, and the key-pair, placement-group and launch-template reapers still use the `awsapi` adapters. A domain-owned cleanup capability for the proven-safe subset of those three kinds (key pairs and empty placement groups no instance, spot request or live launch-template version names, and launch-template residue; never live launch templates) exists as `domains/ec2/accountcleanup` (a Go interface with injected owners and reference readers, no NATS subject); nothing calls it yet. It also holds the StageAccount settings-seal reaper, a consumer of `contracts/ec2/v1`'s `ec2.v1.SealAccountSettings` and per-node `ec2.v1.AccountSettingsSealCapability.<node>` subjects, served by `domains/ec2/account` through the daemon; the reaper reads no KV and imports no domain package, reaching both subjects only through NATS and the configured node set `NewClusterEngine` already receives.
|
||||
- `domains/admission/quota` (dependency, moved from `handlers/quota` unchanged): `live.go` imports `domains/elbv2/awsapi` and `domains/rds/instance` for its live ELBv2 and RDS counters; `records.go` imports `runtime/compute/vm` for the instance-record vCPU sweep; `live.go`, `vcpu.go` and `storage.go` call `domains/ec2/awsapi/{vpc,eip,volume}` describe adapters to count VPCs, subnets, addresses and volumes, as they did under `gateway/ec2`. The ADR's `handlers/quota` current-to-target map row replaces these with reader capabilities once their final locations exist.
|
||||
- `domains/ec2/awsapi/instance` (dependency): imports `domains/iam` (`IAMService` in `DescribeInstances.go`, `RunInstances.go` and `IamInstanceProfileAssociation.go`; `InstanceProfile` in the latter two), `runtime/compute/cache` (`NodeState`, `NodeUnknown`, `NodeLive`) and `runtime/compute/vm` (`VM`), both in `status_synthesis.go`. The IAM edge needs an IAM capability; the runtime edges are domain-to-runtime and need the instance record and node liveness exposed through an EC2-owned projection.
|
||||
- `domains/ec2/awsapi/spotinstance` (dependency): imports `domains/iam.IAMService` and `domains/admission/quota.Service` in `RequestSpotInstances.go`.
|
||||
- `domains/ec2/awsapi/instance` (dependency): imports `domains/iam` for the `InstanceProfile` type that `InstanceProfiles.ResolveInstanceProfile` returns (`RunInstances.go`, `IamInstanceProfileAssociation.go`), `runtime/compute/cache` (`NodeState`, `NodeUnknown`, `NodeLive`) and `runtime/compute/vm` (`VM`), both in `status_synthesis.go`. The IAM edge is now the two-method `InstanceProfiles` interface plus that type; the runtime edges are domain-to-runtime and need the instance record and node liveness exposed through an EC2-owned projection.
|
||||
- Instance adapter consumers outside EC2: `domains/rds/awsapi/enginecatalog.go`, `runtime/daemon/eks_cp_control.go`, `runtime/daemon/rds_deps.go`, `domains/admission/quota`'s `vcpu.go` and `gateway/iam.go` (`CountInstanceProfileAssociations`) call `domains/ec2/awsapi/instance` directly; each needs an EC2 capability rather than the AWS action adapter.
|
||||
- `runtime/daemon` (Q-28 ownership and deployment): the daemon moved whole, keeping its current ownership, deployment and `daemon → gateway → handlers` dependencies. Q-28's write-side consolidation and any daemon decomposition or split remain deferred by ADR-0001 and are not paid by the move.
|
||||
|
||||
|
||||
@@ -15,6 +15,8 @@ import (
|
||||
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
|
||||
awsconfig "github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials/ec2rolecreds"
|
||||
"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
|
||||
)
|
||||
|
||||
// Signer holds a credential provider and signs requests in place.
|
||||
@@ -42,6 +44,16 @@ func NewIMDS(ctx context.Context, region string) (*Signer, error) {
|
||||
return &Signer{provider: cfg.Credentials}, nil
|
||||
}
|
||||
|
||||
// NewInstanceRole returns a Signer whose credentials come only from the
|
||||
// instance metadata service: no environment, profile or static-key fallback.
|
||||
func NewInstanceRole() *Signer {
|
||||
return newInstanceRole(imds.New(imds.Options{}))
|
||||
}
|
||||
|
||||
func newInstanceRole(client *imds.Client) *Signer {
|
||||
return &Signer{provider: aws.NewCredentialsCache(ec2rolecreds.New(func(o *ec2rolecreds.Options) { o.Client = client }))}
|
||||
}
|
||||
|
||||
// EnsureCredentials retrieves once from the provider, surfacing the error so a
|
||||
// caller can warm the IMDS instance-role datapath before the first Sign. The
|
||||
// SDK's IMDS client fast-fails each probe (250ms dial), so a cold per-tap
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
|
||||
"github.com/mulgadc/bluebottle/pkg/sigv4"
|
||||
)
|
||||
|
||||
@@ -123,3 +124,41 @@ func TestNewIMDS_BuildsProvider(t *testing.T) {
|
||||
t.Fatal("expected non-nil credentials provider")
|
||||
}
|
||||
}
|
||||
|
||||
// The instance-role signer ignores environment keys and signs only with the
|
||||
// credentials the metadata service issues.
|
||||
func TestNewInstanceRole_UsesOnlyIMDS(t *testing.T) {
|
||||
t.Setenv("AWS_ACCESS_KEY_ID", testAccessKey)
|
||||
t.Setenv("AWS_SECRET_ACCESS_KEY", testSecretKey)
|
||||
const imdsKey = "ASIAIMDSONLYEXAMPLE1"
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/latest/api/token":
|
||||
_, _ = w.Write([]byte("token"))
|
||||
case "/latest/meta-data/iam/security-credentials/":
|
||||
_, _ = w.Write([]byte("sqsWorkerRole"))
|
||||
case "/latest/meta-data/iam/security-credentials/sqsWorkerRole":
|
||||
_, _ = w.Write([]byte(`{"Code":"Success","AccessKeyId":"` + imdsKey + `","SecretAccessKey":"s","Token":"t","Expiration":"2099-01-01T00:00:00Z"}`))
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
s := newInstanceRole(imds.New(imds.Options{Endpoint: srv.URL}))
|
||||
req := httptest.NewRequest(http.MethodPost, "https://gw.example/", nil)
|
||||
if err := s.Sign(req, emptySHA, testService, testRegion); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if auth := req.Header.Get("Authorization"); !strings.Contains(auth, "Credential="+imdsKey+"/") {
|
||||
t.Fatalf("Authorization = %q, want the IMDS credential", auth)
|
||||
}
|
||||
|
||||
srv.Close()
|
||||
cold := newInstanceRole(imds.New(imds.Options{Endpoint: srv.URL}))
|
||||
if err := cold.EnsureCredentials(context.Background()); err == nil {
|
||||
t.Fatal("environment keys were used when IMDS was unreachable")
|
||||
}
|
||||
}
|
||||
|
||||
const emptySHA = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||
|
||||
@@ -0,0 +1,242 @@
|
||||
// Package sqsworkergw is the shared SQS worker guest's SigV4 client for the
|
||||
// gateway's internal spinifex-sqs service. It sends only public values: the
|
||||
// worker's identity is the signing session, and the store seed never leaves
|
||||
// the guest.
|
||||
package sqsworkergw
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/hex"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/bluebottle/pkg/tlsconfig"
|
||||
"github.com/nats-io/nkeys"
|
||||
)
|
||||
|
||||
const (
|
||||
// The SigV4 credential scope the gateway routes the internal surface on.
|
||||
signingService = "spinifex-sqs"
|
||||
apiVersion = "2026-10-30"
|
||||
callTimeout = 30 * time.Second
|
||||
maxResponse = 64 << 10
|
||||
|
||||
actionClaim = "ClaimFirstStore"
|
||||
actionDescribe = "DescribeStore"
|
||||
actionRecord = "RecordWorkerPublicKey"
|
||||
)
|
||||
|
||||
// Signer signs a request in place for service and region.
|
||||
type Signer interface {
|
||||
Sign(r *http.Request, payloadHash, service, region string) error
|
||||
}
|
||||
|
||||
// Client calls the internal first-store actions for this worker.
|
||||
type Client struct {
|
||||
baseURL string
|
||||
region string
|
||||
signer Signer
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
// New builds a TLS 1.3 client that trusts only caPEM, the cluster CA. An
|
||||
// https URL, a Region and a CA are all required: there is no fallback.
|
||||
func New(baseURL, region string, caPEM []byte, signer Signer) (*Client, error) {
|
||||
u, err := url.Parse(baseURL)
|
||||
if err != nil || u.Scheme != "https" || u.Host == "" {
|
||||
return nil, errors.New("sqsworkergw: gateway URL must be an https URL")
|
||||
}
|
||||
if region == "" {
|
||||
return nil, errors.New("sqsworkergw: Region is required")
|
||||
}
|
||||
if signer == nil {
|
||||
return nil, errors.New("sqsworkergw: signer is required")
|
||||
}
|
||||
pool := x509.NewCertPool()
|
||||
if !pool.AppendCertsFromPEM(caPEM) {
|
||||
return nil, errors.New("sqsworkergw: gateway CA has no usable certificates")
|
||||
}
|
||||
tlsCfg := &tls.Config{MinVersion: tls.VersionTLS13, CurvePreferences: tlsconfig.Curves, RootCAs: pool}
|
||||
return &Client{
|
||||
baseURL: strings.TrimRight(baseURL, "/"),
|
||||
region: region,
|
||||
signer: signer,
|
||||
httpClient: &http.Client{
|
||||
Timeout: callTimeout,
|
||||
Transport: &http.Transport{TLSClientConfig: tlsCfg, MaxIdleConns: 1, IdleConnTimeout: 30 * time.Second},
|
||||
// A redirect would resend a signed request somewhere else.
|
||||
CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// APIError is a gateway refusal; Code is the AWS error code.
|
||||
type APIError struct {
|
||||
Action string
|
||||
StatusCode int
|
||||
Code string
|
||||
}
|
||||
|
||||
func (e *APIError) Error() string {
|
||||
return fmt.Sprintf("spinifex-sqs %s: %s (HTTP %d)", e.Action, e.Code, e.StatusCode)
|
||||
}
|
||||
|
||||
// Retryable reports a refusal that a later identical call may pass:
|
||||
// contention or a store that is not yet available. Every other refusal,
|
||||
// including AccessDenied, is final for this boot.
|
||||
func (e *APIError) Retryable() bool {
|
||||
return e.Code == "ServiceUnavailable" || e.Code == "Throttling" || e.StatusCode == http.StatusServiceUnavailable
|
||||
}
|
||||
|
||||
// Pending reports an AccessDenied, which a worker that has started but is
|
||||
// not yet bound to its identity legitimately receives. It authorizes
|
||||
// nothing; the caller may ask again within its own bound.
|
||||
func (e *APIError) Pending() bool { return e.Code == "AccessDenied" }
|
||||
|
||||
// TransientError is a call that may not have reached the gateway, or whose
|
||||
// reply was lost: signing, sending or reading failed. Both actions are safe
|
||||
// to repeat, so it is retryable.
|
||||
type TransientError struct{ Err error }
|
||||
|
||||
func (e *TransientError) Error() string { return e.Err.Error() }
|
||||
func (e *TransientError) Unwrap() error { return e.Err }
|
||||
func (e *TransientError) Retryable() bool { return true }
|
||||
|
||||
type claimResult struct {
|
||||
CreateKey bool `xml:"CreateKey"`
|
||||
PublicKey string `xml:"PublicKey"`
|
||||
}
|
||||
|
||||
// Claim asks for this worker's current first-store decision. A decision
|
||||
// that is neither "create" nor one valid public key is refused here.
|
||||
func (c *Client) Claim(ctx context.Context) (bool, string, error) {
|
||||
var res claimResult
|
||||
if err := c.call(ctx, actionClaim, nil, &res); err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
switch {
|
||||
case res.CreateKey && res.PublicKey == "":
|
||||
return true, "", nil
|
||||
case !res.CreateKey && nkeys.IsValidPublicUserKey(res.PublicKey):
|
||||
return false, res.PublicKey, nil
|
||||
}
|
||||
return false, "", errors.New("spinifex-sqs ClaimFirstStore: response is not a valid decision")
|
||||
}
|
||||
|
||||
// Store is the Region's store launch as the gateway reports it. Bound means
|
||||
// a VM was launched for it, not that it is ready.
|
||||
type Store struct {
|
||||
Bound bool
|
||||
Address string
|
||||
PublicKey string
|
||||
}
|
||||
|
||||
type describeResult struct {
|
||||
State string `xml:"State"`
|
||||
Address string `xml:"Address"`
|
||||
PublicKey string `xml:"PublicKey"`
|
||||
}
|
||||
|
||||
// DescribeStore reports this worker's store. A bound reply must carry an
|
||||
// IPv4 address and a valid public key; a pending one must carry neither.
|
||||
func (c *Client) DescribeStore(ctx context.Context) (Store, error) {
|
||||
var res describeResult
|
||||
if err := c.call(ctx, actionDescribe, nil, &res); err != nil {
|
||||
return Store{}, err
|
||||
}
|
||||
switch {
|
||||
case res.State == "pending" && res.Address == "" && res.PublicKey == "":
|
||||
return Store{}, nil
|
||||
case res.State == "bound" && isIPv4(res.Address) && nkeys.IsValidPublicUserKey(res.PublicKey):
|
||||
return Store{Bound: true, Address: res.Address, PublicKey: res.PublicKey}, nil
|
||||
}
|
||||
return Store{}, errors.New("spinifex-sqs DescribeStore: response is not a valid store state")
|
||||
}
|
||||
|
||||
func isIPv4(s string) bool {
|
||||
ip := net.ParseIP(s)
|
||||
return ip != nil && ip.To4() != nil && !strings.Contains(s, ":")
|
||||
}
|
||||
|
||||
// RecordPublicKey records the worker's public key. Anything other than a
|
||||
// user public key is refused before it is sent.
|
||||
func (c *Client) RecordPublicKey(ctx context.Context, publicKey string) error {
|
||||
if !nkeys.IsValidPublicUserKey(publicKey) {
|
||||
return errors.New("spinifex-sqs RecordWorkerPublicKey: refusing to send a value that is not a user public key")
|
||||
}
|
||||
return c.call(ctx, actionRecord, url.Values{"PublicKey": {publicKey}}, nil)
|
||||
}
|
||||
|
||||
func (c *Client) call(ctx context.Context, action string, params url.Values, out any) error {
|
||||
form := url.Values{"Action": {action}, "Version": {apiVersion}}
|
||||
maps.Copy(form, params)
|
||||
body := []byte(form.Encode())
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+"/", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return fmt.Errorf("spinifex-sqs %s: build request: %w", action, err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=utf-8")
|
||||
sum := sha256.Sum256(body)
|
||||
if err := c.signer.Sign(req, hex.EncodeToString(sum[:]), signingService, c.region); err != nil {
|
||||
return &TransientError{Err: fmt.Errorf("spinifex-sqs %s: sign: %w", action, err)}
|
||||
}
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return &TransientError{Err: fmt.Errorf("spinifex-sqs %s: send: %w", action, err)}
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
respBody, err := io.ReadAll(io.LimitReader(resp.Body, maxResponse))
|
||||
if err != nil {
|
||||
return &TransientError{Err: fmt.Errorf("spinifex-sqs %s: read response: %w", action, err)}
|
||||
}
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return parseError(action, resp.StatusCode, respBody)
|
||||
}
|
||||
if out == nil {
|
||||
return nil
|
||||
}
|
||||
return decodeResult(action, respBody, out)
|
||||
}
|
||||
|
||||
func decodeResult(action string, body []byte, out any) error {
|
||||
want := action + "Result"
|
||||
dec := xml.NewDecoder(bytes.NewReader(body))
|
||||
for {
|
||||
tok, err := dec.Token()
|
||||
if err != nil {
|
||||
return fmt.Errorf("spinifex-sqs %s: response carries no <%s>", action, want)
|
||||
}
|
||||
if start, ok := tok.(xml.StartElement); ok && start.Name.Local == want {
|
||||
if err := dec.DecodeElement(out, &start); err != nil {
|
||||
return fmt.Errorf("spinifex-sqs %s: decode response: %w", action, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The error body is not echoed: only the parsed code is kept.
|
||||
func parseError(action string, status int, body []byte) error {
|
||||
var env struct {
|
||||
Error struct {
|
||||
Code string `xml:"Code"`
|
||||
} `xml:"Error"`
|
||||
}
|
||||
code := fmt.Sprintf("HTTP%d", status)
|
||||
if xml.Unmarshal(body, &env) == nil && env.Error.Code != "" {
|
||||
code = env.Error.Code
|
||||
}
|
||||
return &APIError{Action: action, StatusCode: status, Code: code}
|
||||
}
|
||||
@@ -0,0 +1,235 @@
|
||||
package sqsworkergw
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/nats-io/nkeys"
|
||||
)
|
||||
|
||||
type nopSigner struct{ err error }
|
||||
|
||||
func (s nopSigner) Sign(r *http.Request, _, service, region string) error {
|
||||
if s.err != nil {
|
||||
return s.err
|
||||
}
|
||||
r.Header.Set("Authorization", "test "+service+"/"+region)
|
||||
return nil
|
||||
}
|
||||
|
||||
func userKey(t *testing.T) (pub, seed string) {
|
||||
t.Helper()
|
||||
kp, err := nkeys.CreateUser()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p, _ := kp.PublicKey()
|
||||
s, _ := kp.Seed()
|
||||
return p, string(s)
|
||||
}
|
||||
|
||||
// server answers with body and status, counting and keeping requests.
|
||||
func server(t *testing.T, status int, body string) (*Client, *atomic.Int32, *[]string) {
|
||||
t.Helper()
|
||||
var n atomic.Int32
|
||||
var seen []string
|
||||
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
n.Add(1)
|
||||
b, _ := io.ReadAll(r.Body)
|
||||
seen = append(seen, r.Header.Get("Authorization")+"|"+string(b))
|
||||
w.WriteHeader(status)
|
||||
_, _ = io.WriteString(w, body)
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
ca := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: srv.Certificate().Raw})
|
||||
c, err := New(srv.URL, "ap-southeast-2", ca, nopSigner{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return c, &n, &seen
|
||||
}
|
||||
|
||||
func TestNew_RefusesWithoutHTTPSRegionCAOrSigner(t *testing.T) {
|
||||
ca := []byte("not a certificate")
|
||||
cases := map[string]func() error{
|
||||
"http": func() error { _, err := New("http://10.0.0.1:9999", "ap-southeast-2", ca, nopSigner{}); return err },
|
||||
"no host": func() error { _, err := New("https://", "ap-southeast-2", ca, nopSigner{}); return err },
|
||||
"no region": func() error { _, err := New("https://10.0.0.1:9999", "", ca, nopSigner{}); return err },
|
||||
"no signer": func() error { _, err := New("https://10.0.0.1:9999", "ap-southeast-2", ca, nil); return err },
|
||||
"no CA": func() error { _, err := New("https://10.0.0.1:9999", "ap-southeast-2", ca, nopSigner{}); return err },
|
||||
}
|
||||
for name, fn := range cases {
|
||||
if fn() == nil {
|
||||
t.Errorf("%s: client built", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestClaim_SignsForTheInternalServiceAndRegion(t *testing.T) {
|
||||
c, _, seen := server(t, 200, `<ClaimFirstStoreResponse><ClaimFirstStoreResult><CreateKey>true</CreateKey></ClaimFirstStoreResult></ClaimFirstStoreResponse>`)
|
||||
create, pub, err := c.Claim(context.Background())
|
||||
if err != nil || !create || pub != "" {
|
||||
t.Fatalf("claim = %v %q %v", create, pub, err)
|
||||
}
|
||||
if got := (*seen)[0]; !strings.HasPrefix(got, "test spinifex-sqs/ap-southeast-2|") || !strings.Contains(got, "Action=ClaimFirstStore") {
|
||||
t.Fatalf("request = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Only "create" or exactly one valid public key is a decision.
|
||||
func TestClaim_RefusesAnythingButOneDecision(t *testing.T) {
|
||||
pub, seed := userKey(t)
|
||||
cases := map[string]string{
|
||||
"create with a key": `<CreateKey>true</CreateKey><PublicKey>` + pub + `</PublicKey>`,
|
||||
"neither": `<CreateKey>false</CreateKey>`,
|
||||
"seed as key": `<CreateKey>false</CreateKey><PublicKey>` + seed + `</PublicKey>`,
|
||||
"no result": ``,
|
||||
}
|
||||
for name, inner := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
body := `<ClaimFirstStoreResponse><ClaimFirstStoreResult>` + inner + `</ClaimFirstStoreResult></ClaimFirstStoreResponse>`
|
||||
if inner == "" {
|
||||
body = `<Other/>`
|
||||
}
|
||||
c, _, _ := server(t, 200, body)
|
||||
_, _, err := c.Claim(context.Background())
|
||||
if err == nil {
|
||||
t.Fatal("accepted a non-decision")
|
||||
}
|
||||
if strings.Contains(err.Error(), seed) {
|
||||
t.Fatal("the error echoes a seed")
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("existing", func(t *testing.T) {
|
||||
c, _, _ := server(t, 200, `<ClaimFirstStoreResponse><ClaimFirstStoreResult><CreateKey>false</CreateKey><PublicKey>`+pub+`</PublicKey></ClaimFirstStoreResult></ClaimFirstStoreResponse>`)
|
||||
create, got, err := c.Claim(context.Background())
|
||||
if err != nil || create || got != pub {
|
||||
t.Fatalf("claim = %v %q %v", create, got, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestRecordPublicKey_SendsOnlyAPublicKey(t *testing.T) {
|
||||
pub, seed := userKey(t)
|
||||
c, n, seen := server(t, 200, `<RecordWorkerPublicKeyResponse/>`)
|
||||
if err := c.RecordPublicKey(context.Background(), seed); err == nil || strings.Contains(err.Error(), seed) {
|
||||
t.Fatalf("seed err = %v", err)
|
||||
}
|
||||
if n.Load() != 0 {
|
||||
t.Fatal("a seed was sent")
|
||||
}
|
||||
if err := c.RecordPublicKey(context.Background(), pub); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains((*seen)[0], "PublicKey="+pub) {
|
||||
t.Fatalf("request = %q", (*seen)[0])
|
||||
}
|
||||
}
|
||||
|
||||
// Refusals carry only the parsed code; ServiceUnavailable is the one
|
||||
// retryable refusal, and transport failures are retryable too.
|
||||
func TestErrors_ClassifiedWithoutEchoingTheBody(t *testing.T) {
|
||||
cases := []struct {
|
||||
status int
|
||||
code string
|
||||
retryable bool
|
||||
}{
|
||||
{403, "AccessDenied", false},
|
||||
{400, "InvalidParameterValue", false},
|
||||
{400, "InvalidState", false},
|
||||
{403, "SignatureDoesNotMatch", false},
|
||||
{503, "ServiceUnavailable", true},
|
||||
{500, "InternalFailure", false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
c, _, _ := server(t, tc.status, `<ErrorResponse><Error><Code>`+tc.code+`</Code><Message>secret-ish detail</Message></Error></ErrorResponse>`)
|
||||
_, _, err := c.Claim(context.Background())
|
||||
api, ok := errors.AsType[*APIError](err)
|
||||
if !ok || api.Code != tc.code || api.Retryable() != tc.retryable || api.Pending() != (tc.code == "AccessDenied") {
|
||||
t.Fatalf("%s: err = %v", tc.code, err)
|
||||
}
|
||||
if strings.Contains(err.Error(), "secret-ish") {
|
||||
t.Fatal("the error echoes the response body")
|
||||
}
|
||||
}
|
||||
|
||||
c, _, _ := server(t, 200, "")
|
||||
c.baseURL = "https://127.0.0.1:1"
|
||||
_, _, err := c.Claim(context.Background())
|
||||
tr, ok := errors.AsType[*TransientError](err)
|
||||
if !ok || !tr.Retryable() {
|
||||
t.Fatalf("unreachable err = %v, want TransientError", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A redirect would resend a signed request elsewhere; it is not followed.
|
||||
func TestCall_DoesNotFollowRedirects(t *testing.T) {
|
||||
var followed atomic.Bool
|
||||
target := httptest.NewTLSServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { followed.Store(true) }))
|
||||
t.Cleanup(target.Close)
|
||||
c, _, _ := server(t, 0, "")
|
||||
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, target.URL, http.StatusTemporaryRedirect)
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
c.baseURL = srv.URL
|
||||
if _, _, err := c.Claim(context.Background()); err == nil {
|
||||
t.Fatal("a redirect was accepted as a decision")
|
||||
}
|
||||
if followed.Load() {
|
||||
t.Fatal("the signed request followed a redirect")
|
||||
}
|
||||
}
|
||||
|
||||
// Only "pending" with nothing else, or "bound" with an IPv4 address and one
|
||||
// valid public key, is a store state.
|
||||
func TestDescribeStore_AcceptsOnlyAValidState(t *testing.T) {
|
||||
pub, seed := userKey(t)
|
||||
wrap := func(inner string) string {
|
||||
return `<DescribeStoreResponse><DescribeStoreResult>` + inner + `</DescribeStoreResult></DescribeStoreResponse>`
|
||||
}
|
||||
t.Run("pending", func(t *testing.T) {
|
||||
c, _, seen := server(t, 200, wrap(`<State>pending</State>`))
|
||||
st, err := c.DescribeStore(context.Background())
|
||||
if err != nil || st.Bound {
|
||||
t.Fatalf("store = %+v %v", st, err)
|
||||
}
|
||||
if !strings.Contains((*seen)[0], "Action=DescribeStore") {
|
||||
t.Fatalf("request = %q", (*seen)[0])
|
||||
}
|
||||
})
|
||||
t.Run("bound", func(t *testing.T) {
|
||||
c, _, _ := server(t, 200, wrap(`<State>bound</State><Address>10.240.0.9</Address><PublicKey>`+pub+`</PublicKey>`))
|
||||
st, err := c.DescribeStore(context.Background())
|
||||
if err != nil || !st.Bound || st.Address != "10.240.0.9" || st.PublicKey != pub {
|
||||
t.Fatalf("store = %+v %v", st, err)
|
||||
}
|
||||
})
|
||||
for name, inner := range map[string]string{
|
||||
"pending with address": `<State>pending</State><Address>10.240.0.9</Address>`,
|
||||
"bound without key": `<State>bound</State><Address>10.240.0.9</Address>`,
|
||||
"bound with seed": `<State>bound</State><Address>10.240.0.9</Address><PublicKey>` + seed + `</PublicKey>`,
|
||||
"bound with hostname": `<State>bound</State><Address>store.example</Address><PublicKey>` + pub + `</PublicKey>`,
|
||||
"bound with IPv6": `<State>bound</State><Address>::ffff:10.240.0.9</Address><PublicKey>` + pub + `</PublicKey>`,
|
||||
"unknown state": `<State>ready</State>`,
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
c, _, _ := server(t, 200, wrap(inner))
|
||||
_, err := c.DescribeStore(context.Background())
|
||||
if err == nil {
|
||||
t.Fatal("accepted an invalid store state")
|
||||
}
|
||||
if strings.Contains(err.Error(), seed) {
|
||||
t.Fatal("the error echoes a seed")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -6,7 +6,7 @@ Node installation, dev-environment lifecycle, guest image builds, and verificati
|
||||
|
||||
| Script | What it does |
|
||||
|---|---|
|
||||
| `setup.sh` | The production binary installer, served from `install.mulgadc.com`. Installs deps, users, sudoers, the host firewall policy, files, systemd units, logrotate and udev rules, and loads `openvswitch`, `geneve`, `vhost_net` and `tun`, listing them in `/etc/modules-load.d/spinifex.conf` for later boots. It masks firewalld wherever it is installed, since the Spinifex nftables policy is the only supported node firewall. An unset `--firewall` keeps the mode recorded in `/etc/spinifex/firewall/mode`, and otherwise arms the policy when firewalld was active. Where systemd-resolved is installed but not running, resolvconf is absent and NetworkManager runs, it hands host DNS to resolved: it saves `/etc/resolv.conf` once to `/var/lib/spinifex-reset/resolv.conf.orig`, links `/etc/resolv.conf` to the resolved stub, and stops unless resolved has upstream servers and the package repository host still resolves. It links `/usr/bin/spx` to `/usr/local/bin/spx` where sudo's PATH leaves out `/usr/local/bin`, and takes the nbdkit plugin directory from `nbdkit --dump-config`, stopping when nbdkit cannot say. `SETUP_STAGES` runs a subset; `ISO_BUILD=1` runs it inside the ISO builder's chroot. |
|
||||
| `setup.sh` | The production binary installer, served from `install.mulgadc.com`. Installs deps, users, sudoers, the host firewall policy, files, systemd units, logrotate and udev rules, and loads `openvswitch`, `geneve`, `vhost_net` and `tun`, listing them in `/etc/modules-load.d/spinifex.conf` for later boots. It masks firewalld wherever it is installed, since the Spinifex nftables policy is the only supported node firewall. It writes a needrestart drop-in so an apt upgrade never restarts the Spinifex, OVS or OVN units; restarting them after an upgrade is left to the operator. An unset `--firewall` keeps the mode recorded in `/etc/spinifex/firewall/mode`, and otherwise arms the policy when firewalld was active. Where systemd-resolved is installed but not running, resolvconf is absent and NetworkManager runs, it hands host DNS to resolved: it saves `/etc/resolv.conf` once to `/var/lib/spinifex-reset/resolv.conf.orig`, links `/etc/resolv.conf` to the resolved stub, and stops unless resolved has upstream servers and the package repository host still resolves. It links `/usr/bin/spx` to `/usr/local/bin/spx` where sudo's PATH leaves out `/usr/local/bin`, and takes the nbdkit plugin directory from `nbdkit --dump-config`, stopping when nbdkit cannot say. `SETUP_STAGES` runs a subset; `ISO_BUILD=1` runs it inside the ISO builder's chroot. |
|
||||
| `setup-ovn.sh` | Bootstraps a compute node for OVN VPC networking: installs OVN/OVS, creates `br-int`, configures the WAN bridge (bridged, DHCP, or routed-NAT uplink), sets the chassis identity, and applies overlay sysctls. It runs OVS and OVN as root, setting `OVS_USER_ID` and `OVN_USER_ID` where `/etc/sysconfig` holds them, and hooks the socket-permission helper onto `ovs-vswitchd`, `ovn-controller`, `openvswitch-ipsec` and, on EL, `ovn-northd`. `--management` also starts the OVN central services, writes their `ovn-ctl` flags to `/etc/default/ovn-central` (Debian) or `/etc/sysconfig/ovn-northd` (EL), and records the role in `/etc/spinifex/ovn-role`. With clustered databases it returns only once this node's NB and SB members are back in their RAFT clusters under a known leader, and fails after `OVN_RAFT_REJOIN_WAIT` seconds (default 60) otherwise, so reruns one node at a time never take two members of a cluster down. A later run without `--management` refuses while that file, or the `OVN_CTL_OPTS` line older nodes carry in `/etc/default/ovn-central`, records the role. It adds `maxsize 100M` to the OVS and OVN logrotate files, plus `rotate 5` where the file sets no count, and rotates as root, matching the daemons. On a host whose IPsec stack is libreswan it enables, stops and masks `openvswitch-ipsec.service` and `ipsec.service` for the daemon to unmask when IPsec is on, unless `openvswitch-ipsec.service` is already running; elsewhere it enables and starts `openvswitch-ipsec.service`, unless the daemon has masked it to keep IPsec off. The links it creates persist through systemd-networkd where that is enabled, and otherwise through `/etc/spinifex/hostnet.json`, which `spinifex-hostnet.service` (`spx admin hostnet apply`) replays at boot and on every OVS start, starting `ovn-controller` again when an OVS stop took it down; the first run records the choice in `/etc/spinifex/hostnet-backend`. Where NetworkManager runs, it leaves the links Spinifex owns unmanaged. |
|
||||
| `install-node.sh` | Forms a multi-node cluster across hosts that are already running Spinifex: rebuilds the OVN database in clustered form, runs `spx admin init` on the first host and `spx admin join` on the rest, then verifies. Before touching any host it refuses hosts of different OS families (`distro_os_family`) or `spx` versions, and `--ipsec on` when any host uses libreswan. The scripted form of [install-multi-node](../docs/install/install-multi-node/README.md). Destroys the joining nodes' logical network state and crypto — see its `--help`. |
|
||||
| `dev-install.sh` | Full local dev setup through the production installer: builds from source, packs a tarball, runs `setup.sh`, initialises the cluster, and starts services under systemd. |
|
||||
|
||||
@@ -5,6 +5,10 @@
|
||||
# sqs-store/sqs-store-lib.sh states its fw_cfg contract. The LB image
|
||||
# (build-microvm-image.sh) is separate and unchanged.
|
||||
#
|
||||
# SQS_MICROVM_PROFILE=worker builds the shared SQS worker guest instead: the
|
||||
# same network /init and kernel, build/microvm/sqs-worker's steps and
|
||||
# $REPO_ROOT/bin/sqs-worker, with no nats-server.
|
||||
#
|
||||
# Inputs:
|
||||
# $REPO_ROOT/build/microvm/sqs-store/init.sh, inittab
|
||||
# $REPO_ROOT/build/microvm/sqs-store/sqs-store-{lib.sh,prepare,run,stop}
|
||||
@@ -26,6 +30,18 @@ INIT_SH="$BUILD_DIR/init.sh"
|
||||
INITTAB="$BUILD_DIR/inittab"
|
||||
NATS_SERVER_BIN="$REPO_ROOT/bin/nats-server"
|
||||
|
||||
PROFILE="${SQS_MICROVM_PROFILE:-store}"
|
||||
case "$PROFILE" in
|
||||
store) ;;
|
||||
worker)
|
||||
BUILD_DIR="$REPO_ROOT/build/microvm/sqs-worker"
|
||||
INITTAB="$BUILD_DIR/inittab"
|
||||
SQS_WORKER_BIN="$REPO_ROOT/bin/sqs-worker"
|
||||
;;
|
||||
*) echo "ERROR: SQS_MICROVM_PROFILE must be store or worker, not $PROFILE" >&2; exit 1 ;;
|
||||
esac
|
||||
TAG="[build-sqs-$PROFILE-microvm-image]"
|
||||
|
||||
ALPINE_VERSION="${ALPINE_VERSION:-3.24}"
|
||||
ALPINE_MIRROR="${ALPINE_MIRROR:-https://dl-cdn.alpinelinux.org/alpine}"
|
||||
# e2fsprogs: mkfs.ext4/e2fsck for a guest that mounts an ext4 data volume.
|
||||
@@ -36,8 +52,8 @@ ALPINE_PACKAGES="busybox linux-virt iproute2 ca-certificates e2fsprogs kmod"
|
||||
# button (QMP system_powerdown) into a signal busybox init handles as poweroff.
|
||||
MICROVM_EXTRA_MODULES="${MICROVM_EXTRA_MODULES:-ext4 tiny_power_button}"
|
||||
|
||||
echo "[build-sqs-store-microvm-image] repo root: $REPO_ROOT"
|
||||
echo "[build-sqs-store-microvm-image] build dir: $BUILD_DIR"
|
||||
echo "$TAG repo root: $REPO_ROOT"
|
||||
echo "$TAG build dir: $BUILD_DIR"
|
||||
mkdir -p "$BUILD_DIR"
|
||||
|
||||
# --- Verify non-negotiable tools ---
|
||||
@@ -52,6 +68,16 @@ for tool in cpio gzip find fakeroot; do
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$PROFILE" = worker ]; then
|
||||
if [ ! -x "$SQS_WORKER_BIN" ]; then
|
||||
echo "ERROR: $SQS_WORKER_BIN not found — run 'make bin/sqs-worker'" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! file -b "$SQS_WORKER_BIN" | grep -q "statically linked"; then
|
||||
echo "ERROR: $SQS_WORKER_BIN is not statically linked — it cannot run on the musl guest" >&2
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
# --- Verify the dedicated nats-server is the go.mod-pinned build ---
|
||||
# A stale or host-installed binary would ship an unreviewed server version.
|
||||
NATS_SERVER_PIN=$(awk '$1 == "github.com/nats-io/nats-server/v2" { print $2; exit }' "$REPO_ROOT/go.mod")
|
||||
@@ -72,7 +98,8 @@ if [ "$nats_server_version" != "$NATS_SERVER_PIN" ]; then
|
||||
echo "ERROR: $NATS_SERVER_BIN reports $nats_server_version, go.mod pins $NATS_SERVER_PIN" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "[build-sqs-store-microvm-image] nats-server: $nats_server_version (matches go.mod)"
|
||||
echo "$TAG nats-server: $nats_server_version (matches go.mod)"
|
||||
fi
|
||||
|
||||
# --- Create chroot ---
|
||||
CHROOT_DIR=$(mktemp -d)
|
||||
@@ -88,11 +115,11 @@ STAGE_DIR=$(mktemp -d "$BUILD_DIR/.stage.XXXXXX")
|
||||
CONTAINER_TOOL=""
|
||||
CONTAINER_CID=""
|
||||
cleanup() {
|
||||
echo "[build-sqs-store-microvm-image] cleaning up chroot: $CHROOT_DIR"
|
||||
echo "$TAG cleaning up chroot: $CHROOT_DIR"
|
||||
rm -rf "$CHROOT_DIR"
|
||||
rm -rf "${STAGE_DIR:?}"
|
||||
if [ -n "$CONTAINER_CID" ] && [ -n "$CONTAINER_TOOL" ]; then
|
||||
echo "[build-sqs-store-microvm-image] cleaning up container: $CONTAINER_CID"
|
||||
echo "$TAG cleaning up container: $CONTAINER_CID"
|
||||
$CONTAINER_TOOL rm -f "$CONTAINER_CID" >/dev/null 2>&1 || true
|
||||
fi
|
||||
}
|
||||
@@ -101,7 +128,7 @@ trap cleanup EXIT
|
||||
# --- Populate rootfs ---
|
||||
# Strategy 1: native apk (Alpine hosts, CI)
|
||||
if command -v apk >/dev/null 2>&1; then
|
||||
echo "[build-sqs-store-microvm-image] rootfs: native apk"
|
||||
echo "$TAG rootfs: native apk"
|
||||
mkdir -p \
|
||||
"$CHROOT_DIR/etc/apk" \
|
||||
"$CHROOT_DIR/lib/apk/db" \
|
||||
@@ -125,7 +152,7 @@ EOF
|
||||
# this grows without bound. `buildx --output type=local` writes the image
|
||||
# filesystem straight to a directory and releases every mount, so nothing leaks.
|
||||
elif command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then
|
||||
echo "[build-sqs-store-microvm-image] rootfs: docker buildx --output type=local (alpine:${ALPINE_VERSION})"
|
||||
echo "$TAG rootfs: docker buildx --output type=local (alpine:${ALPINE_VERSION})"
|
||||
BUILD_CTX=$(mktemp -d)
|
||||
cat > "$BUILD_CTX/Dockerfile" <<DOCKERFILE
|
||||
FROM alpine:${ALPINE_VERSION}
|
||||
@@ -140,7 +167,7 @@ DOCKERFILE
|
||||
# Strategy 3: podman -> run + export. Podman's store does not exhibit the docker
|
||||
# overlay2 export-mount leak, and podman has no buildx.
|
||||
elif command -v podman >/dev/null 2>&1; then
|
||||
echo "[build-sqs-store-microvm-image] rootfs: podman export (alpine:${ALPINE_VERSION})"
|
||||
echo "$TAG rootfs: podman export (alpine:${ALPINE_VERSION})"
|
||||
CONTAINER_TOOL=podman
|
||||
CONTAINER_CID=$(podman run -d \
|
||||
"alpine:${ALPINE_VERSION}" \
|
||||
@@ -170,7 +197,7 @@ rm -rf "${CHROOT_DIR:?}/dev"
|
||||
mkdir "$CHROOT_DIR/dev"
|
||||
|
||||
# --- Place init script ---
|
||||
echo "[build-sqs-store-microvm-image] installing init.sh as /init..."
|
||||
echo "$TAG installing init.sh as /init..."
|
||||
if [ ! -f "$INIT_SH" ]; then
|
||||
echo "ERROR: $INIT_SH not found — cannot build initramfs" >&2
|
||||
exit 1
|
||||
@@ -183,11 +210,26 @@ if [ ! -f "$INITTAB" ]; then
|
||||
echo "ERROR: $INITTAB not found — cannot build initramfs" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "[build-sqs-store-microvm-image] installing inittab..."
|
||||
echo "$TAG installing inittab..."
|
||||
install -m 0644 "$INITTAB" "$CHROOT_DIR/etc/inittab"
|
||||
|
||||
if [ "$PROFILE" = worker ]; then
|
||||
echo "$TAG installing worker guest steps..."
|
||||
for step in sqs-worker-lib.sh sqs-worker-prepare sqs-worker-ready sqs-worker-stop; do
|
||||
if [ ! -f "$BUILD_DIR/$step" ]; then
|
||||
echo "ERROR: $BUILD_DIR/$step not found — cannot build initramfs" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
mkdir -p "$CHROOT_DIR/usr/local/lib/sqs-worker" "$CHROOT_DIR/usr/local/bin"
|
||||
install -m 0755 "$BUILD_DIR/sqs-worker-lib.sh" "$BUILD_DIR/sqs-worker-prepare" "$BUILD_DIR/sqs-worker-ready" "$BUILD_DIR/sqs-worker-stop" \
|
||||
"$CHROOT_DIR/usr/local/lib/sqs-worker/"
|
||||
install -m 0755 "$SQS_WORKER_BIN" "$CHROOT_DIR/usr/local/bin/sqs-worker"
|
||||
# The mountpoint ships empty; the worker proves its disk is mounted here.
|
||||
mkdir -p "$CHROOT_DIR/data"
|
||||
else
|
||||
# --- Place the store's guest steps ---
|
||||
echo "[build-sqs-store-microvm-image] installing store guest steps..."
|
||||
echo "$TAG installing store guest steps..."
|
||||
mkdir -p "$CHROOT_DIR/usr/local/lib/sqs-store"
|
||||
for step in sqs-store-lib.sh sqs-store-prepare sqs-store-run sqs-store-stop; do
|
||||
if [ ! -f "$BUILD_DIR/$step" ]; then
|
||||
@@ -200,9 +242,10 @@ install -m 0755 "$BUILD_DIR/sqs-store-prepare" "$BUILD_DIR/sqs-store-run" "$BUIL
|
||||
"$CHROOT_DIR/usr/local/lib/sqs-store/"
|
||||
|
||||
# --- Copy nats-server binary ---
|
||||
echo "[build-sqs-store-microvm-image] copying nats-server binary..."
|
||||
echo "$TAG copying nats-server binary..."
|
||||
mkdir -p "$CHROOT_DIR/usr/local/bin"
|
||||
install -m 0755 "$NATS_SERVER_BIN" "$CHROOT_DIR/usr/local/bin/nats-server"
|
||||
fi
|
||||
|
||||
# --- Create required directories for init ---
|
||||
mkdir -p \
|
||||
@@ -217,7 +260,7 @@ if [ -z "$KERNEL_IMG" ]; then
|
||||
echo "ERROR: no vmlinuz found in $CHROOT_DIR/boot" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "[build-sqs-store-microvm-image] kernel image: $KERNEL_IMG"
|
||||
echo "$TAG kernel image: $KERNEL_IMG"
|
||||
|
||||
KERNEL_VER=$(basename "$KERNEL_IMG" | sed 's/vmlinuz-//')
|
||||
KERNEL_CONFIG="$CHROOT_DIR/boot/config-${KERNEL_VER}"
|
||||
@@ -231,9 +274,9 @@ if ! find "$CHROOT_DIR/lib/modules" \
|
||||
echo "ERROR: qemu_fw_cfg module missing from initramfs — fw_cfg reads will fail at boot" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "[build-sqs-store-microvm-image] fw_cfg: built-in (CONFIG_FW_CFG_SYSFS=y)"
|
||||
echo "$TAG fw_cfg: built-in (CONFIG_FW_CFG_SYSFS=y)"
|
||||
else
|
||||
echo "[build-sqs-store-microvm-image] fw_cfg: module found in lib/modules"
|
||||
echo "$TAG fw_cfg: module found in lib/modules"
|
||||
fi
|
||||
|
||||
# --- Assert ext4 userspace tools (data volume) ---
|
||||
@@ -243,7 +286,7 @@ for fs_tool in mkfs.ext4 e2fsck; do
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
echo "[build-sqs-store-microvm-image] e2fsprogs: mkfs.ext4 + e2fsck present"
|
||||
echo "$TAG e2fsprogs: mkfs.ext4 + e2fsck present"
|
||||
|
||||
# --- Strip kernel modules (Phase 2 keep-list approach) ---
|
||||
# Only modules needed by a virtio-mmio microvm guest are kept:
|
||||
@@ -282,7 +325,7 @@ image_kmod() {
|
||||
--argv0 "$1" "$CHROOT_DIR/bin/kmod" "${@:2}"
|
||||
}
|
||||
|
||||
echo "[build-sqs-store-microvm-image] stripping kernel modules to virtio+fw_cfg+${MICROVM_EXTRA_MODULES} only..."
|
||||
echo "$TAG stripping kernel modules to virtio+fw_cfg+${MICROVM_EXTRA_MODULES} only..."
|
||||
for kver_dir in "$CHROOT_DIR/lib/modules"/*/; do
|
||||
kver="$(basename "$kver_dir")"
|
||||
kernel_dir="$kver_dir/kernel"
|
||||
@@ -381,8 +424,8 @@ for kver_dir in "$CHROOT_DIR/lib/modules"/*/; do
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
echo "[build-sqs-store-microvm-image] extra module closure: $(echo "$extra_names" | tr '\n' ' ')"
|
||||
echo "[build-sqs-store-microvm-image] modules kept: $(echo "$kept_names" | tr '\n' ' ')"
|
||||
echo "$TAG extra module closure: $(echo "$extra_names" | tr '\n' ' ')"
|
||||
echo "$TAG modules kept: $(echo "$kept_names" | tr '\n' ' ')"
|
||||
|
||||
# Rebuild module metadata with the image's own kmod, the binary the guest's
|
||||
# modprobe runs, so no host depmod is needed. Stock indexes still name .ko.gz
|
||||
@@ -415,12 +458,12 @@ for kver_dir in "$CHROOT_DIR/lib/modules"/*/; do
|
||||
echo "ERROR: modprobe plan for $name names $path, not a kept plain .ko" >&2
|
||||
exit 1
|
||||
done <<< "$plan"
|
||||
echo "[build-sqs-store-microvm-image] modprobe $name: $(awk '{ n = split($2, a, "/"); printf "%s ", a[n] }' <<< "$plan")"
|
||||
echo "$TAG modprobe $name: $(awk '{ n = split($2, a, "/"); printf "%s ", a[n] }' <<< "$plan")"
|
||||
done
|
||||
done
|
||||
|
||||
# --- Strip package-manager artifacts (not needed at runtime) ---
|
||||
echo "[build-sqs-store-microvm-image] stripping package manager artifacts..."
|
||||
echo "$TAG stripping package manager artifacts..."
|
||||
rm -rf \
|
||||
"$CHROOT_DIR/lib/apk" \
|
||||
"$CHROOT_DIR/var/cache/apk" \
|
||||
@@ -434,7 +477,7 @@ rm -rf \
|
||||
# --- Strip debug symbols from binaries ---
|
||||
# nats-server in /usr/local/bin is left as built (-s -w), so the shipped bytes
|
||||
# are the ones whose version was checked above.
|
||||
echo "[build-sqs-store-microvm-image] stripping debug symbols from binaries..."
|
||||
echo "$TAG stripping debug symbols from binaries..."
|
||||
find "$CHROOT_DIR/usr/sbin" "$CHROOT_DIR/usr/bin" \
|
||||
"$CHROOT_DIR/sbin" "$CHROOT_DIR/bin" \
|
||||
-type f 2>/dev/null | while read -r bin; do
|
||||
@@ -443,7 +486,7 @@ done
|
||||
|
||||
# --- Copy vmlinuz out before stripping /boot from chroot ---
|
||||
VMLINUZ_OUT="$STAGE_DIR/vmlinuz"
|
||||
echo "[build-sqs-store-microvm-image] copying kernel: $VMLINUZ_OUT"
|
||||
echo "$TAG copying kernel: $VMLINUZ_OUT"
|
||||
cp "$KERNEL_IMG" "$VMLINUZ_OUT"
|
||||
|
||||
# Drop /boot from the chroot — vmlinuz, Alpine's stock initramfs-virt, System.map
|
||||
@@ -456,7 +499,7 @@ rm -rf "${CHROOT_DIR:?}/boot"
|
||||
|
||||
# --- Build initramfs ---
|
||||
INITRAMFS_OUT="$STAGE_DIR/initramfs.cpio.gz"
|
||||
echo "[build-sqs-store-microvm-image] building initramfs: $INITRAMFS_OUT"
|
||||
echo "$TAG building initramfs: $INITRAMFS_OUT"
|
||||
# Create the static device nodes and pack the archive inside a single fakeroot
|
||||
# session: mknod/cpio see faked device entries and write real char-device
|
||||
# records into the cpio, while the host filesystem never gets an actual node.
|
||||
@@ -470,6 +513,6 @@ fakeroot sh -c '
|
||||
# --- Gate the staged pair, then publish it ---
|
||||
# Nothing reaches $BUILD_DIR until every gate passes, so a failed build cannot
|
||||
# leave a pair the grouped make target would take as up to date.
|
||||
"$REPO_ROOT/scripts/lib/sqs-store-microvm-publish.sh" "$STAGE_DIR" "$BUILD_DIR"
|
||||
"$REPO_ROOT/scripts/lib/sqs-store-microvm-publish.sh" "$STAGE_DIR" "$BUILD_DIR" "$PROFILE"
|
||||
echo ""
|
||||
echo "[build-sqs-store-microvm-image] done."
|
||||
echo "$TAG done."
|
||||
|
||||
@@ -56,6 +56,7 @@ EOF
|
||||
cat > "$TMP/bin/nats-server" <<'EOF'
|
||||
#!/bin/sh
|
||||
echo "nats-server $*" >> "$T/calls.log"
|
||||
[ "$1" = -t ] || echo "GOMEMLIMIT=${GOMEMLIMIT:-unset}" >> "$T/env.log"
|
||||
[ "$1" = -t ] && exit "$(cat "$T/configtest.rc")"
|
||||
exit 0
|
||||
EOF
|
||||
@@ -188,6 +189,8 @@ run_step() {
|
||||
}
|
||||
fresh run-ready; prepare; : > "$T/calls.log"
|
||||
check "a prepared store starts the server" 'run_step && grep -qx "nats-server -c $T/run/nats.conf" "$T/calls.log"'
|
||||
fresh run-memlimit; prepare; : > "$T/env.log"
|
||||
check "the server runs under the ADR-0011:S11 heap limit" 'run_step && grep -qx "GOMEMLIMIT=512MiB" "$T/env.log"'
|
||||
fresh run-unready
|
||||
check "an unprepared store never starts the server" '! run_step && ! grep -q "^nats-server" "$T/calls.log"'
|
||||
fresh run-stopping; prepare; : > "$T/calls.log"; : > "$T/run/stopping"
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
#!/bin/bash
|
||||
# sqs-store-microvm-publish.sh STAGE_DIR OUT_DIR — gate a staged SQS store
|
||||
# microVM kernel + initramfs and move them into OUT_DIR only if every gate passes.
|
||||
# sqs-store-microvm-publish.sh STAGE_DIR OUT_DIR [PROFILE] — gate a staged SQS
|
||||
# store (default) or worker microVM kernel + initramfs and move them into
|
||||
# OUT_DIR only if every gate passes.
|
||||
#
|
||||
# A failed gate leaves OUT_DIR as it was: a previous valid pair stays, and an
|
||||
# older pair stays older than the build inputs, so make rebuilds next time.
|
||||
@@ -10,7 +11,15 @@ STAGE_DIR="${1:?usage: sqs-store-microvm-publish.sh STAGE_DIR OUT_DIR}"
|
||||
OUT_DIR="${2:?usage: sqs-store-microvm-publish.sh STAGE_DIR OUT_DIR}"
|
||||
VMLINUZ="$STAGE_DIR/vmlinuz"
|
||||
INITRAMFS="$STAGE_DIR/initramfs.cpio.gz"
|
||||
tag="[build-sqs-store-microvm-image]"
|
||||
PROFILE="${3:-store}"
|
||||
case "$PROFILE" in
|
||||
store)
|
||||
payload="usr/local/bin/nats-server usr/local/lib/sqs-store/sqs-store-lib.sh usr/local/lib/sqs-store/sqs-store-prepare usr/local/lib/sqs-store/sqs-store-run usr/local/lib/sqs-store/sqs-store-stop" ;;
|
||||
worker)
|
||||
payload="usr/local/bin/sqs-worker usr/local/lib/sqs-worker/sqs-worker-lib.sh usr/local/lib/sqs-worker/sqs-worker-prepare usr/local/lib/sqs-worker/sqs-worker-ready usr/local/lib/sqs-worker/sqs-worker-stop data" ;;
|
||||
*) echo "ERROR: profile must be store or worker, not $PROFILE" >&2; exit 1 ;;
|
||||
esac
|
||||
tag="[build-sqs-$PROFILE-microvm-image]"
|
||||
|
||||
for f in "$VMLINUZ" "$INITRAMFS"; do
|
||||
if [ ! -s "$f" ]; then
|
||||
@@ -24,9 +33,8 @@ if ! archive_list=$(zcat "$INITRAMFS" | cpio --quiet -t); then
|
||||
echo "ERROR: $INITRAMFS is not a readable gzip cpio archive" >&2
|
||||
exit 1
|
||||
fi
|
||||
for want in init etc/inittab usr/local/bin/nats-server dev/console dev/null \
|
||||
usr/local/lib/sqs-store/sqs-store-lib.sh usr/local/lib/sqs-store/sqs-store-prepare \
|
||||
usr/local/lib/sqs-store/sqs-store-run usr/local/lib/sqs-store/sqs-store-stop; do
|
||||
# shellcheck disable=SC2086 # payload is a word list on purpose
|
||||
for want in init etc/inittab dev/console dev/null $payload; do
|
||||
if ! grep -qx "$want" <<< "$archive_list"; then
|
||||
echo "ERROR: $want missing from $INITRAMFS" >&2
|
||||
exit 1
|
||||
@@ -51,6 +59,11 @@ if grep -qE '(^|/)(lb-agent|haproxy|nginx)$' <<< "$archive_list"; then
|
||||
echo "ERROR: LB data-plane binaries found in the SQS store image" >&2
|
||||
exit 1
|
||||
fi
|
||||
# The worker holds the store client key; it must never carry the store server.
|
||||
if [ "$PROFILE" = worker ] && grep -qE '(^|/)nats-server$' <<< "$archive_list"; then
|
||||
echo "ERROR: nats-server found in the SQS worker image" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "$tag content gate: PASS"
|
||||
|
||||
echo ""
|
||||
|
||||
Executable
+163
@@ -0,0 +1,163 @@
|
||||
#!/bin/sh
|
||||
# sqs-worker-guest_test.sh — the SQS worker guest's readiness watcher is
|
||||
# supervised by init, never leaves a store-ready marker behind a dead watcher,
|
||||
# and shutdown unmounts the worker disk only once the watcher is gone.
|
||||
#
|
||||
# Drives build/microvm/sqs-worker/sqs-worker-lib.sh with test-owned processes
|
||||
# and a stub umount. Every signal goes only to this test's own children.
|
||||
# The case settings below are read by the sourced library, not by this file.
|
||||
# shellcheck disable=SC2034
|
||||
set -eu
|
||||
|
||||
ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/../.." && pwd)
|
||||
LIB=${SQSW_GUEST_LIB:-"$ROOT/build/microvm/sqs-worker/sqs-worker-lib.sh"}
|
||||
INITTAB="$ROOT/build/microvm/sqs-worker/inittab"
|
||||
TMP=$(mktemp -d)
|
||||
CHILDREN=""
|
||||
cleanup() {
|
||||
for p in $CHILDREN; do kill -KILL "$p" 2>/dev/null || true; done
|
||||
rm -rf "$TMP"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
fails=0
|
||||
pass() { echo " ok - $1"; }
|
||||
fail() { echo " FAIL - $1"; fails=$((fails + 1)); }
|
||||
check() { if eval "$2"; then pass "$1"; else fail "$1"; fi; }
|
||||
|
||||
echo "-- build/microvm/sqs-worker/sqs-worker-lib.sh"
|
||||
|
||||
mkdir -p "$TMP/bin"
|
||||
cat > "$TMP/bin/umount" <<'EOF'
|
||||
#!/bin/sh
|
||||
echo "umount $*" >> "$T/calls.log"
|
||||
EOF
|
||||
# The fake watcher notes whether a marker was already present when it
|
||||
# started, publishes readiness as the real one does, then stays running.
|
||||
cat > "$TMP/bin/watcher" <<'EOF'
|
||||
#!/bin/sh
|
||||
[ ! -e "$T/run/store-ready" ] || echo stale >> "$T/saw-stale"
|
||||
echo "$$" >> "$T/watchers"
|
||||
echo "10.0.0.5 UPUB" > "$T/run/store-ready"
|
||||
exec sleep 60
|
||||
EOF
|
||||
chmod +x "$TMP/bin"/*
|
||||
|
||||
fresh() {
|
||||
T="$TMP/$1"
|
||||
export T
|
||||
mkdir -p "$T/run" "$T/data"
|
||||
echo UPUBLICKEY > "$T/run/public-key"
|
||||
for f in calls.log console mounts; do : > "$T/$f"; done
|
||||
}
|
||||
gone() { [ ! -r "/proc/$1/status" ] || grep -q '^State:[[:space:]]*Z' "/proc/$1/status" 2>/dev/null; }
|
||||
waitfor() { i=0; while [ "$i" -lt 100 ] && ! eval "$1"; do i=$((i + 1)); sleep 0.05; done; eval "$1"; }
|
||||
# start_run: one init respawn of the readiness step, in the background.
|
||||
start_run() {
|
||||
SQSW_RUN="$T/run" SQSW_CONSOLE="$T/console" SQSW_BIN="$TMP/bin/watcher" SQSW_RESPAWN_GAP="${GAP:-0}" SQSW_PARK_INTERVAL="${PARK:-30}" \
|
||||
sh -c '. "$1"; sqsw_run' sh "$LIB" & pid=$!
|
||||
CHILDREN="$CHILDREN $pid"
|
||||
}
|
||||
# record <pid>: write the identity sqs_run would record for that process.
|
||||
# shellcheck source=../../build/microvm/sqs-worker/sqs-worker-lib.sh
|
||||
record() { echo "$1 $( . "$LIB"; sqsw_starttime "$1" )" > "$T/run/ready.pid"; }
|
||||
# shutdown_step [survivor-pid]: with a pid, that process is reported alive
|
||||
# whatever happens to it, as a watcher stuck past SIGKILL would be.
|
||||
shutdown_step() {
|
||||
(
|
||||
SQSW_RUN="$T/run" SQSW_DATA="$T/data" SQSW_CONSOLE="$T/console" SQSW_MOUNTS="$T/mounts"
|
||||
UMOUNT="$TMP/bin/umount" SQSW_STOP_BUDGET_TENTHS=5
|
||||
# shellcheck source=../../build/microvm/sqs-worker/sqs-worker-lib.sh
|
||||
. "$LIB"
|
||||
if [ -n "${1:-}" ]; then sqsw_same() { [ "$1" = "$survivor" ]; }; survivor=$1; fi
|
||||
sqsw_shutdown
|
||||
) > /dev/null 2>&1
|
||||
}
|
||||
|
||||
check "init respawns the readiness step rather than running it once" \
|
||||
'grep -qx "::respawn:/usr/local/lib/sqs-worker/sqs-worker-ready" "$INITTAB" && ! grep -q "::once:" "$INITTAB"'
|
||||
|
||||
# --- A watcher killed after readiness: its marker must not outlive it ---
|
||||
fresh killed-after-ready
|
||||
start_run; first=$pid
|
||||
check "the watcher publishes store-ready" 'waitfor "[ -s \"\$T/run/store-ready\" ]"'
|
||||
kill -KILL "$first"
|
||||
check "SIGKILL leaves the marker behind, as a crash would" 'waitfor "gone $first" && [ -e "$T/run/store-ready" ]'
|
||||
start_run; second=$pid
|
||||
check "the respawned step starts a new watcher" 'waitfor "[ \"\$(wc -l < \"\$T/watchers\")\" -eq 2 ]"'
|
||||
check "the respawn removed the dead watcher's marker before the new watcher ran" \
|
||||
'[ ! -e "$T/saw-stale" ] && grep -q "removed a store-ready marker left by an earlier watcher" "$T/console"'
|
||||
check "the respawn records the new watcher's identity" 'grep -q "^$second " "$T/run/ready.pid"'
|
||||
kill -KILL "$second" 2>/dev/null || true
|
||||
|
||||
# --- A watcher that keeps exiting is respawned at most once per gap ---
|
||||
fresh respawn-gap
|
||||
date +%s > "$T/run/ready.last"
|
||||
GAP=2 start_run; t0=$(date +%s)
|
||||
waitfor "[ -s \"\$T/watchers\" ]" || true
|
||||
t1=$(date +%s)
|
||||
check "a start right after the previous one waits for the gap" '[ $((t1 - t0)) -ge 1 ]'
|
||||
kill -KILL "$pid" 2>/dev/null || true
|
||||
|
||||
# --- Runs that must not start ---
|
||||
# A run that cannot start parks rather than exiting, so init does not respawn
|
||||
# it in a loop; a second is many park intervals, and it must log only once.
|
||||
fresh run-stopping; : > "$T/run/stopping"; echo stale > "$T/run/store-ready"
|
||||
PARK=0.05 start_run; waitfor "grep -q 'shutdown in progress' \"\$T/console\"" || true
|
||||
sleep 1
|
||||
check "a start during shutdown parks instead of exiting and respawning" '! gone "$pid"'
|
||||
check "a parked start during shutdown starts no watcher and removes the marker" \
|
||||
'[ ! -e "$T/watchers" ] && [ ! -e "$T/run/store-ready" ]'
|
||||
check "a parked start during shutdown logs once" '[ "$(grep -c "shutdown in progress" "$T/console")" -eq 1 ]'
|
||||
shutdown_step
|
||||
check "shutdown stops the parked start" 'waitfor "gone $pid"'
|
||||
|
||||
fresh run-no-key; rm -f "$T/run/public-key"
|
||||
PARK=0.05 start_run; waitfor "grep -q 'no worker key' \"\$T/console\"" || true
|
||||
sleep 1
|
||||
check "a guest without a worker key parks instead of exiting and respawning" '! gone "$pid" && [ ! -e "$T/watchers" ]'
|
||||
check "a parked guest without a worker key logs once" '[ "$(grep -c "no worker key" "$T/console")" -eq 1 ]'
|
||||
echo UPUBLICKEY > "$T/run/public-key"
|
||||
check "a key that appears later starts the watcher" 'waitfor "[ -s \"\$T/watchers\" ]"'
|
||||
kill -KILL "$pid" 2>/dev/null || true
|
||||
|
||||
fresh run-no-key-stop; rm -f "$T/run/public-key"
|
||||
PARK=0.05 start_run; waitfor "grep -q 'no worker key' \"\$T/console\"" || true
|
||||
shutdown_step
|
||||
check "shutdown stops a start parked for a missing key" 'waitfor "gone $pid" && [ ! -e "$T/watchers" ]'
|
||||
|
||||
# --- Shutdown: unmount only once the watcher is confirmed gone ---
|
||||
fresh shutdown-orderly; echo "dev $T/data ext4 rw 0 0" > "$T/mounts"
|
||||
start_run; waitfor "[ -s \"\$T/run/store-ready\" ]"
|
||||
check "orderly shutdown succeeds" 'shutdown_step'
|
||||
check "orderly shutdown confirms the watcher exited, removes the marker, then unmounts" \
|
||||
'gone "$pid" && grep -q "readiness watcher pid=$pid exited" "$T/console" && [ ! -e "$T/run/store-ready" ] &&
|
||||
[ "$(grep "^umount" "$T/calls.log" | tr "\n" "|")" = "umount $T/data|umount -a -r|" ]'
|
||||
check "shutdown blocks any later respawn" '[ -e "$T/run/stopping" ]'
|
||||
|
||||
fresh shutdown-hung; echo "dev $T/data ext4 rw 0 0" > "$T/mounts"
|
||||
(trap '' TERM; exec sleep 60) & pid=$!; CHILDREN="$CHILDREN $pid"; record "$pid"
|
||||
echo "10.0.0.5 UPUB" > "$T/run/store-ready"
|
||||
shutdown_step
|
||||
check "a watcher ignoring SIGTERM is killed after the budget, then the disk is unmounted" \
|
||||
'gone "$pid" && grep -q SIGKILL "$T/console" && grep -qx "umount $T/data" "$T/calls.log"'
|
||||
|
||||
fresh shutdown-survivor; echo "dev $T/data ext4 rw 0 0" > "$T/mounts"
|
||||
(trap '' TERM; exec sleep 60) & pid=$!; CHILDREN="$CHILDREN $pid"; record "$pid"
|
||||
echo "10.0.0.5 UPUB" > "$T/run/store-ready"
|
||||
check "shutdown with a surviving watcher reports failure" '! shutdown_step "$pid"'
|
||||
check "a surviving watcher leaves every mount alone and is never reported stopped" \
|
||||
'! grep -q "^umount" "$T/calls.log" && grep -q "STILL ALIVE" "$T/console" && grep -q "skipping umount -a -r" "$T/console" && ! grep -q "exited" "$T/console"'
|
||||
check "a surviving watcher still leaves no readiness marker" '[ ! -e "$T/run/store-ready" ]'
|
||||
kill -KILL "$pid" 2>/dev/null || true
|
||||
|
||||
fresh shutdown-reused; echo "dev $T/data ext4 rw 0 0" > "$T/mounts"
|
||||
(exec sleep 60) & pid=$!; CHILDREN="$CHILDREN $pid"; echo "$pid 1" > "$T/run/ready.pid"
|
||||
shutdown_step
|
||||
check "a recorded PID with another start time is never signalled" '! gone "$pid" && grep -q "no running readiness watcher" "$T/console"'
|
||||
kill -KILL "$pid" 2>/dev/null || true
|
||||
|
||||
if [ "$fails" -ne 0 ]; then
|
||||
echo " $fails failure(s)"
|
||||
exit 1
|
||||
fi
|
||||
@@ -1,18 +1,24 @@
|
||||
#!/bin/sh
|
||||
# stage-sqs-store-microvm.sh <image-dir> <dist-root>: copy a built SQS store
|
||||
# microVM image into a release tree as <dist-root>/microvm-sqs-store. Without a
|
||||
# complete image it stages nothing and says so, as the load balancer image does.
|
||||
# stage-sqs-store-microvm.sh <image-dir> <dist-root> [sqs-store|sqs-worker]:
|
||||
# copy a built SQS microVM image into a release tree as <dist-root>/microvm-<name>.
|
||||
# Without a complete image it stages nothing and says so, as the LB image does.
|
||||
set -eu
|
||||
src=$1 dest=$2/microvm-sqs-store
|
||||
src=$1 name=${3:-sqs-store}
|
||||
case "$name" in
|
||||
sqs-store) label="SQS store" ;;
|
||||
sqs-worker) label="SQS worker" ;;
|
||||
*) echo "[distro] unknown SQS microVM image '$name'" >&2; exit 2 ;;
|
||||
esac
|
||||
dest=$2/microvm-$name
|
||||
# An image staged by an earlier build must not ship with this one. Only the
|
||||
# two image files are removed; anything else in the directory is left alone.
|
||||
rm -f "$dest/vmlinuz" "$dest/initramfs.cpio.gz"
|
||||
if [ -s "$src/vmlinuz" ] && [ -s "$src/initramfs.cpio.gz" ]; then
|
||||
echo "[distro] staging the SQS store microVM image into the tarball"
|
||||
echo "[distro] staging the $label microVM image into the tarball"
|
||||
mkdir -p "$dest"
|
||||
cp "$src/vmlinuz" "$src/initramfs.cpio.gz" "$dest/"
|
||||
else
|
||||
[ ! -d "$dest" ] || rmdir "$dest" 2> /dev/null || true
|
||||
echo "[distro] WARNING: SQS store microVM image missing; the tarball will not include it"
|
||||
echo "[distro] run 'make build-sqs-store-microvm-image' first"
|
||||
echo "[distro] WARNING: $label microVM image missing; the tarball will not include it"
|
||||
echo "[distro] run 'make build-$name-microvm-image' first"
|
||||
fi
|
||||
|
||||
+42
-13
@@ -16,8 +16,8 @@
|
||||
# VERBOSE Set to 1 to echo "[setup] <stage>" before each top-level step.
|
||||
# SETUP_STAGES Comma-separated subset of stages to run:
|
||||
# deps, aws, dns, users, sudoers, firewall, timesync, files,
|
||||
# directories, env, systemd, sysctl, modules, logrotate,
|
||||
# udev, fixown, migrations
|
||||
# directories, env, systemd, sysctl, needrestart, modules,
|
||||
# logrotate, udev, fixown, migrations
|
||||
# Unset = run every stage appropriate for the current mode.
|
||||
|
||||
set -e
|
||||
@@ -1543,24 +1543,30 @@ install_files() {
|
||||
info " /usr/share/spinifex/microvm/*"
|
||||
fi
|
||||
install_sqs_store_image
|
||||
install_sqs_worker_image
|
||||
}
|
||||
|
||||
# The SQS store image ships beside the load balancer's, not inside it: an older
|
||||
# The SQS images ship beside the load balancer's, not inside it: an older
|
||||
# installer copies microvm/* without -r and would stop at a subdirectory.
|
||||
install_sqs_store_image() {
|
||||
local src="$EXTRACT_DIR/microvm-sqs-store" f
|
||||
local dest="${SPINIFEX_MICROVM_DIR:-/usr/share/spinifex/microvm}/sqs-store"
|
||||
install_sqs_store_image() { install_sqs_microvm_image sqs-store "SQS store"; }
|
||||
install_sqs_worker_image() { install_sqs_microvm_image sqs-worker "SQS worker"; }
|
||||
|
||||
# install_sqs_microvm_image <name> <label>
|
||||
install_sqs_microvm_image() {
|
||||
local name=$1 label=$2 f
|
||||
local src="$EXTRACT_DIR/microvm-$name"
|
||||
local dest="${SPINIFEX_MICROVM_DIR:-/usr/share/spinifex/microvm}/$name"
|
||||
# A previously installed image belongs to an older release, and this
|
||||
# release's daemon must not boot it. Only the two image files are removed.
|
||||
if [ ! -d "$src" ]; then
|
||||
remove_sqs_store_image "$dest"
|
||||
warn "This release has no SQS store microVM image; SQS store launches will be refused"
|
||||
remove_sqs_microvm_image "$dest" "$label"
|
||||
warn "This release has no $label microVM image; $label launches will be refused"
|
||||
return 0
|
||||
fi
|
||||
for f in vmlinuz initramfs.cpio.gz; do
|
||||
if [ ! -s "$src/$f" ]; then
|
||||
remove_sqs_store_image "$dest"
|
||||
fatal "The SQS store microVM image in this release is incomplete: $f is missing"
|
||||
remove_sqs_microvm_image "$dest" "$label"
|
||||
fatal "The $label microVM image in this release is incomplete: $f is missing"
|
||||
fi
|
||||
done
|
||||
# The initramfs goes last, so an interrupted install leaves no pair the
|
||||
@@ -1572,11 +1578,11 @@ install_sqs_store_image() {
|
||||
info " $dest/vmlinuz, $dest/initramfs.cpio.gz"
|
||||
}
|
||||
|
||||
# Running store VMs keep their already-loaded kernel; the files are read at boot.
|
||||
remove_sqs_store_image() {
|
||||
# Running VMs keep their already-loaded kernel; the files are read at boot.
|
||||
remove_sqs_microvm_image() {
|
||||
[ -e "$1/initramfs.cpio.gz" ] || [ -e "$1/vmlinuz" ] || return 0
|
||||
$SUDO rm -f "$1/initramfs.cpio.gz" "$1/vmlinuz"
|
||||
warn "Removed the SQS store microVM image left by an earlier install from $1"
|
||||
warn "Removed the $2 microVM image left by an earlier install from $1"
|
||||
}
|
||||
|
||||
# --- Create directories ---
|
||||
@@ -1930,6 +1936,28 @@ SYSCTL
|
||||
info "Kernel tunables installed (net.core.rmem_max/wmem_max = 16MB)"
|
||||
}
|
||||
|
||||
# --- Keep needrestart off the Spinifex, OVS and OVN units ---
|
||||
# Guest QEMU runs in spinifex-daemon's cgroup, so needrestart charges QEMU's
|
||||
# replaced libraries to the daemon and restarts it after every dpkg run, which
|
||||
# never clears them and ends at the unit's start limit. Restarting the stack
|
||||
# after an upgrade is the operator's call; needrestart still lists the units.
|
||||
install_needrestart_conf() {
|
||||
stage "installing needrestart override"
|
||||
local dir="${DISTRO_ROOT:-}/etc/needrestart/conf.d"
|
||||
$SUDO install -d "$dir"
|
||||
$SUDO tee "$dir/90-spinifex.conf" > /dev/null << 'NEEDRESTART'
|
||||
# Managed by spinifex setup.sh — edits are overwritten.
|
||||
# Never restart these automatically; restart them by hand when it suits.
|
||||
$nrconf{override_rc}{qr(^spinifex-)} = 0;
|
||||
$nrconf{override_rc}{qr(^ovs-)} = 0;
|
||||
$nrconf{override_rc}{qr(^ovsdb-)} = 0;
|
||||
$nrconf{override_rc}{qr(^ovn-)} = 0;
|
||||
$nrconf{override_rc}{qr(^openvswitch-)} = 0;
|
||||
NEEDRESTART
|
||||
$SUDO chmod 0644 "$dir/90-spinifex.conf"
|
||||
info "needrestart will not restart the spinifex, OVS or OVN units"
|
||||
}
|
||||
|
||||
# --- Install logrotate ---
|
||||
install_logrotate() {
|
||||
stage "installing logrotate config"
|
||||
@@ -2124,6 +2152,7 @@ main() {
|
||||
stage_enabled fixown && fix_file_ownership
|
||||
stage_enabled systemd && install_systemd
|
||||
stage_enabled sysctl && install_sysctl
|
||||
stage_enabled needrestart && install_needrestart_conf
|
||||
stage_enabled modules && install_kernel_modules
|
||||
stage_enabled logrotate && install_logrotate
|
||||
stage_enabled udev && install_udev
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
#!/bin/sh
|
||||
# setup_host_test.sh — tests for setup.sh's DNS handover to systemd-resolved,
|
||||
# its chrony drop-in and its nbdkit plugin directory.
|
||||
# its chrony and needrestart drop-ins and its nbdkit plugin directory.
|
||||
#
|
||||
# setup.sh is sourced with INSTALL_SPINIFEX_LIB_ONLY=1 so main never runs. PATH
|
||||
# holds stub commands and links to a few real tools only, so no host resolver,
|
||||
@@ -17,10 +17,10 @@ fails=0
|
||||
pass() { echo " ok - $1"; }
|
||||
fail() { echo " FAIL - $1"; fails=$((fails + 1)); }
|
||||
|
||||
echo "-- setup.sh host DNS, chrony and nbdkit"
|
||||
echo "-- setup.sh host DNS, chrony, needrestart and nbdkit"
|
||||
|
||||
mkdir -p "$TMP/sys"
|
||||
for tool in bash awk sed grep readlink basename dirname cp ln mkdir cat head tail wc tee rm chmod; do
|
||||
for tool in bash awk sed grep readlink basename dirname cp ln mkdir cat head tail wc tee rm chmod install; do
|
||||
ln -s "$(command -v "$tool")" "$TMP/sys/$tool"
|
||||
done
|
||||
|
||||
@@ -249,6 +249,50 @@ case "$RC:$OUT" in
|
||||
*) fail "no chrony: rc=$RC: $OUT" ;;
|
||||
esac
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# needrestart override.
|
||||
# --------------------------------------------------------------------------
|
||||
# restarts <conf> <unit> — how needrestart would treat <unit> under <conf>:
|
||||
# "no" when an override_rc pattern deselects it, "yes" otherwise.
|
||||
restarts() {
|
||||
perl -e 'our %nrconf; do $ARGV[0]; die $@ if $@;
|
||||
for my $re (keys %{$nrconf{override_rc}}) {
|
||||
if ($ARGV[1] =~ $re) { print $nrconf{override_rc}{$re} ? "yes" : "no"; exit }
|
||||
}
|
||||
print "yes"' "$1" "$2"
|
||||
}
|
||||
|
||||
host
|
||||
run install_needrestart_conf
|
||||
conf="$TMP/root/etc/needrestart/conf.d/90-spinifex.conf"
|
||||
if [ "$RC" != 0 ] || [ ! -f "$conf" ]; then
|
||||
fail "needrestart: rc=$RC: $OUT"
|
||||
elif ! command -v perl > /dev/null; then
|
||||
fail "needrestart: perl is not installed, so the override cannot be checked"
|
||||
else
|
||||
wrong=""
|
||||
for unit in spinifex-daemon.service spinifex-viperblock.service ovs-vswitchd.service \
|
||||
ovsdb-server.service ovn-controller.service openvswitch-switch.service; do
|
||||
[ "$(restarts "$conf" "$unit")" = no ] || wrong="$wrong $unit"
|
||||
done
|
||||
for unit in ssh.service rsyslog.service polkit.service; do
|
||||
[ "$(restarts "$conf" "$unit")" = yes ] || wrong="$wrong $unit"
|
||||
done
|
||||
if [ -n "$wrong" ]; then
|
||||
fail "needrestart override treats these wrongly:$wrong"
|
||||
else
|
||||
pass "needrestart keeps off the stack's units and still restarts everything else"
|
||||
fi
|
||||
fi
|
||||
|
||||
before=$(cat "$conf" 2>/dev/null)
|
||||
run install_needrestart_conf
|
||||
if [ "$RC" = 0 ] && [ "$(cat "$conf" 2>/dev/null)" = "$before" ]; then
|
||||
pass "needrestart override is unchanged by a second install"
|
||||
else
|
||||
fail "needrestart override changed on re-run: rc=$RC: $OUT"
|
||||
fi
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# nbdkit plugin directory.
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
#!/bin/sh
|
||||
# setup_sqs_store_image_test.sh — tests for shipping the SQS store microVM
|
||||
# image: staging it into a release tree and installing it with setup.sh.
|
||||
# setup_sqs_store_image_test.sh — tests for shipping the SQS store and worker
|
||||
# microVM images: staging them into a release tree and installing with setup.sh.
|
||||
#
|
||||
# setup.sh is sourced with INSTALL_SPINIFEX_LIB_ONLY=1 so main never runs, and
|
||||
# installs into a temporary SPINIFEX_MICROVM_DIR with SUDO cleared.
|
||||
@@ -16,7 +16,7 @@ fails=0
|
||||
pass() { echo " ok - $1"; }
|
||||
fail() { echo " FAIL - $1"; fails=$((fails + 1)); }
|
||||
|
||||
echo "-- SQS store microVM image release and install"
|
||||
echo "-- SQS store and worker microVM image release and install"
|
||||
|
||||
# image <dir> <tag>: a fake built image whose bytes name its tag.
|
||||
image() {
|
||||
@@ -186,5 +186,59 @@ else
|
||||
fail "interrupted install: rc=$RC initramfs=$([ -e "$TMP/mv/sqs-store/initramfs.cpio.gz" ] && echo present || echo missing)"
|
||||
fi
|
||||
|
||||
# --- SQS worker image: same staging and install path, its own directory ---
|
||||
install_worker() {
|
||||
RC=0
|
||||
OUT=$(EXTRACT_DIR="$1" SPINIFEX_MICROVM_DIR="$2" INSTALL_SPINIFEX_LIB_ONLY=1 \
|
||||
bash -c '. "$1"; SUDO=""; install_sqs_worker_image' _ "$SETUP" 2>&1) || RC=$?
|
||||
}
|
||||
image "$TMP/wbuilt" w1
|
||||
release "$TMP/wrel"
|
||||
sh "$STAGE" "$TMP/built" "$TMP/wrel" > /dev/null
|
||||
sh "$STAGE" "$TMP/wbuilt" "$TMP/wrel" sqs-worker > /dev/null
|
||||
if same "$TMP/wbuilt/vmlinuz" "$TMP/wrel/microvm-sqs-worker/vmlinuz" \
|
||||
&& same "$TMP/wbuilt/initramfs.cpio.gz" "$TMP/wrel/microvm-sqs-worker/initramfs.cpio.gz" \
|
||||
&& same "$TMP/built/vmlinuz" "$TMP/wrel/microvm-sqs-store/vmlinuz"; then
|
||||
pass "the worker image stages beside the store image"
|
||||
else
|
||||
fail "worker staging: $(ls -R "$TMP/wrel")"
|
||||
fi
|
||||
install_worker "$TMP/wrel" "$TMP/wmv"
|
||||
if [ "$RC" = 0 ] && same "$TMP/wbuilt/initramfs.cpio.gz" "$TMP/wmv/sqs-worker/initramfs.cpio.gz" \
|
||||
&& [ ! -e "$TMP/wmv/sqs-store" ]; then
|
||||
pass "the worker image installs into its own directory only"
|
||||
else
|
||||
fail "worker install: rc=$RC $OUT"
|
||||
fi
|
||||
release "$TMP/wrel-none"
|
||||
OUT=$(sh "$STAGE" "$TMP/no-such-build" "$TMP/wrel-none" sqs-worker)
|
||||
if [ ! -e "$TMP/wrel-none/microvm-sqs-worker" ] && echo "$OUT" | grep -q 'SQS worker microVM image missing'; then
|
||||
pass "a missing worker build stages nothing and says so"
|
||||
else
|
||||
fail "missing worker build: $OUT"
|
||||
fi
|
||||
install_worker "$TMP/wrel-none" "$TMP/wmv"
|
||||
if [ "$RC" = 0 ] && [ ! -e "$TMP/wmv/sqs-worker/vmlinuz" ] && [ ! -e "$TMP/wmv/sqs-worker/initramfs.cpio.gz" ] \
|
||||
&& echo "$OUT" | grep -q 'no SQS worker microVM image; SQS worker launches will be refused'; then
|
||||
pass "a release without the worker image removes the earlier one and warns"
|
||||
else
|
||||
fail "release without worker image: rc=$RC $OUT"
|
||||
fi
|
||||
cp -r "$TMP/wrel" "$TMP/wrel-bad"
|
||||
rm "$TMP/wrel-bad/microvm-sqs-worker/vmlinuz"
|
||||
install_worker "$TMP/wrel" "$TMP/wmv"
|
||||
install_worker "$TMP/wrel-bad" "$TMP/wmv"
|
||||
if [ "$RC" != 0 ] && echo "$OUT" | grep -q 'SQS worker microVM image in this release is incomplete: vmlinuz is missing' \
|
||||
&& [ ! -e "$TMP/wmv/sqs-worker/initramfs.cpio.gz" ]; then
|
||||
pass "an incomplete worker image is refused and the earlier pair is removed"
|
||||
else
|
||||
fail "incomplete worker image: rc=$RC $OUT"
|
||||
fi
|
||||
if OUT=$(sh "$STAGE" "$TMP/wbuilt" "$TMP/wrel" lb 2>&1); then
|
||||
fail "an unknown image name was staged"
|
||||
else
|
||||
pass "an unknown image name is refused"
|
||||
fi
|
||||
|
||||
[ "$fails" = 0 ] || { echo "FAIL: $fails"; exit 1; }
|
||||
echo " setup_sqs_store_image_test ok"
|
||||
|
||||
@@ -29,8 +29,9 @@
|
||||
#
|
||||
# /usr/local/bin/spx, the nbdkit plugin, /usr/local/{share,lib}/spinifex,
|
||||
# /usr/share/spinifex, every spinifex systemd unit and slice, the eight
|
||||
# spinifex-* service users, the sudoers grant, the sysctl/logrotate/chrony/udev
|
||||
# drop-ins, the host firewall policy and its loaded nft table, the OVS bridges
|
||||
# spinifex-* service users, the sudoers grant, the sysctl, logrotate, chrony,
|
||||
# needrestart and udev drop-ins, the host firewall policy and its loaded nft
|
||||
# table, the OVS bridges
|
||||
# and OVN databases, the Spinifex resolved drop-in, and the Spinifex CA from the
|
||||
# host trust store. Where setup.sh turned on systemd-resolved, it restores
|
||||
# /etc/resolv.conf and turns resolved off.
|
||||
@@ -457,6 +458,7 @@ rm_path /etc/sysctl.d/*spinifex* \
|
||||
/etc/sudoers.d/spinifex-network \
|
||||
/etc/chrony/conf.d/spinifex.conf \
|
||||
/etc/chrony.d/spinifex.conf \
|
||||
/etc/needrestart/conf.d/90-spinifex.conf \
|
||||
/etc/logrotate.d/spinifex \
|
||||
/etc/tmpfiles.d/spinifex.conf \
|
||||
/etc/udev/rules.d/*spinifex* \
|
||||
@@ -615,6 +617,7 @@ if ! $DRY_RUN; then
|
||||
/usr/share/spinifex "$ETC_DIR" "$LOG_DIR" \
|
||||
/etc/sudoers.d/spinifex-network /etc/logrotate.d/spinifex /etc/tmpfiles.d/spinifex.conf \
|
||||
/etc/chrony/conf.d/spinifex.conf /etc/chrony.d/spinifex.conf "$RESOLV_CONF_SAVED" \
|
||||
/etc/needrestart/conf.d/90-spinifex.conf \
|
||||
"$RESOLVED_DROPIN"; do
|
||||
exists "$p" && LEFT+=("$p")
|
||||
done
|
||||
|
||||
@@ -0,0 +1,676 @@
|
||||
package workerboot
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/hex"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"math/big"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
|
||||
"github.com/mulgadc/bluebottle/pkg/iampolicy"
|
||||
"github.com/mulgadc/spinifex/internal/gwsign"
|
||||
"github.com/mulgadc/spinifex/internal/sqsworkergw"
|
||||
testutil "github.com/mulgadc/spinifex/internal/testkit"
|
||||
"github.com/mulgadc/spinifex/spinifex/agents/sqs/workerkey"
|
||||
sqsworkerapi "github.com/mulgadc/spinifex/spinifex/domains/sqs/awsapi/worker"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/firststore"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/storelaunch"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/workerident"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/workerrole"
|
||||
awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/clustersize"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
|
||||
"github.com/nats-io/nkeys"
|
||||
)
|
||||
|
||||
const (
|
||||
gwInstance = "i-0abc12345def67890"
|
||||
gwSuccessor = "i-0fedcba9876543210"
|
||||
gwVolume = "vol-0a1b2c3d4e5f60718"
|
||||
)
|
||||
|
||||
// session is one instance-role credential: its secret and the instance ID
|
||||
// STS names the session after.
|
||||
type session struct{ secret, instance string }
|
||||
|
||||
// localGateway stands for the gateway's ingress in front of the real
|
||||
// spinifex-sqs registration: it verifies each SigV4 signature against the
|
||||
// issued credential, names the caller from that session alone, and grants
|
||||
// only the worker role's actions. Every request body is kept for inspection.
|
||||
type localGateway struct {
|
||||
t *testing.T
|
||||
srv *httptest.Server
|
||||
reg dispatch.Registration
|
||||
mu sync.Mutex
|
||||
sessions map[string]session
|
||||
bodies []string
|
||||
// dropReply, while set, applies a request and then drops its reply.
|
||||
dropReply func(action string) bool
|
||||
// replied, when set, receives each reply's error code, "" on success.
|
||||
replied chan string
|
||||
}
|
||||
|
||||
func newLocalGateway(t *testing.T, deps sqsworkerapi.Deps) *localGateway {
|
||||
t.Helper()
|
||||
g := &localGateway{t: t, reg: sqsworkerapi.NewRegistration(deps), sessions: map[string]session{}}
|
||||
g.srv = httptest.NewTLSServer(http.HandlerFunc(g.serve))
|
||||
t.Cleanup(g.srv.Close)
|
||||
return g
|
||||
}
|
||||
|
||||
func (g *localGateway) issue(accessKey, secret, instance string) *gwsign.Signer {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
g.sessions[accessKey] = session{secret: secret, instance: instance}
|
||||
return gwsign.NewStatic(accessKey, secret)
|
||||
}
|
||||
|
||||
func (g *localGateway) caPEM() []byte {
|
||||
return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: g.srv.Certificate().Raw})
|
||||
}
|
||||
|
||||
func (g *localGateway) serve(w http.ResponseWriter, r *http.Request) {
|
||||
body, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
g.fail(w, http.StatusBadRequest, "MalformedRequest")
|
||||
return
|
||||
}
|
||||
g.mu.Lock()
|
||||
g.bodies = append(g.bodies, string(body))
|
||||
g.mu.Unlock()
|
||||
|
||||
sess, service, region, ok := g.verify(r, body)
|
||||
if !ok {
|
||||
g.fail(w, http.StatusForbidden, "SignatureDoesNotMatch")
|
||||
return
|
||||
}
|
||||
if service != sqsworkerapi.ServiceName {
|
||||
g.fail(w, http.StatusBadRequest, "InvalidAction")
|
||||
return
|
||||
}
|
||||
r.Body = io.NopCloser(bytes.NewReader(body))
|
||||
action := ""
|
||||
if q, err := parseForm(body); err == nil {
|
||||
action = q
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
err = g.reg.Dispatch(rec, dispatch.Invocation{
|
||||
Request: r,
|
||||
AccountID: awsidentifiers.GlobalAccountID,
|
||||
Service: service,
|
||||
Region: region,
|
||||
Authorize: func(svc, act string, _ []string, _ iampolicy.ConditionKeys) error {
|
||||
if svc != sqsworkerapi.ServiceName || !slices.Contains(workerrole.Actions(), act) {
|
||||
return &dispatch.PolicyDenialError{}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Actor: func() (dispatch.Actor, error) {
|
||||
return dispatch.Actor{
|
||||
AccountID: awsidentifiers.GlobalAccountID,
|
||||
PrincipalType: "assumed-role",
|
||||
PrincipalARN: "arn:aws:iam::" + awsidentifiers.GlobalAccountID + ":role/" + workerrole.RoleName,
|
||||
SessionName: sess.instance,
|
||||
}, nil
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
code := err.Error()
|
||||
status := http.StatusBadRequest
|
||||
switch code {
|
||||
case "AccessDenied":
|
||||
status = http.StatusForbidden
|
||||
case "ServiceUnavailable":
|
||||
status = http.StatusServiceUnavailable
|
||||
}
|
||||
g.fail(w, status, code)
|
||||
return
|
||||
}
|
||||
if g.dropReply != nil && g.dropReply(action) {
|
||||
hj, ok := w.(http.Hijacker)
|
||||
if !ok {
|
||||
g.t.Error("cannot drop a reply")
|
||||
return
|
||||
}
|
||||
conn, _, err := hj.Hijack()
|
||||
if err == nil {
|
||||
_ = conn.Close()
|
||||
}
|
||||
return
|
||||
}
|
||||
g.reply("")
|
||||
maps.Copy(w.Header(), rec.Header())
|
||||
w.WriteHeader(rec.Code)
|
||||
_, _ = w.Write(rec.Body.Bytes())
|
||||
}
|
||||
|
||||
func parseForm(body []byte) (string, error) {
|
||||
r, err := http.NewRequest(http.MethodPost, "/", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
r.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
if err := r.ParseForm(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return r.PostForm.Get("Action"), nil
|
||||
}
|
||||
|
||||
// verify recomputes the signature with the issued secret. It returns the
|
||||
// session, and the service and Region the credential scope names.
|
||||
func (g *localGateway) verify(r *http.Request, body []byte) (session, string, string, bool) {
|
||||
authz := r.Header.Get("Authorization")
|
||||
const prefix = "AWS4-HMAC-SHA256 Credential="
|
||||
if !strings.HasPrefix(authz, prefix) {
|
||||
return session{}, "", "", false
|
||||
}
|
||||
scope, _, _ := strings.Cut(strings.TrimPrefix(authz, prefix), ",")
|
||||
parts := strings.Split(scope, "/")
|
||||
if len(parts) != 5 || parts[4] != "aws4_request" {
|
||||
return session{}, "", "", false
|
||||
}
|
||||
g.mu.Lock()
|
||||
sess, ok := g.sessions[parts[0]]
|
||||
g.mu.Unlock()
|
||||
if !ok {
|
||||
return session{}, "", "", false
|
||||
}
|
||||
sum := sha256.Sum256(body)
|
||||
hash := hex.EncodeToString(sum[:])
|
||||
if r.Header.Get("X-Amz-Content-Sha256") != hash {
|
||||
return session{}, "", "", false
|
||||
}
|
||||
at, err := time.Parse("20060102T150405Z", r.Header.Get("X-Amz-Date"))
|
||||
if err != nil {
|
||||
return session{}, "", "", false
|
||||
}
|
||||
check, err := http.NewRequest(r.Method, "https://"+r.Host+r.URL.RequestURI(), bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return session{}, "", "", false
|
||||
}
|
||||
check.Header.Set("Content-Type", r.Header.Get("Content-Type"))
|
||||
check.Header.Set("X-Amz-Content-Sha256", hash)
|
||||
creds := aws.Credentials{AccessKeyID: parts[0], SecretAccessKey: sess.secret}
|
||||
if err := v4.NewSigner().SignHTTP(context.Background(), creds, check, hash, parts[3], parts[2], at); err != nil {
|
||||
return session{}, "", "", false
|
||||
}
|
||||
return sess, parts[3], parts[2], check.Header.Get("Authorization") == authz
|
||||
}
|
||||
|
||||
func (g *localGateway) fail(w http.ResponseWriter, status int, code string) {
|
||||
g.reply(code)
|
||||
w.Header().Set("Content-Type", "text/xml")
|
||||
w.WriteHeader(status)
|
||||
_, _ = fmt.Fprintf(w, "<ErrorResponse><Error><Code>%s</Code></Error></ErrorResponse>", code)
|
||||
}
|
||||
|
||||
func (g *localGateway) reply(code string) {
|
||||
if g.replied != nil {
|
||||
g.replied <- code
|
||||
}
|
||||
}
|
||||
|
||||
func (g *localGateway) sentBodies() []string {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
return slices.Clone(g.bodies)
|
||||
}
|
||||
|
||||
// gwCluster is the domain's real worker-identity and first-store records.
|
||||
type gwCluster struct {
|
||||
ids workerident.Store
|
||||
fence firststore.Store
|
||||
stores storelaunch.Records
|
||||
gw *localGateway
|
||||
}
|
||||
|
||||
func newGWCluster(t *testing.T) gwCluster {
|
||||
t.Helper()
|
||||
clustersize.DeclareForTest(t, 1)
|
||||
_, _, js := testutil.StartTestJetStream(t)
|
||||
ctx := context.Background()
|
||||
ids, err := workerident.Open(ctx, js)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fence, err := firststore.Open(ctx, js)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stores, err := storelaunch.Open(ctx, js)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return gwCluster{ids: ids, fence: fence, stores: stores, gw: newLocalGateway(t, sqsworkerapi.JetStreamDeps(js))}
|
||||
}
|
||||
|
||||
func (c gwCluster) launch(t *testing.T, claim string, supersedes uint64, inst string) workerident.Intent {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
in, err := c.ids.BeginLaunch(ctx, testRegion, workerident.LaunchRequest{InstanceType: "t3.small",
|
||||
ClaimID: claim, Supersedes: supersedes, NodeID: "node-a", DataVolumeID: gwVolume, DataVolumeEncrypted: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.ids.BindInstance(ctx, in, inst); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return in
|
||||
}
|
||||
|
||||
// guestFor points the fake guest's boot at the local gateway, signing with
|
||||
// the instance's own credential through the real client.
|
||||
func (c gwCluster) guestFor(t *testing.T, g *guest, accessKey, inst string) {
|
||||
t.Helper()
|
||||
signer := c.gw.issue(accessKey, "secret-"+accessKey, inst)
|
||||
g.fw("data-serial", testSerial)
|
||||
g.fw("region", testRegion)
|
||||
g.fw("gateway-url", c.gw.srv.URL)
|
||||
g.cfg.Wait = 10 * time.Millisecond
|
||||
g.cfg.NewClaimer = func(region, url string) (workerkey.Claimer, error) {
|
||||
return sqsworkergw.New(url, region, c.gw.caPEM(), signer)
|
||||
}
|
||||
}
|
||||
|
||||
func (c gwCluster) assertNoSeedSent(t *testing.T, seed string) {
|
||||
t.Helper()
|
||||
seed = strings.TrimSpace(seed)
|
||||
for _, b := range c.gw.sentBodies() {
|
||||
if strings.Contains(b, seed) {
|
||||
t.Fatal("a request carried seed material")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A bound worker boots: the gateway authorizes creation, the guest creates
|
||||
// its key on the mounted disk and records only the public key; reboots get
|
||||
// that same key back from the domain.
|
||||
func TestGateway_FirstBootCreatesKeyAndRebootsReuseIt(t *testing.T) {
|
||||
c := newGWCluster(t)
|
||||
c.launch(t, "claim-0001", 0, gwInstance)
|
||||
g := newGuest(t)
|
||||
c.guestFor(t, g, "ASIAWORKER1", gwInstance)
|
||||
|
||||
pub, err := g.boot()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seed := g.seed()
|
||||
for boot := 2; boot <= 3; boot++ {
|
||||
g.resetMounts()
|
||||
got, err := g.boot()
|
||||
if err != nil || got != pub || g.seed() != seed {
|
||||
t.Fatalf("reboot %d = %q, %v", boot, got, err)
|
||||
}
|
||||
}
|
||||
records := 0
|
||||
for _, b := range c.gw.sentBodies() {
|
||||
if strings.Contains(b, "Action=RecordWorkerPublicKey") {
|
||||
records++
|
||||
if !strings.Contains(b, "PublicKey="+pub) {
|
||||
t.Fatal("a record carried a different key")
|
||||
}
|
||||
}
|
||||
}
|
||||
if records != 1 {
|
||||
t.Fatalf("records = %d, want 1", records)
|
||||
}
|
||||
c.assertNoSeedSent(t, seed)
|
||||
g.assertNoSeedLeak(nil, seed)
|
||||
}
|
||||
|
||||
// The record's reply is lost after it landed: the boot retries, the domain
|
||||
// answers with the recorded key, and the same key is used.
|
||||
func TestGateway_LostRecordReplyKeepsTheKey(t *testing.T) {
|
||||
c := newGWCluster(t)
|
||||
c.launch(t, "claim-0001", 0, gwInstance)
|
||||
g := newGuest(t)
|
||||
c.guestFor(t, g, "ASIAWORKER1", gwInstance)
|
||||
var dropped bool
|
||||
c.gw.dropReply = func(action string) bool {
|
||||
if action == "RecordWorkerPublicKey" && !dropped {
|
||||
dropped = true
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
pub, err := g.boot()
|
||||
if err != nil || !dropped {
|
||||
t.Fatalf("boot = %q, %v (dropped=%v)", pub, err, dropped)
|
||||
}
|
||||
g.resetMounts()
|
||||
if again, err := g.boot(); err != nil || again != pub {
|
||||
t.Fatalf("reboot = %q, %v", again, err)
|
||||
}
|
||||
c.assertNoSeedSent(t, g.seed())
|
||||
}
|
||||
|
||||
// After the key is recorded, a lost guest copy fails closed: the domain
|
||||
// answers with the recorded key and no replacement is created or sent.
|
||||
func TestGateway_KeyLostAfterRecordingFailsClosed(t *testing.T) {
|
||||
c := newGWCluster(t)
|
||||
c.launch(t, "claim-0001", 0, gwInstance)
|
||||
g := newGuest(t)
|
||||
c.guestFor(t, g, "ASIAWORKER1", gwInstance)
|
||||
if _, err := g.boot(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Remove(g.keyFile()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := len(c.gw.sentBodies())
|
||||
g.resetMounts()
|
||||
if _, err := g.boot(); !errors.Is(err, workerkey.ErrKeyMissing) {
|
||||
t.Fatalf("err = %v, want ErrKeyMissing", err)
|
||||
}
|
||||
if _, err := os.Lstat(g.keyFile()); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatal("a replacement key was created")
|
||||
}
|
||||
for _, b := range c.gw.sentBodies()[before:] {
|
||||
if strings.Contains(b, "RecordWorkerPublicKey") {
|
||||
t.Fatal("a replacement key was offered")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// An instance the domain never bound, or one already replaced, is refused
|
||||
// at the first call and creates no key; AccessDenied is not retried.
|
||||
func TestGateway_UnboundOrReplacedInstanceIsRefused(t *testing.T) {
|
||||
t.Run("unbound", func(t *testing.T) {
|
||||
c := newGWCluster(t)
|
||||
c.launch(t, "claim-0001", 0, gwInstance)
|
||||
g := newGuest(t)
|
||||
c.guestFor(t, g, "ASIAOTHER", gwSuccessor)
|
||||
_, err := g.boot()
|
||||
api, ok := errors.AsType[*sqsworkergw.APIError](err)
|
||||
if !ok || api.Code != "AccessDenied" {
|
||||
t.Fatalf("err = %v, want AccessDenied", err)
|
||||
}
|
||||
if n := len(c.gw.sentBodies()); n != g.cfg.Attempts {
|
||||
t.Fatalf("calls = %d, want the %d attempts and no more", n, g.cfg.Attempts)
|
||||
}
|
||||
g.assertNoKey()
|
||||
})
|
||||
t.Run("replaced", func(t *testing.T) {
|
||||
c := newGWCluster(t)
|
||||
first := c.launch(t, "claim-0001", 0, gwInstance)
|
||||
ctx := context.Background()
|
||||
if err := c.ids.RecordStopped(ctx, first, gwInstance); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.fence.Fence(ctx, testRegion, first.Generation+1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.launch(t, "claim-0002", first.Generation, gwSuccessor)
|
||||
g := newGuest(t)
|
||||
c.guestFor(t, g, "ASIAOLD", gwInstance)
|
||||
if _, err := g.boot(); err == nil {
|
||||
t.Fatal("the replaced instance booted")
|
||||
}
|
||||
g.assertNoKey()
|
||||
})
|
||||
}
|
||||
|
||||
// A bad signature is refused before any decision, and creates no key.
|
||||
func TestGateway_BadSignatureCreatesNoKey(t *testing.T) {
|
||||
c := newGWCluster(t)
|
||||
c.launch(t, "claim-0001", 0, gwInstance)
|
||||
g := newGuest(t)
|
||||
c.guestFor(t, g, "ASIAWORKER1", gwInstance)
|
||||
forged := gwsign.NewStatic("ASIAWORKER1", "not-the-secret")
|
||||
g.cfg.NewClaimer = func(region, url string) (workerkey.Claimer, error) {
|
||||
return sqsworkergw.New(url, region, c.gw.caPEM(), forged)
|
||||
}
|
||||
if _, err := g.boot(); err == nil {
|
||||
t.Fatal("a forged signature booted")
|
||||
}
|
||||
g.assertNoKey()
|
||||
}
|
||||
|
||||
// The client trusts only the cluster CA it was given: a gateway whose
|
||||
// certificate another CA signed is never sent a request.
|
||||
func TestGateway_UntrustedServerIsNotContacted(t *testing.T) {
|
||||
c := newGWCluster(t)
|
||||
c.launch(t, "claim-0001", 0, gwInstance)
|
||||
g := newGuest(t)
|
||||
c.guestFor(t, g, "ASIAWORKER1", gwInstance)
|
||||
g.cfg.Attempts = 1
|
||||
signer := c.gw.issue("ASIAWORKER1", "secret-ASIAWORKER1", gwInstance)
|
||||
g.cfg.NewClaimer = func(region, url string) (workerkey.Claimer, error) {
|
||||
return sqsworkergw.New(url, region, otherCA(t), signer)
|
||||
}
|
||||
if _, err := g.boot(); err == nil || !strings.Contains(err.Error(), "certificate") {
|
||||
t.Fatalf("err = %v, want a TLS refusal", err)
|
||||
}
|
||||
if len(c.gw.sentBodies()) != 0 {
|
||||
t.Fatal("a request reached a server outside the trusted CA")
|
||||
}
|
||||
g.assertNoKey()
|
||||
}
|
||||
|
||||
// otherCA is a fresh self-signed CA unrelated to the test server's.
|
||||
func otherCA(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tmpl := &x509.Certificate{
|
||||
SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "other"},
|
||||
NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour),
|
||||
IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign,
|
||||
}
|
||||
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
|
||||
}
|
||||
|
||||
// The worker VM starts before its launch binds the instance, so its first
|
||||
// calls are denied. It creates nothing while denied, and boots once bound.
|
||||
func TestGateway_BootBeforeBindSucceedsAfterBinding(t *testing.T) {
|
||||
c := newGWCluster(t)
|
||||
ctx := context.Background()
|
||||
intent, err := c.ids.BeginLaunch(ctx, testRegion, workerident.LaunchRequest{InstanceType: "t3.small",
|
||||
ClaimID: "claim-0001", NodeID: "node-a", DataVolumeID: gwVolume, DataVolumeEncrypted: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
g := newGuest(t)
|
||||
c.guestFor(t, g, "ASIAWORKER1", gwInstance)
|
||||
g.cfg.Attempts = 1000
|
||||
c.gw.replied = make(chan string, 2000)
|
||||
|
||||
type result struct {
|
||||
pub string
|
||||
err error
|
||||
}
|
||||
done := make(chan result, 1)
|
||||
go func() {
|
||||
pub, err := g.boot()
|
||||
done <- result{pub, err}
|
||||
}()
|
||||
if code := <-c.gw.replied; code != "AccessDenied" {
|
||||
t.Fatalf("first reply = %q, want AccessDenied before binding", code)
|
||||
}
|
||||
g.assertNoKey()
|
||||
if err := c.ids.BindInstance(ctx, intent, gwInstance); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := <-done
|
||||
if r.err != nil || r.pub == "" {
|
||||
t.Fatalf("boot after binding = %q, %v", r.pub, r.err)
|
||||
}
|
||||
if g.seed() == "" {
|
||||
t.Fatal("booted without a key on the worker disk")
|
||||
}
|
||||
}
|
||||
|
||||
// A replaced worker is never bound again: it keeps being denied, creates
|
||||
// nothing and gives up within its attempts.
|
||||
func TestGateway_StaleWorkerGivesUpWithinTheBound(t *testing.T) {
|
||||
c := newGWCluster(t)
|
||||
ctx := context.Background()
|
||||
first := c.launch(t, "claim-0001", 0, gwInstance)
|
||||
if err := c.ids.RecordStopped(ctx, first, gwInstance); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.fence.Fence(ctx, testRegion, first.Generation+1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.launch(t, "claim-0002", first.Generation, gwSuccessor)
|
||||
g := newGuest(t)
|
||||
c.guestFor(t, g, "ASIAOLD", gwInstance)
|
||||
g.cfg.Attempts = 5
|
||||
_, err := g.boot()
|
||||
if api, ok := errors.AsType[*sqsworkergw.APIError](err); !ok || !api.Pending() {
|
||||
t.Fatalf("err = %v, want the denial", err)
|
||||
}
|
||||
if n := len(c.gw.sentBodies()); n != 5 {
|
||||
t.Fatalf("calls = %d, want exactly the 5 attempts", n)
|
||||
}
|
||||
g.assertNoKey()
|
||||
}
|
||||
|
||||
// bindStore runs the store launch record through to bound for pub at address,
|
||||
// as the operator's EnsureSQSStore would.
|
||||
func (c gwCluster) bindStore(t *testing.T, pub, address string) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
const claim = "store-claim-0001"
|
||||
steps := []func() error{
|
||||
func() error { return c.stores.Begin(ctx, testRegion, claim, "node-a", pub, "t3.medium", 8) },
|
||||
func() error { return c.stores.MarkVolumeAttempted(ctx, testRegion, claim) },
|
||||
func() error { return c.stores.RecordVolume(ctx, testRegion, claim, "vol-0b1c2d3e4f5a60718") },
|
||||
func() error { return c.stores.MarkNetworkAttempted(ctx, testRegion, claim) },
|
||||
func() error { return c.stores.MarkDispatched(ctx, testRegion, claim) },
|
||||
func() error { return c.stores.Bind(ctx, testRegion, claim, "i-0123456789abcdef0", address) },
|
||||
}
|
||||
for i, step := range steps {
|
||||
if err := step(); err != nil {
|
||||
t.Fatalf("store launch step %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// readyFor points the guest's readiness at the local gateway and a store on port.
|
||||
func (c gwCluster) readyFor(g *guest, accessKey, inst string, port int) ReadyConfig {
|
||||
signer := c.gw.issue(accessKey, "secret-"+accessKey, inst)
|
||||
return ReadyConfig{
|
||||
Config: g.cfg,
|
||||
NewDescriber: func(region, url string) (StoreDescriber, error) {
|
||||
cl, err := sqsworkergw.New(url, region, c.gw.caPEM(), signer)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return gwDescriber{cl}, nil
|
||||
},
|
||||
Checker: NATSChecker{Port: port, Dial: time.Second},
|
||||
}
|
||||
}
|
||||
|
||||
type gwDescriber struct{ c *sqsworkergw.Client }
|
||||
|
||||
func (d gwDescriber) DescribeStore(ctx context.Context) (Store, error) {
|
||||
st, err := d.c.DescribeStore(ctx)
|
||||
return Store(st), err
|
||||
}
|
||||
|
||||
// The whole path with real components: the worker boots and records its key,
|
||||
// waits while the operator has not launched the store, then reaches the real
|
||||
// store's JetStream with its own key once the store is bound; after a reboot
|
||||
// it is ready again with the same key. No request carries the seed.
|
||||
func TestGateway_ReadyAfterADelayedStoreLaunch(t *testing.T) {
|
||||
c := newGWCluster(t)
|
||||
c.launch(t, "claim-0001", 0, gwInstance)
|
||||
g := newGuest(t)
|
||||
c.guestFor(t, g, "ASIAWORKER1", gwInstance)
|
||||
pub, err := g.boot()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
port := storeServer(t, pub)
|
||||
err = g.watch(c.readyFor(g, "ASIAWORKER1", gwInstance, port), func(call int, ready bool) bool {
|
||||
if ready {
|
||||
if call < 4 {
|
||||
t.Fatalf("ready at call %d, before the store launch", call)
|
||||
}
|
||||
return true
|
||||
}
|
||||
if call == 3 {
|
||||
c.bindStore(t, pub, "127.0.0.1")
|
||||
}
|
||||
return false
|
||||
})
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("watch = %v", err)
|
||||
}
|
||||
for boot := 1; boot <= 2; boot++ {
|
||||
g.resetMounts()
|
||||
if again, err := g.boot(); err != nil || again != pub {
|
||||
t.Fatalf("reboot %d = %q %v", boot, again, err)
|
||||
}
|
||||
if err := g.watch(c.readyFor(g, "ASIAWORKER1", gwInstance, port), func(_ int, ready bool) bool { return ready }); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("ready after reboot %d: %v", boot, err)
|
||||
}
|
||||
}
|
||||
c.assertNoSeedSent(t, g.seed())
|
||||
g.assertNoSeedLeak(nil, g.seed())
|
||||
}
|
||||
|
||||
// A store bound to a key other than this worker's is never checked, and a
|
||||
// replaced worker cannot read the store at all; neither is ever ready.
|
||||
func TestGateway_ReadyRefusesWrongKeyAndReplacedWorker(t *testing.T) {
|
||||
c := newGWCluster(t)
|
||||
first := c.launch(t, "claim-0001", 0, gwInstance)
|
||||
g := newGuest(t)
|
||||
c.guestFor(t, g, "ASIAWORKER1", gwInstance)
|
||||
if _, err := g.boot(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
other, _ := nkeys.CreateUser()
|
||||
otherPub, _ := other.PublicKey()
|
||||
c.bindStore(t, otherPub, "127.0.0.1")
|
||||
port := storeServer(t, otherPub)
|
||||
never := func(call int, ready bool) bool {
|
||||
if ready {
|
||||
t.Fatal("ready against a store for another key")
|
||||
}
|
||||
return call == 3
|
||||
}
|
||||
_ = g.watch(c.readyFor(g, "ASIAWORKER1", gwInstance, port), never)
|
||||
if !slices.ContainsFunc(g.logs, func(l string) bool { return strings.Contains(l, ErrStoreKeyMismatch.Error()) }) {
|
||||
t.Fatalf("no mismatch reported: %v", g.logs)
|
||||
}
|
||||
|
||||
if err := c.ids.RecordStopped(context.Background(), first, gwInstance); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = g.watch(c.readyFor(g, "ASIAWORKER1", gwInstance, port), never)
|
||||
if !strings.Contains(g.logs[len(g.logs)-1], "AccessDenied") {
|
||||
t.Fatalf("stopped worker: %v", g.logs[len(g.logs)-1])
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
package workerboot
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/nats-io/nkeys"
|
||||
)
|
||||
|
||||
// StorePort is the port the store guest's NATS listens on.
|
||||
const StorePort = 4222
|
||||
|
||||
// NATSChecker connects with the worker key and asks JetStream for its account
|
||||
// info. It authenticates the client only: there is no TLS, so a pass says
|
||||
// something answering at the address accepted the key.
|
||||
type NATSChecker struct {
|
||||
// Port overrides StorePort, for tests against a local server.
|
||||
Port int
|
||||
// Dial bounds each connection attempt; zero is five seconds.
|
||||
Dial time.Duration
|
||||
}
|
||||
|
||||
var _ Checker = NATSChecker{}
|
||||
|
||||
// StoreUnreachableError is a store that did not answer yet; it is retried.
|
||||
type StoreUnreachableError struct{ Err error }
|
||||
|
||||
func (e *StoreUnreachableError) Error() string { return "store not answering: " + e.Err.Error() }
|
||||
func (e *StoreUnreachableError) Unwrap() error { return e.Err }
|
||||
func (e *StoreUnreachableError) Retryable() bool { return true }
|
||||
|
||||
// ErrStoreRefusedKey is a store that answered but refused the worker key.
|
||||
var ErrStoreRefusedKey = errors.New("the store refused the worker key")
|
||||
|
||||
// Check passes once JetStream answers for the worker key.
|
||||
func (c NATSChecker) Check(ctx context.Context, address string, kp nkeys.KeyPair) error {
|
||||
if ip := net.ParseIP(address); ip == nil || ip.To4() == nil {
|
||||
return errors.New("store address is not an IPv4 address")
|
||||
}
|
||||
pub, err := kp.PublicKey()
|
||||
if err != nil {
|
||||
return fmt.Errorf("derive worker public key: %w", err)
|
||||
}
|
||||
port := c.Port
|
||||
if port == 0 {
|
||||
port = StorePort
|
||||
}
|
||||
dial := c.Dial
|
||||
if dial == 0 {
|
||||
dial = 5 * time.Second
|
||||
}
|
||||
nc, err := nats.Connect("nats://"+net.JoinHostPort(address, strconv.Itoa(port)),
|
||||
nats.Nkey(pub, kp.Sign),
|
||||
nats.Timeout(dial),
|
||||
nats.NoReconnect(),
|
||||
nats.Name("spinifex-sqs-worker-readiness"),
|
||||
)
|
||||
if err != nil {
|
||||
if errors.Is(err, nats.ErrAuthorization) || errors.Is(err, nats.ErrAuthExpired) {
|
||||
return fmt.Errorf("%w: %w", ErrStoreRefusedKey, err)
|
||||
}
|
||||
return &StoreUnreachableError{Err: err}
|
||||
}
|
||||
defer nc.Close()
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
return &StoreUnreachableError{Err: err}
|
||||
}
|
||||
callCtx, cancel := context.WithTimeout(ctx, dial)
|
||||
defer cancel()
|
||||
if _, err := js.AccountInfo(callCtx); err != nil {
|
||||
return &StoreUnreachableError{Err: fmt.Errorf("jetstream account info: %w", err)}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
package workerboot
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/agents/sqs/workerkey"
|
||||
"github.com/nats-io/nkeys"
|
||||
)
|
||||
|
||||
// Store is the Region's store launch as the gateway reports it. Bound means a
|
||||
// VM was launched for the store, not that it serves.
|
||||
type Store struct {
|
||||
Bound bool
|
||||
Address string
|
||||
PublicKey string
|
||||
}
|
||||
|
||||
// StoreDescriber asks the gateway for this worker's store.
|
||||
type StoreDescriber interface {
|
||||
DescribeStore(ctx context.Context) (Store, error)
|
||||
}
|
||||
|
||||
// NewDescriber builds the gateway client the store's address comes from.
|
||||
type NewDescriber func(region, gatewayURL string) (StoreDescriber, error)
|
||||
|
||||
// Checker proves the store's JetStream answers this worker's key at address.
|
||||
// Only an error with Retryable() true is asked again.
|
||||
type Checker interface {
|
||||
Check(ctx context.Context, address string, kp nkeys.KeyPair) error
|
||||
}
|
||||
|
||||
// ReadyConfig is Config plus the readiness capabilities and cadence.
|
||||
type ReadyConfig struct {
|
||||
Config
|
||||
|
||||
NewDescriber NewDescriber
|
||||
Checker Checker
|
||||
// CallTimeout bounds one describe-and-check pass; zero is 30 seconds.
|
||||
CallTimeout time.Duration
|
||||
// Refresh is the wait after a pass before checking again; zero is 30s.
|
||||
Refresh time.Duration
|
||||
// Refused is the wait after a failure that cannot clear by itself, such
|
||||
// as a refused or mismatched key; zero is five minutes.
|
||||
Refused time.Duration
|
||||
}
|
||||
|
||||
// ErrStoreKeyMismatch is a bound store launched for a key other than the one
|
||||
// on this worker's disk; it is asked again only at the Refused cadence.
|
||||
var ErrStoreKeyMismatch = errors.New("the store was launched for a different worker key")
|
||||
|
||||
const readyFile = "store-ready"
|
||||
|
||||
// Watch keeps the readiness marker true until ctx ends: it re-proves the disk,
|
||||
// then checks the store with the worker's existing key every pass, writing the
|
||||
// marker on a pass and removing it on any failure and on return. It never
|
||||
// creates a key. NKey authenticates this client only, not the store server.
|
||||
func Watch(ctx context.Context, cfg ReadyConfig) error {
|
||||
marker := filepath.Join(cfg.RunDir, readyFile)
|
||||
defer func() { _ = os.Remove(marker) }()
|
||||
if err := os.Remove(marker); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return fmt.Errorf("clear stale store readiness: %w", err)
|
||||
}
|
||||
c, err := readContract(cfg.FwCfgDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := mountWorkerDisk(cfg.Config, c.serial); err != nil {
|
||||
return err
|
||||
}
|
||||
if cfg.NewDescriber == nil || cfg.Checker == nil {
|
||||
return errors.New("no gateway client or store checker is configured")
|
||||
}
|
||||
d, err := cfg.NewDescriber(c.region, c.gatewayURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("build gateway client: %w", err)
|
||||
}
|
||||
keys := workerkey.Store{Dir: filepath.Join(cfg.DataDir, keyDirName)}
|
||||
callTimeout := orDefault(cfg.CallTimeout, 30*time.Second)
|
||||
refresh := orDefault(cfg.Refresh, 30*time.Second)
|
||||
refused := orDefault(cfg.Refused, 5*time.Minute)
|
||||
|
||||
last := ""
|
||||
for {
|
||||
callCtx, cancel := context.WithTimeout(ctx, callTimeout)
|
||||
st, err := readyOnce(callCtx, cfg, d, keys)
|
||||
cancel()
|
||||
delay, state := refresh, ""
|
||||
if err == nil {
|
||||
err = writeReady(cfg.RunDir, st)
|
||||
}
|
||||
switch {
|
||||
case ctx.Err() != nil:
|
||||
return ctx.Err()
|
||||
case err == nil:
|
||||
state = fmt.Sprintf("store at %s answered the worker key (client-authenticated only; the store server is not authenticated)", st.Address)
|
||||
default:
|
||||
_ = os.Remove(marker)
|
||||
delay = cfg.Wait
|
||||
if !pending(err) && !retryable(err) && !errors.Is(err, errStorePending) {
|
||||
delay = refused
|
||||
}
|
||||
state = fmt.Sprintf("store not ready: %v", err)
|
||||
}
|
||||
// Only a change is logged, so a long wait does not flood the console.
|
||||
if state != last {
|
||||
cfg.Log(state)
|
||||
last = state
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(max(delay, time.Millisecond)):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func orDefault(d, def time.Duration) time.Duration {
|
||||
if d <= 0 {
|
||||
return def
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
var errStorePending = errors.New("the store is not yet bound")
|
||||
|
||||
func readyOnce(ctx context.Context, cfg ReadyConfig, d StoreDescriber, keys workerkey.Store) (Store, error) {
|
||||
st, err := d.DescribeStore(ctx)
|
||||
if err != nil {
|
||||
return Store{}, err
|
||||
}
|
||||
if !st.Bound {
|
||||
return Store{}, errStorePending
|
||||
}
|
||||
kp, err := keys.ForExistingStore(st.PublicKey)
|
||||
if err != nil {
|
||||
if errors.Is(err, workerkey.ErrKeyMismatch) {
|
||||
return Store{}, fmt.Errorf("%w: %w", ErrStoreKeyMismatch, err)
|
||||
}
|
||||
return Store{}, err
|
||||
}
|
||||
defer kp.Wipe()
|
||||
if err := cfg.Checker.Check(ctx, st.Address, kp); err != nil {
|
||||
return Store{}, err
|
||||
}
|
||||
return st, nil
|
||||
}
|
||||
|
||||
// writeReady publishes only the address and public key, never the seed.
|
||||
func writeReady(runDir string, st Store) error {
|
||||
if err := os.MkdirAll(runDir, 0o750); err != nil {
|
||||
return fmt.Errorf("create %s: %w", runDir, err)
|
||||
}
|
||||
body := "address=" + st.Address + "\npublic=" + st.PublicKey + "\n"
|
||||
if err := os.WriteFile(filepath.Join(runDir, readyFile), []byte(body), 0o600); err != nil {
|
||||
return fmt.Errorf("publish store readiness: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,438 @@
|
||||
package workerboot
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nkeys"
|
||||
)
|
||||
|
||||
// describer is the gateway's DescribeStore, answering from a script.
|
||||
type describer struct {
|
||||
answers []describeAnswer
|
||||
calls int
|
||||
}
|
||||
|
||||
type describeAnswer struct {
|
||||
st Store
|
||||
err error
|
||||
}
|
||||
|
||||
func (d *describer) DescribeStore(context.Context) (Store, error) {
|
||||
d.calls++
|
||||
a := d.answers[min(d.calls, len(d.answers))-1]
|
||||
return a.st, a.err
|
||||
}
|
||||
|
||||
type checker struct {
|
||||
errs []error
|
||||
calls int
|
||||
seen []string
|
||||
}
|
||||
|
||||
func (c *checker) Check(_ context.Context, address string, kp nkeys.KeyPair) error {
|
||||
c.calls++
|
||||
pub, _ := kp.PublicKey()
|
||||
c.seen = append(c.seen, address+"|"+pub)
|
||||
if len(c.errs) > 0 {
|
||||
err := c.errs[0]
|
||||
c.errs = c.errs[1:]
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// bootedGuest has run the boot step, so its disk holds a recorded key.
|
||||
func bootedGuest(t *testing.T) (*guest, string) {
|
||||
t.Helper()
|
||||
g := newGuest(t)
|
||||
g.contract()
|
||||
pub, err := g.boot()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return g, pub
|
||||
}
|
||||
|
||||
func (g *guest) readyCfg(d StoreDescriber, c Checker) ReadyConfig {
|
||||
return ReadyConfig{
|
||||
Config: g.cfg,
|
||||
NewDescriber: func(string, string) (StoreDescriber, error) { return d, nil },
|
||||
Checker: c,
|
||||
}
|
||||
}
|
||||
|
||||
// hooked asks stop before each describe, after the previous pass has written
|
||||
// or removed the marker, and ends the watch when stop says so.
|
||||
type hooked struct {
|
||||
d StoreDescriber
|
||||
marker string
|
||||
stop func(call int, ready bool) bool
|
||||
cancel context.CancelFunc
|
||||
calls int
|
||||
}
|
||||
|
||||
func (h *hooked) DescribeStore(ctx context.Context) (Store, error) {
|
||||
h.calls++
|
||||
_, err := os.Stat(h.marker)
|
||||
if h.stop(h.calls, err == nil) {
|
||||
h.cancel()
|
||||
return Store{}, context.Canceled
|
||||
}
|
||||
return h.d.DescribeStore(ctx)
|
||||
}
|
||||
|
||||
// watch runs Watch with millisecond cadences until stop ends it, then checks
|
||||
// the marker was removed on the way out.
|
||||
func (g *guest) watch(cfg ReadyConfig, stop func(call int, ready bool) bool) error {
|
||||
g.t.Helper()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
inner := cfg.NewDescriber
|
||||
cfg.NewDescriber = func(region, url string) (StoreDescriber, error) {
|
||||
d, err := inner(region, url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &hooked{d: d, marker: filepath.Join(cfg.RunDir, readyFile), stop: stop, cancel: cancel}, nil
|
||||
}
|
||||
for _, p := range []*time.Duration{&cfg.Wait, &cfg.Refresh, &cfg.Refused} {
|
||||
if *p == 0 {
|
||||
*p = time.Millisecond
|
||||
}
|
||||
}
|
||||
err := Watch(ctx, cfg)
|
||||
if errors.Is(ctx.Err(), context.DeadlineExceeded) {
|
||||
g.t.Fatal("the watch never reached its stop condition")
|
||||
}
|
||||
g.assertNotReady()
|
||||
return err
|
||||
}
|
||||
|
||||
func bound(pub string) describeAnswer {
|
||||
return describeAnswer{st: Store{Bound: true, Address: "10.240.0.9", PublicKey: pub}}
|
||||
}
|
||||
|
||||
func (g *guest) marker() string {
|
||||
g.t.Helper()
|
||||
b, _ := os.ReadFile(filepath.Join(g.cfg.RunDir, readyFile))
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// A store launched long after the worker booted, far past the old 15-minute
|
||||
// bound of 180 attempts, still becomes ready; nothing is published before.
|
||||
func TestWatch_StoreLaunchedLateBecomesReady(t *testing.T) {
|
||||
g, pub := bootedGuest(t)
|
||||
answers := make([]describeAnswer, 201)
|
||||
answers[200] = bound(pub)
|
||||
d := &describer{answers: answers}
|
||||
c := &checker{errs: []error{&StoreUnreachableError{Err: errors.New("connection refused")}}}
|
||||
published := ""
|
||||
err := g.watch(g.readyCfg(d, c), func(call int, ready bool) bool {
|
||||
if ready {
|
||||
published = g.marker()
|
||||
return true
|
||||
}
|
||||
if call > 202 {
|
||||
t.Fatalf("not ready by call %d", call)
|
||||
}
|
||||
return false
|
||||
})
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("watch = %v", err)
|
||||
}
|
||||
if d.calls != 202 || c.calls != 2 || c.seen[0] != "10.240.0.9|"+pub {
|
||||
t.Fatalf("describe %d, check %d, seen %v", d.calls, c.calls, c.seen)
|
||||
}
|
||||
if published != "address=10.240.0.9\npublic="+pub+"\n" {
|
||||
t.Fatalf("published %q", published)
|
||||
}
|
||||
notReady := 0
|
||||
for _, l := range g.logs {
|
||||
if strings.HasPrefix(l, "store not ready") {
|
||||
notReady++
|
||||
}
|
||||
}
|
||||
if notReady > 2 {
|
||||
t.Fatalf("%d not-ready lines for two distinct states", notReady)
|
||||
}
|
||||
}
|
||||
|
||||
// After a pass, the marker goes as soon as a later check fails, and returns
|
||||
// once the store answers again.
|
||||
func TestWatch_LossAfterReadyRemovesTheMarker(t *testing.T) {
|
||||
cases := map[string]func(pub string) (*describer, *checker){
|
||||
"store stops answering": func(pub string) (*describer, *checker) {
|
||||
return &describer{answers: []describeAnswer{bound(pub)}},
|
||||
&checker{errs: []error{nil, &StoreUnreachableError{Err: errors.New("connection refused")}, nil}}
|
||||
},
|
||||
"gateway reply lost": func(pub string) (*describer, *checker) {
|
||||
return &describer{answers: []describeAnswer{bound(pub), {err: transient{}}, bound(pub)}}, &checker{}
|
||||
},
|
||||
"store no longer bound": func(pub string) (*describer, *checker) {
|
||||
return &describer{answers: []describeAnswer{bound(pub), {}, bound(pub)}}, &checker{}
|
||||
},
|
||||
"store now refuses the key": func(pub string) (*describer, *checker) {
|
||||
return &describer{answers: []describeAnswer{bound(pub)}}, &checker{errs: []error{nil, ErrStoreRefusedKey, nil}}
|
||||
},
|
||||
}
|
||||
for name, mk := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
g, pub := bootedGuest(t)
|
||||
d, c := mk(pub)
|
||||
var seen []bool
|
||||
if err := g.watch(g.readyCfg(d, c), func(call int, ready bool) bool {
|
||||
seen = append(seen, ready)
|
||||
return call == 4
|
||||
}); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("watch = %v", err)
|
||||
}
|
||||
if want := []bool{false, true, false, true}; !slices.Equal(seen, want) {
|
||||
t.Fatalf("marker before each pass = %v, want %v", seen, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A marker left by an earlier run is cleared before the first pass.
|
||||
func TestWatch_ClearsAStaleMarker(t *testing.T) {
|
||||
g, pub := bootedGuest(t)
|
||||
mustWrite(t, filepath.Join(g.cfg.RunDir, readyFile), "address=10.0.0.1\n")
|
||||
d := &describer{answers: []describeAnswer{{}}}
|
||||
_ = g.watch(g.readyCfg(d, &checker{}), func(call int, ready bool) bool {
|
||||
if call == 1 && ready {
|
||||
t.Fatal("a stale marker survived into the first pass")
|
||||
}
|
||||
return call == 2
|
||||
})
|
||||
_ = pub
|
||||
}
|
||||
|
||||
// Failures that cannot clear by themselves wait the Refused cadence: in the
|
||||
// window, each is tried once and never published.
|
||||
func TestWatch_FinalFailuresDoNotSpin(t *testing.T) {
|
||||
other, _ := nkeys.CreateUser()
|
||||
otherPub, _ := other.PublicKey()
|
||||
cases := map[string]struct {
|
||||
d func(pub string) *describer
|
||||
c *checker
|
||||
checks int
|
||||
}{
|
||||
"store refuses the key": {func(pub string) *describer { return &describer{answers: []describeAnswer{bound(pub)}} },
|
||||
&checker{errs: []error{ErrStoreRefusedKey}}, 1},
|
||||
"store launched for another key": {func(string) *describer { return &describer{answers: []describeAnswer{bound(otherPub)}} },
|
||||
&checker{}, 0},
|
||||
"unclassified gateway refusal": {func(string) *describer {
|
||||
return &describer{answers: []describeAnswer{{err: errors.New("InvalidParameterValue")}}}
|
||||
}, &checker{}, 0},
|
||||
}
|
||||
for name, tc := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
g, pub := bootedGuest(t)
|
||||
g.cfg.Wait = time.Millisecond
|
||||
d := tc.d(pub)
|
||||
cfg := g.readyCfg(d, tc.c)
|
||||
cfg.Refresh, cfg.Refused = time.Millisecond, time.Hour
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond)
|
||||
defer cancel()
|
||||
if err := Watch(ctx, cfg); !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("watch = %v", err)
|
||||
}
|
||||
if d.calls != 1 || tc.c.calls != tc.checks {
|
||||
t.Fatalf("describe %d, check %d in the window", d.calls, tc.c.calls)
|
||||
}
|
||||
g.assertNotReady()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A mismatched store is asked again at the Refused cadence, so a store the
|
||||
// operator relaunches for this worker's key is picked up.
|
||||
func TestWatch_RecoversWhenTheStoreIsRelaunchedForThisKey(t *testing.T) {
|
||||
g, pub := bootedGuest(t)
|
||||
other, _ := nkeys.CreateUser()
|
||||
otherPub, _ := other.PublicKey()
|
||||
d := &describer{answers: []describeAnswer{bound(otherPub), bound(pub)}}
|
||||
c := &checker{}
|
||||
if err := g.watch(g.readyCfg(d, c), func(_ int, ready bool) bool { return ready }); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("watch = %v", err)
|
||||
}
|
||||
if d.calls != 2 || c.calls != 1 {
|
||||
t.Fatalf("describe %d, check %d", d.calls, c.calls)
|
||||
}
|
||||
}
|
||||
|
||||
// A key missing from the disk is never replaced, however often it is asked.
|
||||
func TestWatch_LostKeyIsNeverReplaced(t *testing.T) {
|
||||
g, pub := bootedGuest(t)
|
||||
dir := filepath.Join(g.cfg.DataDir, keyDirName)
|
||||
entries, _ := os.ReadDir(dir)
|
||||
for _, e := range entries {
|
||||
if err := os.Remove(filepath.Join(dir, e.Name())); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
d := &describer{answers: []describeAnswer{bound(pub)}}
|
||||
c := &checker{}
|
||||
_ = g.watch(g.readyCfg(d, c), func(call int, ready bool) bool {
|
||||
if ready {
|
||||
t.Fatal("ready without a key")
|
||||
}
|
||||
return call == 5
|
||||
})
|
||||
if c.calls != 0 {
|
||||
t.Fatalf("checked %d times without a key", c.calls)
|
||||
}
|
||||
if after, _ := os.ReadDir(dir); len(after) != 0 {
|
||||
t.Fatal("a replacement key was written")
|
||||
}
|
||||
}
|
||||
|
||||
// Readiness never runs on a disk that is not proven mounted.
|
||||
func TestWatch_RefusesWithoutTheMountedWorkerDisk(t *testing.T) {
|
||||
g, pub := bootedGuest(t)
|
||||
g.resetMounts()
|
||||
g.disk.noopMount = true
|
||||
d := &describer{answers: []describeAnswer{bound(pub)}}
|
||||
if err := Watch(context.Background(), g.readyCfg(d, &checker{})); err == nil || d.calls != 0 {
|
||||
t.Fatalf("err = %v, describe %d", err, d.calls)
|
||||
}
|
||||
}
|
||||
|
||||
// After a restart the same key reaches readiness again with no new key.
|
||||
func TestWatch_SurvivesARestart(t *testing.T) {
|
||||
g, pub := bootedGuest(t)
|
||||
for range 2 {
|
||||
d := &describer{answers: []describeAnswer{bound(pub)}}
|
||||
if err := g.watch(g.readyCfg(d, &checker{}), func(_ int, ready bool) bool { return ready }); !errors.Is(err, context.Canceled) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
g.resetMounts()
|
||||
again, err := g.boot()
|
||||
if err != nil || again != pub {
|
||||
t.Fatalf("reboot = %q %v", again, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (g *guest) assertNotReady() {
|
||||
g.t.Helper()
|
||||
if _, err := os.Stat(filepath.Join(g.cfg.RunDir, readyFile)); !os.IsNotExist(err) {
|
||||
g.t.Fatalf("readiness was published: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func startStore(t *testing.T, pub string, port int) *server.Server {
|
||||
t.Helper()
|
||||
opts := &server.Options{Host: "127.0.0.1", Port: port, JetStream: true, StoreDir: t.TempDir(), NoLog: true,
|
||||
NoSigs: true, Nkeys: []*server.NkeyUser{{Nkey: pub}}}
|
||||
ns, err := server.NewServer(opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go ns.Start()
|
||||
if !ns.ReadyForConnections(10 * time.Second) {
|
||||
t.Fatal("store server did not start")
|
||||
}
|
||||
t.Cleanup(ns.Shutdown)
|
||||
return ns
|
||||
}
|
||||
|
||||
// End to end on the guest with a real store: not ready before it serves,
|
||||
// ready once it does, not ready again once it stops, and the marker never
|
||||
// holds the seed.
|
||||
func TestWatch_RealStoreStartsLateThenStops(t *testing.T) {
|
||||
g, pub := bootedGuest(t)
|
||||
seedFile, _ := os.ReadDir(filepath.Join(g.cfg.DataDir, keyDirName))
|
||||
key, _ := os.ReadFile(filepath.Join(g.cfg.DataDir, keyDirName, seedFile[0].Name()))
|
||||
l, _ := net.Listen("tcp", "127.0.0.1:0")
|
||||
port := l.Addr().(*net.TCPAddr).Port
|
||||
_ = l.Close()
|
||||
d := &describer{answers: []describeAnswer{{st: Store{Bound: true, Address: "127.0.0.1", PublicKey: pub}}}}
|
||||
cfg := g.readyCfg(d, NATSChecker{Port: port, Dial: time.Second})
|
||||
var ns *server.Server
|
||||
phase := 0
|
||||
err := g.watch(cfg, func(call int, ready bool) bool {
|
||||
switch {
|
||||
case phase == 0 && call == 3:
|
||||
if ready {
|
||||
t.Fatal("ready before the store served")
|
||||
}
|
||||
ns, phase = startStore(t, pub, port), 1
|
||||
case phase == 1 && ready:
|
||||
if strings.Contains(g.marker(), strings.TrimSpace(string(key))) {
|
||||
t.Fatal("the seed was published")
|
||||
}
|
||||
ns.Shutdown()
|
||||
ns.WaitForShutdown()
|
||||
phase = 2
|
||||
case phase == 2:
|
||||
if ready {
|
||||
t.Fatal("still ready after the store stopped")
|
||||
}
|
||||
return true
|
||||
}
|
||||
return false
|
||||
})
|
||||
if !errors.Is(err, context.Canceled) || phase != 2 {
|
||||
t.Fatalf("watch = %v, phase %d", err, phase)
|
||||
}
|
||||
}
|
||||
|
||||
// storeServer is a real NATS server with JetStream that admits only pub.
|
||||
func storeServer(t *testing.T, pub string) int {
|
||||
t.Helper()
|
||||
opts := &server.Options{
|
||||
Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true,
|
||||
Nkeys: []*server.NkeyUser{{Nkey: pub}},
|
||||
}
|
||||
ns, err := server.NewServer(opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go ns.Start()
|
||||
if !ns.ReadyForConnections(10 * time.Second) {
|
||||
t.Fatal("store server did not start")
|
||||
}
|
||||
t.Cleanup(ns.Shutdown)
|
||||
return ns.Addr().(*net.TCPAddr).Port
|
||||
}
|
||||
|
||||
// The real checker passes only with the admitted key; a wrong key is refused
|
||||
// as final, and nothing listening is retryable.
|
||||
func TestNATSChecker_AgainstARealStore(t *testing.T) {
|
||||
kp, _ := nkeys.CreateUser()
|
||||
pub, _ := kp.PublicKey()
|
||||
port := storeServer(t, pub)
|
||||
c := NATSChecker{Port: port, Dial: 2 * time.Second}
|
||||
if err := c.Check(context.Background(), "127.0.0.1", kp); err != nil {
|
||||
t.Fatalf("admitted key: %v", err)
|
||||
}
|
||||
other, _ := nkeys.CreateUser()
|
||||
err := c.Check(context.Background(), "127.0.0.1", other)
|
||||
if !errors.Is(err, ErrStoreRefusedKey) || retryable(err) {
|
||||
t.Fatalf("wrong key: %v", err)
|
||||
}
|
||||
seed, _ := kp.Seed()
|
||||
if strings.Contains(err.Error(), string(seed)) {
|
||||
t.Fatal("the error echoes a seed")
|
||||
}
|
||||
l, _ := net.Listen("tcp", "127.0.0.1:0")
|
||||
closed := l.Addr().(*net.TCPAddr).Port
|
||||
_ = l.Close()
|
||||
err = NATSChecker{Port: closed, Dial: time.Second}.Check(context.Background(), "127.0.0.1", kp)
|
||||
if !retryable(err) {
|
||||
t.Fatalf("nothing listening: %v is not retryable", err)
|
||||
}
|
||||
if err := c.Check(context.Background(), "store.example", kp); err == nil || retryable(err) {
|
||||
t.Fatalf("hostname accepted: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package workerboot
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// SystemDisk checks with e2fsck and mounts with the mount syscall.
|
||||
type SystemDisk struct {
|
||||
E2fsck string
|
||||
}
|
||||
|
||||
var _ Disk = SystemDisk{}
|
||||
|
||||
// Check runs a preen check: 0 is clean and 1 corrected; anything higher,
|
||||
// including a disk that is not ext4, refuses. Nothing formats.
|
||||
func (d SystemDisk) Check(dev string) error {
|
||||
out, err := exec.Command(d.E2fsck, "-p", dev).CombinedOutput()
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if exitErr, ok := errors.AsType[*exec.ExitError](err); ok && exitErr.ExitCode() == 1 {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("e2fsck refused: %w: %s", err, strings.Join(strings.Fields(string(out)), " "))
|
||||
}
|
||||
|
||||
// Mount mounts the disk ext4 with no device nodes, setuid or execution.
|
||||
func (SystemDisk) Mount(dev, dir string) error {
|
||||
return syscall.Mount(dev, dir, "ext4", syscall.MS_NODEV|syscall.MS_NOSUID|syscall.MS_NOEXEC, "")
|
||||
}
|
||||
|
||||
// IsBlockDevice reports whether path is a block (not character) device node.
|
||||
func IsBlockDevice(path string) bool {
|
||||
fi, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
m := fi.Mode()
|
||||
return m&os.ModeDevice != 0 && m&os.ModeCharDevice == 0
|
||||
}
|
||||
@@ -0,0 +1,361 @@
|
||||
// Package workerboot prepares the shared SQS worker guest: it selects exactly
|
||||
// the expected data disk by serial, checks and mounts it, proves that disk is
|
||||
// what is mounted, and only then loads or creates the worker's store key on it
|
||||
// under the current first-store decision the gateway returns.
|
||||
package workerboot
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/agents/sqs/workerkey"
|
||||
"github.com/nats-io/nkeys"
|
||||
)
|
||||
|
||||
// Guest contract, read from fw_cfg under opt/spinifex/sqs-worker/. None of it
|
||||
// is a decision: whether a store key may be created is asked of the gateway.
|
||||
//
|
||||
// data-serial virtio-blk serial of the worker's own ext4 data disk
|
||||
// region the Region the worker signs its gateway calls for
|
||||
// gateway-url https URL of the gateway on this node's mgmt bridge
|
||||
const (
|
||||
keyDirName = "sqs-worker-key"
|
||||
maxFwCfg = 128
|
||||
)
|
||||
|
||||
var regionPattern = regexp.MustCompile(`^[a-z]{2}(-[a-z]+)+-[0-9]+$`)
|
||||
|
||||
// NewClaimer builds the gateway client the first-store decision comes from.
|
||||
type NewClaimer func(region, gatewayURL string) (workerkey.Claimer, error)
|
||||
|
||||
// Disk checks and mounts the worker data disk. The guest never formats.
|
||||
type Disk interface {
|
||||
Check(dev string) error
|
||||
Mount(dev, dir string) error
|
||||
}
|
||||
|
||||
// Config holds every path the boot step reads or writes, so tests can run it
|
||||
// against a fake fw_cfg, sysfs and mount table.
|
||||
type Config struct {
|
||||
FwCfgDir string
|
||||
SysBlock string
|
||||
DevDir string
|
||||
MountInfo string
|
||||
DataDir string
|
||||
RunDir string
|
||||
Disk Disk
|
||||
// IsBlockDevice reports whether a /dev node is a block device.
|
||||
IsBlockDevice func(path string) bool
|
||||
NewClaimer NewClaimer
|
||||
// Attempts bounds the decision calls, Wait apart. Only a lost reply, a
|
||||
// retryable refusal or a pending authorization is asked again.
|
||||
Attempts int
|
||||
Wait time.Duration
|
||||
Log func(msg string)
|
||||
}
|
||||
|
||||
type contract struct {
|
||||
serial, region, gatewayURL string
|
||||
}
|
||||
|
||||
// Boot runs the boot step and returns the worker's public key. The key is
|
||||
// loaded or created only under the gateway's current decision; any missing,
|
||||
// malformed or ambiguous input or answer refuses.
|
||||
func Boot(ctx context.Context, cfg Config) (string, error) {
|
||||
c, err := readContract(cfg.FwCfgDir)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := mountWorkerDisk(cfg, c.serial); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if cfg.NewClaimer == nil {
|
||||
return "", errors.New("no gateway client is configured")
|
||||
}
|
||||
claimer, err := cfg.NewClaimer(c.region, c.gatewayURL)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("build gateway client: %w", err)
|
||||
}
|
||||
kp, err := acquire(ctx, cfg, workerkey.Store{Dir: filepath.Join(cfg.DataDir, keyDirName)}, claimer)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer kp.Wipe()
|
||||
pub, err := kp.PublicKey()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("derive worker public key: %w", err)
|
||||
}
|
||||
if err := writePublicKey(cfg.RunDir, pub); err != nil {
|
||||
return "", err
|
||||
}
|
||||
cfg.Log("store key ready: public=" + pub)
|
||||
return pub, nil
|
||||
}
|
||||
|
||||
// mountWorkerDisk selects the disk with serial, checks and mounts it at
|
||||
// DataDir unless it is already there, and proves from the mount table that
|
||||
// exactly that disk is what is mounted.
|
||||
func mountWorkerDisk(cfg Config, serial string) error {
|
||||
name, err := findDisk(cfg, serial)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dev := filepath.Join(cfg.DevDir, name)
|
||||
majMin, err := readTrimmed(filepath.Join(cfg.SysBlock, name, "dev"))
|
||||
if err != nil {
|
||||
return fmt.Errorf("read device number of %s: %w", name, err)
|
||||
}
|
||||
|
||||
mounted, err := mountedDevice(cfg.MountInfo, cfg.DataDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch mounted {
|
||||
case majMin:
|
||||
cfg.Log(fmt.Sprintf("%s already mounted at %s", dev, cfg.DataDir))
|
||||
case "":
|
||||
if err := cfg.Disk.Check(dev); err != nil {
|
||||
return fmt.Errorf("check %s: %w", dev, err)
|
||||
}
|
||||
if err := os.MkdirAll(cfg.DataDir, 0o750); err != nil {
|
||||
return fmt.Errorf("create mountpoint %s: %w", cfg.DataDir, err)
|
||||
}
|
||||
if err := cfg.Disk.Mount(dev, cfg.DataDir); err != nil {
|
||||
return fmt.Errorf("mount %s at %s: %w", dev, cfg.DataDir, err)
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("%s holds device %s, not the worker disk %s", cfg.DataDir, mounted, majMin)
|
||||
}
|
||||
|
||||
// Proven from the mount table, not inferred from the mount call: an empty
|
||||
// mountpoint directory must never receive a key.
|
||||
mounted, err = mountedDevice(cfg.MountInfo, cfg.DataDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if mounted != majMin {
|
||||
return fmt.Errorf("worker disk %s (%s) is not mounted at %s", dev, majMin, cfg.DataDir)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func readContract(dir string) (contract, error) {
|
||||
var c contract
|
||||
var err error
|
||||
if c.serial, err = readFwCfg(dir, "data-serial"); err != nil {
|
||||
return c, err
|
||||
}
|
||||
if !validSerial(c.serial) {
|
||||
return c, errors.New("data-serial is not a volume serial")
|
||||
}
|
||||
if c.region, err = readFwCfg(dir, "region"); err != nil {
|
||||
return c, err
|
||||
}
|
||||
if !regionPattern.MatchString(c.region) {
|
||||
return c, errors.New("region is not a Region name")
|
||||
}
|
||||
if c.gatewayURL, err = readFwCfg(dir, "gateway-url"); err != nil {
|
||||
return c, err
|
||||
}
|
||||
if u, err := url.Parse(c.gatewayURL); err != nil || u.Scheme != "https" || u.Host == "" {
|
||||
return c, errors.New("gateway-url is not an https URL")
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// acquire runs the decision and key step, repeating it only on an error that
|
||||
// says it is safe to repeat, including a denial a not-yet-bound worker gets.
|
||||
// A denial creates nothing, and each repeat re-asks the gateway.
|
||||
func acquire(ctx context.Context, cfg Config, store workerkey.Store, c workerkey.Claimer) (kp nkeys.KeyPair, err error) {
|
||||
attempts := max(cfg.Attempts, 1)
|
||||
for i := range attempts {
|
||||
kp, err = store.Acquire(ctx, c)
|
||||
if err == nil || i == attempts-1 {
|
||||
return kp, err
|
||||
}
|
||||
switch {
|
||||
case pending(err):
|
||||
cfg.Log(fmt.Sprintf("worker not yet authorized, asking again: %v", err))
|
||||
case retryable(err):
|
||||
cfg.Log(fmt.Sprintf("first-store decision not settled, retrying: %v", err))
|
||||
default:
|
||||
return kp, err
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case <-time.After(cfg.Wait):
|
||||
}
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
func pending(err error) bool {
|
||||
p, ok := errors.AsType[interface {
|
||||
error
|
||||
Pending() bool
|
||||
}](err)
|
||||
return ok && p.Pending()
|
||||
}
|
||||
|
||||
func retryable(err error) bool {
|
||||
r, ok := errors.AsType[interface {
|
||||
error
|
||||
Retryable() bool
|
||||
}](err)
|
||||
return ok && r.Retryable()
|
||||
}
|
||||
|
||||
// readFwCfg returns one fw_cfg value: a single token with at most one trailing
|
||||
// newline. The value is never echoed in an error.
|
||||
func readFwCfg(dir, name string) (string, error) {
|
||||
path := filepath.Join(dir, name, "raw")
|
||||
fi, err := os.Stat(path)
|
||||
if err != nil {
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return "", fmt.Errorf("fw_cfg %s not supplied: %w", name, fs.ErrNotExist)
|
||||
}
|
||||
return "", fmt.Errorf("fw_cfg %s unreadable", name)
|
||||
}
|
||||
if fi.Size() > maxFwCfg {
|
||||
return "", fmt.Errorf("fw_cfg %s is oversized", name)
|
||||
}
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("fw_cfg %s unreadable", name)
|
||||
}
|
||||
v := strings.TrimSuffix(string(b), "\n")
|
||||
if v == "" || strings.ContainsAny(v, " \t\r\n\v\f") {
|
||||
return "", fmt.Errorf("fw_cfg %s must be exactly one value", name)
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
|
||||
// A volume serial is the volume ID with its dash removed: vol + 17 hex.
|
||||
func validSerial(s string) bool {
|
||||
if len(s) != 20 || !strings.HasPrefix(s, "vol") {
|
||||
return false
|
||||
}
|
||||
for _, r := range s[3:] {
|
||||
if (r < '0' || r > '9') && (r < 'a' || r > 'f') {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// findDisk returns the block device name whose serial matches exactly once.
|
||||
func findDisk(cfg Config, serial string) (string, error) {
|
||||
entries, err := os.ReadDir(cfg.SysBlock)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("list block devices: %w", err)
|
||||
}
|
||||
var found []string
|
||||
for _, e := range entries {
|
||||
s, err := readTrimmed(filepath.Join(cfg.SysBlock, e.Name(), "serial"))
|
||||
if err == nil && s == serial {
|
||||
found = append(found, e.Name())
|
||||
}
|
||||
}
|
||||
if len(found) != 1 {
|
||||
return "", fmt.Errorf("expected one disk with serial %s, found %d", serial, len(found))
|
||||
}
|
||||
if !cfg.IsBlockDevice(filepath.Join(cfg.DevDir, found[0])) {
|
||||
return "", fmt.Errorf("disk %s has no block device node", found[0])
|
||||
}
|
||||
return found[0], nil
|
||||
}
|
||||
|
||||
// mountedDevice returns the major:minor of the topmost ext4 mount of a whole
|
||||
// filesystem at dir, or "" when nothing is mounted there.
|
||||
func mountedDevice(mountInfo, dir string) (string, error) {
|
||||
f, err := os.Open(mountInfo)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read mount table: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
dir = filepath.Clean(dir)
|
||||
var majMin string
|
||||
found := false
|
||||
sc := bufio.NewScanner(f)
|
||||
for sc.Scan() {
|
||||
fields := strings.Fields(sc.Text())
|
||||
sep := -1
|
||||
for i, fld := range fields {
|
||||
if fld == "-" {
|
||||
sep = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if sep < 6 || sep+1 >= len(fields) {
|
||||
continue
|
||||
}
|
||||
if unescapeMountPath(fields[4]) != dir {
|
||||
continue
|
||||
}
|
||||
found = true
|
||||
majMin = ""
|
||||
if fields[3] == "/" && fields[sep+1] == "ext4" {
|
||||
majMin = fields[2]
|
||||
}
|
||||
}
|
||||
if err := sc.Err(); err != nil {
|
||||
return "", fmt.Errorf("read mount table: %w", err)
|
||||
}
|
||||
if found && majMin == "" {
|
||||
return "", fmt.Errorf("%s is mounted, but not as a whole ext4 filesystem", dir)
|
||||
}
|
||||
return majMin, nil
|
||||
}
|
||||
|
||||
// unescapeMountPath decodes the octal escapes mountinfo uses for space, tab,
|
||||
// newline and backslash.
|
||||
func unescapeMountPath(s string) string {
|
||||
r := strings.NewReplacer(`\040`, " ", `\011`, "\t", `\012`, "\n", `\134`, `\`)
|
||||
return r.Replace(s)
|
||||
}
|
||||
|
||||
func readTrimmed(path string) (string, error) {
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return strings.TrimSpace(string(b)), nil
|
||||
}
|
||||
|
||||
// writePublicKey publishes the public key for later reporting; it is not secret.
|
||||
func writePublicKey(runDir, pub string) error {
|
||||
if err := os.MkdirAll(runDir, 0o750); err != nil {
|
||||
return fmt.Errorf("create %s: %w", runDir, err)
|
||||
}
|
||||
tmp, err := os.CreateTemp(runDir, ".public-key-*")
|
||||
if err != nil {
|
||||
return fmt.Errorf("write public key: %w", err)
|
||||
}
|
||||
defer os.Remove(tmp.Name())
|
||||
if _, err := tmp.WriteString(pub + "\n"); err != nil {
|
||||
tmp.Close()
|
||||
return fmt.Errorf("write public key: %w", err)
|
||||
}
|
||||
if err := tmp.Chmod(0o644); err != nil {
|
||||
tmp.Close()
|
||||
return fmt.Errorf("write public key: %w", err)
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
return fmt.Errorf("write public key: %w", err)
|
||||
}
|
||||
if err := os.Rename(tmp.Name(), filepath.Join(runDir, "public-key")); err != nil {
|
||||
return fmt.Errorf("write public key: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,583 @@
|
||||
package workerboot
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/agents/sqs/workerkey"
|
||||
"github.com/nats-io/nkeys"
|
||||
)
|
||||
|
||||
const (
|
||||
testSerial = "vol0a1b2c3d4e5f60718"
|
||||
testMajMin = "253:16"
|
||||
testRegion = "ap-southeast-2"
|
||||
testGatewayURL = "https://10.15.8.1:9999"
|
||||
)
|
||||
|
||||
// guest is a fake guest: fw_cfg, sysfs, /dev and a mount table the fake disk
|
||||
// edits the way a real mount would.
|
||||
type guest struct {
|
||||
t *testing.T
|
||||
root string
|
||||
cfg Config
|
||||
disk *fakeDisk
|
||||
auth *authority
|
||||
logs []string
|
||||
}
|
||||
|
||||
// authority stands for the domain's first-store record behind the gateway:
|
||||
// it authorizes creation until a key is recorded, then only that key.
|
||||
type authority struct {
|
||||
recorded string
|
||||
claimErrs []error
|
||||
// recordErrs fail RecordPublicKey; landed ones record before failing,
|
||||
// as a reply lost after the write.
|
||||
recordErrs []recordErr
|
||||
claims int
|
||||
sent []string
|
||||
builds int
|
||||
buildErr error
|
||||
}
|
||||
|
||||
type recordErr struct {
|
||||
err error
|
||||
landed bool
|
||||
}
|
||||
|
||||
// transient is a refusal the boot may repeat, as a lost reply.
|
||||
type transient struct{}
|
||||
|
||||
func (transient) Error() string { return "connection reset" }
|
||||
func (transient) Retryable() bool { return true }
|
||||
|
||||
// notBoundError is the denial a started but not yet bound worker receives.
|
||||
type notBoundError struct{}
|
||||
|
||||
func (notBoundError) Error() string { return "AccessDenied" }
|
||||
func (notBoundError) Pending() bool { return true }
|
||||
|
||||
func (a *authority) Claim(context.Context) (bool, string, error) {
|
||||
a.claims++
|
||||
if len(a.claimErrs) > 0 {
|
||||
err := a.claimErrs[0]
|
||||
a.claimErrs = a.claimErrs[1:]
|
||||
return false, "", err
|
||||
}
|
||||
return a.recorded == "", a.recorded, nil
|
||||
}
|
||||
|
||||
func (a *authority) RecordPublicKey(_ context.Context, pub string) error {
|
||||
a.sent = append(a.sent, pub)
|
||||
if a.recorded != "" && a.recorded != pub {
|
||||
return errors.New("InvalidParameterValue")
|
||||
}
|
||||
if len(a.recordErrs) > 0 {
|
||||
e := a.recordErrs[0]
|
||||
a.recordErrs = a.recordErrs[1:]
|
||||
if e.landed {
|
||||
a.recorded = pub
|
||||
}
|
||||
return e.err
|
||||
}
|
||||
a.recorded = pub
|
||||
return nil
|
||||
}
|
||||
|
||||
type fakeDisk struct {
|
||||
g *guest
|
||||
checkErr error
|
||||
mountErr error
|
||||
noopMount bool
|
||||
checks int
|
||||
mounts int
|
||||
}
|
||||
|
||||
func (d *fakeDisk) Check(string) error {
|
||||
d.checks++
|
||||
return d.checkErr
|
||||
}
|
||||
|
||||
func (d *fakeDisk) Mount(dev, dir string) error {
|
||||
d.mounts++
|
||||
if d.mountErr != nil {
|
||||
return d.mountErr
|
||||
}
|
||||
if !d.noopMount {
|
||||
d.g.appendMount(testMajMin, "/", dir, "ext4", dev)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func newGuest(t *testing.T) *guest {
|
||||
t.Helper()
|
||||
g := &guest{t: t, root: t.TempDir(), auth: &authority{}}
|
||||
g.disk = &fakeDisk{g: g}
|
||||
g.cfg = Config{
|
||||
FwCfgDir: filepath.Join(g.root, "fwcfg"),
|
||||
SysBlock: filepath.Join(g.root, "sys", "block"),
|
||||
DevDir: filepath.Join(g.root, "dev"),
|
||||
MountInfo: filepath.Join(g.root, "mountinfo"),
|
||||
DataDir: filepath.Join(g.root, "data"),
|
||||
RunDir: filepath.Join(g.root, "run"),
|
||||
Disk: g.disk,
|
||||
IsBlockDevice: func(p string) bool { _, err := os.Stat(p); return err == nil },
|
||||
NewClaimer: func(region, url string) (workerkey.Claimer, error) {
|
||||
g.auth.builds++
|
||||
if region != testRegion || url != testGatewayURL {
|
||||
t.Errorf("claimer built for %q %q", region, url)
|
||||
}
|
||||
if g.auth.buildErr != nil {
|
||||
return nil, g.auth.buildErr
|
||||
}
|
||||
return g.auth, nil
|
||||
},
|
||||
Attempts: 3,
|
||||
Log: func(msg string) { g.logs = append(g.logs, msg) },
|
||||
}
|
||||
for _, d := range []string{g.cfg.SysBlock, g.cfg.DevDir} {
|
||||
mustMkdir(t, d)
|
||||
}
|
||||
g.resetMounts()
|
||||
// An empty mountpoint, as the initramfs ships one.
|
||||
mustMkdir(t, g.cfg.DataDir)
|
||||
g.addDisk("vda", testSerial, testMajMin)
|
||||
return g
|
||||
}
|
||||
|
||||
func mustMkdir(t *testing.T, d string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(d, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func mustWrite(t *testing.T, path, content string) {
|
||||
t.Helper()
|
||||
mustMkdir(t, filepath.Dir(path))
|
||||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (g *guest) addDisk(name, serial, majMin string) {
|
||||
mustWrite(g.t, filepath.Join(g.cfg.SysBlock, name, "serial"), serial+"\n")
|
||||
mustWrite(g.t, filepath.Join(g.cfg.SysBlock, name, "dev"), majMin+"\n")
|
||||
mustWrite(g.t, filepath.Join(g.cfg.DevDir, name), "")
|
||||
}
|
||||
|
||||
func (g *guest) fw(name, value string) {
|
||||
mustWrite(g.t, filepath.Join(g.cfg.FwCfgDir, name, "raw"), value)
|
||||
}
|
||||
|
||||
// contract supplies the whole fw_cfg contract.
|
||||
func (g *guest) contract() {
|
||||
g.fw("data-serial", testSerial+"\n")
|
||||
g.fw("region", testRegion+"\n")
|
||||
g.fw("gateway-url", testGatewayURL+"\n")
|
||||
}
|
||||
|
||||
func (g *guest) boot() (string, error) {
|
||||
g.t.Helper()
|
||||
return Boot(context.Background(), g.cfg)
|
||||
}
|
||||
|
||||
// assertSentOnlyPublicKeys checks nothing but valid public keys left the guest.
|
||||
func (g *guest) assertSentOnlyPublicKeys(seed string) {
|
||||
g.t.Helper()
|
||||
for _, v := range g.auth.sent {
|
||||
if !nkeys.IsValidPublicUserKey(v) || (seed != "" && strings.Contains(v, seed)) {
|
||||
g.t.Fatal("the guest sent something other than a public key")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// seedKey stands for a worker key created when the store was first made and
|
||||
// recorded by the domain; it returns that key's public half.
|
||||
func (g *guest) seedKey() string {
|
||||
g.t.Helper()
|
||||
kp, err := workerkey.Store{Dir: g.keyDir()}.ForNewStore()
|
||||
if err != nil {
|
||||
g.t.Fatalf("seed worker key: %v", err)
|
||||
}
|
||||
defer kp.Wipe()
|
||||
pub, err := kp.PublicKey()
|
||||
if err != nil {
|
||||
g.t.Fatal(err)
|
||||
}
|
||||
g.auth.recorded = pub
|
||||
return pub
|
||||
}
|
||||
|
||||
func (g *guest) keyFile() string { return filepath.Join(g.keyDir(), "store-client.nk") }
|
||||
|
||||
// resetMounts is a reboot: only the initramfs root is mounted.
|
||||
func (g *guest) resetMounts() {
|
||||
mustWrite(g.t, g.cfg.MountInfo, "1 0 0:2 / / rw - rootfs rootfs rw\n")
|
||||
}
|
||||
|
||||
func (g *guest) appendMount(majMin, root, dir, fstype, source string) {
|
||||
f, err := os.OpenFile(g.cfg.MountInfo, os.O_APPEND|os.O_WRONLY, 0)
|
||||
if err != nil {
|
||||
g.t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
if _, err := fmt.Fprintf(f, "30 1 %s %s %s rw,nodev - %s %s rw\n", majMin, root, strings.ReplaceAll(dir, " ", `\040`), fstype, source); err != nil {
|
||||
g.t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (g *guest) keyDir() string { return filepath.Join(g.cfg.DataDir, keyDirName) }
|
||||
|
||||
func (g *guest) seed() string {
|
||||
b, err := os.ReadFile(filepath.Join(g.keyDir(), "store-client.nk"))
|
||||
if err != nil {
|
||||
g.t.Fatalf("read key: %v", err)
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func (g *guest) assertNoKey() {
|
||||
g.t.Helper()
|
||||
if _, err := os.Lstat(g.keyDir()); !errors.Is(err, os.ErrNotExist) {
|
||||
g.t.Fatalf("a key directory exists although boot refused: %v", err)
|
||||
}
|
||||
if _, err := os.Lstat(filepath.Join(g.cfg.RunDir, "public-key")); !errors.Is(err, os.ErrNotExist) {
|
||||
g.t.Fatal("a public key was published although boot refused")
|
||||
}
|
||||
}
|
||||
|
||||
func (g *guest) assertNoSeedLeak(err error, seed string) {
|
||||
g.t.Helper()
|
||||
if seed == "" {
|
||||
return
|
||||
}
|
||||
if err != nil && strings.Contains(err.Error(), seed) {
|
||||
g.t.Fatal("an error exposes the seed")
|
||||
}
|
||||
for _, l := range g.logs {
|
||||
if strings.Contains(l, seed) {
|
||||
g.t.Fatal("a log line exposes the seed")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// otherPub is a valid store key the worker does not hold.
|
||||
func otherPub(t *testing.T) string {
|
||||
t.Helper()
|
||||
pub, _ := userKey(t)
|
||||
return pub
|
||||
}
|
||||
|
||||
func userKey(t *testing.T) (pub, seed string) {
|
||||
t.Helper()
|
||||
kp, err := nkeys.CreateUser()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p, err := kp.PublicKey()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s, err := kp.Seed()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p, string(s)
|
||||
}
|
||||
|
||||
func TestBoot_FirstStoreCreatesKeyThenReusesItAcrossReboots(t *testing.T) {
|
||||
g := newGuest(t)
|
||||
g.contract()
|
||||
pub, err := g.boot()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seed := g.seed()
|
||||
if g.auth.recorded != pub || len(g.auth.sent) != 1 {
|
||||
t.Fatalf("recorded %q after %d sends, want %s once", g.auth.recorded, len(g.auth.sent), pub)
|
||||
}
|
||||
for boot := 2; boot <= 3; boot++ {
|
||||
g.resetMounts()
|
||||
got, err := g.boot()
|
||||
if err != nil || got != pub || g.seed() != seed {
|
||||
t.Fatalf("boot %d = %q, %v; the worker key must not change", boot, got, err)
|
||||
}
|
||||
}
|
||||
if len(g.auth.sent) != 1 {
|
||||
t.Fatalf("a recorded key was sent again: %d sends", len(g.auth.sent))
|
||||
}
|
||||
if g.disk.mounts != 3 {
|
||||
t.Fatalf("mounts = %d, want one per boot", g.disk.mounts)
|
||||
}
|
||||
published, err := os.ReadFile(filepath.Join(g.cfg.RunDir, "public-key"))
|
||||
if err != nil || strings.TrimSpace(string(published)) != pub {
|
||||
t.Fatalf("published key = %q, %v", published, err)
|
||||
}
|
||||
g.assertSentOnlyPublicKeys(seed)
|
||||
g.assertNoSeedLeak(nil, seed)
|
||||
}
|
||||
|
||||
// Once the domain holds a key, losing the guest's copy never yields a new
|
||||
// one, whatever a replayed first-start fw_cfg says.
|
||||
func TestBoot_KeyLostAfterRecordingIsNeverReplaced(t *testing.T) {
|
||||
g := newGuest(t)
|
||||
g.contract()
|
||||
g.seedKey()
|
||||
if err := os.Remove(g.keyFile()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
g.fw("store-status", "new\n")
|
||||
g.resetMounts()
|
||||
_, err := g.boot()
|
||||
if !errors.Is(err, workerkey.ErrKeyMissing) {
|
||||
t.Fatalf("err = %v, want ErrKeyMissing", err)
|
||||
}
|
||||
if _, err := os.Lstat(g.keyFile()); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatal("a replacement key was created for an existing store")
|
||||
}
|
||||
if len(g.auth.sent) != 0 || g.auth.claims != 1 {
|
||||
t.Fatalf("sends=%d claims=%d; a final refusal must not be retried or answered", len(g.auth.sent), g.auth.claims)
|
||||
}
|
||||
if _, err := os.Lstat(filepath.Join(g.cfg.RunDir, "public-key")); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatal("a public key was published")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBoot_RecordedKeyMismatchRefuses(t *testing.T) {
|
||||
g := newGuest(t)
|
||||
g.contract()
|
||||
g.seedKey()
|
||||
seed := g.seed()
|
||||
g.auth.recorded = otherPub(t)
|
||||
_, err := g.boot()
|
||||
if !errors.Is(err, workerkey.ErrKeyMismatch) {
|
||||
t.Fatalf("err = %v, want ErrKeyMismatch", err)
|
||||
}
|
||||
if g.seed() != seed || len(g.auth.sent) != 0 {
|
||||
t.Fatal("a mismatch changed or sent the worker key")
|
||||
}
|
||||
g.assertNoSeedLeak(err, seed)
|
||||
}
|
||||
|
||||
// A reply lost after the key was recorded is retried, and the retry is
|
||||
// answered with that same key: one key, recorded once.
|
||||
func TestBoot_LostRecordReplyKeepsTheSameKey(t *testing.T) {
|
||||
for _, landed := range []bool{true, false} {
|
||||
t.Run(fmt.Sprintf("landed=%v", landed), func(t *testing.T) {
|
||||
g := newGuest(t)
|
||||
g.contract()
|
||||
g.auth.recordErrs = []recordErr{{err: transient{}, landed: landed}}
|
||||
pub, err := g.boot()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if g.auth.recorded != pub || g.auth.claims != 2 {
|
||||
t.Fatalf("recorded=%q claims=%d", g.auth.recorded, g.auth.claims)
|
||||
}
|
||||
for _, sent := range g.auth.sent {
|
||||
if sent != pub {
|
||||
t.Fatal("a second key was sent")
|
||||
}
|
||||
}
|
||||
g.assertSentOnlyPublicKeys(g.seed())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBoot_RetriesOnlyRetryableRefusals(t *testing.T) {
|
||||
t.Run("transient then decided", func(t *testing.T) {
|
||||
g := newGuest(t)
|
||||
g.contract()
|
||||
g.auth.claimErrs = []error{transient{}, transient{}}
|
||||
if _, err := g.boot(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if g.auth.claims != 3 {
|
||||
t.Fatalf("claims = %d, want 3", g.auth.claims)
|
||||
}
|
||||
})
|
||||
t.Run("exhausted", func(t *testing.T) {
|
||||
g := newGuest(t)
|
||||
g.contract()
|
||||
g.auth.claimErrs = []error{transient{}, transient{}, transient{}, transient{}}
|
||||
if _, err := g.boot(); err == nil {
|
||||
t.Fatal("boot passed without a decision")
|
||||
}
|
||||
if g.auth.claims != 3 {
|
||||
t.Fatalf("claims = %d, want the 3 attempts", g.auth.claims)
|
||||
}
|
||||
g.assertNoKey()
|
||||
})
|
||||
t.Run("final", func(t *testing.T) {
|
||||
g := newGuest(t)
|
||||
g.contract()
|
||||
denied := errors.New("AccessDenied")
|
||||
g.auth.claimErrs = []error{denied}
|
||||
if _, err := g.boot(); !errors.Is(err, denied) {
|
||||
t.Fatalf("err = %v, want the denial", err)
|
||||
}
|
||||
if g.auth.claims != 1 {
|
||||
t.Fatalf("claims = %d; a denial must not be retried", g.auth.claims)
|
||||
}
|
||||
g.assertNoKey()
|
||||
})
|
||||
t.Run("pending then authorized", func(t *testing.T) {
|
||||
g := newGuest(t)
|
||||
g.contract()
|
||||
g.auth.claimErrs = []error{notBoundError{}, notBoundError{}}
|
||||
if _, err := g.boot(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if g.auth.claims != 3 {
|
||||
t.Fatalf("claims = %d, want 3", g.auth.claims)
|
||||
}
|
||||
})
|
||||
t.Run("pending never authorized", func(t *testing.T) {
|
||||
g := newGuest(t)
|
||||
g.contract()
|
||||
g.auth.claimErrs = []error{notBoundError{}, notBoundError{}, notBoundError{}, notBoundError{}}
|
||||
if _, err := g.boot(); !errors.Is(err, notBoundError{}) {
|
||||
t.Fatalf("err = %v, want the denial", err)
|
||||
}
|
||||
if g.auth.claims != 3 {
|
||||
t.Fatalf("claims = %d, want the 3 attempts", g.auth.claims)
|
||||
}
|
||||
g.assertNoKey()
|
||||
})
|
||||
t.Run("cancelled while waiting", func(t *testing.T) {
|
||||
g := newGuest(t)
|
||||
g.contract()
|
||||
g.cfg.Wait = time.Hour
|
||||
g.auth.claimErrs = []error{transient{}}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
g.cfg.Log = func(string) { cancel() }
|
||||
if _, err := Boot(ctx, g.cfg); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("err = %v, want context.Canceled", err)
|
||||
}
|
||||
g.assertNoKey()
|
||||
})
|
||||
}
|
||||
|
||||
// The gateway is asked only once the worker disk is proven mounted, so a
|
||||
// disk refusal never reaches it and never creates a key.
|
||||
func TestBoot_DiskRefusalsNeverAskTheGateway(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
setup func(g *guest)
|
||||
want string
|
||||
}{
|
||||
{"wrong serial", func(g *guest) { g.fw("data-serial", "vol0ffffffffffffffff") }, "found 0"},
|
||||
{"no disk", func(g *guest) {
|
||||
if err := os.RemoveAll(filepath.Join(g.cfg.SysBlock, "vda")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}, "found 0"},
|
||||
{"ambiguous", func(g *guest) { g.addDisk("vdb", testSerial, "253:32") }, "found 2"},
|
||||
{"no device node", func(g *guest) {
|
||||
if err := os.Remove(filepath.Join(g.cfg.DevDir, "vda")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}, "no block device node"},
|
||||
{"check fails", func(g *guest) { g.disk.checkErr = errors.New("e2fsck refused") }, "e2fsck refused"},
|
||||
{"mount fails", func(g *guest) { g.disk.mountErr = errors.New("invalid argument") }, "invalid argument"},
|
||||
{"empty mountpoint", func(g *guest) { g.disk.noopMount = true }, "is not mounted at"},
|
||||
{"foreign device", func(g *guest) { g.appendMount("253:48", "/", g.cfg.DataDir, "ext4", "x") }, "not the worker disk"},
|
||||
{"bind of a subdirectory", func(g *guest) { g.appendMount(testMajMin, "/sub", g.cfg.DataDir, "ext4", "x") }, "not as a whole ext4 filesystem"},
|
||||
{"tmpfs", func(g *guest) { g.appendMount("0:40", "/", g.cfg.DataDir, "tmpfs", "x") }, "not as a whole ext4 filesystem"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
g := newGuest(t)
|
||||
g.contract()
|
||||
tc.setup(g)
|
||||
_, err := g.boot()
|
||||
if err == nil || !strings.Contains(err.Error(), tc.want) {
|
||||
t.Fatalf("err = %v, want %q", err, tc.want)
|
||||
}
|
||||
if g.auth.builds != 0 || g.auth.claims != 0 {
|
||||
t.Fatal("the gateway was asked before the disk was proven")
|
||||
}
|
||||
g.assertNoKey()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBoot_AlreadyMountedWorkerDiskIsReused(t *testing.T) {
|
||||
g := newGuest(t)
|
||||
g.contract()
|
||||
g.seedKey()
|
||||
g.appendMount(testMajMin, "/", g.cfg.DataDir, "ext4", "/dev/vda")
|
||||
if _, err := g.boot(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if g.disk.checks != 0 || g.disk.mounts != 0 {
|
||||
t.Fatal("a mounted worker disk was checked or mounted again")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBoot_ContractRefusalsTouchNothing(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
setup func(g *guest)
|
||||
want string
|
||||
}{
|
||||
{"no serial", func(g *guest) { mustRemove(t, filepath.Join(g.cfg.FwCfgDir, "data-serial")) }, "data-serial not supplied"},
|
||||
{"bad serial", func(g *guest) { g.fw("data-serial", "vol-0a1b2c3d4e5f6071") }, "not a volume serial"},
|
||||
{"no region", func(g *guest) { mustRemove(t, filepath.Join(g.cfg.FwCfgDir, "region")) }, "region not supplied"},
|
||||
{"empty region", func(g *guest) { g.fw("region", "") }, "exactly one value"},
|
||||
{"bad region", func(g *guest) { g.fw("region", "Sydney") }, "not a Region name"},
|
||||
{"no gateway", func(g *guest) { mustRemove(t, filepath.Join(g.cfg.FwCfgDir, "gateway-url")) }, "gateway-url not supplied"},
|
||||
{"plain http gateway", func(g *guest) { g.fw("gateway-url", "http://10.15.8.1:9999") }, "not an https URL"},
|
||||
{"no claimer", func(g *guest) { g.cfg.NewClaimer = nil }, "no gateway client"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
g := newGuest(t)
|
||||
g.contract()
|
||||
tc.setup(g)
|
||||
_, err := g.boot()
|
||||
if err == nil || !strings.Contains(err.Error(), tc.want) {
|
||||
t.Fatalf("err = %v, want %q", err, tc.want)
|
||||
}
|
||||
if g.auth.claims != 0 {
|
||||
t.Fatal("a refused contract asked the gateway")
|
||||
}
|
||||
g.assertNoKey()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBoot_ClaimerBuildFailureCreatesNoKey(t *testing.T) {
|
||||
g := newGuest(t)
|
||||
g.contract()
|
||||
g.auth.buildErr = errors.New("no IMDS credentials")
|
||||
if _, err := g.boot(); err == nil || !strings.Contains(err.Error(), "no IMDS credentials") {
|
||||
t.Fatalf("err = %v", err)
|
||||
}
|
||||
g.assertNoKey()
|
||||
}
|
||||
|
||||
func mustRemove(t *testing.T, path string) {
|
||||
t.Helper()
|
||||
if err := os.RemoveAll(path); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMountedDevice_UsesTopmostMountAndUnescapes(t *testing.T) {
|
||||
g := newGuest(t)
|
||||
dir := filepath.Join(g.root, "data dir")
|
||||
g.appendMount("253:48", "/", dir, "ext4", "/dev/vdb")
|
||||
g.appendMount(testMajMin, "/", dir, "ext4", "/dev/vda")
|
||||
got, err := mountedDevice(g.cfg.MountInfo, dir)
|
||||
if err != nil || got != testMajMin {
|
||||
t.Fatalf("mountedDevice = %q, %v; want %s", got, err, testMajMin)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package workerkey
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/nats-io/nkeys"
|
||||
)
|
||||
|
||||
// Claimer is the SQS domain's first-store decision for this worker identity.
|
||||
type Claimer interface {
|
||||
// Claim returns createKey, or the one public key the store authorizes.
|
||||
Claim(ctx context.Context) (createKey bool, publicKey string, err error)
|
||||
RecordPublicKey(ctx context.Context, publicKey string) error
|
||||
}
|
||||
|
||||
// Acquire returns the worker's store key under a current domain decision. A
|
||||
// new key is usable only once its public key is recorded; a recorded key is
|
||||
// never replaced, so a missing or different key on disk fails closed.
|
||||
func (s Store) Acquire(ctx context.Context, c Claimer) (nkeys.KeyPair, error) {
|
||||
create, authorized, err := c.Claim(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !create {
|
||||
return s.ForExistingStore(authorized)
|
||||
}
|
||||
if authorized != "" {
|
||||
return nil, errors.New("first-store decision is contradictory")
|
||||
}
|
||||
// After a crash before recording, this loads the key already on disk.
|
||||
kp, err := s.ForNewStore()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pub, err := kp.PublicKey()
|
||||
if err == nil {
|
||||
err = c.RecordPublicKey(ctx, pub)
|
||||
}
|
||||
if err != nil {
|
||||
kp.Wipe()
|
||||
return nil, err
|
||||
}
|
||||
return kp, nil
|
||||
}
|
||||
@@ -0,0 +1,266 @@
|
||||
package workerkey
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
testutil "github.com/mulgadc/spinifex/internal/testkit"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/firststore"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/nats-io/nkeys"
|
||||
)
|
||||
|
||||
const region = "ap-southeast-2"
|
||||
|
||||
// domainClaimer drives the real first-store state machine for one worker
|
||||
// identity. A failing record either drops the write or applies it and loses
|
||||
// the reply.
|
||||
type domainClaimer struct {
|
||||
s firststore.Store
|
||||
id firststore.Identity
|
||||
failRecord bool
|
||||
applied bool
|
||||
}
|
||||
|
||||
func (c *domainClaimer) Claim(ctx context.Context) (bool, string, error) {
|
||||
d, err := c.s.Claim(ctx, region, c.id)
|
||||
return d.CreateKey, d.PublicKey, err
|
||||
}
|
||||
|
||||
func (c *domainClaimer) RecordPublicKey(ctx context.Context, pub string) error {
|
||||
if c.failRecord {
|
||||
c.failRecord = false
|
||||
if c.applied {
|
||||
c.s.RecordPublicKey(ctx, region, c.id, pub)
|
||||
}
|
||||
return errors.New("nats: timeout")
|
||||
}
|
||||
return c.s.RecordPublicKey(ctx, region, c.id, pub)
|
||||
}
|
||||
|
||||
func domain(t *testing.T) (firststore.Store, jetstream.KeyValue) {
|
||||
t.Helper()
|
||||
_, _, js := testutil.StartTestJetStream(t)
|
||||
kv, err := js.CreateKeyValue(context.Background(), firststore.BucketConfig(1))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return firststore.Store{JS: js, KV: kv}, kv
|
||||
}
|
||||
|
||||
func worker(gen uint64, n byte) firststore.Identity {
|
||||
return firststore.Identity{InstanceID: "i-0000000000000000" + string('0'+n), ClaimID: "claim-" + string('a'+n) + "-0001", Generation: gen}
|
||||
}
|
||||
|
||||
func workerDisk(t *testing.T) Store {
|
||||
return Store{Dir: filepath.Join(t.TempDir(), "sqs-worker-key")}
|
||||
}
|
||||
|
||||
func keyPath(s Store) string { return filepath.Join(s.Dir, keyFile) }
|
||||
|
||||
func pubOf(t *testing.T, kp nkeys.KeyPair) string {
|
||||
t.Helper()
|
||||
p, err := kp.PublicKey()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func recorded(t *testing.T, s firststore.Store, id firststore.Identity) string {
|
||||
t.Helper()
|
||||
d, err := s.Claim(context.Background(), region, id)
|
||||
if err != nil || d.CreateKey {
|
||||
t.Fatalf("domain decision = %+v, %v; want a recorded key", d, err)
|
||||
}
|
||||
return d.PublicKey
|
||||
}
|
||||
|
||||
func TestAcquire_CrashBeforeRecordingReusesTheKeyOnDisk(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
fs, _ := domain(t)
|
||||
disk := workerDisk(t)
|
||||
c := &domainClaimer{s: fs, id: worker(1, 1), failRecord: true}
|
||||
if _, err := disk.Acquire(ctx, c); err == nil {
|
||||
t.Fatal("a key was returned before its public key was recorded")
|
||||
}
|
||||
before, err := os.ReadFile(keyPath(disk))
|
||||
if err != nil {
|
||||
t.Fatalf("key not synced to disk before recording: %v", err)
|
||||
}
|
||||
kp, err := disk.Acquire(ctx, c)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer kp.Wipe()
|
||||
after, _ := os.ReadFile(keyPath(disk))
|
||||
if string(before) != string(after) {
|
||||
t.Fatal("the retry replaced the key")
|
||||
}
|
||||
if recorded(t, fs, c.id) != pubOf(t, kp) {
|
||||
t.Fatal("the recorded key is not the key on disk")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcquire_LostRecordReplyIsRecoveredOnReboot(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
fs, _ := domain(t)
|
||||
disk := workerDisk(t)
|
||||
c := &domainClaimer{s: fs, id: worker(1, 1), failRecord: true, applied: true}
|
||||
if _, err := disk.Acquire(ctx, c); err == nil {
|
||||
t.Fatal("expected the lost reply to surface")
|
||||
}
|
||||
kp, err := disk.Acquire(ctx, c)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer kp.Wipe()
|
||||
if recorded(t, fs, c.id) != pubOf(t, kp) {
|
||||
t.Fatal("reboot did not use the recorded key")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcquire_AfterRecordingKeyLossOrMismatchFailsClosed(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
setup := func(t *testing.T, launch bool) (firststore.Store, Store, *domainClaimer, string) {
|
||||
fs, _ := domain(t)
|
||||
disk := workerDisk(t)
|
||||
c := &domainClaimer{s: fs, id: worker(1, 1)}
|
||||
kp, err := disk.Acquire(ctx, c)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pub := pubOf(t, kp)
|
||||
kp.Wipe()
|
||||
if launch {
|
||||
if err := fs.RecordLaunchIntent(ctx, region, c.id, pub); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return fs, disk, c, pub
|
||||
}
|
||||
for _, launch := range []bool{false, true} {
|
||||
name := map[bool]string{false: "key recorded", true: "launch intent"}[launch]
|
||||
t.Run(name+"/key lost then reboot", func(t *testing.T) {
|
||||
fs, disk, c, pub := setup(t, launch)
|
||||
os.Remove(keyPath(disk))
|
||||
// A relaunched worker generation gets the same answer.
|
||||
for _, id := range []firststore.Identity{c.id, worker(2, 2)} {
|
||||
c.id = id
|
||||
if _, err := disk.Acquire(ctx, c); !errors.Is(err, ErrKeyMissing) {
|
||||
t.Fatalf("acquire = %v, want ErrKeyMissing", err)
|
||||
}
|
||||
if _, err := os.Lstat(keyPath(disk)); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatal("a replacement key was created")
|
||||
}
|
||||
}
|
||||
if recorded(t, fs, c.id) != pub {
|
||||
t.Fatal("the recorded key changed")
|
||||
}
|
||||
})
|
||||
t.Run(name+"/different key on disk", func(t *testing.T) {
|
||||
_, _, c, _ := setup(t, launch)
|
||||
other := workerDisk(t)
|
||||
kp, err := other.ForNewStore()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kp.Wipe()
|
||||
before, _ := os.ReadFile(keyPath(other))
|
||||
if _, err := other.Acquire(ctx, c); !errors.Is(err, ErrKeyMismatch) {
|
||||
t.Fatalf("acquire = %v, want ErrKeyMismatch", err)
|
||||
}
|
||||
after, _ := os.ReadFile(keyPath(other))
|
||||
if string(before) != string(after) {
|
||||
t.Fatal("the mismatched key was replaced")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcquire_RefusedClaimCreatesNoKey(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cases := map[string]func(t *testing.T, fs firststore.Store, kv jetstream.KeyValue) firststore.Identity{
|
||||
"rival holds the claim": func(t *testing.T, fs firststore.Store, _ jetstream.KeyValue) firststore.Identity {
|
||||
fs.Claim(ctx, region, worker(1, 1))
|
||||
return worker(1, 2)
|
||||
},
|
||||
"stale replayed generation": func(t *testing.T, fs firststore.Store, _ jetstream.KeyValue) firststore.Identity {
|
||||
fs.Claim(ctx, region, worker(1, 1))
|
||||
fs.Claim(ctx, region, worker(2, 2))
|
||||
return worker(1, 1)
|
||||
},
|
||||
"unreadable record": func(t *testing.T, _ firststore.Store, kv jetstream.KeyValue) firststore.Identity {
|
||||
kv.Put(ctx, "region."+region, []byte("{"))
|
||||
return worker(1, 1)
|
||||
},
|
||||
}
|
||||
for name, setup := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
fs, kv := domain(t)
|
||||
id := setup(t, fs, kv)
|
||||
disk := workerDisk(t)
|
||||
if _, err := disk.Acquire(ctx, &domainClaimer{s: fs, id: id}); err == nil {
|
||||
t.Fatal("a refused claim returned a key")
|
||||
}
|
||||
if _, err := os.Lstat(keyPath(disk)); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatal("a refused claim created a key")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type contradictoryClaimer struct{ pub string }
|
||||
|
||||
func (c contradictoryClaimer) Claim(context.Context) (bool, string, error) { return true, c.pub, nil }
|
||||
func (contradictoryClaimer) RecordPublicKey(context.Context, string) error {
|
||||
return errors.New("unexpected")
|
||||
}
|
||||
|
||||
func TestAcquire_ContradictoryDecisionCreatesNoKey(t *testing.T) {
|
||||
kp, _ := nkeys.CreateUser()
|
||||
disk := workerDisk(t)
|
||||
if _, err := disk.Acquire(context.Background(), contradictoryClaimer{pub: pubOf(t, kp)}); err == nil {
|
||||
t.Fatal("a contradictory decision returned a key")
|
||||
}
|
||||
if _, err := os.Lstat(keyPath(disk)); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatal("a contradictory decision created a key")
|
||||
}
|
||||
}
|
||||
|
||||
// A replica that has applied the claim but not the key record must not lead a
|
||||
// worker whose key is lost to create a replacement.
|
||||
func TestAcquire_StaleClaimedReadAfterRecordingCreatesNoKey(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
fs, kv := domain(t)
|
||||
disk := workerDisk(t)
|
||||
id := worker(1, 1)
|
||||
fs.Claim(ctx, region, id)
|
||||
claimed, err := kv.Get(ctx, "region."+region)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kp, err := disk.Acquire(ctx, &domainClaimer{s: fs, id: id})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kp.Wipe()
|
||||
os.Remove(keyPath(disk))
|
||||
|
||||
_, _, replicaJS := testutil.StartTestJetStream(t)
|
||||
replica, err := replicaJS.CreateKeyValue(ctx, firststore.BucketConfig(1))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
replica.Put(ctx, "region."+region, claimed.Value())
|
||||
stale := firststore.Store{JS: replicaJS, KV: fs.KV}
|
||||
if _, err := disk.Acquire(ctx, &domainClaimer{s: stale, id: id}); err == nil {
|
||||
t.Fatal("a stale claimed read returned a key")
|
||||
}
|
||||
if _, err := os.Lstat(keyPath(disk)); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatal("a stale claimed read created a replacement key")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
// Package workerkey keeps the shared SQS worker's store-client NKey on the
|
||||
// worker's own persistent disk. The seed is generated and held only inside the
|
||||
// worker guest; the platform learns the public key alone.
|
||||
package workerkey
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
|
||||
"github.com/nats-io/nkeys"
|
||||
)
|
||||
|
||||
const (
|
||||
keyFile = "store-client.nk"
|
||||
tempPrefix = ".store-client-"
|
||||
tempSuffix = ".tmp"
|
||||
// A user seed is 58 bytes; anything much larger is not one.
|
||||
maxKeyFileSize = 256
|
||||
)
|
||||
|
||||
// Each refusal is fail-closed: the worker never replaces a key it cannot use.
|
||||
var (
|
||||
ErrKeyMissing = errors.New("worker store key is missing but the store already exists")
|
||||
ErrKeyMismatch = errors.New("worker store key does not match the store's authorized key")
|
||||
ErrUnsafe = errors.New("worker store key path is not owner-only")
|
||||
ErrCorrupt = errors.New("worker store key file does not hold a user NKey seed")
|
||||
)
|
||||
|
||||
// Store is the key directory on the worker disk. Workerkey cannot tell a
|
||||
// mounted disk from an empty mountpoint, so the caller must first verify the
|
||||
// expected data volume is mounted there.
|
||||
type Store struct {
|
||||
Dir string
|
||||
}
|
||||
|
||||
// ForNewStore returns the worker's store-client key for a store that the
|
||||
// caller has positively established does not exist yet, generating it on first
|
||||
// use. An unreadable or absent store record is not that proof.
|
||||
func (s Store) ForNewStore() (nkeys.KeyPair, error) {
|
||||
return s.ensure("")
|
||||
}
|
||||
|
||||
// ForExistingStore returns the worker's key only if it is already on disk and
|
||||
// is the one the store authorizes; it never generates a key.
|
||||
func (s Store) ForExistingStore(authorized string) (nkeys.KeyPair, error) {
|
||||
if !nkeys.IsValidPublicUserKey(authorized) {
|
||||
return nil, errors.New("authorized store key is not a user public NKey")
|
||||
}
|
||||
return s.ensure(authorized)
|
||||
}
|
||||
|
||||
func (s Store) ensure(expected string) (nkeys.KeyPair, error) {
|
||||
unlock, err := s.prepareDir()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer unlock()
|
||||
kp, err := s.load()
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
if expected != "" {
|
||||
return nil, ErrKeyMissing
|
||||
}
|
||||
kp, err = s.create()
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if expected != "" {
|
||||
pub, err := kp.PublicKey()
|
||||
if err != nil {
|
||||
kp.Wipe()
|
||||
return nil, ErrCorrupt
|
||||
}
|
||||
if pub != expected {
|
||||
kp.Wipe()
|
||||
return nil, fmt.Errorf("%w: have %s, store authorizes %s", ErrKeyMismatch, pub, expected)
|
||||
}
|
||||
}
|
||||
// Re-synced on every return, so a key whose earlier sync failed is never
|
||||
// handed out before its name and directory are durable.
|
||||
if err := s.syncNames(); err != nil {
|
||||
kp.Wipe()
|
||||
return nil, err
|
||||
}
|
||||
return kp, nil
|
||||
}
|
||||
|
||||
func (s Store) path() string { return filepath.Join(s.Dir, keyFile) }
|
||||
|
||||
// prepareDir creates only the key directory, inside a parent that must already
|
||||
// exist, refuses one that is not owner-only, locks it and removes temporary
|
||||
// files a crash left behind.
|
||||
func (s Store) prepareDir() (unlock func(), err error) {
|
||||
if s.Dir == "" {
|
||||
return nil, errors.New("worker store key directory is not set")
|
||||
}
|
||||
parent, err := os.Stat(filepath.Dir(s.Dir))
|
||||
if err != nil || !parent.IsDir() {
|
||||
return nil, errors.New("worker key directory parent is missing")
|
||||
}
|
||||
if err := os.Mkdir(s.Dir, 0o700); err != nil && !errors.Is(err, fs.ErrExist) {
|
||||
return nil, fmt.Errorf("create worker key directory: %w", err)
|
||||
}
|
||||
fi, err := os.Lstat(s.Dir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stat worker key directory: %w", err)
|
||||
}
|
||||
if !fi.IsDir() || fi.Mode().Perm()&0o077 != 0 {
|
||||
return nil, ErrUnsafe
|
||||
}
|
||||
d, err := os.Open(s.Dir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open worker key directory: %w", err)
|
||||
}
|
||||
if err := syscall.Flock(int(d.Fd()), syscall.LOCK_EX); err != nil {
|
||||
d.Close()
|
||||
return nil, fmt.Errorf("lock worker key directory: %w", err)
|
||||
}
|
||||
unlock = func() { d.Close() }
|
||||
stale, err := filepath.Glob(filepath.Join(s.Dir, tempPrefix+"*"+tempSuffix))
|
||||
if err != nil {
|
||||
unlock()
|
||||
return nil, fmt.Errorf("list stale worker key files: %w", err)
|
||||
}
|
||||
for _, f := range stale {
|
||||
if err := os.Remove(f); err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||
unlock()
|
||||
return nil, fmt.Errorf("remove stale worker key file: %w", err)
|
||||
}
|
||||
}
|
||||
return unlock, nil
|
||||
}
|
||||
|
||||
// load reads an existing key. Any file that is present but unusable refuses
|
||||
// rather than being replaced.
|
||||
func (s Store) load() (nkeys.KeyPair, error) {
|
||||
fi, err := os.Lstat(s.path())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !fi.Mode().IsRegular() || fi.Mode().Perm()&0o077 != 0 {
|
||||
return nil, ErrUnsafe
|
||||
}
|
||||
if fi.Size() > maxKeyFileSize {
|
||||
return nil, ErrCorrupt
|
||||
}
|
||||
seed, err := os.ReadFile(s.path())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read worker store key: %w", err)
|
||||
}
|
||||
kp, err := nkeys.FromSeed(bytes.TrimSpace(seed))
|
||||
if err != nil {
|
||||
return nil, ErrCorrupt
|
||||
}
|
||||
pub, err := kp.PublicKey()
|
||||
if err != nil || !nkeys.IsValidPublicUserKey(pub) {
|
||||
kp.Wipe()
|
||||
return nil, ErrCorrupt
|
||||
}
|
||||
return kp, nil
|
||||
}
|
||||
|
||||
// create writes a new seed to a synced temporary file and links it into place
|
||||
// without replacing anything; the caller syncs the directory names.
|
||||
func (s Store) create() (nkeys.KeyPair, error) {
|
||||
kp, err := nkeys.CreateUser()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("generate worker store key: %w", err)
|
||||
}
|
||||
seed, err := kp.Seed()
|
||||
if err != nil {
|
||||
kp.Wipe()
|
||||
return nil, fmt.Errorf("encode worker store key: %w", err)
|
||||
}
|
||||
|
||||
tmp, err := os.CreateTemp(s.Dir, tempPrefix+"*"+tempSuffix)
|
||||
if err != nil {
|
||||
kp.Wipe()
|
||||
return nil, fmt.Errorf("create worker store key file: %w", err)
|
||||
}
|
||||
tmpName := tmp.Name()
|
||||
defer os.Remove(tmpName)
|
||||
if err := writeSynced(tmp, seed); err != nil {
|
||||
kp.Wipe()
|
||||
return nil, err
|
||||
}
|
||||
if err := os.Link(tmpName, s.path()); err != nil {
|
||||
kp.Wipe()
|
||||
if errors.Is(err, fs.ErrExist) {
|
||||
return s.load()
|
||||
}
|
||||
return nil, fmt.Errorf("install worker store key file: %w", err)
|
||||
}
|
||||
return kp, nil
|
||||
}
|
||||
|
||||
// syncNames makes the key's directory entry and the key directory's own entry
|
||||
// in its parent durable.
|
||||
func (s Store) syncNames() error {
|
||||
if err := syncDirFn(s.Dir); err != nil {
|
||||
return err
|
||||
}
|
||||
return syncDirFn(filepath.Dir(s.Dir))
|
||||
}
|
||||
|
||||
func writeSynced(f *os.File, data []byte) error {
|
||||
if err := f.Chmod(0o600); err != nil {
|
||||
f.Close()
|
||||
return fmt.Errorf("restrict worker store key file: %w", err)
|
||||
}
|
||||
if _, err := f.Write(data); err != nil {
|
||||
f.Close()
|
||||
return fmt.Errorf("write worker store key file: %w", err)
|
||||
}
|
||||
if err := f.Sync(); err != nil {
|
||||
f.Close()
|
||||
return fmt.Errorf("sync worker store key file: %w", err)
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
return fmt.Errorf("close worker store key file: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// syncDirFn is replaced in tests to fail a directory sync deterministically.
|
||||
var syncDirFn = syncDir
|
||||
|
||||
func syncDir(dir string) error {
|
||||
d, err := os.Open(dir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open worker key directory: %w", err)
|
||||
}
|
||||
defer d.Close()
|
||||
if err := d.Sync(); err != nil {
|
||||
return fmt.Errorf("sync worker key directory: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,423 @@
|
||||
package workerkey
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nkeys"
|
||||
)
|
||||
|
||||
func newStore(t *testing.T) Store {
|
||||
t.Helper()
|
||||
return Store{Dir: filepath.Join(t.TempDir(), "keys")}
|
||||
}
|
||||
|
||||
func publicKey(t *testing.T, kp nkeys.KeyPair) string {
|
||||
t.Helper()
|
||||
pub, err := kp.PublicKey()
|
||||
if err != nil {
|
||||
t.Fatalf("public key: %v", err)
|
||||
}
|
||||
return pub
|
||||
}
|
||||
|
||||
func otherUserKey(t *testing.T) string {
|
||||
t.Helper()
|
||||
kp, err := nkeys.CreateUser()
|
||||
if err != nil {
|
||||
t.Fatalf("create user: %v", err)
|
||||
}
|
||||
return publicKey(t, kp)
|
||||
}
|
||||
|
||||
func readFile(t *testing.T, path string) []byte {
|
||||
t.Helper()
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read %s: %v", path, err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func assertNoSeed(t *testing.T, err error, seed []byte) {
|
||||
t.Helper()
|
||||
if err != nil && strings.Contains(err.Error(), string(seed)) {
|
||||
t.Fatalf("error exposes the seed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsure_FirstBootPersistsAndRestartReusesKey(t *testing.T) {
|
||||
s := newStore(t)
|
||||
kp, err := s.ForNewStore()
|
||||
if err != nil {
|
||||
t.Fatalf("first boot: %v", err)
|
||||
}
|
||||
pub := publicKey(t, kp)
|
||||
|
||||
dirInfo, err := os.Stat(s.Dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := dirInfo.Mode().Perm(); got != 0o700 {
|
||||
t.Fatalf("directory mode = %o, want 700", got)
|
||||
}
|
||||
fileInfo, err := os.Stat(s.path())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := fileInfo.Mode().Perm(); got != 0o600 {
|
||||
t.Fatalf("key file mode = %o, want 600", got)
|
||||
}
|
||||
onDisk, err := nkeys.FromSeed(readFile(t, s.path()))
|
||||
if err != nil {
|
||||
t.Fatalf("persisted file is not a seed: %v", err)
|
||||
}
|
||||
if publicKey(t, onDisk) != pub {
|
||||
t.Fatal("persisted seed does not match the returned key")
|
||||
}
|
||||
|
||||
// A restart is a fresh Store over the same disk; neither path may
|
||||
// generate a replacement.
|
||||
again, err := Store{Dir: s.Dir}.ForExistingStore(pub)
|
||||
if err != nil {
|
||||
t.Fatalf("restart for the existing store: %v", err)
|
||||
}
|
||||
if publicKey(t, again) != pub {
|
||||
t.Fatal("restart for the existing store changed the key")
|
||||
}
|
||||
again, err = Store{Dir: s.Dir}.ForNewStore()
|
||||
if err != nil {
|
||||
t.Fatalf("repeated first start: %v", err)
|
||||
}
|
||||
if publicKey(t, again) != pub {
|
||||
t.Fatal("repeated first start replaced the key")
|
||||
}
|
||||
entries, err := os.ReadDir(s.Dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(entries) != 1 {
|
||||
t.Fatalf("key directory holds %d entries, want only the key", len(entries))
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsure_ExistingStoreWithMissingKeyFailsClosed(t *testing.T) {
|
||||
s := newStore(t)
|
||||
_, err := s.ForExistingStore(otherUserKey(t))
|
||||
if !errors.Is(err, ErrKeyMissing) {
|
||||
t.Fatalf("err = %v, want ErrKeyMissing", err)
|
||||
}
|
||||
if _, err := os.Lstat(s.path()); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("a key was generated for an existing store: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsure_MismatchedKeyFailsClosedAndKeepsFile(t *testing.T) {
|
||||
s := newStore(t)
|
||||
if _, err := s.ForNewStore(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := readFile(t, s.path())
|
||||
|
||||
_, err := Store{Dir: s.Dir}.ForExistingStore(otherUserKey(t))
|
||||
if !errors.Is(err, ErrKeyMismatch) {
|
||||
t.Fatalf("err = %v, want ErrKeyMismatch", err)
|
||||
}
|
||||
assertNoSeed(t, err, before)
|
||||
if string(readFile(t, s.path())) != string(before) {
|
||||
t.Fatal("mismatch replaced the key file")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsure_UnusableKeyFileIsNeverReplaced(t *testing.T) {
|
||||
seedKP, err := nkeys.CreateUser()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
userSeed, err := seedKP.Seed()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
accountKP, err := nkeys.CreateAccount()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
accountSeed, err := accountKP.Seed()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
content []byte
|
||||
mode os.FileMode
|
||||
want error
|
||||
}{
|
||||
{"empty", nil, 0o600, ErrCorrupt},
|
||||
{"garbage", []byte("not a seed"), 0o600, ErrCorrupt},
|
||||
{"account seed", accountSeed, 0o600, ErrCorrupt},
|
||||
{"oversized", make([]byte, maxKeyFileSize+1), 0o600, ErrCorrupt},
|
||||
{"group readable", userSeed, 0o640, ErrUnsafe},
|
||||
{"world readable", userSeed, 0o604, ErrUnsafe},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
s := newStore(t)
|
||||
if err := os.MkdirAll(s.Dir, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(s.path(), tc.content, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Chmod(s.path(), tc.mode); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err := s.ForNewStore()
|
||||
if !errors.Is(err, tc.want) {
|
||||
t.Fatalf("err = %v, want %v", err, tc.want)
|
||||
}
|
||||
assertNoSeed(t, err, userSeed)
|
||||
assertNoSeed(t, err, accountSeed)
|
||||
if string(readFile(t, s.path())) != string(tc.content) {
|
||||
t.Fatal("an unusable key file was replaced")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsure_RefusesUnsafeDirectoryAndSymlink(t *testing.T) {
|
||||
t.Run("open directory", func(t *testing.T) {
|
||||
s := newStore(t)
|
||||
if err := os.MkdirAll(s.Dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Chmod(s.Dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.ForNewStore(); !errors.Is(err, ErrUnsafe) {
|
||||
t.Fatalf("err = %v, want ErrUnsafe", err)
|
||||
}
|
||||
if _, err := os.Lstat(s.path()); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatal("a key was written into an open directory")
|
||||
}
|
||||
})
|
||||
t.Run("symlinked key", func(t *testing.T) {
|
||||
s := newStore(t)
|
||||
target := newStore(t)
|
||||
if _, err := target.ForNewStore(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.MkdirAll(s.Dir, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Symlink(target.path(), s.path()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.ForNewStore(); !errors.Is(err, ErrUnsafe) {
|
||||
t.Fatalf("err = %v, want ErrUnsafe", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestEnsure_CrashLeftoverTempIsNotAKey(t *testing.T) {
|
||||
s := newStore(t)
|
||||
if err := os.MkdirAll(s.Dir, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stale := filepath.Join(s.Dir, tempPrefix+"123"+tempSuffix)
|
||||
if err := os.WriteFile(stale, []byte("partial"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// A crash before the link means no store was authorized for it.
|
||||
if _, err := s.ForExistingStore(otherUserKey(t)); !errors.Is(err, ErrKeyMissing) {
|
||||
t.Fatalf("err = %v, want ErrKeyMissing", err)
|
||||
}
|
||||
if _, err := os.Lstat(stale); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatal("stale temporary key file was kept")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsure_ConcurrentFirstBootAgreesOnOneKey(t *testing.T) {
|
||||
const rounds, n = 20, 16
|
||||
for range rounds {
|
||||
s := newStore(t)
|
||||
pubs := make([]string, n)
|
||||
errs := make([]error, n)
|
||||
start := make(chan struct{})
|
||||
var wg sync.WaitGroup
|
||||
for i := range n {
|
||||
wg.Go(func() {
|
||||
<-start
|
||||
kp, err := Store{Dir: s.Dir}.ForNewStore()
|
||||
if err != nil {
|
||||
errs[i] = err
|
||||
return
|
||||
}
|
||||
pubs[i], errs[i] = kp.PublicKey()
|
||||
})
|
||||
}
|
||||
close(start)
|
||||
wg.Wait()
|
||||
onDisk, err := nkeys.FromSeed(readFile(t, s.path()))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := publicKey(t, onDisk)
|
||||
for i := range n {
|
||||
if errs[i] != nil {
|
||||
t.Fatalf("call %d: %v", i, errs[i])
|
||||
}
|
||||
if pubs[i] != want {
|
||||
t.Fatalf("call %d returned a key that is not the persisted one", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestForExistingStore_RejectsInvalidAuthorizedKey(t *testing.T) {
|
||||
kp, err := nkeys.CreateUser()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seed, err := kp.Seed()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// An empty key is what an unreadable store record would yield; it must not
|
||||
// fall through to first-start generation.
|
||||
for _, authorized := range []string{"", string(seed)} {
|
||||
s := newStore(t)
|
||||
_, err := s.ForExistingStore(authorized)
|
||||
if err == nil {
|
||||
t.Fatalf("authorized key %q was accepted", authorized[:min(len(authorized), 2)])
|
||||
}
|
||||
assertNoSeed(t, err, seed)
|
||||
if _, err := os.Lstat(s.Dir); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatal("an invalid authorized key still touched the disk")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestForNewStore_RefusesMissingParent(t *testing.T) {
|
||||
s := Store{Dir: filepath.Join(t.TempDir(), "missing-parent", "keys")}
|
||||
if _, err := s.ForNewStore(); err == nil {
|
||||
t.Fatal("a key directory was created without its parent")
|
||||
}
|
||||
if _, err := os.Lstat(filepath.Dir(s.Dir)); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatal("the missing parent was created")
|
||||
}
|
||||
}
|
||||
|
||||
// hookSyncs records every directory sync and fails call number failAt
|
||||
// (counted from one); zero fails none.
|
||||
func hookSyncs(t *testing.T, failAt int) *[]string {
|
||||
t.Helper()
|
||||
var synced []string
|
||||
orig := syncDirFn
|
||||
t.Cleanup(func() { syncDirFn = orig })
|
||||
syncDirFn = func(dir string) error {
|
||||
synced = append(synced, dir)
|
||||
if len(synced) == failAt {
|
||||
return errors.New("injected directory sync failure")
|
||||
}
|
||||
return orig(dir)
|
||||
}
|
||||
return &synced
|
||||
}
|
||||
|
||||
func TestEnsure_RetryAfterFailedDirectorySyncKeepsKeyAndSyncs(t *testing.T) {
|
||||
retries := map[string]func(s Store, pub string) (nkeys.KeyPair, error){
|
||||
"first start": func(s Store, _ string) (nkeys.KeyPair, error) { return s.ForNewStore() },
|
||||
"existing store": func(s Store, pub string) (nkeys.KeyPair, error) {
|
||||
return s.ForExistingStore(pub)
|
||||
},
|
||||
}
|
||||
for name, retry := range retries {
|
||||
for failAt := 1; failAt <= 2; failAt++ {
|
||||
t.Run(fmt.Sprintf("%s after sync %d failed", name, failAt), func(t *testing.T) {
|
||||
s := newStore(t)
|
||||
hookSyncs(t, failAt)
|
||||
if _, err := s.ForNewStore(); err == nil {
|
||||
t.Fatal("a key was returned although a directory sync failed")
|
||||
}
|
||||
first := readFile(t, s.path())
|
||||
onDisk, err := nkeys.FromSeed(first)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pub := publicKey(t, onDisk)
|
||||
|
||||
synced := hookSyncs(t, 0)
|
||||
kp, err := retry(Store{Dir: s.Dir}, pub)
|
||||
if err != nil {
|
||||
t.Fatalf("retry: %v", err)
|
||||
}
|
||||
if string(readFile(t, s.path())) != string(first) {
|
||||
t.Fatal("retry replaced the key")
|
||||
}
|
||||
if publicKey(t, kp) != pub {
|
||||
t.Fatal("retry returned a key other than the persisted one")
|
||||
}
|
||||
want := []string{s.Dir, filepath.Dir(s.Dir)}
|
||||
if len(*synced) != 2 || (*synced)[0] != want[0] || (*synced)[1] != want[1] {
|
||||
t.Fatalf("retry synced %v, want %v", *synced, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Another holder of the directory lock, such as a second worker process on
|
||||
// the same disk, must finish before this one reads, cleans or creates a key.
|
||||
func TestEnsure_WaitsForDirectoryLock(t *testing.T) {
|
||||
s := newStore(t)
|
||||
if err := os.MkdirAll(s.Dir, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
holder, err := os.Open(s.Dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer holder.Close()
|
||||
if err := syscall.Flock(int(holder.Fd()), syscall.LOCK_EX); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := Store{Dir: s.Dir}.ForNewStore()
|
||||
done <- err
|
||||
}()
|
||||
deadline := time.Now().Add(300 * time.Millisecond)
|
||||
for time.Now().Before(deadline) {
|
||||
select {
|
||||
case err := <-done:
|
||||
t.Fatalf("Ensure returned while another holder had the lock: %v", err)
|
||||
default:
|
||||
}
|
||||
if _, err := os.Lstat(s.path()); err == nil {
|
||||
t.Fatal("a key was created while another holder had the lock")
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
|
||||
if err := syscall.Flock(int(holder.Fd()), syscall.LOCK_UN); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Fatalf("Ensure after unlock: %v", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("Ensure did not proceed after the lock was released")
|
||||
}
|
||||
}
|
||||
@@ -247,6 +247,7 @@ type Config struct {
|
||||
ACM ACMConfig `json:"ACM" mapstructure:"acm"`
|
||||
Bedrock BedrockConfig `json:"Bedrock" mapstructure:"bedrock"`
|
||||
OchreVector OchreVectorConfig `json:"OchreVector" mapstructure:"ochre_vector"`
|
||||
SQS SQSConfig `json:"SQS" mapstructure:"sqs"`
|
||||
|
||||
BaseDir string `json:"BaseDir" mapstructure:"base_dir"`
|
||||
WalDir string `json:"WalDir" mapstructure:"wal_dir"`
|
||||
@@ -447,6 +448,19 @@ type BedrockConfig struct {
|
||||
// (10.248.0.0/14) and EKS control-plane (10.252.0.0/14) supernets.
|
||||
const BedrockDefaultSystemVPCSupernet = "10.244.0.0/14"
|
||||
|
||||
// SQSConfig is a node's [sqs] section. The SQS worker and store VMs share one
|
||||
// private system VPC per region, with no internet or NAT gateway.
|
||||
type SQSConfig struct {
|
||||
// The IPv4 /14 the system VPC's /22 is carved from. It must not overlap the
|
||||
// RDS, Bedrock or EKS control-plane supernets or any customer VPC CIDR.
|
||||
SystemVPCSupernet string `json:"SystemVPCSupernet" mapstructure:"system_vpc_supernet"`
|
||||
}
|
||||
|
||||
// SQSDefaultSystemVPCSupernet anchors the SQS system VPC address space at
|
||||
// 10.240.0.0/14, immediately below and disjoint from the Bedrock, RDS and EKS
|
||||
// control-plane supernets.
|
||||
const SQSDefaultSystemVPCSupernet = "10.240.0.0/14"
|
||||
|
||||
// OchreVectorConfig gates the Ochre vector store's platform Postgres
|
||||
// appliance and VectorService NATS surface. Disabled by default: an unset
|
||||
// section constructs nothing and registers no ochre.vector.* subjects, so an
|
||||
|
||||
@@ -0,0 +1,631 @@
|
||||
// Package awsapi serves the EC2 query-protocol actions over composed capabilities.
|
||||
package awsapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"reflect"
|
||||
"strings"
|
||||
"uuid"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
ec2accountapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/account"
|
||||
ec2capacityreservationapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/capacityreservation"
|
||||
ec2eigwapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/eigw"
|
||||
ec2eipapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/eip"
|
||||
ec2igwapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/igw"
|
||||
ec2imageapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/image"
|
||||
ec2instanceapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/instance"
|
||||
ec2keyapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/key"
|
||||
ec2launchtemplateapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/launchtemplate"
|
||||
ec2natgwapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/natgw"
|
||||
ec2placementgroupapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/placementgroup"
|
||||
ec2routetableapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/routetable"
|
||||
ec2snapshotapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/snapshot"
|
||||
ec2spotinstanceapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/spotinstance"
|
||||
ec2tagsapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/tags"
|
||||
ec2volumeapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/volume"
|
||||
ec2vpcapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/vpc"
|
||||
ec2zoneapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/zone"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
awsxml "github.com/mulgadc/spinifex/spinifex/foundation/aws/xml"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/lifecycle/idempotency"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/query"
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// ec2Action parses once so authorization and dispatch share one typed input.
|
||||
type ec2Action struct {
|
||||
parse func(map[string]string) (any, error)
|
||||
dispatch func(ctx context.Context, action string, parsed any, c *call, accountID string) ([]byte, error)
|
||||
// idempotent records that this action honours ClientToken, so the set of
|
||||
// actions that do is assertable rather than folklore.
|
||||
idempotent bool
|
||||
}
|
||||
|
||||
func (h ec2Action) withQueryPreprocessor(preprocess func(map[string]string)) ec2Action {
|
||||
parse := h.parse
|
||||
h.parse = func(q map[string]string) (any, error) {
|
||||
preprocess(q)
|
||||
return parse(q)
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
// requestContext returns r's context, or Background for a nil request
|
||||
// (some callers and tests dispatch without an *http.Request). The SDK's
|
||||
// invocation id rides along: it is repeated across retries of one call, so it
|
||||
// is what lets a mutation recognise its own resend instead of doing the work
|
||||
// twice.
|
||||
func requestContext(r *http.Request) context.Context {
|
||||
if r == nil {
|
||||
return context.Background()
|
||||
}
|
||||
return idempotency.WithKey(r.Context(), r.Header.Get(idempotency.SDKInvocationIDHeader))
|
||||
}
|
||||
|
||||
// parseEC2Input parses query args into an action's input type. Shared by every
|
||||
// ec2Action constructor so they all consume the same parse.
|
||||
func parseEC2Input[In any](q map[string]string) (any, error) {
|
||||
input := new(In)
|
||||
if err := query.QueryParamsToStruct(q, input); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return input, nil
|
||||
}
|
||||
|
||||
// ec2Handler creates a type-safe EC2Handler. Parsing is separate from dispatch
|
||||
// so authorization and execution consume the exact same input.
|
||||
func ec2Handler[In any](handler func(ctx context.Context, input *In, c *call, accountID string) (any, error)) ec2Action {
|
||||
return ec2Action{
|
||||
parse: parseEC2Input[In],
|
||||
dispatch: func(ctx context.Context, action string, parsed any, c *call, accountID string) ([]byte, error) {
|
||||
input, ok := parsed.(*In)
|
||||
if !ok {
|
||||
return nil, errors.New(awserrors.ErrorInternalError)
|
||||
}
|
||||
output, err := handler(ctx, input, c, accountID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
xmlOutput, err := marshalEC2Response(action, output)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return xmlOutput, nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// ec2ListsAsSet names the list fields AWS omits when empty instead of
|
||||
// rendering them. Each is rendered as its handler set it, so nil omits it.
|
||||
var ec2ListsAsSet = map[reflect.Type][]string{
|
||||
reflect.TypeFor[ec2.CreateKeyPairOutput](): {"Tags"},
|
||||
reflect.TypeFor[ec2.Image](): {"Tags", "ProductCodes"},
|
||||
reflect.TypeFor[ec2.CreateSecurityGroupOutput](): {"Tags"},
|
||||
reflect.TypeFor[ec2.SecurityGroup](): {"Tags"},
|
||||
reflect.TypeFor[ec2.SecurityGroupRule](): {"Tags"},
|
||||
reflect.TypeFor[ec2.RevokeSecurityGroupIngressOutput](): {"UnknownIpPermissions"},
|
||||
reflect.TypeFor[ec2.RevokeSecurityGroupEgressOutput](): {"UnknownIpPermissions"},
|
||||
reflect.TypeFor[ec2.NetworkInterface](): {"Ipv4Prefixes", "Ipv6Prefixes"},
|
||||
reflect.TypeFor[ec2.Vpc](): {"Ipv6CidrBlockAssociationSet"},
|
||||
reflect.TypeFor[ec2.Subnet](): {"Tags"},
|
||||
}
|
||||
|
||||
// marshalEC2Response renders an EC2 handler's output into the action's XML
|
||||
// envelope. Every EC2 action funnels through here (via ec2Handler and
|
||||
// ec2HandlerWithReq), so wire-format fixes belong here, not in each handler.
|
||||
func marshalEC2Response(action string, output any) ([]byte, error) {
|
||||
// BuildXML omits a nil slice's container element entirely but renders an
|
||||
// empty one for a non-nil empty slice; AWS mostly renders the latter.
|
||||
normalized := awsxml.NormalizeOutput(output, ec2ListsAsSet)
|
||||
// The SDK's generated output structs never carry a RequestId field.
|
||||
withRequestID := awsxml.WithRequestID(normalized, uuid.NewV4().String())
|
||||
payload := awsxml.ResponsePayload(action+"Response", withRequestID)
|
||||
xmlOutput, err := awsxml.Marshal(payload)
|
||||
if err != nil {
|
||||
return nil, errors.New("failed to marshal response to XML")
|
||||
}
|
||||
return xmlOutput, nil
|
||||
}
|
||||
|
||||
var ec2Actions = map[string]ec2Action{
|
||||
"DescribeInstances": ec2Handler(func(ctx context.Context, input *ec2.DescribeInstancesInput, c *call, accountID string) (any, error) {
|
||||
out, err := ec2instanceapi.DescribeInstancesChecked(ctx, input, c.NATS, 0, c.NodeIDs, accountID)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
ec2instanceapi.EnrichInstanceProfileIDs(out, c.IAM, accountID)
|
||||
return out, nil
|
||||
}),
|
||||
"RunInstances": ec2Handler(func(ctx context.Context, input *ec2.RunInstancesInput, c *call, accountID string) (any, error) {
|
||||
passRoleCheck := func(roleARN string) error {
|
||||
return c.passRole(roleARN)
|
||||
}
|
||||
launchQuotaCheck := func() error {
|
||||
return c.Quota.EnforceLaunch(ctx, accountID, aws.StringValue(input.InstanceType), int(aws.Int64Value(input.MaxCount)))
|
||||
}
|
||||
reservation, err := ec2instanceapi.RunInstances(ctx, input, c.NATS, c.IAM, accountID, passRoleCheck, launchQuotaCheck, c.ExpectedNodes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Charge the actual launched vCPUs; a counter write failure is drift for
|
||||
// reconcile to correct, so it must not fail the already-successful launch.
|
||||
if err := c.Quota.ChargeLaunch(ctx, accountID, &reservation); err != nil {
|
||||
slog.WarnContext(ctx, "RunInstances: vcpu quota charge failed, reconcile will correct", "account", accountID, "err", err)
|
||||
}
|
||||
return reservation, nil
|
||||
}),
|
||||
"AssociateIamInstanceProfile": ec2Handler(func(ctx context.Context, input *ec2.AssociateIamInstanceProfileInput, c *call, accountID string) (any, error) {
|
||||
passRoleCheck := func(roleARN string) error {
|
||||
return c.passRole(roleARN)
|
||||
}
|
||||
return ec2instanceapi.AssociateIamInstanceProfile(ctx, input, c.NATS, c.IAM, accountID, passRoleCheck)
|
||||
}),
|
||||
"DisassociateIamInstanceProfile": ec2Handler(func(ctx context.Context, input *ec2.DisassociateIamInstanceProfileInput, c *call, accountID string) (any, error) {
|
||||
return ec2instanceapi.DisassociateIamInstanceProfile(ctx, input, c.NATS, c.ActiveNodes(ctx), accountID)
|
||||
}),
|
||||
"ReplaceIamInstanceProfileAssociation": ec2Handler(func(ctx context.Context, input *ec2.ReplaceIamInstanceProfileAssociationInput, c *call, accountID string) (any, error) {
|
||||
passRoleCheck := func(roleARN string) error {
|
||||
return c.passRole(roleARN)
|
||||
}
|
||||
return ec2instanceapi.ReplaceIamInstanceProfileAssociation(ctx, input, c.NATS, c.IAM, c.ActiveNodes(ctx), accountID, passRoleCheck)
|
||||
}),
|
||||
"DescribeIamInstanceProfileAssociations": ec2Handler(func(ctx context.Context, input *ec2.DescribeIamInstanceProfileAssociationsInput, c *call, accountID string) (any, error) {
|
||||
return ec2instanceapi.DescribeIamInstanceProfileAssociations(ctx, input, c.NATS, c.ActiveNodes(ctx), accountID)
|
||||
}),
|
||||
"StartInstances": ec2Handler(func(ctx context.Context, input *ec2.StartInstancesInput, c *call, accountID string) (any, error) {
|
||||
return ec2instanceapi.StartInstances(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"StopInstances": ec2Handler(func(ctx context.Context, input *ec2.StopInstancesInput, c *call, accountID string) (any, error) {
|
||||
return ec2instanceapi.StopInstances(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"RebootInstances": ec2Handler(func(ctx context.Context, input *ec2.RebootInstancesInput, c *call, accountID string) (any, error) {
|
||||
return ec2instanceapi.RebootInstances(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"MonitorInstances": ec2Handler(func(ctx context.Context, input *ec2.MonitorInstancesInput, c *call, accountID string) (any, error) {
|
||||
return ec2instanceapi.MonitorInstances(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"UnmonitorInstances": ec2Handler(func(ctx context.Context, input *ec2.UnmonitorInstancesInput, c *call, accountID string) (any, error) {
|
||||
return ec2instanceapi.UnmonitorInstances(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"TerminateInstances": ec2Handler(func(ctx context.Context, input *ec2.TerminateInstancesInput, c *call, accountID string) (any, error) {
|
||||
return ec2instanceapi.TerminateInstances(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DescribeInstanceTypes": ec2Handler(func(ctx context.Context, input *ec2.DescribeInstanceTypesInput, c *call, accountID string) (any, error) {
|
||||
return ec2instanceapi.DescribeInstanceTypes(ctx, input, c.NATS, c.ExpectedNodes, c.NodeIDs, accountID)
|
||||
}),
|
||||
"DescribeInstanceTypeOfferings": ec2Handler(func(ctx context.Context, input *ec2.DescribeInstanceTypeOfferingsInput, c *call, accountID string) (any, error) {
|
||||
return ec2instanceapi.DescribeInstanceTypeOfferings(ctx, input, c.NATS, c.ExpectedNodes, accountID, c.Region, c.AZ)
|
||||
}),
|
||||
"DescribeInstanceStatus": ec2Handler(func(ctx context.Context, input *ec2.DescribeInstanceStatusInput, c *call, accountID string) (any, error) {
|
||||
return ec2instanceapi.DescribeInstanceStatus(ctx, input, c.NATS, c.ActiveNodes(ctx), accountID, c.AZ, c.InstanceStatus)
|
||||
}),
|
||||
"GetConsoleOutput": ec2Handler(func(ctx context.Context, input *ec2.GetConsoleOutputInput, c *call, accountID string) (any, error) {
|
||||
return ec2instanceapi.GetConsoleOutput(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"GetPasswordData": ec2Handler(func(ctx context.Context, input *ec2.GetPasswordDataInput, c *call, accountID string) (any, error) {
|
||||
return ec2instanceapi.GetPasswordData(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"ModifyInstanceAttribute": ec2Handler(func(ctx context.Context, input *ec2.ModifyInstanceAttributeInput, c *call, accountID string) (any, error) {
|
||||
var delta int
|
||||
if input.InstanceType != nil {
|
||||
d, err := c.Quota.EnforceRetype(ctx, c.NATS, func() []string { return c.NodeIDs }, accountID, aws.StringValue(input.InstanceId), aws.StringValue(input.InstanceType.Value))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
delta = d
|
||||
}
|
||||
out, err := ec2instanceapi.ModifyInstanceAttribute(ctx, input, c.NATS, accountID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Charge the retype's vCPU growth; a counter write failure is drift for
|
||||
// reconcile to correct, so it must not fail the applied retype.
|
||||
if err := c.Quota.AddVCPU(ctx, accountID, delta); err != nil {
|
||||
slog.WarnContext(ctx, "ModifyInstanceAttribute: vcpu quota charge failed, reconcile will correct", "account", accountID, "err", err)
|
||||
}
|
||||
return out, nil
|
||||
}),
|
||||
"DescribeInstanceAttribute": ec2Handler(func(ctx context.Context, input *ec2.DescribeInstanceAttributeInput, c *call, accountID string) (any, error) {
|
||||
return ec2instanceapi.DescribeInstanceAttribute(ctx, input, c.NATS, c.ActiveNodes(ctx), accountID)
|
||||
}),
|
||||
"ModifyInstanceMetadataOptions": ec2Handler(func(ctx context.Context, input *ec2.ModifyInstanceMetadataOptionsInput, c *call, accountID string) (any, error) {
|
||||
return ec2instanceapi.ModifyInstanceMetadataOptions(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DescribeInstanceCreditSpecifications": ec2Handler(func(ctx context.Context, input *ec2.DescribeInstanceCreditSpecificationsInput, c *call, accountID string) (any, error) {
|
||||
return ec2instanceapi.DescribeInstanceCreditSpecifications(input)
|
||||
}),
|
||||
"CreateKeyPair": ec2Handler(func(ctx context.Context, input *ec2.CreateKeyPairInput, c *call, accountID string) (any, error) {
|
||||
return ec2keyapi.CreateKeyPair(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DeleteKeyPair": ec2Handler(func(ctx context.Context, input *ec2.DeleteKeyPairInput, c *call, accountID string) (any, error) {
|
||||
return ec2keyapi.DeleteKeyPair(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DescribeKeyPairs": ec2Handler(func(ctx context.Context, input *ec2.DescribeKeyPairsInput, c *call, accountID string) (any, error) {
|
||||
return ec2keyapi.DescribeKeyPairs(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"ImportKeyPair": ec2Handler(func(ctx context.Context, input *ec2.ImportKeyPairInput, c *call, accountID string) (any, error) {
|
||||
return ec2keyapi.ImportKeyPair(ctx, input, c.NATS, accountID)
|
||||
}).withQueryPreprocessor(func(q map[string]string) {
|
||||
// Parser leaves Base64 padding URL-encoded; decode it before dispatch.
|
||||
if strings.HasSuffix(q["PublicKeyMaterial"], "%3D%3D") {
|
||||
q["PublicKeyMaterial"] = strings.Replace(q["PublicKeyMaterial"], "%3D%3D", "==", 1)
|
||||
}
|
||||
}),
|
||||
"DescribeImages": ec2Handler(func(ctx context.Context, input *ec2.DescribeImagesInput, c *call, accountID string) (any, error) {
|
||||
return ec2imageapi.DescribeImages(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"CreateImage": ec2Handler(func(ctx context.Context, input *ec2.CreateImageInput, c *call, accountID string) (any, error) {
|
||||
return ec2imageapi.CreateImage(ctx, input, c.NATS, c.ActiveNodes(ctx), accountID)
|
||||
}),
|
||||
"DeregisterImage": ec2Handler(func(ctx context.Context, input *ec2.DeregisterImageInput, c *call, accountID string) (any, error) {
|
||||
return ec2imageapi.DeregisterImage(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"RegisterImage": ec2Handler(func(ctx context.Context, input *ec2.RegisterImageInput, c *call, accountID string) (any, error) {
|
||||
return ec2imageapi.RegisterImage(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"CopyImage": ec2IdempotentHandler(func(ctx context.Context, input *ec2.CopyImageInput, c *call, accountID string) (ec2.CopyImageOutput, error) {
|
||||
return ec2imageapi.CopyImage(ctx, input, c.NATS, c.Region, accountID)
|
||||
}),
|
||||
"DescribeImageAttribute": ec2Handler(func(ctx context.Context, input *ec2.DescribeImageAttributeInput, c *call, accountID string) (any, error) {
|
||||
return ec2imageapi.DescribeImageAttribute(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"ModifyImageAttribute": ec2Handler(func(ctx context.Context, input *ec2.ModifyImageAttributeInput, c *call, accountID string) (any, error) {
|
||||
return ec2imageapi.ModifyImageAttribute(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"ResetImageAttribute": ec2Handler(func(ctx context.Context, input *ec2.ResetImageAttributeInput, c *call, accountID string) (any, error) {
|
||||
return ec2imageapi.ResetImageAttribute(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DescribeRegions": ec2Handler(func(ctx context.Context, input *ec2.DescribeRegionsInput, c *call, accountID string) (any, error) {
|
||||
return ec2zoneapi.DescribeRegions(input, c.Region, c.Endpoint)
|
||||
}),
|
||||
"DescribeAvailabilityZones": ec2Handler(func(ctx context.Context, input *ec2.DescribeAvailabilityZonesInput, c *call, accountID string) (any, error) {
|
||||
return ec2zoneapi.DescribeAvailabilityZones(input, c.Region, c.AZ)
|
||||
}),
|
||||
"DescribeVolumes": ec2Handler(func(ctx context.Context, input *ec2.DescribeVolumesInput, c *call, accountID string) (any, error) {
|
||||
return ec2volumeapi.DescribeVolumes(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"ModifyVolume": ec2Handler(func(ctx context.Context, input *ec2.ModifyVolumeInput, c *call, accountID string) (any, error) {
|
||||
if err := c.Quota.EnforceVolumeModify(ctx, c.NATS, accountID, aws.StringValue(input.VolumeId), int(aws.Int64Value(input.Size))); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ec2volumeapi.ModifyVolume(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"CreateVolume": ec2IdempotentHandler(func(ctx context.Context, input *ec2.CreateVolumeInput, c *call, accountID string) (ec2.Volume, error) {
|
||||
if err := c.Quota.EnforceVolumeCreate(ctx, c.NATS, accountID, int(aws.Int64Value(input.Size))); err != nil {
|
||||
return ec2.Volume{}, err
|
||||
}
|
||||
return ec2volumeapi.CreateVolume(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DeleteVolume": ec2Handler(func(ctx context.Context, input *ec2.DeleteVolumeInput, c *call, accountID string) (any, error) {
|
||||
return ec2volumeapi.DeleteVolume(ctx, input, c.NATS, c.NodeIDs, accountID)
|
||||
}),
|
||||
"AttachVolume": ec2Handler(func(ctx context.Context, input *ec2.AttachVolumeInput, c *call, accountID string) (any, error) {
|
||||
return ec2volumeapi.AttachVolume(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DescribeVolumeStatus": ec2Handler(func(ctx context.Context, input *ec2.DescribeVolumeStatusInput, c *call, accountID string) (any, error) {
|
||||
return ec2volumeapi.DescribeVolumeStatus(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DescribeVolumesModifications": ec2Handler(func(ctx context.Context, input *ec2.DescribeVolumesModificationsInput, c *call, accountID string) (any, error) {
|
||||
return ec2volumeapi.DescribeVolumesModifications(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DetachVolume": ec2Handler(func(ctx context.Context, input *ec2.DetachVolumeInput, c *call, accountID string) (any, error) {
|
||||
return ec2volumeapi.DetachVolume(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DescribeAccountAttributes": ec2Handler(func(ctx context.Context, input *ec2.DescribeAccountAttributesInput, c *call, accountID string) (any, error) {
|
||||
return ec2accountapi.DescribeAccountAttributes(input, func() (string, error) {
|
||||
return defaultVPCID(ctx, c.NATS, accountID)
|
||||
})
|
||||
}),
|
||||
"EnableEbsEncryptionByDefault": ec2Handler(func(ctx context.Context, input *ec2.EnableEbsEncryptionByDefaultInput, c *call, accountID string) (any, error) {
|
||||
return ec2accountapi.EnableEbsEncryptionByDefault(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DisableEbsEncryptionByDefault": ec2Handler(func(ctx context.Context, input *ec2.DisableEbsEncryptionByDefaultInput, c *call, accountID string) (any, error) {
|
||||
return ec2accountapi.DisableEbsEncryptionByDefault(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"GetEbsEncryptionByDefault": ec2Handler(func(ctx context.Context, input *ec2.GetEbsEncryptionByDefaultInput, c *call, accountID string) (any, error) {
|
||||
return ec2accountapi.GetEbsEncryptionByDefault(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"GetSerialConsoleAccessStatus": ec2Handler(func(ctx context.Context, input *ec2.GetSerialConsoleAccessStatusInput, c *call, accountID string) (any, error) {
|
||||
return ec2accountapi.GetSerialConsoleAccessStatus(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"EnableSerialConsoleAccess": ec2Handler(func(ctx context.Context, input *ec2.EnableSerialConsoleAccessInput, c *call, accountID string) (any, error) {
|
||||
return ec2accountapi.EnableSerialConsoleAccess(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DisableSerialConsoleAccess": ec2Handler(func(ctx context.Context, input *ec2.DisableSerialConsoleAccessInput, c *call, accountID string) (any, error) {
|
||||
return ec2accountapi.DisableSerialConsoleAccess(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"CreateTags": ec2Handler(func(ctx context.Context, input *ec2.CreateTagsInput, c *call, accountID string) (any, error) {
|
||||
return ec2tagsapi.CreateTags(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DeleteTags": ec2Handler(func(ctx context.Context, input *ec2.DeleteTagsInput, c *call, accountID string) (any, error) {
|
||||
return ec2tagsapi.DeleteTags(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DescribeTags": ec2Handler(func(ctx context.Context, input *ec2.DescribeTagsInput, c *call, accountID string) (any, error) {
|
||||
return ec2tagsapi.DescribeTags(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"CreateSnapshot": ec2Handler(func(ctx context.Context, input *ec2.CreateSnapshotInput, c *call, accountID string) (any, error) {
|
||||
return ec2snapshotapi.CreateSnapshot(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DeleteSnapshot": ec2Handler(func(ctx context.Context, input *ec2.DeleteSnapshotInput, c *call, accountID string) (any, error) {
|
||||
return ec2snapshotapi.DeleteSnapshot(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DescribeSnapshots": ec2Handler(func(ctx context.Context, input *ec2.DescribeSnapshotsInput, c *call, accountID string) (any, error) {
|
||||
return ec2snapshotapi.DescribeSnapshots(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"CopySnapshot": ec2Handler(func(ctx context.Context, input *ec2.CopySnapshotInput, c *call, accountID string) (any, error) {
|
||||
return ec2snapshotapi.CopySnapshot(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"CreateInternetGateway": ec2Handler(func(ctx context.Context, input *ec2.CreateInternetGatewayInput, c *call, accountID string) (any, error) {
|
||||
return ec2igwapi.CreateInternetGateway(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DeleteInternetGateway": ec2Handler(func(ctx context.Context, input *ec2.DeleteInternetGatewayInput, c *call, accountID string) (any, error) {
|
||||
return ec2igwapi.DeleteInternetGateway(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DescribeInternetGateways": ec2Handler(func(ctx context.Context, input *ec2.DescribeInternetGatewaysInput, c *call, accountID string) (any, error) {
|
||||
return ec2igwapi.DescribeInternetGateways(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"AttachInternetGateway": ec2Handler(func(ctx context.Context, input *ec2.AttachInternetGatewayInput, c *call, accountID string) (any, error) {
|
||||
return ec2igwapi.AttachInternetGateway(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DetachInternetGateway": ec2Handler(func(ctx context.Context, input *ec2.DetachInternetGatewayInput, c *call, accountID string) (any, error) {
|
||||
return ec2igwapi.DetachInternetGateway(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"CreateEgressOnlyInternetGateway": ec2IdempotentHandler(func(ctx context.Context, input *ec2.CreateEgressOnlyInternetGatewayInput, c *call, accountID string) (ec2.CreateEgressOnlyInternetGatewayOutput, error) {
|
||||
return ec2eigwapi.CreateEgressOnlyInternetGateway(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DeleteEgressOnlyInternetGateway": ec2Handler(func(ctx context.Context, input *ec2.DeleteEgressOnlyInternetGatewayInput, c *call, accountID string) (any, error) {
|
||||
return ec2eigwapi.DeleteEgressOnlyInternetGateway(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DescribeEgressOnlyInternetGateways": ec2Handler(func(ctx context.Context, input *ec2.DescribeEgressOnlyInternetGatewaysInput, c *call, accountID string) (any, error) {
|
||||
return ec2eigwapi.DescribeEgressOnlyInternetGateways(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"CreatePlacementGroup": ec2Handler(func(ctx context.Context, input *ec2.CreatePlacementGroupInput, c *call, accountID string) (any, error) {
|
||||
return ec2placementgroupapi.CreatePlacementGroup(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DeletePlacementGroup": ec2Handler(func(ctx context.Context, input *ec2.DeletePlacementGroupInput, c *call, accountID string) (any, error) {
|
||||
return ec2placementgroupapi.DeletePlacementGroup(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DescribePlacementGroups": ec2Handler(func(ctx context.Context, input *ec2.DescribePlacementGroupsInput, c *call, accountID string) (any, error) {
|
||||
return ec2placementgroupapi.DescribePlacementGroups(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"CreateCapacityReservation": ec2IdempotentHandler(func(ctx context.Context, input *ec2.CreateCapacityReservationInput, c *call, accountID string) (ec2.CreateCapacityReservationOutput, error) {
|
||||
return ec2capacityreservationapi.CreateCapacityReservation(ctx, input, c.NATS, c.ExpectedNodes, accountID)
|
||||
}),
|
||||
"DescribeCapacityReservations": ec2Handler(func(ctx context.Context, input *ec2.DescribeCapacityReservationsInput, c *call, accountID string) (any, error) {
|
||||
return ec2capacityreservationapi.DescribeCapacityReservations(ctx, input, c.NATS, c.ExpectedNodes, accountID)
|
||||
}),
|
||||
"CancelCapacityReservation": ec2Handler(func(ctx context.Context, input *ec2.CancelCapacityReservationInput, c *call, accountID string) (any, error) {
|
||||
return ec2capacityreservationapi.CancelCapacityReservation(ctx, input, c.NATS, c.ExpectedNodes, accountID)
|
||||
}),
|
||||
"CreateLaunchTemplate": ec2Handler(func(ctx context.Context, input *ec2.CreateLaunchTemplateInput, c *call, accountID string) (any, error) {
|
||||
return ec2launchtemplateapi.CreateLaunchTemplate(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"CreateLaunchTemplateVersion": ec2IdempotentHandler(func(ctx context.Context, input *ec2.CreateLaunchTemplateVersionInput, c *call, accountID string) (ec2.CreateLaunchTemplateVersionOutput, error) {
|
||||
return ec2launchtemplateapi.CreateLaunchTemplateVersion(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DeleteLaunchTemplate": ec2Handler(func(ctx context.Context, input *ec2.DeleteLaunchTemplateInput, c *call, accountID string) (any, error) {
|
||||
return ec2launchtemplateapi.DeleteLaunchTemplate(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DeleteLaunchTemplateVersions": ec2Handler(func(ctx context.Context, input *ec2.DeleteLaunchTemplateVersionsInput, c *call, accountID string) (any, error) {
|
||||
return ec2launchtemplateapi.DeleteLaunchTemplateVersions(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"ModifyLaunchTemplate": ec2Handler(func(ctx context.Context, input *ec2.ModifyLaunchTemplateInput, c *call, accountID string) (any, error) {
|
||||
return ec2launchtemplateapi.ModifyLaunchTemplate(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DescribeLaunchTemplates": ec2Handler(func(ctx context.Context, input *ec2.DescribeLaunchTemplatesInput, c *call, accountID string) (any, error) {
|
||||
return ec2launchtemplateapi.DescribeLaunchTemplates(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DescribeLaunchTemplateVersions": ec2Handler(func(ctx context.Context, input *ec2.DescribeLaunchTemplateVersionsInput, c *call, accountID string) (any, error) {
|
||||
return ec2launchtemplateapi.DescribeLaunchTemplateVersions(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"RequestSpotInstances": ec2Handler(func(ctx context.Context, input *ec2.RequestSpotInstancesInput, c *call, accountID string) (any, error) {
|
||||
passRoleCheck := func(roleARN string) error {
|
||||
return c.passRole(roleARN)
|
||||
}
|
||||
return ec2spotinstanceapi.RequestSpotInstances(ctx, input, c.NATS, c.IAM, accountID, c.AZ, passRoleCheck, c.Quota, c.ExpectedNodes)
|
||||
}),
|
||||
"DescribeSpotInstanceRequests": ec2Handler(func(ctx context.Context, input *ec2.DescribeSpotInstanceRequestsInput, c *call, accountID string) (any, error) {
|
||||
return ec2spotinstanceapi.DescribeSpotInstanceRequests(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"CancelSpotInstanceRequests": ec2Handler(func(ctx context.Context, input *ec2.CancelSpotInstanceRequestsInput, c *call, accountID string) (any, error) {
|
||||
return ec2spotinstanceapi.CancelSpotInstanceRequests(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"CreateVpc": ec2Handler(func(ctx context.Context, input *ec2.CreateVpcInput, c *call, accountID string) (any, error) {
|
||||
if err := c.Quota.EnforceVPCs(ctx, c.NATS, accountID, 1); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ec2vpcapi.CreateVpc(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DeleteVpc": ec2Handler(func(ctx context.Context, input *ec2.DeleteVpcInput, c *call, accountID string) (any, error) {
|
||||
return ec2vpcapi.DeleteVpc(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DescribeVpcs": ec2Handler(func(ctx context.Context, input *ec2.DescribeVpcsInput, c *call, accountID string) (any, error) {
|
||||
return ec2vpcapi.DescribeVpcs(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"ModifyVpcAttribute": ec2Handler(func(ctx context.Context, input *ec2.ModifyVpcAttributeInput, c *call, accountID string) (any, error) {
|
||||
return ec2vpcapi.ModifyVpcAttribute(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DescribeVpcAttribute": ec2Handler(func(ctx context.Context, input *ec2.DescribeVpcAttributeInput, c *call, accountID string) (any, error) {
|
||||
return ec2vpcapi.DescribeVpcAttribute(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
// No VPC endpoint can exist here, because CreateVpcEndpoint is not served, so
|
||||
// the empty list is the true answer rather than a stand-in for one. It has to
|
||||
// be answered all the same: the Terraform AWS provider calls this on every VPC
|
||||
// delete to sweep GuardDuty-managed endpoints, and an InvalidAction there fails
|
||||
// the destroy and strands the VPC.
|
||||
"DescribeVpcEndpoints": ec2Handler(func(_ context.Context, _ *ec2.DescribeVpcEndpointsInput, _ *call, _ string) (any, error) {
|
||||
return ec2.DescribeVpcEndpointsOutput{VpcEndpoints: []*ec2.VpcEndpoint{}}, nil
|
||||
}),
|
||||
"CreateSubnet": ec2Handler(func(ctx context.Context, input *ec2.CreateSubnetInput, c *call, accountID string) (any, error) {
|
||||
if err := c.Quota.EnforceSubnets(ctx, c.NATS, accountID, 1); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ec2vpcapi.CreateSubnet(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DeleteSubnet": ec2Handler(func(ctx context.Context, input *ec2.DeleteSubnetInput, c *call, accountID string) (any, error) {
|
||||
return ec2vpcapi.DeleteSubnet(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DescribeSubnets": ec2Handler(func(ctx context.Context, input *ec2.DescribeSubnetsInput, c *call, accountID string) (any, error) {
|
||||
return ec2vpcapi.DescribeSubnets(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"ModifySubnetAttribute": ec2Handler(func(ctx context.Context, input *ec2.ModifySubnetAttributeInput, c *call, accountID string) (any, error) {
|
||||
return ec2vpcapi.ModifySubnetAttribute(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"CreateRouteTable": ec2IdempotentHandler(func(ctx context.Context, input *ec2.CreateRouteTableInput, c *call, accountID string) (ec2.CreateRouteTableOutput, error) {
|
||||
return ec2routetableapi.CreateRouteTable(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DeleteRouteTable": ec2Handler(func(ctx context.Context, input *ec2.DeleteRouteTableInput, c *call, accountID string) (any, error) {
|
||||
return ec2routetableapi.DeleteRouteTable(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DescribeRouteTables": ec2Handler(func(ctx context.Context, input *ec2.DescribeRouteTablesInput, c *call, accountID string) (any, error) {
|
||||
return ec2routetableapi.DescribeRouteTables(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"CreateRoute": ec2Handler(func(ctx context.Context, input *ec2.CreateRouteInput, c *call, accountID string) (any, error) {
|
||||
return ec2routetableapi.CreateRoute(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DeleteRoute": ec2Handler(func(ctx context.Context, input *ec2.DeleteRouteInput, c *call, accountID string) (any, error) {
|
||||
return ec2routetableapi.DeleteRoute(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"ReplaceRoute": ec2Handler(func(ctx context.Context, input *ec2.ReplaceRouteInput, c *call, accountID string) (any, error) {
|
||||
return ec2routetableapi.ReplaceRoute(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"AssociateRouteTable": ec2Handler(func(ctx context.Context, input *ec2.AssociateRouteTableInput, c *call, accountID string) (any, error) {
|
||||
return ec2routetableapi.AssociateRouteTable(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DisassociateRouteTable": ec2Handler(func(ctx context.Context, input *ec2.DisassociateRouteTableInput, c *call, accountID string) (any, error) {
|
||||
return ec2routetableapi.DisassociateRouteTable(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"ReplaceRouteTableAssociation": ec2Handler(func(ctx context.Context, input *ec2.ReplaceRouteTableAssociationInput, c *call, accountID string) (any, error) {
|
||||
return ec2routetableapi.ReplaceRouteTableAssociation(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"CreateNetworkInterface": ec2IdempotentHandler(func(ctx context.Context, input *ec2.CreateNetworkInterfaceInput, c *call, accountID string) (ec2.CreateNetworkInterfaceOutput, error) {
|
||||
return ec2vpcapi.CreateNetworkInterface(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DeleteNetworkInterface": ec2Handler(func(ctx context.Context, input *ec2.DeleteNetworkInterfaceInput, c *call, accountID string) (any, error) {
|
||||
return ec2vpcapi.DeleteNetworkInterface(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DescribeNetworkInterfaces": ec2Handler(func(ctx context.Context, input *ec2.DescribeNetworkInterfacesInput, c *call, accountID string) (any, error) {
|
||||
return ec2vpcapi.DescribeNetworkInterfaces(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"ModifyNetworkInterfaceAttribute": ec2Handler(func(ctx context.Context, input *ec2.ModifyNetworkInterfaceAttributeInput, c *call, accountID string) (any, error) {
|
||||
return ec2vpcapi.ModifyNetworkInterfaceAttribute(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"AttachNetworkInterface": ec2Handler(func(ctx context.Context, input *ec2.AttachNetworkInterfaceInput, c *call, accountID string) (any, error) {
|
||||
return ec2vpcapi.AttachNetworkInterface(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DetachNetworkInterface": ec2Handler(func(ctx context.Context, input *ec2.DetachNetworkInterfaceInput, c *call, accountID string) (any, error) {
|
||||
return ec2vpcapi.DetachNetworkInterface(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"CreateSecurityGroup": ec2Handler(func(ctx context.Context, input *ec2.CreateSecurityGroupInput, c *call, accountID string) (any, error) {
|
||||
return ec2vpcapi.CreateSecurityGroup(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DeleteSecurityGroup": ec2Handler(func(ctx context.Context, input *ec2.DeleteSecurityGroupInput, c *call, accountID string) (any, error) {
|
||||
return ec2vpcapi.DeleteSecurityGroup(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DescribeSecurityGroups": ec2Handler(func(ctx context.Context, input *ec2.DescribeSecurityGroupsInput, c *call, accountID string) (any, error) {
|
||||
return ec2vpcapi.DescribeSecurityGroups(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DescribeSecurityGroupRules": ec2Handler(func(ctx context.Context, input *ec2.DescribeSecurityGroupRulesInput, c *call, accountID string) (any, error) {
|
||||
return ec2vpcapi.DescribeSecurityGroupRules(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"GetSecurityGroupsForVpc": ec2Handler(func(ctx context.Context, input *ec2.GetSecurityGroupsForVpcInput, c *call, accountID string) (any, error) {
|
||||
return ec2vpcapi.GetSecurityGroupsForVpc(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"AuthorizeSecurityGroupIngress": ec2Handler(func(ctx context.Context, input *ec2.AuthorizeSecurityGroupIngressInput, c *call, accountID string) (any, error) {
|
||||
return ec2vpcapi.AuthorizeSecurityGroupIngress(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"AuthorizeSecurityGroupEgress": ec2Handler(func(ctx context.Context, input *ec2.AuthorizeSecurityGroupEgressInput, c *call, accountID string) (any, error) {
|
||||
return ec2vpcapi.AuthorizeSecurityGroupEgress(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"RevokeSecurityGroupIngress": ec2Handler(func(ctx context.Context, input *ec2.RevokeSecurityGroupIngressInput, c *call, accountID string) (any, error) {
|
||||
return ec2vpcapi.RevokeSecurityGroupIngress(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"RevokeSecurityGroupEgress": ec2Handler(func(ctx context.Context, input *ec2.RevokeSecurityGroupEgressInput, c *call, accountID string) (any, error) {
|
||||
return ec2vpcapi.RevokeSecurityGroupEgress(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"UpdateSecurityGroupRuleDescriptionsIngress": ec2Handler(func(ctx context.Context, input *ec2.UpdateSecurityGroupRuleDescriptionsIngressInput, c *call, accountID string) (any, error) {
|
||||
return ec2vpcapi.UpdateSecurityGroupRuleDescriptionsIngress(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"UpdateSecurityGroupRuleDescriptionsEgress": ec2Handler(func(ctx context.Context, input *ec2.UpdateSecurityGroupRuleDescriptionsEgressInput, c *call, accountID string) (any, error) {
|
||||
return ec2vpcapi.UpdateSecurityGroupRuleDescriptionsEgress(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"ModifySecurityGroupRules": ec2Handler(func(ctx context.Context, input *ec2.ModifySecurityGroupRulesInput, c *call, accountID string) (any, error) {
|
||||
return ec2vpcapi.ModifySecurityGroupRules(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"AllocateAddress": ec2Handler(func(ctx context.Context, input *ec2.AllocateAddressInput, c *call, accountID string) (any, error) {
|
||||
if err := c.Quota.EnforceEIPs(ctx, c.NATS, accountID, 1); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ec2eipapi.AllocateAddress(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"ReleaseAddress": ec2Handler(func(ctx context.Context, input *ec2.ReleaseAddressInput, c *call, accountID string) (any, error) {
|
||||
return ec2eipapi.ReleaseAddress(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"AssociateAddress": ec2Handler(func(ctx context.Context, input *ec2.AssociateAddressInput, c *call, accountID string) (any, error) {
|
||||
return ec2eipapi.AssociateAddress(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DisassociateAddress": ec2Handler(func(ctx context.Context, input *ec2.DisassociateAddressInput, c *call, accountID string) (any, error) {
|
||||
return ec2eipapi.DisassociateAddress(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DescribeAddresses": ec2Handler(func(ctx context.Context, input *ec2.DescribeAddressesInput, c *call, accountID string) (any, error) {
|
||||
return ec2eipapi.DescribeAddresses(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DescribeAddressesAttribute": ec2Handler(func(ctx context.Context, input *ec2.DescribeAddressesAttributeInput, c *call, accountID string) (any, error) {
|
||||
return ec2eipapi.DescribeAddressesAttribute(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"CreateNatGateway": ec2IdempotentHandler(func(ctx context.Context, input *ec2.CreateNatGatewayInput, c *call, accountID string) (ec2.CreateNatGatewayOutput, error) {
|
||||
return ec2natgwapi.CreateNatGateway(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DeleteNatGateway": ec2Handler(func(ctx context.Context, input *ec2.DeleteNatGatewayInput, c *call, accountID string) (any, error) {
|
||||
return ec2natgwapi.DeleteNatGateway(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
"DescribeNatGateways": ec2Handler(func(ctx context.Context, input *ec2.DescribeNatGatewaysInput, c *call, accountID string) (any, error) {
|
||||
return ec2natgwapi.DescribeNatGateways(ctx, input, c.NATS, accountID)
|
||||
}),
|
||||
}
|
||||
|
||||
// defaultVPCID returns the ID of accountID's default VPC, or "" when it has
|
||||
// none, as the account's own DescribeVpcs sees it.
|
||||
func defaultVPCID(ctx context.Context, natsConn *nats.Conn, accountID string) (string, error) {
|
||||
if natsConn == nil {
|
||||
return "", errors.New(awserrors.ErrorServerInternal)
|
||||
}
|
||||
out, err := ec2vpcapi.DescribeVpcs(ctx, &ec2.DescribeVpcsInput{
|
||||
Filters: []*ec2.Filter{{Name: aws.String("is-default"), Values: aws.StringSlice([]string{"true"})}},
|
||||
}, natsConn, accountID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(out.Vpcs) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
return aws.StringValue(out.Vpcs[0].VpcId), nil
|
||||
}
|
||||
|
||||
// ec2LocalActions are actions that don't require NATS. DescribeAccountAttributes
|
||||
// needs it only for default-vpc.
|
||||
var ec2LocalActions = map[string]bool{
|
||||
"DescribeRegions": true,
|
||||
"DescribeAvailabilityZones": true,
|
||||
"DescribeAccountAttributes": true,
|
||||
}
|
||||
|
||||
// dryRunRequested reports whether a parsed EC2 input carries DryRun=true. Only
|
||||
// actions whose SDK input models the parameter can answer DryRunOperation.
|
||||
func dryRunRequested(input any) bool {
|
||||
v := reflect.ValueOf(input)
|
||||
if v.Kind() != reflect.Pointer || v.IsNil() || v.Elem().Kind() != reflect.Struct {
|
||||
return false
|
||||
}
|
||||
field := v.Elem().FieldByName("DryRun")
|
||||
if !field.IsValid() {
|
||||
return false
|
||||
}
|
||||
dryRun, ok := reflect.TypeAssert[*bool](field)
|
||||
return ok && aws.BoolValue(dryRun)
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
//test:in-package — reads the unexported ec2Actions route table.
|
||||
|
||||
package awsapi
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestImportKeyPair_Base64PaddingWorkaround(t *testing.T) {
|
||||
// The ImportKeyPair handler decodes URL-encoded Base64 padding (%3D%3D → ==)
|
||||
// before passing to the generic handler.
|
||||
handler := ec2Actions["ImportKeyPair"]
|
||||
require.NotNil(t, handler)
|
||||
|
||||
q := map[string]string{
|
||||
"Action": "ImportKeyPair",
|
||||
"KeyName": "test-key",
|
||||
"PublicKeyMaterial": "c3NoLXJzYSBBQUFBQjNOemFDMXljMkVBQUFBREFRQUJBQUFCZ1FD%3D%3D",
|
||||
}
|
||||
|
||||
_, err := handler.parse(q)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.True(t, strings.HasSuffix(q["PublicKeyMaterial"], "=="),
|
||||
"Expected PublicKeyMaterial to end with == but got: %s", q["PublicKeyMaterial"])
|
||||
assert.NotContains(t, q["PublicKeyMaterial"], "%3D",
|
||||
"Expected no URL-encoded padding remaining")
|
||||
}
|
||||
+5
-6
@@ -1,12 +1,11 @@
|
||||
//test:in-package — ec2Actions is unexported here, and the completeness test
|
||||
// exists to compare it against the scope table.
|
||||
//test:in-package — ec2Actions is unexported, and the completeness test exists
|
||||
// to compare it against the scope table.
|
||||
|
||||
package gateway
|
||||
package awsapi
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
ec2awsapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
@@ -15,11 +14,11 @@ import (
|
||||
// behind by a deleted or renamed action fails too.
|
||||
func TestEC2ScopeTableIsExhaustive(t *testing.T) {
|
||||
for action := range ec2Actions {
|
||||
assert.True(t, ec2awsapi.HasScope(action),
|
||||
assert.True(t, HasScope(action),
|
||||
"ec2 action %q has no resource scope entry: add one to ec2Scopes in domains/ec2/awsapi/authz.go", action)
|
||||
}
|
||||
|
||||
for _, action := range ec2awsapi.ScopedActions() {
|
||||
for _, action := range ScopedActions() {
|
||||
_, ok := ec2Actions[action]
|
||||
assert.True(t, ok,
|
||||
"ec2Scopes has an entry for %q, which the dispatch table does not serve: remove it from domains/ec2/awsapi/authz.go", action)
|
||||
@@ -0,0 +1,65 @@
|
||||
package awsapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
ec2instanceapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/instance"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// Quota is the account quota the EC2 actions consult and charge. It is declared
|
||||
// here because domains/admission/quota imports the EC2 action packages to count
|
||||
// live resources, and its retype gate takes a resolver type this package must
|
||||
// not name.
|
||||
type Quota interface {
|
||||
EnforceLaunch(ctx context.Context, accountID, instanceType string, maxCount int) error
|
||||
ChargeLaunch(ctx context.Context, accountID string, reservation *ec2.Reservation) error
|
||||
// EnforceRetype returns the vCPU delta AddVCPU charges once the retype applies.
|
||||
EnforceRetype(ctx context.Context, nc *nats.Conn, nodeIDs func() []string, accountID, instanceID, newType string) (int, error)
|
||||
AddVCPU(ctx context.Context, accountID string, delta int) error
|
||||
EnforceVolumeCreate(ctx context.Context, nc *nats.Conn, accountID string, requestedGiB int) error
|
||||
EnforceVolumeModify(ctx context.Context, nc *nats.Conn, accountID, volumeID string, newGiB int) error
|
||||
EnforceEIPs(ctx context.Context, nc *nats.Conn, accountID string, want int) error
|
||||
EnforceSubnets(ctx context.Context, nc *nats.Conn, accountID string, want int) error
|
||||
EnforceVPCs(ctx context.Context, nc *nats.Conn, accountID string, want int) error
|
||||
}
|
||||
|
||||
// Deps are the composed capabilities the EC2 control plane dispatches to.
|
||||
type Deps struct {
|
||||
NATS *nats.Conn
|
||||
IAM ec2instanceapi.InstanceProfiles
|
||||
Quota Quota
|
||||
// ActiveNodes is the number of daemons currently answering.
|
||||
ActiveNodes func(ctx context.Context) int
|
||||
ExpectedNodes int
|
||||
NodeIDs []string
|
||||
AZ string
|
||||
// Endpoint is the gateway's own dialable base URL DescribeRegions advertises.
|
||||
Endpoint string
|
||||
InstanceStatus ec2instanceapi.StatusSynthesis
|
||||
}
|
||||
|
||||
// Handler serves EC2 requests over its Deps. It binds the client-token store on
|
||||
// first use and is safe for concurrent requests.
|
||||
type Handler struct {
|
||||
Deps
|
||||
|
||||
tokenOnce sync.Once
|
||||
tokenKV jetstream.KeyValue
|
||||
tokenErr error
|
||||
}
|
||||
|
||||
// NewHandler returns the handler over deps.
|
||||
func NewHandler(deps Deps) *Handler { return &Handler{Deps: deps} }
|
||||
|
||||
// call is one request's view of the handler: the handler's capabilities plus the
|
||||
// region and the PassRole gate of the authenticated caller.
|
||||
type call struct {
|
||||
*Handler
|
||||
|
||||
Region string
|
||||
passRole func(roleARN string) error
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
//test:in-package — drives the unexported action table without a request.
|
||||
|
||||
package awsapi
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// DescribeRegions must advertise the endpoint the handler was composed with, so
|
||||
// a caller on a concrete-host deployment is never told to dial its own loopback.
|
||||
func TestDescribeRegions_UsesAdvertisedEndpoint(t *testing.T) {
|
||||
c := &call{Handler: NewHandler(Deps{Endpoint: "https://10.0.0.5:9999"}), Region: "ap-southeast-2"}
|
||||
h := ec2Actions["DescribeRegions"]
|
||||
input, err := h.parse(map[string]string{})
|
||||
require.NoError(t, err)
|
||||
xmlOutput, err := h.dispatch(t.Context(), "DescribeRegions", input, c, "acct-123")
|
||||
require.NoError(t, err)
|
||||
|
||||
body := string(xmlOutput)
|
||||
assert.Contains(t, body, "<regionEndpoint>https://10.0.0.5:9999</regionEndpoint>")
|
||||
assert.NotContains(t, body, "localhost")
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
//test:in-package — drives the unexported action table through Dispatch.
|
||||
|
||||
package awsapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/mulgadc/bluebottle/pkg/iampolicy"
|
||||
"github.com/mulgadc/spinifex/internal/testkit"
|
||||
iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type allowQuota struct{}
|
||||
|
||||
var _ Quota = allowQuota{}
|
||||
|
||||
func (allowQuota) EnforceLaunch(context.Context, string, string, int) error { return nil }
|
||||
|
||||
func (allowQuota) ChargeLaunch(context.Context, string, *ec2.Reservation) error { return nil }
|
||||
|
||||
func (allowQuota) EnforceRetype(context.Context, *nats.Conn, func() []string, string, string, string) (int, error) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func (allowQuota) AddVCPU(context.Context, string, int) error { return nil }
|
||||
|
||||
func (allowQuota) EnforceVolumeCreate(context.Context, *nats.Conn, string, int) error { return nil }
|
||||
|
||||
func (allowQuota) EnforceVolumeModify(context.Context, *nats.Conn, string, string, int) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (allowQuota) EnforceEIPs(context.Context, *nats.Conn, string, int) error { return nil }
|
||||
|
||||
func (allowQuota) EnforceSubnets(context.Context, *nats.Conn, string, int) error { return nil }
|
||||
|
||||
func (allowQuota) EnforceVPCs(context.Context, *nats.Conn, string, int) error { return nil }
|
||||
|
||||
type noProfiles struct{}
|
||||
|
||||
func (noProfiles) ResolveInstanceProfile(string, string) (*iamdomain.InstanceProfile, error) {
|
||||
return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
|
||||
}
|
||||
|
||||
func (noProfiles) CanonicalResourceARN(string, arn.IAMResourceType, string) (string, error) {
|
||||
return "", errors.New(awserrors.ErrorIAMNoSuchEntity)
|
||||
}
|
||||
|
||||
func allowAll(string, string, []string, iampolicy.ConditionKeys) error { return nil }
|
||||
|
||||
func formRequest(body string) *http.Request {
|
||||
r := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body))
|
||||
r.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
return r
|
||||
}
|
||||
|
||||
func newTestHandler(t *testing.T) *Handler {
|
||||
t.Helper()
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
// A daemon that answers every EC2 subject with an empty result, so each
|
||||
// action reaches its domain call and returns instead of waiting.
|
||||
sub, err := nc.Subscribe("ec2.>", func(m *nats.Msg) { _ = m.Respond([]byte("{}")) })
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
|
||||
return NewHandler(Deps{
|
||||
NATS: nc,
|
||||
IAM: noProfiles{},
|
||||
Quota: allowQuota{},
|
||||
ActiveNodes: func(context.Context) int { return 1 },
|
||||
ExpectedNodes: 1,
|
||||
NodeIDs: []string{"node-1"},
|
||||
AZ: "ap-southeast-2a",
|
||||
Endpoint: "https://localhost:9999",
|
||||
})
|
||||
}
|
||||
|
||||
// Every routed action must reach its handler through Dispatch. Inputs are empty,
|
||||
// so the domain call may reject them, but the wrapper must run and Dispatch must
|
||||
// answer with a result or an error rather than panic.
|
||||
func TestDispatch_EveryActionReachesItsHandler(t *testing.T) {
|
||||
h := newTestHandler(t)
|
||||
for _, action := range ActionNames() {
|
||||
t.Run(action, func(t *testing.T) {
|
||||
w := httptest.NewRecorder()
|
||||
inv := dispatch.Invocation{
|
||||
Request: formRequest("Action=" + action),
|
||||
AccountID: testAccountID,
|
||||
Service: ServiceName,
|
||||
Region: testRegion,
|
||||
Authorize: allowAll,
|
||||
}
|
||||
assert.NotPanics(t, func() { _ = h.Dispatch(w, inv) })
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDispatch_RequestErrors(t *testing.T) {
|
||||
h := newTestHandler(t)
|
||||
deny := func(string, string, []string, iampolicy.ConditionKeys) error {
|
||||
return errors.New(awserrors.ErrorAccessDenied)
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
body string
|
||||
accountID string
|
||||
authorize dispatch.AuthorizeFunc
|
||||
want string
|
||||
}{
|
||||
{"missing action", "Foo=bar", testAccountID, allowAll, awserrors.ErrorMissingAction},
|
||||
{"unknown action", "Action=NoSuchAction", testAccountID, allowAll, awserrors.ErrorInvalidAction},
|
||||
{"no account", "Action=DescribeVpcs", "", allowAll, awserrors.ErrorInternalError},
|
||||
{"policy denial", "Action=DescribeVpcs", testAccountID, deny, awserrors.ErrorUnauthorizedOperation},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
err := h.Dispatch(httptest.NewRecorder(), dispatch.Invocation{
|
||||
Request: formRequest(tc.body), AccountID: tc.accountID, Service: ServiceName,
|
||||
Region: testRegion, Authorize: tc.authorize,
|
||||
})
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, tc.want, err.Error())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDispatch_SuccessWritesXML(t *testing.T) {
|
||||
h := newTestHandler(t)
|
||||
w := httptest.NewRecorder()
|
||||
err := h.Dispatch(w, dispatch.Invocation{
|
||||
Request: formRequest("Action=DescribeRegions"), AccountID: testAccountID,
|
||||
Service: ServiceName, Region: testRegion, Authorize: allowAll,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, http.StatusOK, w.Code)
|
||||
assert.Equal(t, "text/xml", w.Header().Get("Content-Type"))
|
||||
assert.Contains(t, w.Body.String(), "ap-southeast-2")
|
||||
}
|
||||
|
||||
// Without a NATS connection the gateway-local actions still answer and every
|
||||
// other action reports ServerInternal.
|
||||
func TestDispatch_NoNATS(t *testing.T) {
|
||||
h := NewHandler(Deps{AZ: "ap-southeast-2a", Endpoint: "https://localhost:9999"})
|
||||
inv := func(body string) dispatch.Invocation {
|
||||
return dispatch.Invocation{
|
||||
Request: formRequest(body), AccountID: testAccountID, Service: ServiceName,
|
||||
Region: testRegion, Authorize: allowAll,
|
||||
}
|
||||
}
|
||||
require.NoError(t, h.Dispatch(httptest.NewRecorder(), inv("Action=DescribeRegions")))
|
||||
err := h.Dispatch(httptest.NewRecorder(), inv("Action=DescribeVpcs"))
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
|
||||
}
|
||||
|
||||
func TestInputHasField(t *testing.T) {
|
||||
assert.True(t, InputHasField("CreateVpc", "DryRun"))
|
||||
assert.False(t, InputHasField("CreateVpc", "NoSuchField"))
|
||||
assert.False(t, InputHasField("NoSuchAction", "DryRun"))
|
||||
}
|
||||
+4
-4
@@ -1,4 +1,4 @@
|
||||
package gateway
|
||||
package awsapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -13,17 +13,17 @@ import (
|
||||
)
|
||||
|
||||
// render drives a single EC2 action through ec2Handler exactly as the real
|
||||
// dispatch table does (see ec2Actions in ec2.go), with a stub inner handler
|
||||
// dispatch table does (see ec2Actions in actions.go), with a stub inner handler
|
||||
// standing in for the NATS round trip. This exercises the real marshal path
|
||||
// (currently marshalEC2Response) without needing a live NATS connection.
|
||||
func render[In any](t *testing.T, action string, out any) string {
|
||||
t.Helper()
|
||||
h := ec2Handler(func(_ context.Context, _ *In, _ *GatewayConfig, _ string) (any, error) {
|
||||
h := ec2Handler(func(_ context.Context, _ *In, _ *call, _ string) (any, error) {
|
||||
return out, nil
|
||||
})
|
||||
input, err := h.parse(map[string]string{})
|
||||
require.NoError(t, err)
|
||||
xmlOutput, err := h.dispatch(action, input, &GatewayConfig{}, "acct-123", nil)
|
||||
xmlOutput, err := h.dispatch(t.Context(), action, input, &call{Handler: NewHandler(Deps{})}, "acct-123")
|
||||
require.NoError(t, err)
|
||||
return string(xmlOutput)
|
||||
}
|
||||
@@ -1,11 +1,10 @@
|
||||
package gateway
|
||||
package awsapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
|
||||
ec2idemapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/idem"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
@@ -13,40 +12,39 @@ import (
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// ec2ClientTokenBucket lazily binds the EC2 token bucket, once per gateway
|
||||
// clientTokenBucket lazily binds the EC2 token bucket, once per handler
|
||||
// rather than once per request.
|
||||
func (gw *GatewayConfig) ec2ClientTokenBucket(ctx context.Context) (jetstream.KeyValue, error) {
|
||||
gw.ec2TokenOnce.Do(func() {
|
||||
js, err := jetstream.New(gw.NATSConn)
|
||||
func (h *Handler) clientTokenBucket(ctx context.Context) (jetstream.KeyValue, error) {
|
||||
h.tokenOnce.Do(func() {
|
||||
js, err := jetstream.New(h.NATS)
|
||||
if err != nil {
|
||||
gw.ec2TokenErr = fmt.Errorf("clienttoken jetstream: %w", err)
|
||||
h.tokenErr = fmt.Errorf("clienttoken jetstream: %w", err)
|
||||
return
|
||||
}
|
||||
// The bind happens once, so it must not inherit the first caller's
|
||||
// cancellation: a client that disconnects mid-open would poison the
|
||||
// bucket for every later request. Deadline-free, so the open falls back
|
||||
// to the JetStream API's own timeout.
|
||||
gw.ec2TokenKV, gw.ec2TokenErr = idempotency.OpenBucket(
|
||||
h.tokenKV, h.tokenErr = idempotency.OpenBucket(
|
||||
context.WithoutCancel(ctx), js, ec2idemapi.KVBucket, ec2idemapi.TTL)
|
||||
})
|
||||
return gw.ec2TokenKV, gw.ec2TokenErr
|
||||
return h.tokenKV, h.tokenErr
|
||||
}
|
||||
|
||||
// ec2IdempotentHandler is ec2Handler for creates that honour ClientToken: a
|
||||
// repeat of the same token returns the first call's result instead of creating
|
||||
// a second resource. Out is pinned so the result can be stored and replayed.
|
||||
func ec2IdempotentHandler[In, Out any](handler func(ctx context.Context, input *In, gw *GatewayConfig, accountID string) (Out, error)) ec2Action {
|
||||
func ec2IdempotentHandler[In, Out any](handler func(ctx context.Context, input *In, c *call, accountID string) (Out, error)) ec2Action {
|
||||
return ec2Action{
|
||||
idempotent: true,
|
||||
parse: parseEC2Input[In],
|
||||
dispatch: func(action string, parsed any, gw *GatewayConfig, accountID string, r *http.Request) ([]byte, error) {
|
||||
dispatch: func(ctx context.Context, action string, parsed any, c *call, accountID string) ([]byte, error) {
|
||||
input, ok := parsed.(*In)
|
||||
if !ok {
|
||||
return nil, errors.New(awserrors.ErrorInternalError)
|
||||
}
|
||||
ctx := requestContext(r)
|
||||
output, err := runEC2Idempotent(ctx, action, gw, accountID, input, func() (Out, error) {
|
||||
return handler(ctx, input, gw, accountID)
|
||||
output, err := runEC2Idempotent(ctx, action, c, accountID, input, func() (Out, error) {
|
||||
return handler(ctx, input, c, accountID)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -61,7 +59,7 @@ func ec2IdempotentHandler[In, Out any](handler func(ctx context.Context, input *
|
||||
func runEC2Idempotent[In, Out any](
|
||||
ctx context.Context,
|
||||
action string,
|
||||
gw *GatewayConfig,
|
||||
c *call,
|
||||
accountID string,
|
||||
input *In,
|
||||
work func() (Out, error),
|
||||
@@ -74,7 +72,7 @@ func runEC2Idempotent[In, Out any](
|
||||
return work()
|
||||
}
|
||||
|
||||
kv, kerr := gw.ec2ClientTokenBucket(ctx)
|
||||
kv, kerr := c.clientTokenBucket(ctx)
|
||||
if kerr != nil {
|
||||
// Running anyway would create the duplicate the token was sent to
|
||||
// prevent, so this fails rather than degrading to no idempotency.
|
||||
+44
-45
@@ -1,8 +1,7 @@
|
||||
//test:in-package — the decorator, the action table and the gateway's token
|
||||
//bucket field are all unexported, and the point of these tests is that the
|
||||
//table's entries are wired to the decorator.
|
||||
//test:in-package — the decorator and the action table are unexported, and the
|
||||
// point of these tests is that the table entries are wired to the decorator.
|
||||
|
||||
package gateway
|
||||
package awsapi
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
@@ -60,32 +59,32 @@ func responseFields(t *testing.T, doc []byte) []string {
|
||||
return fields
|
||||
}
|
||||
|
||||
// newTokenGateway returns a gateway whose client-token bucket is already bound
|
||||
// newTokenCall returns a gateway whose client-token bucket is already bound
|
||||
// to a test JetStream, so dispatch never reaches a real NATS connection.
|
||||
func newTokenGateway(t *testing.T) *GatewayConfig {
|
||||
func newTokenCall(t *testing.T) *call {
|
||||
t.Helper()
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
kv, err := idempotency.OpenBucket(t.Context(), testutil.NewJetStream(t, nc), "gw-ec2-tokens", time.Minute)
|
||||
require.NoError(t, err)
|
||||
|
||||
gw := &GatewayConfig{}
|
||||
// Burns the Once as it binds, so ec2ClientTokenBucket returns this bucket
|
||||
h := NewHandler(Deps{})
|
||||
// Burns the Once as it binds, so clientTokenBucket returns this bucket
|
||||
// rather than dialing.
|
||||
gw.ec2TokenOnce.Do(func() { gw.ec2TokenKV = kv })
|
||||
return gw
|
||||
h.tokenOnce.Do(func() { h.tokenKV = kv })
|
||||
return &call{Handler: h}
|
||||
}
|
||||
|
||||
// dispatchTwice runs one action twice through the decorator with the same input
|
||||
// and reports how many times the create actually ran.
|
||||
func dispatchTwice[In, Out any](t *testing.T, gw *GatewayConfig, action string, input *In, result Out) (calls int, first, second []byte) {
|
||||
func dispatchTwice[In, Out any](t *testing.T, c *call, action string, input *In, result Out) (calls int, first, second []byte) {
|
||||
t.Helper()
|
||||
handler := ec2IdempotentHandler(func(ctx context.Context, in *In, gw *GatewayConfig, accountID string) (Out, error) {
|
||||
handler := ec2IdempotentHandler(func(ctx context.Context, in *In, c *call, accountID string) (Out, error) {
|
||||
calls++
|
||||
return result, nil
|
||||
})
|
||||
first, err := handler.dispatch(action, input, gw, idemTestAccount, nil)
|
||||
first, err := handler.dispatch(t.Context(), action, input, c, idemTestAccount)
|
||||
require.NoError(t, err)
|
||||
second, err = handler.dispatch(action, input, gw, idemTestAccount, nil)
|
||||
second, err = handler.dispatch(t.Context(), action, input, c, idemTestAccount)
|
||||
require.NoError(t, err)
|
||||
return calls, first, second
|
||||
}
|
||||
@@ -98,8 +97,8 @@ func TestIdempotentHandler_SameTokenCreatesOneResource(t *testing.T) {
|
||||
|
||||
t.Run("CreateVolume", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
gw := newTokenGateway(t)
|
||||
calls, first, second := dispatchTwice(t, gw, "CreateVolume",
|
||||
c := newTokenCall(t)
|
||||
calls, first, second := dispatchTwice(t, c, "CreateVolume",
|
||||
&ec2.CreateVolumeInput{Size: aws.Int64(8), ClientToken: tok},
|
||||
ec2.Volume{VolumeId: aws.String("vol-1")})
|
||||
assert.Equal(t, 1, calls)
|
||||
@@ -109,8 +108,8 @@ func TestIdempotentHandler_SameTokenCreatesOneResource(t *testing.T) {
|
||||
|
||||
t.Run("CreateNetworkInterface", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
gw := newTokenGateway(t)
|
||||
calls, first, second := dispatchTwice(t, gw, "CreateNetworkInterface",
|
||||
c := newTokenCall(t)
|
||||
calls, first, second := dispatchTwice(t, c, "CreateNetworkInterface",
|
||||
&ec2.CreateNetworkInterfaceInput{ClientToken: tok},
|
||||
ec2.CreateNetworkInterfaceOutput{
|
||||
NetworkInterface: &ec2.NetworkInterface{NetworkInterfaceId: aws.String("eni-1")},
|
||||
@@ -123,8 +122,8 @@ func TestIdempotentHandler_SameTokenCreatesOneResource(t *testing.T) {
|
||||
// Both IDs have to survive the replay: re-creating either one leaks it.
|
||||
t.Run("CreateNatGateway", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
gw := newTokenGateway(t)
|
||||
calls, first, second := dispatchTwice(t, gw, "CreateNatGateway",
|
||||
c := newTokenCall(t)
|
||||
calls, first, second := dispatchTwice(t, c, "CreateNatGateway",
|
||||
&ec2.CreateNatGatewayInput{ClientToken: tok},
|
||||
ec2.CreateNatGatewayOutput{
|
||||
NatGateway: &ec2.NatGateway{
|
||||
@@ -142,8 +141,8 @@ func TestIdempotentHandler_SameTokenCreatesOneResource(t *testing.T) {
|
||||
|
||||
t.Run("CreateRouteTable", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
gw := newTokenGateway(t)
|
||||
calls, first, second := dispatchTwice(t, gw, "CreateRouteTable",
|
||||
c := newTokenCall(t)
|
||||
calls, first, second := dispatchTwice(t, c, "CreateRouteTable",
|
||||
&ec2.CreateRouteTableInput{ClientToken: tok},
|
||||
ec2.CreateRouteTableOutput{
|
||||
RouteTable: &ec2.RouteTable{
|
||||
@@ -161,8 +160,8 @@ func TestIdempotentHandler_SameTokenCreatesOneResource(t *testing.T) {
|
||||
|
||||
t.Run("CreateEgressOnlyInternetGateway", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
gw := newTokenGateway(t)
|
||||
calls, first, second := dispatchTwice(t, gw, "CreateEgressOnlyInternetGateway",
|
||||
c := newTokenCall(t)
|
||||
calls, first, second := dispatchTwice(t, c, "CreateEgressOnlyInternetGateway",
|
||||
&ec2.CreateEgressOnlyInternetGatewayInput{ClientToken: tok},
|
||||
ec2.CreateEgressOnlyInternetGatewayOutput{
|
||||
EgressOnlyInternetGateway: &ec2.EgressOnlyInternetGateway{
|
||||
@@ -176,8 +175,8 @@ func TestIdempotentHandler_SameTokenCreatesOneResource(t *testing.T) {
|
||||
|
||||
t.Run("CopyImage", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
gw := newTokenGateway(t)
|
||||
calls, first, second := dispatchTwice(t, gw, "CopyImage",
|
||||
c := newTokenCall(t)
|
||||
calls, first, second := dispatchTwice(t, c, "CopyImage",
|
||||
&ec2.CopyImageInput{Name: aws.String("copy"), ClientToken: tok},
|
||||
ec2.CopyImageOutput{ImageId: aws.String("ami-1")})
|
||||
assert.Equal(t, 1, calls)
|
||||
@@ -189,8 +188,8 @@ func TestIdempotentHandler_SameTokenCreatesOneResource(t *testing.T) {
|
||||
// client that thought it made one call.
|
||||
t.Run("CreateLaunchTemplateVersion", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
gw := newTokenGateway(t)
|
||||
calls, first, second := dispatchTwice(t, gw, "CreateLaunchTemplateVersion",
|
||||
c := newTokenCall(t)
|
||||
calls, first, second := dispatchTwice(t, c, "CreateLaunchTemplateVersion",
|
||||
&ec2.CreateLaunchTemplateVersionInput{LaunchTemplateId: aws.String("lt-1"), ClientToken: tok},
|
||||
ec2.CreateLaunchTemplateVersionOutput{
|
||||
LaunchTemplateVersion: &ec2.LaunchTemplateVersion{
|
||||
@@ -205,8 +204,8 @@ func TestIdempotentHandler_SameTokenCreatesOneResource(t *testing.T) {
|
||||
|
||||
t.Run("CreateCapacityReservation", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
gw := newTokenGateway(t)
|
||||
calls, first, second := dispatchTwice(t, gw, "CreateCapacityReservation",
|
||||
c := newTokenCall(t)
|
||||
calls, first, second := dispatchTwice(t, c, "CreateCapacityReservation",
|
||||
&ec2.CreateCapacityReservationInput{ClientToken: tok},
|
||||
ec2.CreateCapacityReservationOutput{
|
||||
CapacityReservation: &ec2.CapacityReservation{
|
||||
@@ -223,14 +222,14 @@ func TestIdempotentHandler_SameTokenCreatesOneResource(t *testing.T) {
|
||||
// separate pieces of work rather than one replaying the other's response.
|
||||
func TestIdempotentHandler_TokensAreScopedPerAction(t *testing.T) {
|
||||
t.Parallel()
|
||||
gw := newTokenGateway(t)
|
||||
c := newTokenCall(t)
|
||||
tok := aws.String("shared-token")
|
||||
|
||||
volumeCalls, _, _ := dispatchTwice(t, gw, "CreateVolume",
|
||||
volumeCalls, _, _ := dispatchTwice(t, c, "CreateVolume",
|
||||
&ec2.CreateVolumeInput{ClientToken: tok}, ec2.Volume{VolumeId: aws.String("vol-1")})
|
||||
require.Equal(t, 1, volumeCalls)
|
||||
|
||||
eigwCalls, _, second := dispatchTwice(t, gw, "CreateEgressOnlyInternetGateway",
|
||||
eigwCalls, _, second := dispatchTwice(t, c, "CreateEgressOnlyInternetGateway",
|
||||
&ec2.CreateEgressOnlyInternetGatewayInput{ClientToken: tok},
|
||||
ec2.CreateEgressOnlyInternetGatewayOutput{
|
||||
EgressOnlyInternetGateway: &ec2.EgressOnlyInternetGateway{
|
||||
@@ -246,19 +245,19 @@ func TestIdempotentHandler_TokensAreScopedPerAction(t *testing.T) {
|
||||
// must not run.
|
||||
func TestIdempotentHandler_ParamMismatchIsRejected(t *testing.T) {
|
||||
t.Parallel()
|
||||
gw := newTokenGateway(t)
|
||||
c := newTokenCall(t)
|
||||
calls := 0
|
||||
handler := ec2IdempotentHandler(func(ctx context.Context, in *ec2.CreateVolumeInput, gw *GatewayConfig, accountID string) (ec2.Volume, error) {
|
||||
handler := ec2IdempotentHandler(func(ctx context.Context, in *ec2.CreateVolumeInput, c *call, accountID string) (ec2.Volume, error) {
|
||||
calls++
|
||||
return ec2.Volume{VolumeId: aws.String("vol-1")}, nil
|
||||
})
|
||||
|
||||
_, err := handler.dispatch("CreateVolume",
|
||||
&ec2.CreateVolumeInput{Size: aws.Int64(8), ClientToken: aws.String("tok-mm")}, gw, idemTestAccount, nil)
|
||||
_, err := handler.dispatch(t.Context(), "CreateVolume",
|
||||
&ec2.CreateVolumeInput{Size: aws.Int64(8), ClientToken: aws.String("tok-mm")}, c, idemTestAccount)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = handler.dispatch("CreateVolume",
|
||||
&ec2.CreateVolumeInput{Size: aws.Int64(16), ClientToken: aws.String("tok-mm")}, gw, idemTestAccount, nil)
|
||||
_, err = handler.dispatch(t.Context(), "CreateVolume",
|
||||
&ec2.CreateVolumeInput{Size: aws.Int64(16), ClientToken: aws.String("tok-mm")}, c, idemTestAccount)
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorIdempotentParameterMismatch, err.Error())
|
||||
assert.Equal(t, 1, calls, "a mismatched token must not create anything")
|
||||
@@ -267,8 +266,8 @@ func TestIdempotentHandler_ParamMismatchIsRejected(t *testing.T) {
|
||||
// Without a token the client did not ask for idempotency, so each call creates.
|
||||
func TestIdempotentHandler_NoTokenCreatesEachTime(t *testing.T) {
|
||||
t.Parallel()
|
||||
gw := newTokenGateway(t)
|
||||
calls, _, _ := dispatchTwice(t, gw, "CreateVolume",
|
||||
c := newTokenCall(t)
|
||||
calls, _, _ := dispatchTwice(t, c, "CreateVolume",
|
||||
&ec2.CreateVolumeInput{Size: aws.Int64(8)}, ec2.Volume{VolumeId: aws.String("vol-1")})
|
||||
assert.Equal(t, 2, calls)
|
||||
}
|
||||
@@ -277,10 +276,10 @@ func TestIdempotentHandler_NoTokenCreatesEachTime(t *testing.T) {
|
||||
// again rather than replay the failure.
|
||||
func TestIdempotentHandler_FailedCreateIsRetryable(t *testing.T) {
|
||||
t.Parallel()
|
||||
gw := newTokenGateway(t)
|
||||
c := newTokenCall(t)
|
||||
boom := errors.New("create failed")
|
||||
calls := 0
|
||||
handler := ec2IdempotentHandler(func(ctx context.Context, in *ec2.CreateVolumeInput, gw *GatewayConfig, accountID string) (ec2.Volume, error) {
|
||||
handler := ec2IdempotentHandler(func(ctx context.Context, in *ec2.CreateVolumeInput, c *call, accountID string) (ec2.Volume, error) {
|
||||
calls++
|
||||
if calls == 1 {
|
||||
return ec2.Volume{}, boom
|
||||
@@ -289,10 +288,10 @@ func TestIdempotentHandler_FailedCreateIsRetryable(t *testing.T) {
|
||||
})
|
||||
input := &ec2.CreateVolumeInput{Size: aws.Int64(8), ClientToken: aws.String("tok-retry")}
|
||||
|
||||
_, err := handler.dispatch("CreateVolume", input, gw, idemTestAccount, nil)
|
||||
_, err := handler.dispatch(t.Context(), "CreateVolume", input, c, idemTestAccount)
|
||||
require.ErrorIs(t, err, boom, "the handler's own error reaches the caller unchanged")
|
||||
|
||||
out, err := handler.dispatch("CreateVolume", input, gw, idemTestAccount, nil)
|
||||
out, err := handler.dispatch(t.Context(), "CreateVolume", input, c, idemTestAccount)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 2, calls)
|
||||
assert.Contains(t, string(out), "vol-1")
|
||||
@@ -14,7 +14,6 @@ import (
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
@@ -465,7 +464,7 @@ func judgeIdentityCompleteness(ctx context.Context, frames []natsmsg.Frame, sum
|
||||
// that carries only an ARN. Results are cached per ARN to avoid repeated RPCs.
|
||||
// Misses are warn-logged and leave Id empty (graceful degradation for deleted profiles).
|
||||
// Safe to call with a nil output or nil IAMService (no-op).
|
||||
func EnrichInstanceProfileIDs(out *ec2.DescribeInstancesOutput, iamSvc iamdomain.IAMService, accountID string) {
|
||||
func EnrichInstanceProfileIDs(out *ec2.DescribeInstancesOutput, iamSvc InstanceProfiles, accountID string) {
|
||||
if out == nil || iamSvc == nil {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -24,7 +24,7 @@ const fanOutTimeout = 3 * time.Second
|
||||
|
||||
// resolveAndAuthorizeProfile resolves a profile (name or ARN) and enforces iam:PassRole.
|
||||
// Profiles without a role skip PassRole; missing profile → InvalidIamInstanceProfile.NotFound.
|
||||
func resolveAndAuthorizeProfile(spec *ec2.IamInstanceProfileSpecification, iamSvc iamdomain.IAMService, accountID string, passRoleCheck PassRoleChecker) (*iamdomain.InstanceProfile, error) {
|
||||
func resolveAndAuthorizeProfile(spec *ec2.IamInstanceProfileSpecification, iamSvc InstanceProfiles, accountID string, passRoleCheck PassRoleChecker) (*iamdomain.InstanceProfile, error) {
|
||||
nameOrARN := profileNameOrARN(spec)
|
||||
if nameOrARN == "" {
|
||||
return nil, errors.New(awserrors.ErrorMissingParameter)
|
||||
@@ -55,7 +55,7 @@ func resolveAndAuthorizeProfile(spec *ec2.IamInstanceProfileSpecification, iamSv
|
||||
// AssociateIamInstanceProfile attaches a profile to a running instance.
|
||||
// The gateway resolves the profile and enforces PassRole; the daemon validates
|
||||
// no existing profile and generates a fresh iip-assoc-… ID.
|
||||
func AssociateIamInstanceProfile(ctx context.Context, input *ec2.AssociateIamInstanceProfileInput, natsConn *nats.Conn, iamSvc iamdomain.IAMService, accountID string, passRoleCheck PassRoleChecker) (*ec2.AssociateIamInstanceProfileOutput, error) {
|
||||
func AssociateIamInstanceProfile(ctx context.Context, input *ec2.AssociateIamInstanceProfileInput, natsConn *nats.Conn, iamSvc InstanceProfiles, accountID string, passRoleCheck PassRoleChecker) (*ec2.AssociateIamInstanceProfileOutput, error) {
|
||||
if input == nil || input.InstanceId == nil || *input.InstanceId == "" {
|
||||
return nil, errors.New(awserrors.ErrorMissingParameter)
|
||||
}
|
||||
@@ -107,7 +107,7 @@ func DisassociateIamInstanceProfile(ctx context.Context, input *ec2.Disassociate
|
||||
|
||||
// ReplaceIamInstanceProfileAssociation swaps the profile for an existing association.
|
||||
// The gateway resolves the new profile and enforces PassRole; the daemon swaps atomically.
|
||||
func ReplaceIamInstanceProfileAssociation(ctx context.Context, input *ec2.ReplaceIamInstanceProfileAssociationInput, natsConn *nats.Conn, iamSvc iamdomain.IAMService, expectedNodes int, accountID string, passRoleCheck PassRoleChecker) (*ec2.ReplaceIamInstanceProfileAssociationOutput, error) {
|
||||
func ReplaceIamInstanceProfileAssociation(ctx context.Context, input *ec2.ReplaceIamInstanceProfileAssociationInput, natsConn *nats.Conn, iamSvc InstanceProfiles, expectedNodes int, accountID string, passRoleCheck PassRoleChecker) (*ec2.ReplaceIamInstanceProfileAssociationOutput, error) {
|
||||
if input == nil || input.AssociationId == nil || *input.AssociationId == "" {
|
||||
return nil, errors.New(awserrors.ErrorMissingParameter)
|
||||
}
|
||||
@@ -245,3 +245,10 @@ func broadcastDescribeAssociations(ctx context.Context, natsConn *nats.Conn, inp
|
||||
}
|
||||
return associations, nil
|
||||
}
|
||||
|
||||
// InstanceProfiles is the IAM capability the instance-profile actions consume:
|
||||
// resolving a profile and naming the role it holds.
|
||||
type InstanceProfiles interface {
|
||||
ResolveInstanceProfile(accountID, nameOrARN string) (*iamdomain.InstanceProfile, error)
|
||||
CanonicalResourceARN(accountID string, kind arn.IAMResourceType, name string) (string, error)
|
||||
}
|
||||
|
||||
@@ -131,7 +131,7 @@ func hasPinnedPrivateIP(input *ec2.RunInstancesInput) bool {
|
||||
// passRoleCheck may be nil to skip PassRole enforcement.
|
||||
// launchQuotaCheck may be nil to skip quota enforcement; when present it runs
|
||||
// after validation and PassRole authorization, but before any launch dispatch.
|
||||
func RunInstances(ctx context.Context, input *ec2.RunInstancesInput, natsConn *nats.Conn, iamSvc iamdomain.IAMService, accountID string, passRoleCheck PassRoleChecker, launchQuotaCheck LaunchQuotaChecker, expectedNodes int) (reservation ec2.Reservation, err error) {
|
||||
func RunInstances(ctx context.Context, input *ec2.RunInstancesInput, natsConn *nats.Conn, iamSvc InstanceProfiles, accountID string, passRoleCheck PassRoleChecker, launchQuotaCheck LaunchQuotaChecker, expectedNodes int) (reservation ec2.Reservation, err error) {
|
||||
// Expand a referenced launch template before validation so validation, quota,
|
||||
// and downstream routing all see the effective (post-merge) parameters.
|
||||
if err = ExpandLaunchTemplate(ctx, natsConn, input, accountID); err != nil {
|
||||
@@ -163,7 +163,7 @@ func RunInstances(ctx context.Context, input *ec2.RunInstancesInput, natsConn *n
|
||||
|
||||
// runInstancesInner performs the actual launch: profile resolution, placement
|
||||
// routing, and capacity-aware distribution. Wrapped by RunInstances for idempotency.
|
||||
func runInstancesInner(ctx context.Context, input *ec2.RunInstancesInput, natsConn *nats.Conn, iamSvc iamdomain.IAMService, accountID string, passRoleCheck PassRoleChecker, launchQuotaCheck LaunchQuotaChecker, expectedNodes int) (reservation ec2.Reservation, err error) {
|
||||
func runInstancesInner(ctx context.Context, input *ec2.RunInstancesInput, natsConn *nats.Conn, iamSvc InstanceProfiles, accountID string, passRoleCheck PassRoleChecker, launchQuotaCheck LaunchQuotaChecker, expectedNodes int) (reservation ec2.Reservation, err error) {
|
||||
resolvedProfile, err := resolveAndAuthorizeInstanceProfile(input, iamSvc, accountID, passRoleCheck)
|
||||
if err != nil {
|
||||
return reservation, err
|
||||
@@ -236,7 +236,7 @@ func runInstancesInner(ctx context.Context, input *ec2.RunInstancesInput, natsCo
|
||||
|
||||
// resolveAndAuthorizeInstanceProfile resolves an optional instance profile,
|
||||
// enforces PassRole, and normalises input to the canonical ARN only.
|
||||
func resolveAndAuthorizeInstanceProfile(input *ec2.RunInstancesInput, iamSvc iamdomain.IAMService, accountID string, passRoleCheck PassRoleChecker) (*iamdomain.InstanceProfile, error) {
|
||||
func resolveAndAuthorizeInstanceProfile(input *ec2.RunInstancesInput, iamSvc InstanceProfiles, accountID string, passRoleCheck PassRoleChecker) (*iamdomain.InstanceProfile, error) {
|
||||
if input.IamInstanceProfile == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
package awsapi
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"maps"
|
||||
"net/http"
|
||||
"reflect"
|
||||
"slices"
|
||||
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/mulgadc/bluebottle/pkg/iampolicy"
|
||||
"github.com/mulgadc/bluebottle/pkg/sigv4"
|
||||
ec2instanceapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/instance"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/query"
|
||||
)
|
||||
|
||||
const (
|
||||
// ServiceName is the SigV4 credential-scope service EC2 clients sign for.
|
||||
ServiceName = "ec2"
|
||||
|
||||
ec2ServicePrincipal = "ec2.amazonaws.com"
|
||||
)
|
||||
|
||||
// readQueryArgs parses the Query body and rewinds it, so the same bytes can be
|
||||
// read again. Any read or parse failure is a malformed query string.
|
||||
func readQueryArgs(r *http.Request) (map[string]string, error) {
|
||||
body, err := io.ReadAll(io.LimitReader(r.Body, sigv4.MaxPayloadLen+1))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r.Body = io.NopCloser(bytes.NewReader(body))
|
||||
if int64(len(body)) > sigv4.MaxPayloadLen {
|
||||
return nil, errors.New(awserrors.ErrorRequestEntityTooLarge)
|
||||
}
|
||||
return query.ParseArgs(string(body))
|
||||
}
|
||||
|
||||
// Dispatch parses, authorizes and runs one EC2 request, then writes the XML
|
||||
// response. A returned error is an awserrors code the gateway renders.
|
||||
func (h *Handler) Dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
|
||||
r := inv.Request
|
||||
queryArgs, err := readQueryArgs(r)
|
||||
if err != nil {
|
||||
slog.Debug("EC2: malformed query string", "err", err)
|
||||
return errors.New(awserrors.ErrorMalformedQueryString)
|
||||
}
|
||||
|
||||
action := queryArgs["Action"]
|
||||
if action == "" {
|
||||
return errors.New(awserrors.ErrorMissingAction)
|
||||
}
|
||||
handler, ok := ec2Actions[action]
|
||||
if !ok {
|
||||
return errors.New(awserrors.ErrorInvalidAction)
|
||||
}
|
||||
|
||||
// Hoisted above the policy check because the resolver builds ARNs from it.
|
||||
// inv.Region is the gateway's own region, never the caller-supplied
|
||||
// credential-scope region, which would let a caller sign for another region
|
||||
// and slide out from under a Deny.
|
||||
accountID := inv.AccountID
|
||||
if accountID == "" {
|
||||
slog.Error("EC2 dispatch: no account ID in auth context")
|
||||
// InternalError, not ServerInternal: the policy gate used to reach this
|
||||
// case first and that is the code the caller has always seen.
|
||||
return errors.New(awserrors.ErrorInternalError)
|
||||
}
|
||||
|
||||
input, err := handler.parse(queryArgs)
|
||||
if err != nil {
|
||||
if errors.Is(err, query.ErrSliceTooLarge) {
|
||||
return errors.New(awserrors.ErrorMalformedQueryString)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Launch templates supply effective RunInstances resources. Resolve them
|
||||
// before authorization, and dispatch the same expanded input afterwards.
|
||||
if runInput, ok := input.(*ec2.RunInstancesInput); ok && action == "RunInstances" {
|
||||
if err := ec2instanceapi.ExpandLaunchTemplate(r.Context(), h.NATS, runInput, accountID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
resources, err := ResourceARNs(action, inv.Region, accountID, input)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := inv.Authorize(ServiceName, action, resources, nil); err != nil {
|
||||
// EC2 answers a policy denial with UnauthorizedOperation, not AccessDenied.
|
||||
if code, _ := awserrors.ResolveErrorCode(err); code == awserrors.ErrorAccessDenied {
|
||||
return errors.New(awserrors.ErrorUnauthorizedOperation)
|
||||
}
|
||||
return err
|
||||
}
|
||||
// AWS refuses a bad tag resource type even on a DryRun request.
|
||||
if err := validateTagSpecificationTypes(action, input); err != nil {
|
||||
return err
|
||||
}
|
||||
if dryRunRequested(input) {
|
||||
return errors.New(awserrors.ErrorDryRunOperation)
|
||||
}
|
||||
if err := validateMaxResults(action, input); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if h.NATS == nil && !ec2LocalActions[action] {
|
||||
return errors.New(awserrors.ErrorServerInternal)
|
||||
}
|
||||
|
||||
c := &call{
|
||||
Handler: h,
|
||||
Region: inv.Region,
|
||||
passRole: func(roleARN string) error {
|
||||
return inv.Authorize("iam", "PassRole", []string{roleARN},
|
||||
iampolicy.ConditionKeys{iampolicy.KeyPassedToService: ec2ServicePrincipal})
|
||||
},
|
||||
}
|
||||
xmlOutput, err := handler.dispatch(requestContext(r), action, input, c, accountID)
|
||||
if err != nil {
|
||||
// A gateway iam:PassRole denial is EC2's UnauthorizedOperation too; an
|
||||
// AccessDenied a daemon returns is left as it is.
|
||||
if _, denied := errors.AsType[*dispatch.PolicyDenialError](err); denied {
|
||||
return errors.New(awserrors.ErrorUnauthorizedOperation)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "text/xml")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
if _, err := w.Write(xmlOutput); err != nil {
|
||||
slog.Error("Failed to write EC2 response", "err", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ActionNames lists every action the handler dispatches, sorted.
|
||||
func ActionNames() []string {
|
||||
return slices.Sorted(maps.Keys(ec2Actions))
|
||||
}
|
||||
|
||||
// InputHasField reports whether the SDK input type an action parses into has
|
||||
// the named field, so callers can ask what the model declares for an action
|
||||
// without reaching the route table.
|
||||
func InputHasField(action, field string) bool {
|
||||
entry, ok := ec2Actions[action]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
input, err := entry.parse(map[string]string{"Action": action})
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
_, has := reflect.TypeOf(input).Elem().FieldByName(field)
|
||||
return has
|
||||
}
|
||||
@@ -14,10 +14,8 @@ import (
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
admissionquota "github.com/mulgadc/spinifex/spinifex/domains/admission/quota"
|
||||
ec2instanceapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/instance"
|
||||
ec2spotinstance "github.com/mulgadc/spinifex/spinifex/domains/ec2/spotinstance"
|
||||
iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
|
||||
awstags "github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
|
||||
@@ -64,7 +62,7 @@ func ValidateRequestSpotInstancesInput(input *ec2.RequestSpotInstancesInput) err
|
||||
// ClientToken passed through, same vCPU quota gate), then builds and persists one
|
||||
// active/fulfilled SpotInstanceRequest per launched instance. On a launch failure
|
||||
// (including InsufficientInstanceCapacity) it returns the error and persists nothing.
|
||||
func RequestSpotInstances(ctx context.Context, input *ec2.RequestSpotInstancesInput, natsConn *nats.Conn, iamSvc iamdomain.IAMService, accountID, az string, passRoleCheck ec2instanceapi.PassRoleChecker, quota *admissionquota.Service, expectedNodes int) (ec2.RequestSpotInstancesOutput, error) {
|
||||
func RequestSpotInstances(ctx context.Context, input *ec2.RequestSpotInstancesInput, natsConn *nats.Conn, iamSvc ec2instanceapi.InstanceProfiles, accountID, az string, passRoleCheck ec2instanceapi.PassRoleChecker, quota LaunchQuota, expectedNodes int) (ec2.RequestSpotInstancesOutput, error) {
|
||||
var output ec2.RequestSpotInstancesOutput
|
||||
|
||||
if err := ValidateRequestSpotInstancesInput(input); err != nil {
|
||||
@@ -224,3 +222,9 @@ func buildSpotRequests(input *ec2.RequestSpotInstancesInput, runInput *ec2.RunIn
|
||||
}
|
||||
return requests
|
||||
}
|
||||
|
||||
// LaunchQuota is the vCPU quota a spot request launches under.
|
||||
type LaunchQuota interface {
|
||||
EnforceLaunch(ctx context.Context, accountID, instanceType string, maxCount int) error
|
||||
ChargeLaunch(ctx context.Context, accountID string, reservation *ec2.Reservation) error
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package gateway
|
||||
package awsapi
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
@@ -0,0 +1,26 @@
|
||||
//test:in-package — walks the unexported ec2Actions route table.
|
||||
|
||||
package awsapi
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// An action added to the route table with TagSpecifications must say which
|
||||
// resource types it tags, or a wrong type would be silently dropped again.
|
||||
func TestEC2TaggableTypes_CoverEveryAction(t *testing.T) {
|
||||
for action, handler := range ec2Actions {
|
||||
input, err := handler.parse(map[string]string{"Action": action})
|
||||
require.NoError(t, err, action)
|
||||
_, tags := reflect.TypeOf(input).Elem().FieldByName("TagSpecifications")
|
||||
_, listed := ec2TaggableTypes[action]
|
||||
assert.Equal(t, tags, listed, "%s: TagSpecifications modelled=%v, listed in ec2TaggableTypes=%v", action, tags, listed)
|
||||
}
|
||||
for action := range ec2TaggableTypes {
|
||||
assert.Contains(t, ec2Actions, action, "ec2TaggableTypes lists an action the registration does not route")
|
||||
}
|
||||
}
|
||||
@@ -111,6 +111,10 @@ func TestCurrentBehaviour_AccountTermination_CreateImageSnapshotDoesNotPinSource
|
||||
imageID := s.createImage(t, "vol-imaged", "ct-image")
|
||||
require.NoError(t, s.deleteVolume("vol-imaged"), "a CreateImage snapshot does not block deleting its source volume")
|
||||
|
||||
// DeleteVolume records the obligation and returns, so the document is gone
|
||||
// once the reclaimer has converged it rather than when the call returns.
|
||||
_, err = s.volumes.NewVolumeReclaimReaper().Sweep(ctx)
|
||||
require.NoError(t, err)
|
||||
_, err = s.metadata.GetVolume(ctx, ctAccountID, "vol-imaged")
|
||||
require.Error(t, err, "the source volume's document is gone")
|
||||
|
||||
|
||||
@@ -88,9 +88,11 @@ type SystemInstanceInput struct {
|
||||
IamInstanceProfileArn string `json:"iam_instance_profile_arn,omitempty"`
|
||||
|
||||
// Kind selects a non-load-balancer direct-boot guest; empty is the load
|
||||
// balancer. SQSStore carries the KindSQSStore guest's volume and key.
|
||||
Kind Kind `json:"kind,omitempty"`
|
||||
SQSStore *SQSStoreInput `json:"sqs_store,omitempty"`
|
||||
// balancer. SQSStore carries the KindSQSStore guest's volume and key, and
|
||||
// SQSWorker the KindSQSWorker guest's volume and store decision.
|
||||
Kind Kind `json:"kind,omitempty"`
|
||||
SQSStore *SQSStoreInput `json:"sqs_store,omitempty"`
|
||||
SQSWorker *SQSWorkerInput `json:"sqs_worker,omitempty"`
|
||||
}
|
||||
|
||||
// ExtraENIInput describes an additional pre-created ENI to attach to a system
|
||||
|
||||
@@ -54,6 +54,8 @@ func (in *SystemInstanceInput) ValidateSQSStore() error {
|
||||
return fmt.Errorf("%w: boot mode %q, the store boots direct only", ErrInvalidSQSStoreInput, in.BootMode)
|
||||
case in.SQSStore == nil:
|
||||
return fmt.Errorf("%w: no data volume or client key", ErrInvalidSQSStoreInput)
|
||||
case in.SQSWorker != nil:
|
||||
return fmt.Errorf("%w: SQS worker settings on an SQS store launch", ErrInvalidSQSStoreInput)
|
||||
case !volumeIDPattern.MatchString(in.SQSStore.DataVolumeID):
|
||||
return fmt.Errorf("%w: data volume ID %q is malformed", ErrInvalidSQSStoreInput, in.SQSStore.DataVolumeID)
|
||||
case !nkeys.IsValidPublicUserKey(in.SQSStore.ClientNKey):
|
||||
@@ -69,7 +71,7 @@ func (in *SystemInstanceInput) ValidateSQSStore() error {
|
||||
// RefuseNonLoadBalancerKind keeps every non-load-balancer guest off the
|
||||
// legacy launch path, which would boot the load balancer image instead.
|
||||
func (in *SystemInstanceInput) RefuseNonLoadBalancerKind() error {
|
||||
if in.Kind != "" || in.SQSStore != nil {
|
||||
if in.Kind != "" || in.SQSStore != nil || in.SQSWorker != nil {
|
||||
return fmt.Errorf("%w: kind %q must use its own launch subject", ErrInvalidSQSStoreInput, in.Kind)
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
package systeminstance
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
)
|
||||
|
||||
// KindSQSWorker is the shared SQS worker guest. It holds the store's private
|
||||
// key on its own data volume and reaches the gateway over its mgmt NIC.
|
||||
const KindSQSWorker Kind = "sqs-worker"
|
||||
|
||||
// SQSWorkerInput is what an SQS worker guest needs beyond its network: its
|
||||
// existing encrypted data volume and the Region it signs gateway calls for.
|
||||
// The first-store decision is not here: the guest asks the gateway at boot.
|
||||
type SQSWorkerInput struct {
|
||||
DataVolumeID string `json:"data_volume_id"`
|
||||
Region string `json:"region"`
|
||||
}
|
||||
|
||||
var regionPattern = regexp.MustCompile(`^[a-z]{2}(-[a-z]+)+-[0-9]+$`)
|
||||
|
||||
// ErrInvalidSQSWorkerInput is returned for a malformed SQS worker launch.
|
||||
var ErrInvalidSQSWorkerInput = errors.New("sysinstance: invalid SQS worker launch")
|
||||
|
||||
// ValidateSQSWorker checks an SQS worker launch's shape. The worker must have
|
||||
// an instance profile and a mgmt NIC placeholder: it has no other path to the
|
||||
// gateway and no static credential.
|
||||
func (in *SystemInstanceInput) ValidateSQSWorker() error {
|
||||
switch {
|
||||
case in.Kind != KindSQSWorker:
|
||||
return fmt.Errorf("%w: kind %q is not %q", ErrInvalidSQSWorkerInput, in.Kind, KindSQSWorker)
|
||||
case in.BootMode != "" && in.BootMode != BootDirect:
|
||||
return fmt.Errorf("%w: boot mode %q, the worker boots direct only", ErrInvalidSQSWorkerInput, in.BootMode)
|
||||
case in.SQSWorker == nil:
|
||||
return fmt.Errorf("%w: no data volume or Region", ErrInvalidSQSWorkerInput)
|
||||
case in.SQSStore != nil:
|
||||
return fmt.Errorf("%w: SQS store settings on an SQS worker launch", ErrInvalidSQSWorkerInput)
|
||||
case in.IamInstanceProfileArn == "":
|
||||
return fmt.Errorf("%w: no instance profile", ErrInvalidSQSWorkerInput)
|
||||
case in.LBAgentEnv != "" || in.CACert != "" || in.Scheme != "":
|
||||
// The daemon supplies the CA from its own trust store, never the caller.
|
||||
return fmt.Errorf("%w: load balancer settings on an SQS worker launch", ErrInvalidSQSWorkerInput)
|
||||
case len(in.NICs) != 2 || !in.NICs[0].IsDefault || in.NICs[1].IsDefault || in.NICs[1].MAC != "" || in.NICs[1].CIDR != "":
|
||||
return fmt.Errorf("%w: want a default VPC NIC and an unallocated mgmt NIC", ErrInvalidSQSWorkerInput)
|
||||
}
|
||||
return in.SQSWorker.Validate()
|
||||
}
|
||||
|
||||
// Validate checks the worker's volume and Region. It reads nothing.
|
||||
func (w *SQSWorkerInput) Validate() error {
|
||||
if !volumeIDPattern.MatchString(w.DataVolumeID) {
|
||||
return fmt.Errorf("%w: data volume ID %q is malformed", ErrInvalidSQSWorkerInput, w.DataVolumeID)
|
||||
}
|
||||
if !regionPattern.MatchString(w.Region) {
|
||||
// Not echoed: a caller-supplied value may be anything, a seed included.
|
||||
return fmt.Errorf("%w: Region is malformed", ErrInvalidSQSWorkerInput)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -103,8 +103,10 @@ func (s *racingDeleteStore) DeleteObject(ctx context.Context, input *s3.DeleteOb
|
||||
return s.MemoryObjectStore.DeleteObject(ctx, input)
|
||||
}
|
||||
|
||||
// A document another delete removed after this one read it is deleted, so the
|
||||
// call must succeed rather than answer InternalError for a finished delete.
|
||||
// A document another sweep removed after this one read it is deleted, so the
|
||||
// reclaimer must count it done rather than fail the sweep for a finished
|
||||
// delete. The record is the obligation, so retiring it is the reclaimer's step
|
||||
// and this is where a concurrent remover is now observed.
|
||||
func TestDeleteVolume_ConcurrentlyDeletedDocumentIsSuccess(t *testing.T) {
|
||||
memory := objectstore.NewMemoryObjectStore()
|
||||
svc := newTestVolumeServiceWithStore("ap-southeast-2a", memory)
|
||||
@@ -115,15 +117,19 @@ func TestDeleteVolume_ConcurrentlyDeletedDocumentIsSuccess(t *testing.T) {
|
||||
VolumeID: volumeID, CapacityGiB: 10, State: "available",
|
||||
})
|
||||
|
||||
racing := &racingDeleteStore{MemoryObjectStore: memory}
|
||||
svc.store = racing
|
||||
svc.metadata = ebsmetadata.NewStore(racing, "test-bucket")
|
||||
|
||||
_, err := svc.DeleteVolume(context.Background(), &ec2.DeleteVolumeInput{
|
||||
VolumeId: aws.String(volumeID),
|
||||
}, testVolAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
racing := &racingDeleteStore{MemoryObjectStore: memory}
|
||||
svc.store = racing
|
||||
svc.metadata = ebsmetadata.NewStore(racing, "test-bucket")
|
||||
|
||||
reclaimed, err := svc.NewVolumeReclaimReaper().Sweep(context.Background())
|
||||
require.NoError(t, err, "a document a concurrent sweep removed is a finished delete, not a sweep failure")
|
||||
assert.Equal(t, 1, reclaimed)
|
||||
|
||||
_, getErr := ebsmetadata.NewStore(memory, "test-bucket").GetVolume(context.Background(), testVolAccountID, volumeID)
|
||||
assert.True(t, objectstore.IsNoSuchKeyError(getErr), "the document must be gone, got %v", getErr)
|
||||
}
|
||||
|
||||
@@ -97,6 +97,52 @@ func TestRLC3_VolumeDeleteRequiresDetach(t *testing.T) {
|
||||
"ADR-0003:S2 behaviour; historic PROP-LIFECYCLE-005 §2 mechanism: deleting an attached volume must return VolumeInUse (detach-before-delete, rule #3)")
|
||||
}
|
||||
|
||||
// TestS2_VolumeDeleteRecordsTheObligationBeforeRemovingTheRecord enforces
|
||||
// ADR-0003:S2's delete contract: "Delete first creates a durable obligation and
|
||||
// prevents new dependants ... The active record is removed only when all
|
||||
// required cleanup is durably complete."
|
||||
//
|
||||
// A delete that removes the record as part of the request cannot satisfy this,
|
||||
// because the cleanup it depends on costs one round trip per object and so has
|
||||
// no bound a request can carry. On prod a 100 GiB volume's delete overran its
|
||||
// 30 s budget, the tenant was told it failed, and the teardown that had been
|
||||
// started was left with no record accountable for it.
|
||||
func TestS2_VolumeDeleteRecordsTheObligationBeforeRemovingTheRecord(t *testing.T) {
|
||||
store := objectstore.NewMemoryObjectStore()
|
||||
svc := newTestVolumeServiceWithStore("ap-southeast-2a", store)
|
||||
svc.snapshotKV = setupTestVolumeKV(t)
|
||||
seedVolume(t, svc, "vol-obligation000", "available", "")
|
||||
|
||||
_, err := svc.DeleteVolume(context.Background(), &ec2.DeleteVolumeInput{VolumeId: aws.String("vol-obligation000")}, testVolAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
owed, err := svc.GetVolumeMetadata(testVolAccountID, "vol-obligation000")
|
||||
require.NoErrorf(t, err, "ADR-0003:S2: the active record must survive the request so the obligation stays readable (service_impl.go DeleteVolume: record the state, do not remove the document)")
|
||||
assert.Equalf(t, volumeStateDeleting, owed.State,
|
||||
"ADR-0003:S2: delete must first create a durable obligation (service_impl.go DeleteVolume: write State=%q before returning)", volumeStateDeleting)
|
||||
|
||||
// The obligation is what callers see, so describe must report it rather than
|
||||
// the available state the volume no longer has. The attach refusal reads the
|
||||
// same field (runtime/daemon/daemon_handlers_volume.go: State != "available").
|
||||
described, err := svc.DescribeVolumes(context.Background(), &ec2.DescribeVolumesInput{
|
||||
VolumeIds: []*string{aws.String("vol-obligation000")},
|
||||
}, testVolAccountID)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, described.Volumes, 1)
|
||||
assert.Equalf(t, volumeStateDeleting, aws.StringValue(described.Volumes[0].State),
|
||||
"ADR-0003:S2: a delete must prevent new dependants, and %q is the state every dependant path gates on", volumeStateDeleting)
|
||||
|
||||
// A second delete adds nothing rather than racing the first teardown.
|
||||
_, err = svc.DeleteVolume(context.Background(), &ec2.DeleteVolumeInput{VolumeId: aws.String("vol-obligation000")}, testVolAccountID)
|
||||
require.NoErrorf(t, err, "ADR-0003:S2: the obligation is durable, so a repeated delete is answered from it (service_impl.go DeleteVolume: return early on State==%q)", volumeStateDeleting)
|
||||
|
||||
reclaimed, err := svc.NewVolumeReclaimReaper().Sweep(context.Background())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, reclaimed)
|
||||
_, err = svc.GetVolumeMetadata(testVolAccountID, "vol-obligation000")
|
||||
require.Errorf(t, err, "ADR-0003:S2: the active record is removed once cleanup is durably complete (volume_reclaim_reaper.go Sweep: delete the document last)")
|
||||
}
|
||||
|
||||
// TestVolumeTagMirror_WritesOnlyControlPlaneDocument locks the single-writer
|
||||
// invariant: tag mutations must never rewrite config.json or route an encrypted
|
||||
// volume through the ebs.config keyholder queue.
|
||||
|
||||
@@ -33,6 +33,11 @@ import (
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// volumeStateDeleting is the state a delete records as its durable obligation.
|
||||
// The record carries it until the teardown is complete, which is what makes a
|
||||
// delete survive a restart and what prevents a second one repeating the work.
|
||||
const volumeStateDeleting = string(ebsprovider.VolumeStateDeleting)
|
||||
|
||||
// Ensure VolumeServiceImpl implements VolumeService.
|
||||
var _ VolumeService = (*VolumeServiceImpl)(nil)
|
||||
|
||||
@@ -1078,6 +1083,14 @@ func (s *VolumeServiceImpl) ModifyVolume(ctx context.Context, input *ec2.ModifyV
|
||||
// is always a metadata-only clear, never a QMP call. Errors from either step
|
||||
// are returned, not swallowed.
|
||||
func (s *VolumeServiceImpl) DeleteVolumeOnTerminate(ctx context.Context, volumeID, accountID string) error {
|
||||
// Clearing the attachment would overwrite a recorded deletion, so a volume
|
||||
// already owed is left alone: the reclaimer owns it from here, and resetting
|
||||
// it to available would let a second teardown run against the same data.
|
||||
meta, err := s.metadata.GetVolume(ctx, accountID, volumeID)
|
||||
if err == nil && meta.State == volumeStateDeleting {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := s.UpdateVolumeState(accountID, volumeID, "available", "", ""); err != nil {
|
||||
// No metadata doc means the volume is already gone, per GetVolume's own
|
||||
// contract, so absence here is success, not a teardown failure.
|
||||
@@ -1086,7 +1099,7 @@ func (s *VolumeServiceImpl) DeleteVolumeOnTerminate(ctx context.Context, volumeI
|
||||
}
|
||||
return fmt.Errorf("clear attachment before terminate delete: %w", err)
|
||||
}
|
||||
_, err := s.DeleteVolume(ctx, &ec2.DeleteVolumeInput{VolumeId: &volumeID}, accountID)
|
||||
_, err = s.DeleteVolume(ctx, &ec2.DeleteVolumeInput{VolumeId: &volumeID}, accountID)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -1154,8 +1167,17 @@ func (s *VolumeServiceImpl) ForceDetachVolume(ctx context.Context, input *ec2.De
|
||||
}, nil
|
||||
}
|
||||
|
||||
// DeleteVolume deletes an EBS volume: validates state, asks the EBS provider to
|
||||
// destroy the backing data, then removes the control-plane metadata document.
|
||||
// DeleteVolume records a durable obligation to destroy an EBS volume and
|
||||
// returns, matching AWS: DeleteVolume's output is an empty shape and the
|
||||
// documented state machine has the volume "remain in the deleting state for
|
||||
// several minutes". It validates, decides every AWS-returnable error, moves the
|
||||
// volume to "deleting", and leaves the object teardown to VolumeReclaimReaper.
|
||||
//
|
||||
// The caller never waits on data. Object deletion costs one round trip per
|
||||
// object, so the old synchronous form made this call's latency a function of
|
||||
// the volume's size: no timeout can be both short enough for a control-plane
|
||||
// request and long enough for a large volume, and the overrun contributed no
|
||||
// error a caller could act on.
|
||||
func (s *VolumeServiceImpl) DeleteVolume(ctx context.Context, input *ec2.DeleteVolumeInput, accountID string) (*ec2.DeleteVolumeOutput, error) {
|
||||
if input == nil || input.VolumeId == nil || *input.VolumeId == "" {
|
||||
return nil, errors.New(awserrors.ErrorInvalidParameterValue)
|
||||
@@ -1182,6 +1204,14 @@ func (s *VolumeServiceImpl) DeleteVolume(ctx context.Context, input *ec2.DeleteV
|
||||
return nil, errors.New(awserrors.ErrorInvalidVolumeNotFound)
|
||||
}
|
||||
|
||||
// The obligation is already recorded, so this request has nothing to add.
|
||||
// That makes a tenant's retry and DeleteOnTermination's own delete for the
|
||||
// same volume collapse into one, rather than racing each other's teardown.
|
||||
if meta.State == volumeStateDeleting {
|
||||
slog.InfoContext(ctx, "DeleteVolume: already deleting", "volumeId", volumeID)
|
||||
return &ec2.DeleteVolumeOutput{}, nil
|
||||
}
|
||||
|
||||
// An unattached volume is deletable. State must be "available" OR empty: a
|
||||
// detach/terminate that failed to write back "available" leaves the State
|
||||
// drifted to empty with no attachment, and gating on State=="available"
|
||||
@@ -1196,19 +1226,16 @@ func (s *VolumeServiceImpl) DeleteVolume(ctx context.Context, input *ec2.DeleteV
|
||||
if err := s.checkVolumeHasNoSnapshots(ctx, volumeID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.provider.DeleteVolume(ctx, ebsprovider.DeleteVolumeRequest{
|
||||
Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, Handle: meta.ProviderHandle,
|
||||
}); err != nil {
|
||||
slog.ErrorContext(ctx, "DeleteVolume: provider deletion failed", "volumeId", volumeID, "err", err)
|
||||
// The terminate reaper retries this every two minutes and logs whatever
|
||||
// it gets back, so a bare code makes a persistent fault unreadable.
|
||||
return nil, awserrors.Errorf(awserrors.ErrorServerInternal, "delete volume %s: %v", volumeID, err)
|
||||
}
|
||||
if err := s.metadata.DeleteVolume(ctx, accountID, volumeID); err != nil && !objectstore.IsNoSuchKeyError(err) {
|
||||
return nil, awserrors.Errorf(awserrors.ErrorServerInternal, "delete volume metadata %s: %v", volumeID, err)
|
||||
|
||||
// Last write of the request, and the only one that can fail from here: the
|
||||
// record is the obligation, so until this lands nothing is owed and the
|
||||
// caller must not be told otherwise.
|
||||
if err := s.UpdateVolumeState(accountID, volumeID, volumeStateDeleting, "", ""); err != nil {
|
||||
slog.ErrorContext(ctx, "DeleteVolume: failed to record the deletion", "volumeId", volumeID, "err", err)
|
||||
return nil, awserrors.Errorf(awserrors.ErrorServerInternal, "mark volume %s deleting: %v", volumeID, err)
|
||||
}
|
||||
|
||||
slog.InfoContext(ctx, "DeleteVolume completed", "volumeId", volumeID)
|
||||
slog.InfoContext(ctx, "DeleteVolume accepted", "volumeId", volumeID)
|
||||
|
||||
return &ec2.DeleteVolumeOutput{}, nil
|
||||
}
|
||||
|
||||
@@ -326,6 +326,12 @@ func TestCreateVolume_UsesInjectedProvider(t *testing.T) {
|
||||
require.NoError(t, svc.UpdateVolumeState(testVolAccountID, aws.StringValue(vol.VolumeId), "available", "", ""))
|
||||
_, err = svc.DeleteVolume(context.Background(), &ec2.DeleteVolumeInput{VolumeId: vol.VolumeId}, testVolAccountID)
|
||||
require.NoError(t, err)
|
||||
deleting, err := svc.metadata.GetVolume(context.Background(), testVolAccountID, aws.StringValue(vol.VolumeId))
|
||||
require.NoError(t, err, "the record carries the obligation until the teardown is complete")
|
||||
assert.Equal(t, "deleting", deleting.State)
|
||||
reclaimed, err := svc.NewVolumeReclaimReaper().Sweep(context.Background())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, reclaimed)
|
||||
_, err = svc.metadata.GetVolume(context.Background(), testVolAccountID, aws.StringValue(vol.VolumeId))
|
||||
assert.Error(t, err)
|
||||
}
|
||||
@@ -1764,8 +1770,16 @@ func TestDeleteVolumeOnTerminate_ClearsAttachmentThenDeletes(t *testing.T) {
|
||||
err := svc.DeleteVolumeOnTerminate(context.Background(), volumeID, testVolAccountID)
|
||||
require.NoError(t, err, "terminate implies detach: a stale attachment must not block the terminate delete")
|
||||
|
||||
deleting, err := svc.GetVolumeMetadata(testVolAccountID, volumeID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "deleting", deleting.State, "the volume must be recorded as deleting, not merely detached")
|
||||
assert.Empty(t, deleting.AttachedInstance)
|
||||
|
||||
reclaimed, err := svc.NewVolumeReclaimReaper().Sweep(context.Background())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, reclaimed)
|
||||
_, err = svc.GetVolumeMetadata(testVolAccountID, volumeID)
|
||||
require.Error(t, err, "the volume must actually be deleted, not merely detached")
|
||||
require.Error(t, err, "the volume must actually be deleted once the teardown completes")
|
||||
assert.Contains(t, err.Error(), awserrors.ErrorInvalidVolumeNotFound)
|
||||
}
|
||||
|
||||
@@ -1809,11 +1823,12 @@ func (p *failingDeleteProvider) DeleteVolume(context.Context, ebsprovider.Delete
|
||||
return p.err
|
||||
}
|
||||
|
||||
// TestDeleteVolume_ProviderFailureCarriesItsCause pins that a provider delete
|
||||
// failure keeps its cause on the way out. The terminate reaper retries this
|
||||
// every two minutes and logs whatever it is handed, so a bare ServerInternal
|
||||
// makes a persistent fault indistinguishable from a transient one.
|
||||
func TestDeleteVolume_ProviderFailureCarriesItsCause(t *testing.T) {
|
||||
// TestDeleteVolume_ProviderFailureDoesNotFailTheRequest pins the behaviour a
|
||||
// 100 GiB volume on prod needed: an unreachable blob node must not turn an
|
||||
// accepted delete into an error the tenant retries. The obligation is already
|
||||
// recorded, so the request succeeds and the teardown is the reclaimer's to
|
||||
// retry until it lands.
|
||||
func TestDeleteVolume_ProviderFailureDoesNotFailTheRequest(t *testing.T) {
|
||||
kv := setupTestVolumeKV(t)
|
||||
store := objectstore.NewMemoryObjectStore()
|
||||
svc := newTestVolumeServiceWithStore("ap-southeast-2a", store)
|
||||
@@ -1829,12 +1844,74 @@ func TestDeleteVolume_ProviderFailureCarriesItsCause(t *testing.T) {
|
||||
})
|
||||
|
||||
_, err := svc.DeleteVolume(context.Background(), &ec2.DeleteVolumeInput{VolumeId: aws.String(volumeID)}, testVolAccountID)
|
||||
require.Error(t, err)
|
||||
require.NoError(t, err, "a provider that cannot be reached must not fail a delete the control plane has accepted")
|
||||
|
||||
assert.Equal(t, awserrors.ErrorServerInternal, awserrors.ValidErrorCodeFromError(err),
|
||||
"the code stays ServerInternal so callers classify it unchanged")
|
||||
assert.Contains(t, err.Error(), "dial node 6",
|
||||
"an operator reading the reaper log must learn why the delete keeps failing")
|
||||
meta, err := svc.metadata.GetVolume(context.Background(), testVolAccountID, volumeID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "deleting", meta.State)
|
||||
|
||||
reclaimed, err := svc.NewVolumeReclaimReaper().Sweep(context.Background())
|
||||
require.NoError(t, err, "a failed teardown is retried, not reported as a sweep failure")
|
||||
assert.Equal(t, 0, reclaimed)
|
||||
meta, err = svc.metadata.GetVolume(context.Background(), testVolAccountID, volumeID)
|
||||
require.NoError(t, err, "the record survives so the obligation is still readable")
|
||||
assert.Equal(t, "deleting", meta.State)
|
||||
|
||||
// The provider comes back, and the same record is what drives the retry.
|
||||
svc.SetEBSProvider(&failingDeleteProvider{EBSProvider: svc.provider, err: nil})
|
||||
reclaimed, err = svc.NewVolumeReclaimReaper().Sweep(context.Background())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, reclaimed)
|
||||
_, err = svc.metadata.GetVolume(context.Background(), testVolAccountID, volumeID)
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
// TestDeleteVolume_SecondDeleteIsAcceptedWithoutASecondTeardown pins the dedupe
|
||||
// the deleting state buys. A tenant retry and DeleteOnTermination's own delete
|
||||
// both arrive for the same volume, and on prod three destructive handlers ran
|
||||
// one volume's teardown concurrently because nothing collapsed them.
|
||||
func TestDeleteVolume_SecondDeleteIsAcceptedWithoutASecondTeardown(t *testing.T) {
|
||||
kv := setupTestVolumeKV(t)
|
||||
store := objectstore.NewMemoryObjectStore()
|
||||
svc := newTestVolumeServiceWithStore("ap-southeast-2a", store)
|
||||
svc.snapshotKV = kv
|
||||
|
||||
volumeID := "vol-deleted-twice"
|
||||
createVolumeInStoreWithMeta(t, svc, store, volumeID, ebsmetadata.Volume{
|
||||
VolumeID: volumeID, CapacityGiB: 10, State: "available",
|
||||
})
|
||||
counting := &countingDeleteProvider{EBSProvider: svc.provider}
|
||||
svc.SetEBSProvider(counting)
|
||||
|
||||
for range 3 {
|
||||
_, err := svc.DeleteVolume(context.Background(), &ec2.DeleteVolumeInput{VolumeId: aws.String(volumeID)}, testVolAccountID)
|
||||
require.NoError(t, err, "AWS answers a repeated delete with the same empty success")
|
||||
}
|
||||
assert.Equal(t, 0, counting.calls, "no teardown runs in the request path at all")
|
||||
|
||||
// DeleteOnTermination arriving for a volume already owed must not reset it
|
||||
// to available, which would let a second teardown through.
|
||||
require.NoError(t, svc.DeleteVolumeOnTerminate(context.Background(), volumeID, testVolAccountID))
|
||||
meta, err := svc.metadata.GetVolume(context.Background(), testVolAccountID, volumeID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "deleting", meta.State)
|
||||
|
||||
reclaimed, err := svc.NewVolumeReclaimReaper().Sweep(context.Background())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, reclaimed)
|
||||
assert.Equal(t, 1, counting.calls, "one recorded obligation is one teardown")
|
||||
}
|
||||
|
||||
// countingDeleteProvider counts the deletes that reach the provider boundary.
|
||||
type countingDeleteProvider struct {
|
||||
ebsprovider.EBSProvider
|
||||
|
||||
calls int
|
||||
}
|
||||
|
||||
func (p *countingDeleteProvider) DeleteVolume(context.Context, ebsprovider.DeleteVolumeRequest) error {
|
||||
p.calls++
|
||||
return nil
|
||||
}
|
||||
|
||||
// TestDeleteVolumeOnTerminate_SurfacesDeleteFailure verifies a DeleteVolume
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
package volume
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
|
||||
ebsprovider "github.com/mulgadc/spinifex/spinifex/providers/ebs"
|
||||
"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
|
||||
)
|
||||
|
||||
// VolumeReclaimReaper converges the volumes a tenant has already asked to
|
||||
// delete. DeleteVolume records the obligation as State="deleting" and returns,
|
||||
// so the object teardown runs here: its cost is one round trip per object and
|
||||
// therefore a function of the volume, which is not work a caller can be held
|
||||
// on. The active record is removed only once the teardown reports complete.
|
||||
//
|
||||
// Its licence to delete volume data is narrow and is not VolumeLeakReaper's.
|
||||
// That reaper may only mark and alarm because it would be deciding, on its own
|
||||
// judgement, that a volume is garbage. This one decides nothing: the tenant
|
||||
// decided, the decision is recorded on the volume itself, and the sweep only
|
||||
// finishes what the API accepted.
|
||||
type VolumeReclaimReaper struct {
|
||||
svc *VolumeServiceImpl
|
||||
}
|
||||
|
||||
var _ vm.Reaper = (*VolumeReclaimReaper)(nil)
|
||||
|
||||
// NewVolumeReclaimReaper builds the reclaimer for volumes recorded as deleting.
|
||||
func (s *VolumeServiceImpl) NewVolumeReclaimReaper() *VolumeReclaimReaper {
|
||||
return &VolumeReclaimReaper{svc: s}
|
||||
}
|
||||
|
||||
func (r *VolumeReclaimReaper) Class() string { return "volume-reclaim" }
|
||||
|
||||
// Scope is cluster-wide, which is also what serialises the teardown. A delete
|
||||
// that ran on every node would put several destructive handlers on one volume's
|
||||
// objects at once, which is what the synchronous path did when a caller retried.
|
||||
func (r *VolumeReclaimReaper) Scope() vm.ReaperScope { return vm.ScopeClusterWide }
|
||||
|
||||
// Sweep finishes every recorded deletion it can and reports how many records it
|
||||
// retired. A volume whose teardown fails keeps its record and is retried on the
|
||||
// next sweep: predastore being slow, saturated or down delays reclamation and
|
||||
// must not undo a delete the tenant has already been told succeeded.
|
||||
func (r *VolumeReclaimReaper) Sweep(ctx context.Context) (int, error) {
|
||||
if r.svc.provider == nil {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// The sweep has no caller account, so each document supplies the account its
|
||||
// own key is under.
|
||||
volumes, err := r.svc.metadata.ListAllVolumes(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
reclaimed := 0
|
||||
for _, meta := range volumes {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return reclaimed, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
if meta.State != volumeStateDeleting {
|
||||
continue
|
||||
}
|
||||
|
||||
if err := r.svc.provider.DeleteVolume(ctx, ebsprovider.DeleteVolumeRequest{
|
||||
Versioned: ebsprovider.NewVersioned(),
|
||||
VolumeID: meta.VolumeID,
|
||||
Handle: meta.ProviderHandle,
|
||||
}); err != nil {
|
||||
slog.WarnContext(ctx, "volume-reclaim: teardown incomplete, retrying next sweep",
|
||||
"volumeId", meta.VolumeID, "accountId", meta.TenantID, "err", err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Removing the record is the last step by design: while it exists the
|
||||
// obligation is readable, so a sweep interrupted anywhere above resumes
|
||||
// rather than leaving objects nobody is accountable for.
|
||||
if err := r.svc.metadata.DeleteVolume(ctx, meta.TenantID, meta.VolumeID); err != nil && !objectstore.IsNoSuchKeyError(err) {
|
||||
slog.ErrorContext(ctx, "volume-reclaim: failed to remove the volume record",
|
||||
"volumeId", meta.VolumeID, "accountId", meta.TenantID, "err", err)
|
||||
continue
|
||||
}
|
||||
|
||||
slog.InfoContext(ctx, "volume-reclaim: deletion complete",
|
||||
"volumeId", meta.VolumeID, "accountId", meta.TenantID)
|
||||
reclaimed++
|
||||
}
|
||||
return reclaimed, nil
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package systemvpc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
)
|
||||
|
||||
// PrivateRefs identifies a private-only system VPC and its one subnet.
|
||||
type PrivateRefs struct {
|
||||
VpcID string
|
||||
VpcCIDR string
|
||||
SubnetID string
|
||||
SubnetCIDR string
|
||||
}
|
||||
|
||||
// EnsurePrivate idempotently builds a VPC with one private subnet and nothing
|
||||
// else. It takes only the VPC and subnet surface, so it cannot create an IGW,
|
||||
// route table, NAT gateway or EIP. Use a Spec with either this or Ensure.
|
||||
func EnsurePrivate(ctx context.Context, vpcp VPCProvisioner, spec Spec, accountID string) (*PrivateRefs, error) {
|
||||
if err := spec.validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
vpcCIDR, _, privateCIDRs, err := spec.cidrs()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
refs := &PrivateRefs{VpcCIDR: vpcCIDR, SubnetCIDR: privateCIDRs[0]}
|
||||
if refs.VpcID, err = ensureVPC(ctx, vpcp, spec, accountID, vpcCIDR); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if refs.SubnetID, err = ensureSubnet(ctx, vpcp, spec, accountID, refs.VpcID, refs.SubnetCIDR,
|
||||
spec.Roles().PrivateSubnet, spec.az(0)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
slog.InfoContext(ctx, "systemvpc: private system VPC ready",
|
||||
"name", spec.Name, "managedBy", spec.ManagedBy, "vpc", refs.VpcID, "subnet", refs.SubnetID)
|
||||
return refs, nil
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
package systemvpc
|
||||
|
||||
import (
|
||||
"maps"
|
||||
"net/netip"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// The private shape is a VPC and one subnet carved from the spec's supernet,
|
||||
// owned and role-tagged like Ensure's, with no gateway, route table or EIP.
|
||||
func TestEnsurePrivateBuildsOnlyAVPCAndOneSubnet(t *testing.T) {
|
||||
f := newFakeEC2()
|
||||
spec := testSpec("sqs-demo")
|
||||
|
||||
refs, err := EnsurePrivate(t.Context(), f, spec, "000000000000")
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, map[string]int{"vpc": 1, "subnet": 1}, f.creates)
|
||||
assert.Empty(t, f.igws)
|
||||
assert.Empty(t, f.rts)
|
||||
assert.Empty(t, f.ngws)
|
||||
assert.Empty(t, f.eips)
|
||||
|
||||
vpcCIDR, _, privateCIDRs, err := spec.cidrs()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, vpcCIDR, refs.VpcCIDR)
|
||||
assert.Equal(t, privateCIDRs[0], refs.SubnetCIDR)
|
||||
sub := f.subnets[refs.SubnetID]
|
||||
require.NotNil(t, sub)
|
||||
assert.Equal(t, refs.VpcID, aws.StringValue(sub.VpcId))
|
||||
assert.Equal(t, refs.SubnetCIDR, aws.StringValue(sub.CidrBlock))
|
||||
assert.Equal(t, spec.Roles().PrivateSubnet, tagValue(sub.Tags, spec.RoleTagKey))
|
||||
assert.Equal(t, spec.Name, tagValue(sub.Tags, spec.OwnerTagKey))
|
||||
assert.Equal(t, spec.Roles().VPC, tagValue(f.vpcs[refs.VpcID].Tags, spec.RoleTagKey))
|
||||
|
||||
super := netip.MustParsePrefix(spec.Supernet)
|
||||
assert.True(t, super.Contains(netip.MustParsePrefix(refs.SubnetCIDR).Addr()), "subnet outside the supernet")
|
||||
}
|
||||
|
||||
func TestEnsurePrivateIsIdempotent(t *testing.T) {
|
||||
f := newFakeEC2()
|
||||
spec := testSpec("sqs-demo")
|
||||
first, err := EnsurePrivate(t.Context(), f, spec, "000000000000")
|
||||
require.NoError(t, err)
|
||||
creates := maps.Clone(f.creates)
|
||||
|
||||
second, err := EnsurePrivate(t.Context(), f, spec, "000000000000")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, first, second)
|
||||
assert.Equal(t, creates, f.creates, "a re-run must create nothing")
|
||||
}
|
||||
|
||||
// A VPC left by a run that failed before its subnet is reused, not duplicated.
|
||||
func TestEnsurePrivateConvergesAfterPartialFailure(t *testing.T) {
|
||||
f := newFakeEC2()
|
||||
spec := testSpec("sqs-demo")
|
||||
vpcCIDR, _, _, err := spec.cidrs()
|
||||
require.NoError(t, err)
|
||||
vpcID, err := ensureVPC(t.Context(), f, spec, "000000000000", vpcCIDR)
|
||||
require.NoError(t, err)
|
||||
|
||||
refs, err := EnsurePrivate(t.Context(), f, spec, "000000000000")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, vpcID, refs.VpcID)
|
||||
assert.Equal(t, 1, f.creates["vpc"])
|
||||
}
|
||||
|
||||
func TestEnsurePrivateRejectsAnUnusableSpec(t *testing.T) {
|
||||
f := newFakeEC2()
|
||||
spec := testSpec("sqs-demo")
|
||||
spec.Supernet = "10.0.0.0/16"
|
||||
_, err := EnsurePrivate(t.Context(), f, spec, "000000000000")
|
||||
require.Error(t, err)
|
||||
assert.Empty(t, f.creates)
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/workerrole"
|
||||
)
|
||||
|
||||
// The registration dispatches exactly the actions the worker role is granted.
|
||||
func TestActionNames_MatchTheRoleGrant(t *testing.T) {
|
||||
if got, want := ActionNames(), workerrole.Actions(); !slices.Equal(got, want) {
|
||||
t.Fatalf("dispatched actions %v, role grants %v", got, want)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,348 @@
|
||||
// Package worker is the SQS worker's private control-plane surface. It is
|
||||
// served under its own SigV4 signing service, apart from the public AmazonSQS
|
||||
// action inventory, and admits only the Region's current SQS worker VM.
|
||||
package worker
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"maps"
|
||||
"net/http"
|
||||
"slices"
|
||||
|
||||
"github.com/mulgadc/bluebottle/pkg/sigv4"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/firststore"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/storelaunch"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/workerident"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/workerrole"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/query"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/nats-io/nkeys"
|
||||
)
|
||||
|
||||
// The contract lives in workerrole so the role grant can read it without
|
||||
// importing this adapter.
|
||||
const (
|
||||
ServiceName = workerrole.ServiceName
|
||||
WorkerRoleName = workerrole.RoleName
|
||||
ActionClaimFirstStore = workerrole.ActionClaimFirstStore
|
||||
ActionDescribeStore = workerrole.ActionDescribeStore
|
||||
ActionRecordWorkerPublicKey = workerrole.ActionRecordWorkerPublicKey
|
||||
)
|
||||
|
||||
const principalTypeAssumedRole = "assumed-role"
|
||||
|
||||
// params lists the only parameters each action accepts. Identity is never a
|
||||
// parameter: it is derived from the session and the current worker record.
|
||||
var params = map[string][]string{
|
||||
ActionClaimFirstStore: {},
|
||||
ActionDescribeStore: {},
|
||||
ActionRecordWorkerPublicKey: {"PublicKey"},
|
||||
}
|
||||
|
||||
// ActionNames lists every action the registration dispatches, sorted.
|
||||
func ActionNames() []string {
|
||||
return slices.Sorted(maps.Keys(params))
|
||||
}
|
||||
|
||||
// Identities resolves a worker session to its current authoritative identity.
|
||||
type Identities interface {
|
||||
Authorize(ctx context.Context, instanceID string) (workerident.Identity, error)
|
||||
}
|
||||
|
||||
// FirstStore is the first-store decision the worker's key handling consults.
|
||||
type FirstStore interface {
|
||||
Claim(ctx context.Context, region string, id firststore.Identity) (firststore.Decision, error)
|
||||
RecordPublicKey(ctx context.Context, region string, id firststore.Identity, pub string) error
|
||||
}
|
||||
|
||||
// Stores reads the Region's store launch record from the stream leader.
|
||||
type Stores interface {
|
||||
Current(ctx context.Context, region string) (storelaunch.Record, error)
|
||||
}
|
||||
|
||||
var (
|
||||
_ Stores = storelaunch.Records{}
|
||||
_ Identities = workerident.Store{}
|
||||
_ FirstStore = firststore.Store{}
|
||||
)
|
||||
|
||||
// Deps are the capabilities the actions run against. Either one nil answers
|
||||
// every authorized request with ServiceUnavailable, never a decision.
|
||||
type Deps struct {
|
||||
Identities Identities
|
||||
FirstStore FirstStore
|
||||
Stores Stores
|
||||
}
|
||||
|
||||
// NewRegistration returns the internal SQS worker Query registration.
|
||||
func NewRegistration(deps Deps) dispatch.Registration {
|
||||
return dispatch.Registration{
|
||||
Service: ServiceName,
|
||||
ResolveAction: resolveAction,
|
||||
Dispatch: deps.dispatch,
|
||||
Errors: dispatch.ErrorEnvelopeXML,
|
||||
Inventory: dispatch.Inventory{Registered: ActionNames()},
|
||||
}
|
||||
}
|
||||
|
||||
func resolveAction(r *http.Request) string {
|
||||
args, err := readQueryArgs(r)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return args["Action"]
|
||||
}
|
||||
|
||||
// readQueryArgs parses the Query body and rewinds it for a second read.
|
||||
func readQueryArgs(r *http.Request) (map[string]string, error) {
|
||||
body, err := io.ReadAll(io.LimitReader(r.Body, sigv4.MaxPayloadLen+1))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r.Body = io.NopCloser(bytes.NewReader(body))
|
||||
if int64(len(body)) > sigv4.MaxPayloadLen {
|
||||
return nil, errors.New(awserrors.ErrorRequestEntityTooLarge)
|
||||
}
|
||||
return query.ParseArgs(string(body))
|
||||
}
|
||||
|
||||
// dispatch gates the caller as a worker session before anything about the
|
||||
// request is evaluated, then the role policy, then resolves the session to
|
||||
// the Region's current worker identity. Errors are awserrors codes.
|
||||
func (d Deps) dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
|
||||
r := inv.Request
|
||||
ctx := r.Context()
|
||||
args, err := readQueryArgs(r)
|
||||
if err != nil {
|
||||
return errors.New(awserrors.ErrorMalformedQueryString)
|
||||
}
|
||||
action := args["Action"]
|
||||
if action == "" {
|
||||
return errors.New(awserrors.ErrorMissingAction)
|
||||
}
|
||||
allowed, ok := params[action]
|
||||
if !ok {
|
||||
return errors.New(awserrors.ErrorInvalidAction)
|
||||
}
|
||||
|
||||
actor, _ := inv.Actor()
|
||||
session, err := requireWorkerPrincipal(ctx, inv.AccountID, actor)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := inv.Authorize(ServiceName, action, []string{"*"}, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
for name := range args {
|
||||
if name != "Action" && name != "Version" && !slices.Contains(allowed, name) {
|
||||
// The name is caller-chosen and may carry a secret, so it is not logged.
|
||||
slog.DebugContext(ctx, "SQS internal: unexpected parameter refused", "action", action)
|
||||
return errors.New(awserrors.ErrorInvalidParameterValue)
|
||||
}
|
||||
}
|
||||
|
||||
if d.Identities == nil || d.FirstStore == nil || d.Stores == nil {
|
||||
return errors.New(awserrors.ErrorServiceUnavailable)
|
||||
}
|
||||
ident, err := d.Identities.Authorize(ctx, session)
|
||||
if err != nil {
|
||||
return mapError(ctx, action, err)
|
||||
}
|
||||
// inv.Region is the gateway's own Region; a worker bound elsewhere is not
|
||||
// this Region's worker, whatever region it signed for.
|
||||
if ident.Region != inv.Region {
|
||||
slog.DebugContext(ctx, "SQS internal: worker bound to another Region", "instanceID", session)
|
||||
return errors.New(awserrors.ErrorAccessDenied)
|
||||
}
|
||||
id := firststore.Identity{InstanceID: ident.InstanceID, ClaimID: ident.ClaimID, Generation: ident.Generation}
|
||||
|
||||
switch action {
|
||||
case ActionDescribeStore:
|
||||
return d.describeStore(ctx, w, ident.Region)
|
||||
case ActionClaimFirstStore:
|
||||
dec, err := d.FirstStore.Claim(ctx, ident.Region, id)
|
||||
if err != nil {
|
||||
return mapError(ctx, action, err)
|
||||
}
|
||||
return writeXML(ctx, w, claimResponse{Result: claimResult{CreateKey: dec.CreateKey, PublicKey: dec.PublicKey}})
|
||||
default:
|
||||
pub := args["PublicKey"]
|
||||
if pub == "" {
|
||||
return errors.New(awserrors.ErrorMissingParameter)
|
||||
}
|
||||
// A seed or any other value is refused without echoing it anywhere.
|
||||
if !nkeys.IsValidPublicUserKey(pub) {
|
||||
return errors.New(awserrors.ErrorInvalidParameterValue)
|
||||
}
|
||||
if err := d.FirstStore.RecordPublicKey(ctx, ident.Region, id, pub); err != nil {
|
||||
return mapError(ctx, action, err)
|
||||
}
|
||||
return writeXML(ctx, w, recordResponse{})
|
||||
}
|
||||
}
|
||||
|
||||
// requireWorkerPrincipal admits only a session of the worker instance role in
|
||||
// the system account, and returns its session name: the instance ID STS set.
|
||||
// It says the caller is an SQS worker VM, not that it is the current one.
|
||||
func requireWorkerPrincipal(ctx context.Context, accountID string, actor dispatch.Actor) (string, error) {
|
||||
if actor.PrincipalType != principalTypeAssumedRole ||
|
||||
accountID != awsidentifiers.GlobalAccountID || actor.AccountID != awsidentifiers.GlobalAccountID ||
|
||||
actor.RoleName() != WorkerRoleName || actor.SessionName == "" {
|
||||
slog.DebugContext(ctx, "SQS internal: rejected non-worker caller",
|
||||
"principalType", actor.PrincipalType, "accountID", accountID)
|
||||
return "", errors.New(awserrors.ErrorAccessDenied)
|
||||
}
|
||||
return actor.SessionName, nil
|
||||
}
|
||||
|
||||
// mapError turns a store refusal into the code the worker acts on: identity
|
||||
// refusals are AccessDenied, contention is retryable, the rest are faults.
|
||||
func mapError(ctx context.Context, action string, err error) error {
|
||||
switch {
|
||||
case errors.Is(err, workerident.ErrUnknownInstance), errors.Is(err, workerident.ErrNotCurrent),
|
||||
errors.Is(err, firststore.ErrClaimHeld), errors.Is(err, firststore.ErrStaleGeneration):
|
||||
return errors.New(awserrors.ErrorAccessDenied)
|
||||
case errors.Is(err, firststore.ErrKeyMismatch):
|
||||
return errors.New(awserrors.ErrorInvalidParameterValue)
|
||||
case errors.Is(err, firststore.ErrNotClaimed):
|
||||
return errors.New(awserrors.ErrorInvalidState)
|
||||
case errors.Is(err, workerident.ErrContended), errors.Is(err, firststore.ErrContended):
|
||||
return errors.New(awserrors.ErrorServiceUnavailable)
|
||||
case errors.Is(err, errUnavailable):
|
||||
slog.WarnContext(ctx, "SQS internal: worker stores unavailable", "action", action, "err", err)
|
||||
return errors.New(awserrors.ErrorServiceUnavailable)
|
||||
}
|
||||
slog.ErrorContext(ctx, "SQS internal: store failure", "action", action, "err", err)
|
||||
return errors.New(awserrors.ErrorServerInternal)
|
||||
}
|
||||
|
||||
// describeStore reports the Region's store launch: pending until it is bound,
|
||||
// then the address and the public key it was launched for. Bound is not
|
||||
// ready; the worker checks the store itself with its own key.
|
||||
func (d Deps) describeStore(ctx context.Context, w http.ResponseWriter, region string) error {
|
||||
rec, err := d.Stores.Current(ctx, region)
|
||||
switch {
|
||||
case errors.Is(err, jetstream.ErrKeyNotFound):
|
||||
return writeXML(ctx, w, describeResponse{Result: describeResult{State: StorePending}})
|
||||
case errors.Is(err, storelaunch.ErrUnreadable), errors.Is(err, storelaunch.ErrInvalidRecording):
|
||||
return mapError(ctx, ActionDescribeStore, err)
|
||||
case err != nil:
|
||||
return mapError(ctx, ActionDescribeStore, fmt.Errorf("%w: %w", errUnavailable, err))
|
||||
case rec.State != storelaunch.StateBound:
|
||||
return writeXML(ctx, w, describeResponse{Result: describeResult{State: StorePending}})
|
||||
}
|
||||
return writeXML(ctx, w, describeResponse{Result: describeResult{
|
||||
State: StoreBound, Address: rec.Address, PublicKey: rec.PublicKey,
|
||||
}})
|
||||
}
|
||||
|
||||
// DescribeStore states.
|
||||
const (
|
||||
StorePending = "pending"
|
||||
StoreBound = "bound"
|
||||
)
|
||||
|
||||
type describeResult struct {
|
||||
State string `xml:"State"`
|
||||
Address string `xml:"Address,omitempty"`
|
||||
PublicKey string `xml:"PublicKey,omitempty"`
|
||||
}
|
||||
|
||||
type describeResponse struct {
|
||||
XMLName xml.Name `xml:"DescribeStoreResponse"`
|
||||
Result describeResult `xml:"DescribeStoreResult"`
|
||||
}
|
||||
|
||||
type claimResult struct {
|
||||
CreateKey bool `xml:"CreateKey"`
|
||||
PublicKey string `xml:"PublicKey,omitempty"`
|
||||
}
|
||||
|
||||
type claimResponse struct {
|
||||
XMLName xml.Name `xml:"ClaimFirstStoreResponse"`
|
||||
Result claimResult `xml:"ClaimFirstStoreResult"`
|
||||
}
|
||||
|
||||
type recordResponse struct {
|
||||
XMLName xml.Name `xml:"RecordWorkerPublicKeyResponse"`
|
||||
}
|
||||
|
||||
func writeXML(ctx context.Context, w http.ResponseWriter, v any) error {
|
||||
out, err := xml.Marshal(v)
|
||||
if err != nil {
|
||||
return errors.New(awserrors.ErrorServerInternal)
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/xml")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
if _, err := w.Write(out); err != nil {
|
||||
slog.ErrorContext(ctx, "Failed to write SQS internal response", "err", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// JetStreamDeps binds the existing worker-identity and first-store buckets per
|
||||
// request and never creates them, so a gateway provisions nothing and refuses
|
||||
// every request until the worker launch path has created them durably.
|
||||
func JetStreamDeps(js jetstream.JetStream) Deps {
|
||||
if js == nil {
|
||||
return Deps{}
|
||||
}
|
||||
return Deps{Identities: boundIdentities{js}, FirstStore: boundFirstStore{js}, Stores: boundStores{js}}
|
||||
}
|
||||
|
||||
// errUnavailable marks a bucket that could not be bound: absent, unreachable
|
||||
// or not durable. It decides nothing, so the worker retries later.
|
||||
var errUnavailable = errors.New("SQS worker stores unavailable")
|
||||
|
||||
type boundIdentities struct{ js jetstream.JetStream }
|
||||
|
||||
func (b boundIdentities) Authorize(ctx context.Context, instanceID string) (workerident.Identity, error) {
|
||||
s, err := workerident.Bind(ctx, b.js)
|
||||
if err != nil {
|
||||
return workerident.Identity{}, fmt.Errorf("%w: %w", errUnavailable, err)
|
||||
}
|
||||
return s.Authorize(ctx, instanceID)
|
||||
}
|
||||
|
||||
type boundFirstStore struct{ js jetstream.JetStream }
|
||||
|
||||
func (b boundFirstStore) Claim(ctx context.Context, region string, id firststore.Identity) (firststore.Decision, error) {
|
||||
s, err := firststore.Bind(ctx, b.js)
|
||||
if err != nil {
|
||||
return firststore.Decision{}, fmt.Errorf("%w: %w", errUnavailable, err)
|
||||
}
|
||||
return s.Claim(ctx, region, id)
|
||||
}
|
||||
|
||||
func (b boundFirstStore) RecordPublicKey(ctx context.Context, region string, id firststore.Identity, pub string) error {
|
||||
s, err := firststore.Bind(ctx, b.js)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %w", errUnavailable, err)
|
||||
}
|
||||
return s.RecordPublicKey(ctx, region, id, pub)
|
||||
}
|
||||
|
||||
type boundStores struct{ js jetstream.JetStream }
|
||||
|
||||
func (b boundStores) Current(ctx context.Context, region string) (storelaunch.Record, error) {
|
||||
r, err := storelaunch.Bind(ctx, b.js)
|
||||
if err != nil {
|
||||
return storelaunch.Record{}, fmt.Errorf("%w: %w", errUnavailable, err)
|
||||
}
|
||||
return r.Current(ctx, region)
|
||||
}
|
||||
|
||||
var (
|
||||
_ Identities = boundIdentities{}
|
||||
_ FirstStore = boundFirstStore{}
|
||||
_ Stores = boundStores{}
|
||||
)
|
||||
@@ -0,0 +1,667 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/bluebottle/pkg/iampolicy"
|
||||
testutil "github.com/mulgadc/spinifex/internal/testkit"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/firststore"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/storelaunch"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/workerident"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/clustersize"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/nats-io/nkeys"
|
||||
)
|
||||
|
||||
const (
|
||||
region = "ap-southeast-2"
|
||||
instance = "i-0abc12345def67890"
|
||||
successor = "i-0fed98765cba43210"
|
||||
volume = "vol-0123456789abcdef0"
|
||||
)
|
||||
|
||||
func workerActor(session string) dispatch.Actor {
|
||||
return dispatch.Actor{
|
||||
AccountID: awsidentifiers.GlobalAccountID,
|
||||
PrincipalType: principalTypeAssumedRole,
|
||||
PrincipalARN: "arn:aws:iam::" + awsidentifiers.GlobalAccountID + ":role/" + WorkerRoleName,
|
||||
SessionName: session,
|
||||
}
|
||||
}
|
||||
|
||||
type call struct {
|
||||
policyChecks int
|
||||
policyErr error
|
||||
}
|
||||
|
||||
func invoke(deps Deps, body string, actor dispatch.Actor, c *call) (*httptest.ResponseRecorder, error) {
|
||||
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body))
|
||||
rec := httptest.NewRecorder()
|
||||
err := deps.dispatch(rec, dispatch.Invocation{
|
||||
Request: req,
|
||||
AccountID: actor.AccountID,
|
||||
Service: ServiceName,
|
||||
Region: region,
|
||||
Authorize: func(service, action string, _ []string, _ iampolicy.ConditionKeys) error {
|
||||
c.policyChecks++
|
||||
if service != ServiceName {
|
||||
return errors.New(awserrors.ErrorAccessDenied)
|
||||
}
|
||||
return c.policyErr
|
||||
},
|
||||
Actor: func() (dispatch.Actor, error) { return actor, nil },
|
||||
})
|
||||
return rec, err
|
||||
}
|
||||
|
||||
func assertCode(t *testing.T, err error, code string) {
|
||||
t.Helper()
|
||||
if err == nil || err.Error() != code {
|
||||
t.Fatalf("got %v, want %s", err, code)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeIdentities struct {
|
||||
ident workerident.Identity
|
||||
err error
|
||||
asked []string
|
||||
}
|
||||
|
||||
func (f *fakeIdentities) Authorize(_ context.Context, instanceID string) (workerident.Identity, error) {
|
||||
f.asked = append(f.asked, instanceID)
|
||||
return f.ident, f.err
|
||||
}
|
||||
|
||||
type fakeFirstStore struct {
|
||||
decision firststore.Decision
|
||||
err error
|
||||
regions []string
|
||||
ids []firststore.Identity
|
||||
keys []string
|
||||
}
|
||||
|
||||
func (f *fakeFirstStore) Claim(_ context.Context, region string, id firststore.Identity) (firststore.Decision, error) {
|
||||
f.regions, f.ids = append(f.regions, region), append(f.ids, id)
|
||||
return f.decision, f.err
|
||||
}
|
||||
|
||||
func (f *fakeFirstStore) RecordPublicKey(_ context.Context, region string, id firststore.Identity, pub string) error {
|
||||
f.regions, f.ids, f.keys = append(f.regions, region), append(f.ids, id), append(f.keys, pub)
|
||||
return f.err
|
||||
}
|
||||
|
||||
func current() workerident.Identity {
|
||||
return workerident.Identity{Region: region, InstanceID: instance, Generation: 3, ClaimID: "claim-0003", NodeID: "node-a", DataVolumeID: volume}
|
||||
}
|
||||
|
||||
func fakes() (*fakeIdentities, *fakeFirstStore, Deps) {
|
||||
ids, fs := &fakeIdentities{ident: current()}, &fakeFirstStore{decision: firststore.Decision{CreateKey: true}}
|
||||
return ids, fs, Deps{Identities: ids, FirstStore: fs, Stores: &fakeStores{err: jetstream.ErrKeyNotFound}}
|
||||
}
|
||||
|
||||
type fakeStores struct {
|
||||
rec storelaunch.Record
|
||||
err error
|
||||
regions []string
|
||||
}
|
||||
|
||||
func (f *fakeStores) Current(_ context.Context, region string) (storelaunch.Record, error) {
|
||||
f.regions = append(f.regions, region)
|
||||
return f.rec, f.err
|
||||
}
|
||||
|
||||
// The first-store identity is the authoritative record's, keyed by the session
|
||||
// STS set, never anything the request carried.
|
||||
func TestClaim_UsesTheCurrentRecordIdentity(t *testing.T) {
|
||||
ids, fs, deps := fakes()
|
||||
rec, err := invoke(deps, "Action=ClaimFirstStore", workerActor(instance), &call{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(ids.asked) != 1 || ids.asked[0] != instance {
|
||||
t.Fatalf("authorized %v, want the session's instance", ids.asked)
|
||||
}
|
||||
want := firststore.Identity{InstanceID: instance, ClaimID: "claim-0003", Generation: 3}
|
||||
if len(fs.ids) != 1 || fs.ids[0] != want || fs.regions[0] != region {
|
||||
t.Fatalf("claimed %v in %v, want %v in %s", fs.ids, fs.regions, want, region)
|
||||
}
|
||||
if !strings.Contains(rec.Body.String(), "<CreateKey>true</CreateKey>") {
|
||||
t.Fatalf("response %q does not authorize key creation", rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestClaim_ReturnsTheRecordedKey(t *testing.T) {
|
||||
_, fs, deps := fakes()
|
||||
pub := newUserKey(t).pub
|
||||
fs.decision = firststore.Decision{PublicKey: pub}
|
||||
rec, err := invoke(deps, "Action=ClaimFirstStore", workerActor(instance), &call{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := rec.Body.String()
|
||||
if !strings.Contains(body, "<CreateKey>false</CreateKey>") || !strings.Contains(body, "<PublicKey>"+pub+"</PublicKey>") {
|
||||
t.Fatalf("response %q does not carry exactly the recorded key", body)
|
||||
}
|
||||
}
|
||||
|
||||
// Identity fields in the request are refused outright rather than ignored, and
|
||||
// before any store is consulted.
|
||||
func TestRequestSuppliedIdentityIsRefused(t *testing.T) {
|
||||
for _, body := range []string{
|
||||
"Action=ClaimFirstStore&InstanceId=" + successor,
|
||||
"Action=ClaimFirstStore&Generation=99",
|
||||
"Action=ClaimFirstStore&ClaimId=claim-9999",
|
||||
"Action=ClaimFirstStore&Region=us-east-1",
|
||||
"Action=RecordWorkerPublicKey&PublicKey=" + newUserKey(t).pub + "&Generation=99",
|
||||
} {
|
||||
ids, fs, deps := fakes()
|
||||
_, err := invoke(deps, body, workerActor(instance), &call{})
|
||||
assertCode(t, err, awserrors.ErrorInvalidParameterValue)
|
||||
if len(ids.asked)+len(fs.ids) != 0 {
|
||||
t.Fatalf("%s: consulted the stores", body)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Nothing but the worker role's own session in the system account passes, and
|
||||
// a refused caller never reaches the policy evaluator or the stores.
|
||||
func TestPrincipalGate(t *testing.T) {
|
||||
const customer = "123456789012"
|
||||
roleIn := func(acct, role string) string { return "arn:aws:iam::" + acct + ":role/" + role }
|
||||
cases := map[string]dispatch.Actor{
|
||||
"customer account role": {AccountID: customer, PrincipalType: principalTypeAssumedRole,
|
||||
PrincipalARN: roleIn(customer, WorkerRoleName), SessionName: instance},
|
||||
"system account, another role": {AccountID: awsidentifiers.GlobalAccountID, PrincipalType: principalTypeAssumedRole,
|
||||
PrincipalARN: roleIn(awsidentifiers.GlobalAccountID, "rdsInstanceRole"), SessionName: instance},
|
||||
"role in another account": {AccountID: awsidentifiers.GlobalAccountID, PrincipalType: principalTypeAssumedRole,
|
||||
PrincipalARN: roleIn(customer, WorkerRoleName), SessionName: instance},
|
||||
"system account user": {AccountID: awsidentifiers.GlobalAccountID, PrincipalType: "user",
|
||||
PrincipalARN: "arn:aws:iam::" + awsidentifiers.GlobalAccountID + ":user/" + WorkerRoleName, UserName: WorkerRoleName},
|
||||
"empty session": {AccountID: awsidentifiers.GlobalAccountID, PrincipalType: principalTypeAssumedRole, PrincipalARN: roleIn(awsidentifiers.GlobalAccountID, WorkerRoleName)},
|
||||
"anonymous caller": {},
|
||||
}
|
||||
for name, actor := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
ids, fs, deps := fakes()
|
||||
c := &call{}
|
||||
_, err := invoke(deps, "Action=ClaimFirstStore", actor, c)
|
||||
assertCode(t, err, awserrors.ErrorAccessDenied)
|
||||
if c.policyChecks != 0 || len(ids.asked)+len(fs.ids) != 0 {
|
||||
t.Fatal("a refused caller reached the policy check or the stores")
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("invocation account differs from the actor", func(t *testing.T) {
|
||||
ids, fs, deps := fakes()
|
||||
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader("Action=ClaimFirstStore"))
|
||||
err := deps.dispatch(httptest.NewRecorder(), dispatch.Invocation{
|
||||
Request: req, AccountID: customer, Region: region,
|
||||
Authorize: func(string, string, []string, iampolicy.ConditionKeys) error { return nil },
|
||||
Actor: func() (dispatch.Actor, error) { return workerActor(instance), nil },
|
||||
})
|
||||
assertCode(t, err, awserrors.ErrorAccessDenied)
|
||||
if len(ids.asked)+len(fs.ids) != 0 {
|
||||
t.Fatal("consulted the stores")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestPolicyDenialStops(t *testing.T) {
|
||||
ids, fs, deps := fakes()
|
||||
c := &call{policyErr: errors.New(awserrors.ErrorAccessDenied)}
|
||||
_, err := invoke(deps, "Action=ClaimFirstStore", workerActor(instance), c)
|
||||
assertCode(t, err, awserrors.ErrorAccessDenied)
|
||||
if c.policyChecks != 1 || len(ids.asked)+len(fs.ids) != 0 {
|
||||
t.Fatal("the role policy was not the gate before the stores")
|
||||
}
|
||||
}
|
||||
|
||||
// An unknown, superseded or foreign-Region worker never reaches the
|
||||
// first-store decision.
|
||||
func TestNonCurrentWorkerIsRefused(t *testing.T) {
|
||||
for name, set := range map[string]func(*fakeIdentities){
|
||||
"unknown instance": func(f *fakeIdentities) { f.err = workerident.ErrUnknownInstance },
|
||||
"superseded instance": func(f *fakeIdentities) { f.err = workerident.ErrNotCurrent },
|
||||
"another Region": func(f *fakeIdentities) { f.ident.Region = "us-east-1" },
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
ids, fs, deps := fakes()
|
||||
set(ids)
|
||||
_, err := invoke(deps, "Action=ClaimFirstStore", workerActor(instance), &call{})
|
||||
assertCode(t, err, awserrors.ErrorAccessDenied)
|
||||
if len(fs.ids) != 0 {
|
||||
t.Fatal("a non-current worker reached the first-store decision")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type userKey struct{ pub, seed string }
|
||||
|
||||
func newUserKey(t *testing.T) userKey {
|
||||
t.Helper()
|
||||
kp, err := nkeys.CreateUser()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pub, _ := kp.PublicKey()
|
||||
seed, _ := kp.Seed()
|
||||
return userKey{pub: pub, seed: string(seed)}
|
||||
}
|
||||
|
||||
// corrupt always changes the last character, which the key's checksum covers.
|
||||
func corrupt(pub string) string {
|
||||
last := "A"
|
||||
if strings.HasSuffix(pub, "A") {
|
||||
last = "B"
|
||||
}
|
||||
return pub[:len(pub)-1] + last
|
||||
}
|
||||
|
||||
func captureLogs(t *testing.T) *bytes.Buffer {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
prev := slog.Default()
|
||||
slog.SetDefault(slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})))
|
||||
t.Cleanup(func() { slog.SetDefault(prev) })
|
||||
return &buf
|
||||
}
|
||||
|
||||
// A seed, or anything else that is not a public user key, is refused without
|
||||
// being recorded, logged or echoed.
|
||||
func TestRecordPublicKey_RefusesSeedsAndNonKeys(t *testing.T) {
|
||||
k := newUserKey(t)
|
||||
acct, _ := nkeys.CreateAccount()
|
||||
acctPub, _ := acct.PublicKey()
|
||||
for name, value := range map[string]string{
|
||||
"user seed": k.seed,
|
||||
"account public key": acctPub,
|
||||
"corrupt key": corrupt(k.pub),
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
logs := captureLogs(t)
|
||||
_, fs, deps := fakes()
|
||||
rec, err := invoke(deps, "Action=RecordWorkerPublicKey&PublicKey="+value, workerActor(instance), &call{})
|
||||
assertCode(t, err, awserrors.ErrorInvalidParameterValue)
|
||||
if len(fs.keys) != 0 {
|
||||
t.Fatal("recorded a value that is not a public user key")
|
||||
}
|
||||
if strings.Contains(rec.Body.String()+err.Error()+logs.String(), value) {
|
||||
t.Fatal("the refused value was echoed or logged")
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("missing key", func(t *testing.T) {
|
||||
_, fs, deps := fakes()
|
||||
_, err := invoke(deps, "Action=RecordWorkerPublicKey", workerActor(instance), &call{})
|
||||
assertCode(t, err, awserrors.ErrorMissingParameter)
|
||||
if len(fs.keys) != 0 {
|
||||
t.Fatal("recorded an empty key")
|
||||
}
|
||||
})
|
||||
t.Run("public key is recorded under the current identity", func(t *testing.T) {
|
||||
_, fs, deps := fakes()
|
||||
if _, err := invoke(deps, "Action=RecordWorkerPublicKey&PublicKey="+k.pub, workerActor(instance), &call{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := firststore.Identity{InstanceID: instance, ClaimID: "claim-0003", Generation: 3}
|
||||
if len(fs.keys) != 1 || fs.keys[0] != k.pub || fs.ids[0] != want {
|
||||
t.Fatalf("recorded %v for %v", fs.keys, fs.ids)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestStoreErrorsMapToWorkerCodes(t *testing.T) {
|
||||
for err, code := range map[error]string{
|
||||
firststore.ErrClaimHeld: awserrors.ErrorAccessDenied,
|
||||
firststore.ErrStaleGeneration: awserrors.ErrorAccessDenied,
|
||||
firststore.ErrKeyMismatch: awserrors.ErrorInvalidParameterValue,
|
||||
firststore.ErrNotClaimed: awserrors.ErrorInvalidState,
|
||||
firststore.ErrContended: awserrors.ErrorServiceUnavailable,
|
||||
workerident.ErrContended: awserrors.ErrorServiceUnavailable,
|
||||
firststore.ErrUnreadable: awserrors.ErrorServerInternal,
|
||||
errors.New("leader unreachable"): awserrors.ErrorServerInternal,
|
||||
} {
|
||||
_, fs, deps := fakes()
|
||||
fs.err = err
|
||||
_, got := invoke(deps, "Action=ClaimFirstStore", workerActor(instance), &call{})
|
||||
assertCode(t, got, code)
|
||||
}
|
||||
}
|
||||
|
||||
// Without its stores the surface refuses every worker, after the gate has
|
||||
// refused everyone else.
|
||||
func TestMissingDependenciesFailClosed(t *testing.T) {
|
||||
for name, deps := range map[string]Deps{
|
||||
"no deps": {},
|
||||
"no first store": {Identities: &fakeIdentities{ident: current()}, Stores: &fakeStores{}},
|
||||
"no identities": {FirstStore: &fakeFirstStore{decision: firststore.Decision{CreateKey: true}}, Stores: &fakeStores{}},
|
||||
"no stores": {Identities: &fakeIdentities{ident: current()}, FirstStore: &fakeFirstStore{}},
|
||||
"nil JetStream": JetStreamDeps(nil),
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
_, err := invoke(deps, "Action=ClaimFirstStore", workerActor(instance), &call{})
|
||||
assertCode(t, err, awserrors.ErrorServiceUnavailable)
|
||||
_, err = invoke(deps, "Action=ClaimFirstStore", dispatch.Actor{}, &call{})
|
||||
assertCode(t, err, awserrors.ErrorAccessDenied)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnknownAndMissingActions(t *testing.T) {
|
||||
_, _, deps := fakes()
|
||||
_, err := invoke(deps, "Action=SendMessage", workerActor(instance), &call{})
|
||||
assertCode(t, err, awserrors.ErrorInvalidAction)
|
||||
_, err = invoke(deps, "", workerActor(instance), &call{})
|
||||
assertCode(t, err, awserrors.ErrorMissingAction)
|
||||
}
|
||||
|
||||
// --- Against the real stores on embedded JetStream. ---
|
||||
|
||||
type cluster struct {
|
||||
js jetstream.JetStream
|
||||
ids workerident.Store
|
||||
fence firststore.Store
|
||||
deps Deps
|
||||
}
|
||||
|
||||
func newCluster(t *testing.T) cluster {
|
||||
t.Helper()
|
||||
clustersize.DeclareForTest(t, 1)
|
||||
_, _, js := testutil.StartTestJetStream(t)
|
||||
ctx := context.Background()
|
||||
ids, err := workerident.Open(ctx, js)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fence, err := firststore.Open(ctx, js)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return cluster{js: js, ids: ids, fence: fence, deps: JetStreamDeps(js)}
|
||||
}
|
||||
|
||||
func (c cluster) launch(t *testing.T, claim string, supersedes uint64, inst string) workerident.Intent {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
in, err := c.ids.BeginLaunch(ctx, region, workerident.LaunchRequest{InstanceType: "t3.small",
|
||||
ClaimID: claim, Supersedes: supersedes, NodeID: "node-a", DataVolumeID: volume, DataVolumeEncrypted: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.ids.BindInstance(ctx, in, inst); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return in
|
||||
}
|
||||
|
||||
// replace follows the launch path's order once old is confirmed stopped: raise
|
||||
// the first-store fence past old's generation, then begin and bind the successor.
|
||||
func (c cluster) replace(t *testing.T, old workerident.Intent, claim, inst string) workerident.Intent {
|
||||
t.Helper()
|
||||
if err := c.fence.Fence(context.Background(), region, old.Generation+1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return c.launch(t, claim, old.Generation, inst)
|
||||
}
|
||||
|
||||
// The full path: a bound worker claims, records its key, and after a confirmed
|
||||
// replacement the old instance is refused while the new one gets only the key.
|
||||
func TestRealStores_ClaimRecordAndReplace(t *testing.T) {
|
||||
c := newCluster(t)
|
||||
k := newUserKey(t)
|
||||
first := c.launch(t, "claim-0001", 0, instance)
|
||||
|
||||
rec, err := invoke(c.deps, "Action=ClaimFirstStore", workerActor(instance), &call{})
|
||||
if err != nil || !strings.Contains(rec.Body.String(), "<CreateKey>true</CreateKey>") {
|
||||
t.Fatalf("first claim: %v %q", err, rec.Body.String())
|
||||
}
|
||||
if _, err := invoke(c.deps, "Action=RecordWorkerPublicKey&PublicKey="+k.pub, workerActor(instance), &call{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A lost reply is retried with the same key.
|
||||
if _, err := invoke(c.deps, "Action=RecordWorkerPublicKey&PublicKey="+k.pub, workerActor(instance), &call{}); err != nil {
|
||||
t.Fatalf("retried record: %v", err)
|
||||
}
|
||||
other := newUserKey(t)
|
||||
_, err = invoke(c.deps, "Action=RecordWorkerPublicKey&PublicKey="+other.pub, workerActor(instance), &call{})
|
||||
assertCode(t, err, awserrors.ErrorInvalidParameterValue)
|
||||
|
||||
// An instance that was never bound is refused.
|
||||
_, err = invoke(c.deps, "Action=ClaimFirstStore", workerActor(successor), &call{})
|
||||
assertCode(t, err, awserrors.ErrorAccessDenied)
|
||||
|
||||
if err := c.ids.RecordStopped(context.Background(), first, instance); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Stopped but not yet replaced: the old instance is no longer current.
|
||||
_, err = invoke(c.deps, "Action=ClaimFirstStore", workerActor(instance), &call{})
|
||||
assertCode(t, err, awserrors.ErrorAccessDenied)
|
||||
|
||||
c.replace(t, first, "claim-0002", successor)
|
||||
_, err = invoke(c.deps, "Action=ClaimFirstStore", workerActor(instance), &call{})
|
||||
assertCode(t, err, awserrors.ErrorAccessDenied)
|
||||
rec, err = invoke(c.deps, "Action=ClaimFirstStore", workerActor(successor), &call{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := rec.Body.String()
|
||||
if !strings.Contains(body, "<CreateKey>false</CreateKey>") || !strings.Contains(body, k.pub) {
|
||||
t.Fatalf("successor got %q, want only the recorded key", body)
|
||||
}
|
||||
}
|
||||
|
||||
// The race between authorization and the first-store call: a request that
|
||||
// passed Authorize just before a replacement still carries the old generation,
|
||||
// and once the successor has claimed, firststore refuses it.
|
||||
func TestRealStores_InFlightOldGenerationLosesToTheSuccessor(t *testing.T) {
|
||||
c := newCluster(t)
|
||||
first := c.launch(t, "claim-0001", 0, instance)
|
||||
stale, err := c.ids.Authorize(context.Background(), instance)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.ids.RecordStopped(context.Background(), first, instance); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.replace(t, first, "claim-0002", successor)
|
||||
if _, err := invoke(c.deps, "Action=ClaimFirstStore", workerActor(successor), &call{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
inFlight := Deps{Identities: &fakeIdentities{ident: stale}, FirstStore: c.deps.FirstStore, Stores: c.deps.Stores}
|
||||
_, err = invoke(inFlight, "Action=ClaimFirstStore", workerActor(instance), &call{})
|
||||
assertCode(t, err, awserrors.ErrorAccessDenied)
|
||||
_, err = invoke(inFlight, "Action=RecordWorkerPublicKey&PublicKey="+newUserKey(t).pub, workerActor(instance), &call{})
|
||||
assertCode(t, err, awserrors.ErrorAccessDenied)
|
||||
}
|
||||
|
||||
// Concurrent claims from the one current worker decide once.
|
||||
func TestRealStores_ConcurrentClaimsAgree(t *testing.T) {
|
||||
c := newCluster(t)
|
||||
c.launch(t, "claim-0001", 0, instance)
|
||||
var wg sync.WaitGroup
|
||||
bodies := make([]string, 8)
|
||||
errs := make([]error, 8)
|
||||
for i := range bodies {
|
||||
wg.Go(func() {
|
||||
rec, err := invoke(c.deps, "Action=ClaimFirstStore", workerActor(instance), &call{})
|
||||
bodies[i], errs[i] = rec.Body.String(), err
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
for i := range bodies {
|
||||
if errs[i] != nil && errs[i].Error() != awserrors.ErrorServiceUnavailable {
|
||||
t.Fatalf("claim %d: %v", i, errs[i])
|
||||
}
|
||||
if errs[i] == nil && !strings.Contains(bodies[i], "<CreateKey>true</CreateKey>") {
|
||||
t.Fatalf("claim %d: %q", i, bodies[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Absent buckets refuse the worker and stay absent: the gateway provisions
|
||||
// nothing.
|
||||
func TestRealStores_AbsentBucketsFailClosedAndStayAbsent(t *testing.T) {
|
||||
clustersize.DeclareForTest(t, 1)
|
||||
_, _, js := testutil.StartTestJetStream(t)
|
||||
deps := JetStreamDeps(js)
|
||||
_, err := invoke(deps, "Action=ClaimFirstStore", workerActor(instance), &call{})
|
||||
assertCode(t, err, awserrors.ErrorServiceUnavailable)
|
||||
for _, name := range []string{workerident.Bucket, firststore.Bucket} {
|
||||
if _, err := js.KeyValue(context.Background(), name); !errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||
t.Fatalf("the gateway provisioned %s: %v", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// pausingIdentities runs the real Authorize, then runs between before the
|
||||
// request reaches firststore, standing in for a request delayed in flight.
|
||||
type pausingIdentities struct {
|
||||
real Identities
|
||||
between func()
|
||||
}
|
||||
|
||||
func (p pausingIdentities) Authorize(ctx context.Context, instanceID string) (workerident.Identity, error) {
|
||||
id, err := p.real.Authorize(ctx, instanceID)
|
||||
if err == nil {
|
||||
p.between()
|
||||
}
|
||||
return id, err
|
||||
}
|
||||
|
||||
// The other order: the old request passes Authorize, the old worker is stopped
|
||||
// and its successor bound, and the old request reaches firststore before the
|
||||
// successor has claimed. The fence refuses it, so the successor still decides.
|
||||
func TestRealStores_InFlightOldGenerationCannotPreemptTheSuccessor(t *testing.T) {
|
||||
for _, action := range []string{"Action=ClaimFirstStore", "Action=RecordWorkerPublicKey&PublicKey="} {
|
||||
t.Run(strings.TrimSuffix(strings.TrimPrefix(action, "Action="), "&PublicKey="), func(t *testing.T) {
|
||||
c := newCluster(t)
|
||||
oldKey := newUserKey(t)
|
||||
first := c.launch(t, "claim-0001", 0, instance)
|
||||
if strings.Contains(action, "Record") {
|
||||
// The old worker has claimed, so only its key record is in flight.
|
||||
if _, err := invoke(c.deps, "Action=ClaimFirstStore", workerActor(instance), &call{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
action += oldKey.pub
|
||||
}
|
||||
replaced := false
|
||||
inFlight := Deps{
|
||||
Identities: pausingIdentities{real: c.deps.Identities, between: func() {
|
||||
if err := c.ids.RecordStopped(context.Background(), first, instance); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.replace(t, first, "claim-0002", successor)
|
||||
replaced = true
|
||||
}},
|
||||
FirstStore: c.deps.FirstStore,
|
||||
Stores: c.deps.Stores,
|
||||
}
|
||||
_, err := invoke(inFlight, action, workerActor(instance), &call{})
|
||||
if !replaced {
|
||||
t.Fatal("the replacement did not run inside the request")
|
||||
}
|
||||
assertCode(t, err, awserrors.ErrorAccessDenied)
|
||||
|
||||
rec, err := invoke(c.deps, "Action=ClaimFirstStore", workerActor(successor), &call{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := rec.Body.String()
|
||||
if !strings.Contains(body, "<CreateKey>true</CreateKey>") || strings.Contains(body, oldKey.pub) {
|
||||
t.Fatalf("successor got %q, want to create its own key", body)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A caller-chosen parameter name never reaches the log.
|
||||
func TestUnexpectedParameterNameIsNotLogged(t *testing.T) {
|
||||
logs := captureLogs(t)
|
||||
_, _, deps := fakes()
|
||||
secret := newUserKey(t).seed
|
||||
_, err := invoke(deps, "Action=ClaimFirstStore&"+secret+"=1", workerActor(instance), &call{})
|
||||
assertCode(t, err, awserrors.ErrorInvalidParameterValue)
|
||||
if !strings.Contains(logs.String(), "unexpected parameter refused") {
|
||||
t.Fatalf("refusal not logged: %q", logs.String())
|
||||
}
|
||||
if strings.Contains(logs.String(), secret) || strings.Contains(logs.String(), secret[2:12]) {
|
||||
t.Fatal("parameter name reached the log")
|
||||
}
|
||||
}
|
||||
|
||||
// DescribeStore reports only the current worker's own Region, says pending
|
||||
// until the store is bound, and never echoes anything but the public key.
|
||||
func TestDescribeStore(t *testing.T) {
|
||||
k := newUserKey(t)
|
||||
bound := storelaunch.Record{Region: region, State: storelaunch.StateBound, PublicKey: k.pub,
|
||||
InstanceID: successor, Address: "10.240.0.9"}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
stores *fakeStores
|
||||
want []string
|
||||
wantErr string
|
||||
}{
|
||||
{"no record", &fakeStores{err: jetstream.ErrKeyNotFound}, []string{"<State>pending</State>"}, ""},
|
||||
{"launching", &fakeStores{rec: storelaunch.Record{Region: region, State: storelaunch.StateLaunching}},
|
||||
[]string{"<State>pending</State>"}, ""},
|
||||
{"bound", &fakeStores{rec: bound},
|
||||
[]string{"<State>bound</State>", "<Address>10.240.0.9</Address>", "<PublicKey>" + k.pub + "</PublicKey>"}, ""},
|
||||
{"leader unavailable", &fakeStores{err: errors.New("leader lost")}, nil, awserrors.ErrorServiceUnavailable},
|
||||
{"corrupt record", &fakeStores{err: storelaunch.ErrUnreadable}, nil, awserrors.ErrorServerInternal},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
ids, _, deps := fakes()
|
||||
deps.Stores = tc.stores
|
||||
rec, err := invoke(deps, "Action=DescribeStore", workerActor(instance), &call{})
|
||||
if tc.wantErr != "" {
|
||||
assertCode(t, err, tc.wantErr)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, w := range tc.want {
|
||||
if !strings.Contains(rec.Body.String(), w) {
|
||||
t.Fatalf("body %q lacks %q", rec.Body.String(), w)
|
||||
}
|
||||
}
|
||||
if tc.name != "bound" && strings.Contains(rec.Body.String(), "<Address>") {
|
||||
t.Fatalf("an unbound store reported an address: %q", rec.Body.String())
|
||||
}
|
||||
if len(tc.stores.regions) != 1 || tc.stores.regions[0] != ids.ident.Region {
|
||||
t.Fatalf("read Regions %v, want only the bound worker's", tc.stores.regions)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A caller that is not the current worker never reads the store record.
|
||||
func TestDescribeStore_RefusesNonCurrentWorkers(t *testing.T) {
|
||||
ids, _, deps := fakes()
|
||||
ids.err = workerident.ErrNotCurrent
|
||||
stores := &fakeStores{}
|
||||
deps.Stores = stores
|
||||
_, err := invoke(deps, "Action=DescribeStore", workerActor(instance), &call{})
|
||||
assertCode(t, err, awserrors.ErrorAccessDenied)
|
||||
_, err = invoke(deps, "Action=DescribeStore&Region=ap-southeast-2", workerActor(instance), &call{})
|
||||
if err == nil {
|
||||
t.Fatal("an identity parameter was accepted")
|
||||
}
|
||||
if len(stores.regions) != 0 {
|
||||
t.Fatalf("store record read for a refused caller: %v", stores.regions)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,390 @@
|
||||
// Package firststore is the SQS domain's durable decision on whether the shared
|
||||
// SQS worker may create the store's client key. One record per Region moves
|
||||
// claimed -> key-recorded -> launch-intent under compare-and-swap, behind a
|
||||
// generation fence the launch path raises before it replaces a worker.
|
||||
package firststore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
|
||||
sqsjs "github.com/mulgadc/spinifex/spinifex/domains/sqs/message/jetstream"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/nats-io/nkeys"
|
||||
)
|
||||
|
||||
// Bucket is the SQS domain's KV bucket for first-store records.
|
||||
const Bucket = "spinifex-sqs-first-store"
|
||||
|
||||
// BucketConfig is the bucket's configuration for test fixtures; production
|
||||
// opens it through Open, which takes the replica count from the cluster.
|
||||
func BucketConfig(replicas int) jetstream.KeyValueConfig {
|
||||
return jetstream.KeyValueConfig{
|
||||
Bucket: Bucket,
|
||||
History: 1,
|
||||
Storage: jetstream.FileStorage,
|
||||
Replicas: replicas,
|
||||
}
|
||||
}
|
||||
|
||||
type State string
|
||||
|
||||
const (
|
||||
// Only a fence is recorded: no worker has claimed since it was raised.
|
||||
StateOpen State = "open"
|
||||
// A worker identity holds the claim; no key is recorded, so no store can
|
||||
// have been launched with any key.
|
||||
StateClaimed State = "claimed"
|
||||
// The worker's public key is recorded; from here a missing or different
|
||||
// worker key fails closed.
|
||||
StateKeyRecorded State = "key-recorded"
|
||||
// A store launch may use the recorded key.
|
||||
StateLaunchIntent State = "launch-intent"
|
||||
)
|
||||
|
||||
// Identity is one worker VM incarnation. Generation increases each time the
|
||||
// worker is relaunched. A caller must derive it from authoritative instance
|
||||
// state, never from the worker's request: a newer generation may take over.
|
||||
type Identity struct {
|
||||
InstanceID string `json:"instanceID"`
|
||||
ClaimID string `json:"claimID"`
|
||||
Generation uint64 `json:"generation"`
|
||||
}
|
||||
|
||||
// Record is the stored decision. It never holds private key material.
|
||||
type Record struct {
|
||||
Region string `json:"region"`
|
||||
State State `json:"state"`
|
||||
Worker Identity `json:"worker"`
|
||||
PublicKey string `json:"publicKey,omitempty"`
|
||||
// Fence is the lowest generation still accepted. It only rises, so a
|
||||
// request from a replaced worker that was already in flight is refused.
|
||||
Fence uint64 `json:"fence,omitempty"`
|
||||
}
|
||||
|
||||
// Decision is what a claim authorizes: create a key, or use exactly the
|
||||
// recorded one.
|
||||
type Decision struct {
|
||||
CreateKey bool
|
||||
PublicKey string
|
||||
}
|
||||
|
||||
var (
|
||||
ErrClaimHeld = errors.New("first-store claim is held by another worker identity")
|
||||
ErrStaleGeneration = errors.New("worker generation is older than the recorded claim")
|
||||
ErrNotClaimed = errors.New("no first-store claim is recorded")
|
||||
ErrKeyNotRecorded = errors.New("worker public key is not recorded")
|
||||
ErrKeyMismatch = errors.New("worker public key does not match the recorded key")
|
||||
ErrUnreadable = errors.New("first-store record is unreadable")
|
||||
ErrContended = errors.New("first-store record changed concurrently; retry")
|
||||
)
|
||||
|
||||
const maxAttempts = 8
|
||||
|
||||
var (
|
||||
regionPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,62}$`)
|
||||
instancePattern = regexp.MustCompile(`^i-[0-9a-f]{8,17}$`)
|
||||
claimPattern = regexp.MustCompile(`^[A-Za-z0-9-]{8,64}$`)
|
||||
)
|
||||
|
||||
// Store is the first-store state machine over the domain's KV bucket. Reads
|
||||
// go to the stream leader, and every authorizing answer is also confirmed by a
|
||||
// compare-and-swap at the revision read, so a stale read authorizes nothing.
|
||||
type Store struct {
|
||||
JS jetstream.JetStream
|
||||
KV jetstream.KeyValue
|
||||
}
|
||||
|
||||
// Open opens or creates the bucket and refuses one that could lose records.
|
||||
func Open(ctx context.Context, js jetstream.JetStream) (Store, error) {
|
||||
kv, err := sqsjs.OpenDurableKV(ctx, js, Bucket, "SQS first-store decisions: who may create the store key per Region")
|
||||
if err != nil {
|
||||
return Store{}, err
|
||||
}
|
||||
return Store{JS: js, KV: kv}, nil
|
||||
}
|
||||
|
||||
// Bind binds the existing bucket without creating it, refusing one that could
|
||||
// lose records; a reader that must not provision state uses it.
|
||||
func Bind(ctx context.Context, js jetstream.JetStream) (Store, error) {
|
||||
kv, err := sqsjs.BindDurableKV(ctx, js, Bucket)
|
||||
if err != nil {
|
||||
return Store{}, err
|
||||
}
|
||||
return Store{JS: js, KV: kv}, nil
|
||||
}
|
||||
|
||||
func key(region string) string { return "region." + region }
|
||||
|
||||
func validate(region string, id Identity) error {
|
||||
if !regionPattern.MatchString(region) {
|
||||
return errors.New("invalid region")
|
||||
}
|
||||
if !instancePattern.MatchString(id.InstanceID) {
|
||||
return errors.New("invalid worker instance ID")
|
||||
}
|
||||
if !claimPattern.MatchString(id.ClaimID) {
|
||||
return errors.New("invalid claim ID")
|
||||
}
|
||||
if id.Generation == 0 {
|
||||
return errors.New("worker generation must be positive")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Claim decides for this worker identity. Absent a record it creates a claim
|
||||
// and authorizes key creation. A claim with no recorded key may pass to a newer
|
||||
// generation; once a key is recorded only that key is ever authorized.
|
||||
func (s Store) Claim(ctx context.Context, region string, id Identity) (Decision, error) {
|
||||
if err := validate(region, id); err != nil {
|
||||
return Decision{}, err
|
||||
}
|
||||
for range maxAttempts {
|
||||
rec, rev, err := s.read(ctx, region)
|
||||
if errors.Is(err, ErrNotClaimed) {
|
||||
// A failed create may still have applied; the re-read decides.
|
||||
if s.create(ctx, region, Record{Region: region, State: StateClaimed, Worker: id}) == nil {
|
||||
return Decision{CreateKey: true}, nil
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return Decision{}, err
|
||||
}
|
||||
if id.Generation < rec.Fence {
|
||||
return Decision{}, ErrStaleGeneration
|
||||
}
|
||||
if rec.State == StateOpen {
|
||||
next := *rec
|
||||
next.State, next.Worker = StateClaimed, id
|
||||
if s.update(ctx, region, next, rev) == nil {
|
||||
return Decision{CreateKey: true}, nil
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err := checkGeneration(rec.Worker, id); err != nil {
|
||||
return Decision{}, err
|
||||
}
|
||||
// Rebinds a strictly newer generation, keeping state and key; for the
|
||||
// same identity it rewrites the record to prove the read was current.
|
||||
next := *rec
|
||||
next.Worker = id
|
||||
if s.update(ctx, region, next, rev) != nil {
|
||||
continue
|
||||
}
|
||||
if rec.State == StateClaimed {
|
||||
return Decision{CreateKey: true}, nil
|
||||
}
|
||||
return Decision{PublicKey: rec.PublicKey}, nil
|
||||
}
|
||||
return Decision{}, ErrContended
|
||||
}
|
||||
|
||||
// RecordPublicKey binds the worker's public key to its current claim. It is
|
||||
// idempotent for the same key, so a lost reply can be retried.
|
||||
func (s Store) RecordPublicKey(ctx context.Context, region string, id Identity, pub string) error {
|
||||
return s.advance(ctx, region, id, pub, func(rec *Record) (*Record, error) {
|
||||
switch rec.State {
|
||||
case StateClaimed:
|
||||
next := *rec
|
||||
next.State, next.PublicKey = StateKeyRecorded, pub
|
||||
return &next, nil
|
||||
default:
|
||||
return nil, nil
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// RecordLaunchIntent permits a store launch with the recorded key. It refuses
|
||||
// until the worker's public key is recorded.
|
||||
func (s Store) RecordLaunchIntent(ctx context.Context, region string, id Identity, pub string) error {
|
||||
return s.advance(ctx, region, id, pub, func(rec *Record) (*Record, error) {
|
||||
switch rec.State {
|
||||
case StateClaimed:
|
||||
return nil, ErrKeyNotRecorded
|
||||
case StateKeyRecorded:
|
||||
next := *rec
|
||||
next.State = StateLaunchIntent
|
||||
return &next, nil
|
||||
default:
|
||||
return nil, nil
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// RecordedKey is the Region's recorded worker public key and the identity
|
||||
// that recorded it, read from the stream leader. It authorizes nothing by
|
||||
// itself: RecordLaunchIntent confirms the same key under compare-and-swap.
|
||||
func (s Store) RecordedKey(ctx context.Context, region string) (Identity, string, error) {
|
||||
if !regionPattern.MatchString(region) {
|
||||
return Identity{}, "", errors.New("invalid region")
|
||||
}
|
||||
rec, _, err := s.read(ctx, region)
|
||||
if err != nil {
|
||||
return Identity{}, "", err
|
||||
}
|
||||
if rec.State != StateKeyRecorded && rec.State != StateLaunchIntent {
|
||||
return Identity{}, "", ErrKeyNotRecorded
|
||||
}
|
||||
return rec.Worker, rec.PublicKey, nil
|
||||
}
|
||||
|
||||
// advance applies step to the current record of the exact claiming identity.
|
||||
// A nil next record means the transition has already happened.
|
||||
func (s Store) advance(ctx context.Context, region string, id Identity, pub string, step func(*Record) (*Record, error)) error {
|
||||
if err := validate(region, id); err != nil {
|
||||
return err
|
||||
}
|
||||
if !nkeys.IsValidPublicUserKey(pub) {
|
||||
return errors.New("worker key is not a user public NKey")
|
||||
}
|
||||
for range maxAttempts {
|
||||
rec, rev, err := s.read(ctx, region)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if id.Generation < rec.Fence {
|
||||
return ErrStaleGeneration
|
||||
}
|
||||
if rec.State == StateOpen {
|
||||
return ErrNotClaimed
|
||||
}
|
||||
if rec.Worker != id {
|
||||
if err := checkGeneration(rec.Worker, id); err != nil {
|
||||
return err
|
||||
}
|
||||
// A newer generation must claim before it records.
|
||||
return ErrClaimHeld
|
||||
}
|
||||
if rec.State != StateClaimed && rec.PublicKey != pub {
|
||||
return ErrKeyMismatch
|
||||
}
|
||||
next, err := step(rec)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if next == nil {
|
||||
// Already applied: rewrite unchanged to prove the read was current.
|
||||
next = rec
|
||||
}
|
||||
if s.update(ctx, region, *next, rev) == nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return ErrContended
|
||||
}
|
||||
|
||||
// Fence raises the lowest accepted generation to gen and never lowers it. The
|
||||
// launch path calls it before it replaces a worker, so a request from the old
|
||||
// generation that is still in flight decides nothing once the fence is up.
|
||||
func (s Store) Fence(ctx context.Context, region string, gen uint64) error {
|
||||
if !regionPattern.MatchString(region) || gen == 0 {
|
||||
return errors.New("invalid fence")
|
||||
}
|
||||
for range maxAttempts {
|
||||
rec, rev, err := s.read(ctx, region)
|
||||
if errors.Is(err, ErrNotClaimed) {
|
||||
if s.create(ctx, region, Record{Region: region, State: StateOpen, Fence: gen}) == nil {
|
||||
return nil
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
next := *rec
|
||||
next.Fence = max(rec.Fence, gen)
|
||||
// Unchanged when already raised: the rewrite proves the read was current.
|
||||
if s.update(ctx, region, next, rev) == nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return ErrContended
|
||||
}
|
||||
|
||||
// FencedAt is the Region's current fence, zero when none is recorded. The
|
||||
// fence only rises, so a value read from the leader stays a lower bound.
|
||||
func (s Store) FencedAt(ctx context.Context, region string) (uint64, error) {
|
||||
rec, _, err := s.read(ctx, region)
|
||||
if errors.Is(err, ErrNotClaimed) {
|
||||
return 0, nil
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return rec.Fence, nil
|
||||
}
|
||||
|
||||
// checkGeneration refuses an older generation, and a different identity at
|
||||
// the same generation; only a strictly newer generation may supersede.
|
||||
func checkGeneration(held, id Identity) error {
|
||||
switch {
|
||||
case held == id:
|
||||
return nil
|
||||
case id.Generation < held.Generation:
|
||||
return ErrStaleGeneration
|
||||
case id.Generation == held.Generation:
|
||||
return ErrClaimHeld
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (s Store) read(ctx context.Context, region string) (*Record, uint64, error) {
|
||||
value, rev, err := sqsjs.LeaderKVGet(ctx, s.JS, Bucket, key(region))
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil, 0, ErrNotClaimed
|
||||
}
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("read first-store record: %w", err)
|
||||
}
|
||||
var rec Record
|
||||
if err := json.Unmarshal(value, &rec); err != nil {
|
||||
return nil, 0, ErrUnreadable
|
||||
}
|
||||
if !wellFormed(&rec, region) {
|
||||
return nil, 0, ErrUnreadable
|
||||
}
|
||||
return &rec, rev, nil
|
||||
}
|
||||
|
||||
func wellFormed(rec *Record, region string) bool {
|
||||
if rec.Region != region {
|
||||
return false
|
||||
}
|
||||
if rec.State == StateOpen {
|
||||
return rec.Worker == Identity{} && rec.PublicKey == "" && rec.Fence > 0
|
||||
}
|
||||
if validate(region, rec.Worker) != nil {
|
||||
return false
|
||||
}
|
||||
switch rec.State {
|
||||
case StateClaimed:
|
||||
return rec.PublicKey == ""
|
||||
case StateKeyRecorded, StateLaunchIntent:
|
||||
return nkeys.IsValidPublicUserKey(rec.PublicKey)
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (s Store) create(ctx context.Context, region string, rec Record) error {
|
||||
b, err := json.Marshal(rec)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = s.KV.Create(ctx, key(region), b)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s Store) update(ctx context.Context, region string, rec Record, rev uint64) error {
|
||||
b, err := json.Marshal(rec)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = s.KV.Update(ctx, key(region), b, rev)
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,602 @@
|
||||
package firststore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
testutil "github.com/mulgadc/spinifex/internal/testkit"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/nats-io/nkeys"
|
||||
)
|
||||
|
||||
const region = "ap-southeast-2"
|
||||
|
||||
func newStore(t *testing.T) (Store, jetstream.KeyValue) {
|
||||
t.Helper()
|
||||
_, _, js := testutil.StartTestJetStream(t)
|
||||
kv, err := js.CreateKeyValue(context.Background(), BucketConfig(1))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return Store{JS: js, KV: kv}, kv
|
||||
}
|
||||
|
||||
func ident(n int, gen uint64) Identity {
|
||||
return Identity{InstanceID: fmt.Sprintf("i-%017x", n), ClaimID: fmt.Sprintf("claim-%04d-%d", n, gen), Generation: gen}
|
||||
}
|
||||
|
||||
func userPub(t *testing.T) (pub, seed string) {
|
||||
t.Helper()
|
||||
kp, err := nkeys.CreateUser()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p, _ := kp.PublicKey()
|
||||
s, _ := kp.Seed()
|
||||
return p, string(s)
|
||||
}
|
||||
|
||||
func stored(t *testing.T, kv jetstream.KeyValue) string {
|
||||
t.Helper()
|
||||
e, err := kv.Get(context.Background(), key(region))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return string(e.Value())
|
||||
}
|
||||
|
||||
func TestClaim_FirstClaimCreatesKeyThenOnlyTheRecordedKey(t *testing.T) {
|
||||
s, _ := newStore(t)
|
||||
ctx := context.Background()
|
||||
w := ident(1, 1)
|
||||
d, err := s.Claim(ctx, region, w)
|
||||
if err != nil || !d.CreateKey {
|
||||
t.Fatalf("first claim = %+v, %v; want CreateKey", d, err)
|
||||
}
|
||||
pub, _ := userPub(t)
|
||||
if err := s.RecordPublicKey(ctx, region, w, pub); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d, err = s.Claim(ctx, region, w)
|
||||
if err != nil || d.CreateKey || d.PublicKey != pub {
|
||||
t.Fatalf("after record = %+v, %v; want only the recorded key", d, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClaim_ConcurrentClaimsHaveOneWinner(t *testing.T) {
|
||||
s, _ := newStore(t)
|
||||
const n = 16
|
||||
var created, held atomic.Int32
|
||||
var wg sync.WaitGroup
|
||||
for i := range n {
|
||||
wg.Go(func() {
|
||||
d, err := s.Claim(context.Background(), region, ident(i+1, 1))
|
||||
switch {
|
||||
case err == nil && d.CreateKey:
|
||||
created.Add(1)
|
||||
case errors.Is(err, ErrClaimHeld):
|
||||
held.Add(1)
|
||||
default:
|
||||
t.Errorf("claim %d = %+v, %v", i, d, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
if created.Load() != 1 || held.Load() != n-1 {
|
||||
t.Fatalf("created=%d held=%d, want exactly one winner", created.Load(), held.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestClaim_GenerationRules(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
t.Run("newer generation takes over a claim with no key", func(t *testing.T) {
|
||||
s, _ := newStore(t)
|
||||
old, next := ident(1, 1), ident(2, 2)
|
||||
if _, err := s.Claim(ctx, region, old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d, err := s.Claim(ctx, region, next)
|
||||
if err != nil || !d.CreateKey {
|
||||
t.Fatalf("takeover = %+v, %v", d, err)
|
||||
}
|
||||
pub, _ := userPub(t)
|
||||
if err := s.RecordPublicKey(ctx, region, old, pub); !errors.Is(err, ErrStaleGeneration) {
|
||||
t.Fatalf("superseded record = %v, want ErrStaleGeneration", err)
|
||||
}
|
||||
if _, err := s.Claim(ctx, region, old); !errors.Is(err, ErrStaleGeneration) {
|
||||
t.Fatalf("superseded claim = %v, want ErrStaleGeneration", err)
|
||||
}
|
||||
})
|
||||
t.Run("newer generation after recording gets only the recorded key", func(t *testing.T) {
|
||||
s, _ := newStore(t)
|
||||
old, next := ident(1, 1), ident(2, 2)
|
||||
s.Claim(ctx, region, old)
|
||||
pub, _ := userPub(t)
|
||||
if err := s.RecordPublicKey(ctx, region, old, pub); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d, err := s.Claim(ctx, region, next)
|
||||
if err != nil || d.CreateKey || d.PublicKey != pub {
|
||||
t.Fatalf("relaunched worker = %+v, %v; want the recorded key", d, err)
|
||||
}
|
||||
other, _ := userPub(t)
|
||||
if err := s.RecordPublicKey(ctx, region, next, other); !errors.Is(err, ErrKeyMismatch) {
|
||||
t.Fatalf("new key after recording = %v, want ErrKeyMismatch", err)
|
||||
}
|
||||
if _, err := s.Claim(ctx, region, old); !errors.Is(err, ErrStaleGeneration) {
|
||||
t.Fatalf("replayed old boot = %v, want ErrStaleGeneration", err)
|
||||
}
|
||||
})
|
||||
t.Run("different identity at the same generation is refused", func(t *testing.T) {
|
||||
s, _ := newStore(t)
|
||||
s.Claim(ctx, region, ident(1, 3))
|
||||
rival := ident(2, 3)
|
||||
if _, err := s.Claim(ctx, region, rival); !errors.Is(err, ErrClaimHeld) {
|
||||
t.Fatalf("rival claim = %v, want ErrClaimHeld", err)
|
||||
}
|
||||
pub, _ := userPub(t)
|
||||
if err := s.RecordPublicKey(ctx, region, rival, pub); !errors.Is(err, ErrClaimHeld) {
|
||||
t.Fatalf("rival record = %v, want ErrClaimHeld", err)
|
||||
}
|
||||
sameInstance := ident(1, 3)
|
||||
sameInstance.ClaimID = "claim-replayed-x"
|
||||
if _, err := s.Claim(ctx, region, sameInstance); !errors.Is(err, ErrClaimHeld) {
|
||||
t.Fatalf("replayed claim ID = %v, want ErrClaimHeld", err)
|
||||
}
|
||||
})
|
||||
t.Run("newer generation must claim before recording", func(t *testing.T) {
|
||||
s, _ := newStore(t)
|
||||
s.Claim(ctx, region, ident(1, 1))
|
||||
pub, _ := userPub(t)
|
||||
if err := s.RecordPublicKey(ctx, region, ident(2, 2), pub); !errors.Is(err, ErrClaimHeld) {
|
||||
t.Fatalf("unclaimed newer record = %v, want ErrClaimHeld", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestRecordPublicKey_IdempotentAndFailClosed(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s, kv := newStore(t)
|
||||
w := ident(1, 1)
|
||||
pub, seed := userPub(t)
|
||||
if err := s.RecordPublicKey(ctx, region, w, pub); !errors.Is(err, ErrNotClaimed) {
|
||||
t.Fatalf("record without claim = %v, want ErrNotClaimed", err)
|
||||
}
|
||||
s.Claim(ctx, region, w)
|
||||
err := s.RecordPublicKey(ctx, region, w, seed)
|
||||
if err == nil || strings.Contains(err.Error(), seed) {
|
||||
t.Fatalf("seed as public key = %v; want a refusal without the seed", err)
|
||||
}
|
||||
if err := s.RecordPublicKey(ctx, region, w, pub); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.RecordPublicKey(ctx, region, w, pub); err != nil {
|
||||
t.Fatalf("retried record = %v, want idempotent success", err)
|
||||
}
|
||||
other, _ := userPub(t)
|
||||
if err := s.RecordPublicKey(ctx, region, w, other); !errors.Is(err, ErrKeyMismatch) {
|
||||
t.Fatalf("second key = %v, want ErrKeyMismatch", err)
|
||||
}
|
||||
if strings.Contains(stored(t, kv), seed) {
|
||||
t.Fatal("the record holds a seed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecordLaunchIntent_NeedsRecordedKey(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s, _ := newStore(t)
|
||||
w := ident(1, 1)
|
||||
pub, _ := userPub(t)
|
||||
if err := s.RecordLaunchIntent(ctx, region, w, pub); !errors.Is(err, ErrNotClaimed) {
|
||||
t.Fatalf("launch without claim = %v", err)
|
||||
}
|
||||
s.Claim(ctx, region, w)
|
||||
if err := s.RecordLaunchIntent(ctx, region, w, pub); !errors.Is(err, ErrKeyNotRecorded) {
|
||||
t.Fatalf("launch before key = %v, want ErrKeyNotRecorded", err)
|
||||
}
|
||||
s.RecordPublicKey(ctx, region, w, pub)
|
||||
other, _ := userPub(t)
|
||||
if err := s.RecordLaunchIntent(ctx, region, w, other); !errors.Is(err, ErrKeyMismatch) {
|
||||
t.Fatalf("launch with another key = %v, want ErrKeyMismatch", err)
|
||||
}
|
||||
for range 2 {
|
||||
if err := s.RecordLaunchIntent(ctx, region, w, pub); err != nil {
|
||||
t.Fatalf("launch intent = %v", err)
|
||||
}
|
||||
}
|
||||
d, err := s.Claim(ctx, region, w)
|
||||
if err != nil || d.CreateKey || d.PublicKey != pub {
|
||||
t.Fatalf("claim after launch intent = %+v, %v", d, err)
|
||||
}
|
||||
if err := s.RecordPublicKey(ctx, region, w, other); !errors.Is(err, ErrKeyMismatch) {
|
||||
t.Fatalf("new key after launch intent = %v, want ErrKeyMismatch", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClaim_UnreadableRecordAuthorizesNothing(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
pub, _ := userPub(t)
|
||||
for name, raw := range map[string]string{
|
||||
"not json": "{",
|
||||
"unknown state": `{"region":"ap-southeast-2","state":"ready","worker":{"instanceID":"i-00000000000000001","claimID":"claim-0001-1","generation":1}}`,
|
||||
"keyed without key": `{"region":"ap-southeast-2","state":"key-recorded","worker":{"instanceID":"i-00000000000000001","claimID":"claim-0001-1","generation":1}}`,
|
||||
"claimed with a key": `{"region":"ap-southeast-2","state":"claimed","worker":{"instanceID":"i-00000000000000001","claimID":"claim-0001-1","generation":1},"publicKey":"` + pub + `"}`,
|
||||
"other region": `{"region":"us-east-1","state":"claimed","worker":{"instanceID":"i-00000000000000001","claimID":"claim-0001-1","generation":1}}`,
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
s, kv := newStore(t)
|
||||
if _, err := kv.Put(ctx, key(region), []byte(raw)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d, err := s.Claim(ctx, region, ident(1, 1))
|
||||
if !errors.Is(err, ErrUnreadable) || d.CreateKey {
|
||||
t.Fatalf("claim = %+v, %v; want ErrUnreadable", d, err)
|
||||
}
|
||||
if err := s.RecordPublicKey(ctx, region, ident(1, 1), pub); !errors.Is(err, ErrUnreadable) {
|
||||
t.Fatalf("record = %v, want ErrUnreadable", err)
|
||||
}
|
||||
if stored(t, kv) != raw {
|
||||
t.Fatal("an unreadable record was overwritten")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// lossyKV applies or drops a write and then reports failure, as a write whose
|
||||
// reply was lost or that never reached the server.
|
||||
type lossyKV struct {
|
||||
jetstream.KeyValue
|
||||
|
||||
failCreate, failUpdate int
|
||||
apply bool
|
||||
beforeCreate func()
|
||||
}
|
||||
|
||||
func (k *lossyKV) Create(ctx context.Context, key string, value []byte, opts ...jetstream.KVCreateOpt) (uint64, error) {
|
||||
if k.beforeCreate != nil {
|
||||
k.beforeCreate()
|
||||
k.beforeCreate = nil
|
||||
}
|
||||
if k.failCreate > 0 {
|
||||
k.failCreate--
|
||||
if k.apply {
|
||||
k.KeyValue.Create(ctx, key, value, opts...)
|
||||
}
|
||||
return 0, errors.New("nats: timeout")
|
||||
}
|
||||
return k.KeyValue.Create(ctx, key, value, opts...)
|
||||
}
|
||||
|
||||
func (k *lossyKV) Update(ctx context.Context, key string, value []byte, rev uint64) (uint64, error) {
|
||||
if k.failUpdate > 0 {
|
||||
k.failUpdate--
|
||||
if k.apply {
|
||||
k.KeyValue.Update(ctx, key, value, rev)
|
||||
}
|
||||
return 0, errors.New("nats: timeout")
|
||||
}
|
||||
return k.KeyValue.Update(ctx, key, value, rev)
|
||||
}
|
||||
|
||||
func TestAmbiguousWrites(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
t.Run("applied claim with a lost reply is recovered", func(t *testing.T) {
|
||||
base, kv := newStore(t)
|
||||
s := Store{JS: base.JS, KV: &lossyKV{KeyValue: kv, failCreate: 1, apply: true}}
|
||||
d, err := s.Claim(ctx, region, ident(1, 1))
|
||||
if err != nil || !d.CreateKey {
|
||||
t.Fatalf("claim = %+v, %v", d, err)
|
||||
}
|
||||
})
|
||||
t.Run("dropped claim that a rival won is refused", func(t *testing.T) {
|
||||
base, kv := newStore(t)
|
||||
lossy := &lossyKV{KeyValue: kv, failCreate: 1}
|
||||
lossy.beforeCreate = func() { base.Claim(ctx, region, ident(2, 1)) }
|
||||
if _, err := (Store{JS: base.JS, KV: lossy}).Claim(ctx, region, ident(1, 1)); !errors.Is(err, ErrClaimHeld) {
|
||||
t.Fatalf("claim = %v, want ErrClaimHeld", err)
|
||||
}
|
||||
})
|
||||
t.Run("applied key record with a lost reply is confirmed on retry", func(t *testing.T) {
|
||||
base, kv := newStore(t)
|
||||
w := ident(1, 1)
|
||||
base.Claim(ctx, region, w)
|
||||
pub, _ := userPub(t)
|
||||
lossy := &lossyKV{KeyValue: kv, failUpdate: 1, apply: true}
|
||||
s := Store{JS: base.JS, KV: lossy}
|
||||
// The update applies but its reply is lost; the re-read sees it recorded.
|
||||
if err := s.RecordPublicKey(ctx, region, w, pub); err != nil {
|
||||
t.Fatalf("record = %v, want recovered success", err)
|
||||
}
|
||||
d, err := s.Claim(ctx, region, w)
|
||||
if err != nil || d.CreateKey || d.PublicKey != pub {
|
||||
t.Fatalf("claim after lost reply = %+v, %v", d, err)
|
||||
}
|
||||
})
|
||||
t.Run("persistently dropped writes authorize nothing", func(t *testing.T) {
|
||||
base, kv := newStore(t)
|
||||
s := Store{JS: base.JS, KV: &lossyKV{KeyValue: kv, failCreate: maxAttempts}}
|
||||
d, err := s.Claim(ctx, region, ident(1, 1))
|
||||
if !errors.Is(err, ErrContended) || d.CreateKey {
|
||||
t.Fatalf("claim = %+v, %v; want ErrContended", d, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestValidation(t *testing.T) {
|
||||
s, _ := newStore(t)
|
||||
ctx := context.Background()
|
||||
bad := []Identity{
|
||||
{InstanceID: "", ClaimID: "claim-0001", Generation: 1},
|
||||
{InstanceID: "i-00000000000000001", ClaimID: "", Generation: 1},
|
||||
{InstanceID: "i-00000000000000001", ClaimID: "claim-0001", Generation: 0},
|
||||
{InstanceID: "i-00000000000000001.x", ClaimID: "claim-0001", Generation: 1},
|
||||
}
|
||||
for _, id := range bad {
|
||||
if _, err := s.Claim(ctx, region, id); err == nil {
|
||||
t.Errorf("claim accepted %+v", id)
|
||||
}
|
||||
}
|
||||
if _, err := s.Claim(ctx, "Bad.Region", ident(1, 1)); err == nil {
|
||||
t.Error("claim accepted an invalid region")
|
||||
}
|
||||
}
|
||||
|
||||
// laggingReplica serves reads from a second server whose bucket holds an
|
||||
// older history of the record, so its revisions match the leader's past ones,
|
||||
// while writes still go to the real leader.
|
||||
func laggingReplica(t *testing.T, leader Store, history ...string) Store {
|
||||
t.Helper()
|
||||
_, _, js := testutil.StartTestJetStream(t)
|
||||
kv, err := js.CreateKeyValue(context.Background(), BucketConfig(1))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, v := range history {
|
||||
if _, err := kv.Put(context.Background(), key(region), []byte(v)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return Store{JS: js, KV: leader.KV}
|
||||
}
|
||||
|
||||
func TestStaleRead_RecordedKeyNeverReauthorizesCreation(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s, kv := newStore(t)
|
||||
w := ident(1, 1)
|
||||
s.Claim(ctx, region, w)
|
||||
claimed := stored(t, kv)
|
||||
pub, _ := userPub(t)
|
||||
if err := s.RecordPublicKey(ctx, region, w, pub); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The replica has applied the claim but not the key record.
|
||||
stale := laggingReplica(t, s, claimed)
|
||||
d, err := stale.Claim(ctx, region, w)
|
||||
if d.CreateKey || !errors.Is(err, ErrContended) {
|
||||
t.Fatalf("stale claim = %+v, %v; want no authorization", d, err)
|
||||
}
|
||||
if err := stale.RecordLaunchIntent(ctx, region, w, pub); !errors.Is(err, ErrKeyNotRecorded) {
|
||||
t.Fatalf("stale launch intent = %v, want a refusal", err)
|
||||
}
|
||||
if d, err := s.Claim(ctx, region, w); err != nil || d.CreateKey || d.PublicKey != pub {
|
||||
t.Fatalf("leader claim = %+v, %v; want the recorded key", d, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStaleRead_OldGenerationIsNotRevivedAfterRebind(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
for _, keyed := range []bool{false, true} {
|
||||
t.Run(map[bool]string{false: "claimed", true: "key recorded"}[keyed], func(t *testing.T) {
|
||||
s, kv := newStore(t)
|
||||
old, next := ident(1, 1), ident(2, 2)
|
||||
s.Claim(ctx, region, old)
|
||||
history := []string{stored(t, kv)}
|
||||
pub, _ := userPub(t)
|
||||
if keyed {
|
||||
s.RecordPublicKey(ctx, region, old, pub)
|
||||
history = append(history, stored(t, kv))
|
||||
}
|
||||
if _, err := s.Claim(ctx, region, next); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The replica still shows the old generation as the holder.
|
||||
stale := laggingReplica(t, s, history...)
|
||||
if d, err := stale.Claim(ctx, region, old); err == nil || d.CreateKey || d.PublicKey != "" {
|
||||
t.Fatalf("stale old-generation claim = %+v, %v; want refusal", d, err)
|
||||
}
|
||||
if err := stale.RecordPublicKey(ctx, region, old, pub); err == nil {
|
||||
t.Fatal("stale old-generation record was accepted")
|
||||
}
|
||||
if err := stale.RecordLaunchIntent(ctx, region, old, pub); err == nil {
|
||||
t.Fatal("stale old-generation launch intent was accepted")
|
||||
}
|
||||
var rec Record
|
||||
json.Unmarshal([]byte(stored(t, kv)), &rec)
|
||||
if rec.Worker != next {
|
||||
t.Fatalf("leader record holder = %+v, want the newer generation", rec.Worker)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Reads must be stream message gets, which only the leader answers, never
|
||||
// direct gets, which any replica answers.
|
||||
func TestReads_UseLeaderMessageGet(t *testing.T) {
|
||||
_, nc, js := testutil.StartTestJetStream(t)
|
||||
kv, err := js.CreateKeyValue(context.Background(), BucketConfig(1))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var direct, leader atomic.Int32
|
||||
sub1, _ := nc.Subscribe("$JS.API.DIRECT.GET.KV_"+Bucket+".>", func(*nats.Msg) { direct.Add(1) })
|
||||
sub2, _ := nc.Subscribe("$JS.API.DIRECT.GET.KV_"+Bucket, func(*nats.Msg) { direct.Add(1) })
|
||||
sub3, _ := nc.Subscribe("$JS.API.STREAM.MSG.GET.KV_"+Bucket, func(*nats.Msg) { leader.Add(1) })
|
||||
defer sub1.Unsubscribe()
|
||||
defer sub2.Unsubscribe()
|
||||
defer sub3.Unsubscribe()
|
||||
nc.Flush()
|
||||
s := Store{JS: js, KV: kv}
|
||||
ctx := context.Background()
|
||||
w := ident(1, 1)
|
||||
pub, _ := userPub(t)
|
||||
s.Claim(ctx, region, w)
|
||||
s.RecordPublicKey(ctx, region, w, pub)
|
||||
s.Claim(ctx, region, w)
|
||||
nc.Flush()
|
||||
if direct.Load() != 0 || leader.Load() < 3 {
|
||||
t.Fatalf("direct gets=%d leader gets=%d; want only leader reads", direct.Load(), leader.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func mustFence(t *testing.T, s Store, gen uint64) {
|
||||
t.Helper()
|
||||
if err := s.Fence(context.Background(), region, gen); err != nil {
|
||||
t.Fatalf("fence %d: %v", gen, err)
|
||||
}
|
||||
}
|
||||
|
||||
func fencedAt(t *testing.T, s Store) uint64 {
|
||||
t.Helper()
|
||||
f, err := s.FencedAt(context.Background(), region)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// A fence raised before any claim leaves an open record: an older generation
|
||||
// can neither claim nor record, and the fenced generation claims as first.
|
||||
func TestFence_BeforeAnyClaim(t *testing.T) {
|
||||
s, _ := newStore(t)
|
||||
ctx := context.Background()
|
||||
if f := fencedAt(t, s); f != 0 {
|
||||
t.Fatalf("fence with no record = %d, want 0", f)
|
||||
}
|
||||
mustFence(t, s, 2)
|
||||
pub, _ := userPub(t)
|
||||
if _, err := s.Claim(ctx, region, ident(1, 1)); !errors.Is(err, ErrStaleGeneration) {
|
||||
t.Fatalf("old claim behind the fence = %v, want ErrStaleGeneration", err)
|
||||
}
|
||||
if err := s.RecordPublicKey(ctx, region, ident(1, 1), pub); !errors.Is(err, ErrStaleGeneration) {
|
||||
t.Fatalf("old record behind the fence = %v, want ErrStaleGeneration", err)
|
||||
}
|
||||
if err := s.RecordPublicKey(ctx, region, ident(2, 2), pub); !errors.Is(err, ErrNotClaimed) {
|
||||
t.Fatalf("record before any claim = %v, want ErrNotClaimed", err)
|
||||
}
|
||||
d, err := s.Claim(ctx, region, ident(2, 2))
|
||||
if err != nil || !d.CreateKey {
|
||||
t.Fatalf("fenced generation's claim = %+v, %v; want CreateKey", d, err)
|
||||
}
|
||||
if f := fencedAt(t, s); f != 2 {
|
||||
t.Fatalf("claim moved the fence to %d", f)
|
||||
}
|
||||
}
|
||||
|
||||
// Raised over a recorded key, the fence refuses the old generation's every
|
||||
// call and keeps the key, which the successor alone receives.
|
||||
func TestFence_OverARecordedKey(t *testing.T) {
|
||||
s, _ := newStore(t)
|
||||
ctx := context.Background()
|
||||
old, next := ident(1, 1), ident(2, 2)
|
||||
pub, _ := userPub(t)
|
||||
if _, err := s.Claim(ctx, region, old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.RecordPublicKey(ctx, region, old, pub); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mustFence(t, s, 2)
|
||||
if _, err := s.Claim(ctx, region, old); !errors.Is(err, ErrStaleGeneration) {
|
||||
t.Fatalf("old claim = %v, want ErrStaleGeneration", err)
|
||||
}
|
||||
if err := s.RecordPublicKey(ctx, region, old, pub); !errors.Is(err, ErrStaleGeneration) {
|
||||
t.Fatalf("old retried record = %v, want ErrStaleGeneration", err)
|
||||
}
|
||||
if err := s.RecordLaunchIntent(ctx, region, old, pub); !errors.Is(err, ErrStaleGeneration) {
|
||||
t.Fatalf("old launch intent = %v, want ErrStaleGeneration", err)
|
||||
}
|
||||
d, err := s.Claim(ctx, region, next)
|
||||
if err != nil || d.CreateKey || d.PublicKey != pub {
|
||||
t.Fatalf("successor = %+v, %v; want only the recorded key", d, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The fence never falls, and repeating it is a no-op, so a retried or late
|
||||
// lower fence cannot readmit a replaced generation.
|
||||
func TestFence_OnlyRises(t *testing.T) {
|
||||
s, _ := newStore(t)
|
||||
for _, gen := range []uint64{3, 3, 1, 2} {
|
||||
mustFence(t, s, gen)
|
||||
}
|
||||
if f := fencedAt(t, s); f != 3 {
|
||||
t.Fatalf("fence = %d, want 3", f)
|
||||
}
|
||||
if _, err := s.Claim(context.Background(), region, ident(1, 2)); !errors.Is(err, ErrStaleGeneration) {
|
||||
t.Fatalf("claim below the fence = %v, want ErrStaleGeneration", err)
|
||||
}
|
||||
if err := s.Fence(context.Background(), region, 0); err == nil {
|
||||
t.Fatal("accepted a zero fence")
|
||||
}
|
||||
}
|
||||
|
||||
// An old-generation call racing the fence either lands before it, as a
|
||||
// completed write the successor inherits, or is refused; never both.
|
||||
func TestFence_RacesOldGenerationCalls(t *testing.T) {
|
||||
for range 20 {
|
||||
s, _ := newStore(t)
|
||||
ctx := context.Background()
|
||||
old := ident(1, 1)
|
||||
pub, _ := userPub(t)
|
||||
if _, err := s.Claim(ctx, region, old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
var recErr error
|
||||
wg.Go(func() { recErr = s.RecordPublicKey(ctx, region, old, pub) })
|
||||
wg.Go(func() { mustFence(t, s, 2) })
|
||||
wg.Wait()
|
||||
d, err := s.Claim(ctx, region, ident(2, 2))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
switch {
|
||||
case recErr == nil && (d.CreateKey || d.PublicKey != pub):
|
||||
t.Fatalf("old key recorded but successor got %+v", d)
|
||||
case recErr != nil && (!errors.Is(recErr, ErrStaleGeneration) || !d.CreateKey):
|
||||
t.Fatalf("old record refused with %v but successor got %+v", recErr, d)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// RecordedKey reports the recorded key and its recorder, before and after the
|
||||
// launch intent, and refuses when no key is recorded.
|
||||
func TestRecordedKey_ReportsTheRecorderAndKey(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s, _ := newStore(t)
|
||||
w := ident(1, 1)
|
||||
if _, _, err := s.RecordedKey(ctx, region); !errors.Is(err, ErrNotClaimed) {
|
||||
t.Fatalf("before claim = %v", err)
|
||||
}
|
||||
s.Claim(ctx, region, w)
|
||||
if _, _, err := s.RecordedKey(ctx, region); !errors.Is(err, ErrKeyNotRecorded) {
|
||||
t.Fatalf("before key = %v, want ErrKeyNotRecorded", err)
|
||||
}
|
||||
pub, _ := userPub(t)
|
||||
s.RecordPublicKey(ctx, region, w, pub)
|
||||
for range 2 {
|
||||
holder, got, err := s.RecordedKey(ctx, region)
|
||||
if err != nil || holder != w || got != pub {
|
||||
t.Fatalf("recorded key = %+v %q %v", holder, got, err)
|
||||
}
|
||||
if err := s.RecordLaunchIntent(ctx, region, w, pub); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
package jetstream
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/clustersize"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
|
||||
jsapi "github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// OpenDurableKV opens an SQS authority bucket through the shared chokepoint,
|
||||
// creating it with one value per key and no TTL when absent, and refuses one
|
||||
// whose stored configuration could expire, cap or evict records, or holds
|
||||
// fewer replicas than the cluster needs.
|
||||
func OpenDurableKV(ctx context.Context, js jsapi.JetStream, bucket, description string) (jsapi.KeyValue, error) {
|
||||
kv, err := kvutil.GetOrCreateBucketWithOptions(ctx, js, kvutil.BucketOptions{
|
||||
Name: bucket, Description: description, History: 1,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return verifyDurable(ctx, kv, bucket)
|
||||
}
|
||||
|
||||
// BindDurableKV binds an existing SQS authority bucket without ever creating
|
||||
// one, so a reader that must not provision state fails closed when it is absent,
|
||||
// and refuses it on the same durability terms as OpenDurableKV.
|
||||
func BindDurableKV(ctx context.Context, js jsapi.JetStream, bucket string) (jsapi.KeyValue, error) {
|
||||
kv, err := js.KeyValue(ctx, bucket)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("bind bucket %s: %w", bucket, err)
|
||||
}
|
||||
return verifyDurable(ctx, kv, bucket)
|
||||
}
|
||||
|
||||
func verifyDurable(ctx context.Context, kv jsapi.KeyValue, bucket string) (jsapi.KeyValue, error) {
|
||||
replicas, err := clustersize.Replicas()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("bucket %s: %w", bucket, err)
|
||||
}
|
||||
status, err := kv.Status(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read bucket %s configuration: %w", bucket, err)
|
||||
}
|
||||
st, ok := status.(*jsapi.KeyValueBucketStatus)
|
||||
if !ok || st.StreamInfo() == nil {
|
||||
return nil, fmt.Errorf("bucket %s: stream configuration unavailable", bucket)
|
||||
}
|
||||
if err := checkDurable(st.StreamInfo().Config, replicas); err != nil {
|
||||
return nil, fmt.Errorf("bucket %s is not durable: %w", bucket, err)
|
||||
}
|
||||
return kv, nil
|
||||
}
|
||||
|
||||
// checkDurable refuses any effective stream setting under which an accepted
|
||||
// authority write could later expire, be evicted or lost, lag a source, or be
|
||||
// refused; the bucket's records are compared and swapped, never re-derived.
|
||||
func checkDurable(sc jsapi.StreamConfig, replicas int) error {
|
||||
switch {
|
||||
case sc.Retention != jsapi.LimitsPolicy:
|
||||
return fmt.Errorf("retention is %s, not limits", sc.Retention)
|
||||
case sc.MaxAge != 0:
|
||||
return fmt.Errorf("records expire after %s", sc.MaxAge)
|
||||
case sc.AllowMsgTTL:
|
||||
return errors.New("per-message TTLs can expire records")
|
||||
case sc.SubjectDeleteMarkerTTL != 0:
|
||||
return errors.New("delete markers expire")
|
||||
case sc.Storage != jsapi.FileStorage:
|
||||
return errors.New("storage is not file-backed")
|
||||
case sc.PersistMode != jsapi.DefaultPersistMode:
|
||||
return errors.New("writes are flushed asynchronously")
|
||||
case sc.MaxBytes > 0 || sc.MaxMsgs > 0 || sc.MaxMsgSize > 0:
|
||||
return errors.New("a size or message cap can evict or refuse records")
|
||||
case sc.MaxMsgsPerSubject < 1:
|
||||
return errors.New("per-key history is unbounded or zero")
|
||||
case sc.Discard != jsapi.DiscardNew || sc.DiscardNewPerSubject:
|
||||
return errors.New("the discard policy can evict or refuse records")
|
||||
case sc.NoAck:
|
||||
return errors.New("writes are not acknowledged")
|
||||
case sc.Sealed:
|
||||
return errors.New("the bucket is sealed")
|
||||
case sc.Mirror != nil || len(sc.Sources) > 0:
|
||||
return errors.New("the bucket mirrors or sources another stream")
|
||||
case sc.Replicas < replicas:
|
||||
return fmt.Errorf("%d replicas, want at least %d", sc.Replicas, replicas)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -38,6 +38,22 @@ func leaderLastMsg(ctx context.Context, js jsapi.JetStream, stream, subject stri
|
||||
return leaderMsgGet(ctx, js, stream, msgGetRequest{LastFor: subject})
|
||||
}
|
||||
|
||||
// LeaderKVGet reads key from an SQS KV bucket through the stream leader and
|
||||
// returns its value and revision. An absent, deleted or purged key is
|
||||
// jsapi.ErrKeyNotFound; any other failure is returned and decides nothing.
|
||||
func LeaderKVGet(ctx context.Context, js jsapi.JetStream, bucket, key string) ([]byte, uint64, error) {
|
||||
m, err := leaderLastMsg(ctx, js, "KV_"+bucket, "$KV."+bucket+"."+key)
|
||||
switch {
|
||||
case errors.Is(err, jsapi.ErrMsgNotFound):
|
||||
return nil, 0, jsapi.ErrKeyNotFound
|
||||
case err != nil:
|
||||
return nil, 0, err
|
||||
case kvRemoved(m.Header):
|
||||
return nil, 0, jsapi.ErrKeyNotFound
|
||||
}
|
||||
return m.Data, m.Sequence, nil
|
||||
}
|
||||
|
||||
// leaderStateKeys lists the live keys under keyPrefix in a state bucket up
|
||||
// to sequence bound, with one leader message get per record.
|
||||
func leaderStateKeys(ctx context.Context, js jsapi.JetStream, bucket, keyPrefix string, bound uint64) (map[string]struct{}, error) {
|
||||
|
||||
@@ -0,0 +1,301 @@
|
||||
package storelaunch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/firststore"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/storevolume"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/workerident"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/workerlaunch"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// Volumes prepares the store's data volume and reads its own metadata.
|
||||
type Volumes interface {
|
||||
// Prepare creates a new, empty, formatted volume and returns its ID. On
|
||||
// failure it returns any volume it created that it could not discard.
|
||||
Prepare(ctx context.Context, sizeGiB int64) (string, error)
|
||||
// Encrypted reports the volume record's own encryption flag.
|
||||
Encrypted(ctx context.Context, volumeID string) (bool, error)
|
||||
}
|
||||
|
||||
// NetworkAllocator manages the store's one ENI in the worker's subnet. Find
|
||||
// only looks and returns nil when none is visible.
|
||||
type NetworkAllocator interface {
|
||||
FindStoreNetwork(ctx context.Context, region, claimID string) (*workerlaunch.Network, error)
|
||||
CreateStoreNetwork(ctx context.Context, region, claimID string) (*workerlaunch.Network, error)
|
||||
}
|
||||
|
||||
// Spec is one store VM launch. ClientNKey is the worker's recorded public
|
||||
// key; nothing here is private.
|
||||
type Spec struct {
|
||||
InstanceType string
|
||||
Network workerlaunch.Network
|
||||
DataVolumeID string
|
||||
ClientNKey string
|
||||
}
|
||||
|
||||
// Host reports whether this node can boot the store now. It allocates and
|
||||
// records nothing.
|
||||
type Host interface {
|
||||
Ready(ctx context.Context) error
|
||||
}
|
||||
|
||||
// VMLauncher starts the store VM on this node. Any error leaves the outcome
|
||||
// unknown to the caller.
|
||||
type VMLauncher interface {
|
||||
LaunchStore(ctx context.Context, spec Spec) (string, error)
|
||||
}
|
||||
|
||||
var (
|
||||
ErrIncomplete = errors.New("SQS store launcher is missing a capability")
|
||||
ErrHostNotReady = errors.New("this node cannot boot the SQS store yet")
|
||||
ErrWorkerNotBound = errors.New("the Region's SQS worker is not bound on this node")
|
||||
ErrKeyNotRecorded = errors.New("the SQS worker has not recorded its public key")
|
||||
ErrWrongNode = errors.New("the Region's SQS store is pinned to another node")
|
||||
ErrUnencrypted = errors.New("SQS store data volume is not encrypted")
|
||||
// ErrVolumeNotCreated is a volume Prepare that sent no create; a retry may
|
||||
// try again once its cause is fixed.
|
||||
ErrVolumeNotCreated = errors.New("SQS store volume was not created")
|
||||
// ErrOutcomeUnknown is returned for a dispatched launch whose result was
|
||||
// not seen; it is never sent again until reconciled.
|
||||
ErrOutcomeUnknown = errors.New("SQS store launch outcome is unknown; reconcile before relaunching")
|
||||
// ErrVolumeOutcomeUnknown is returned for a volume create that was sent
|
||||
// and not recorded; it is never sent again until reconciled.
|
||||
ErrVolumeOutcomeUnknown = errors.New("SQS store volume create outcome is unknown; reconcile before relaunching")
|
||||
// ErrNetworkOutcomeUnknown is returned for an ENI create that was sent
|
||||
// with no ENI visible; it is never sent again until reconciled.
|
||||
ErrNetworkOutcomeUnknown = errors.New("SQS store network create outcome is unknown; reconcile before relaunching")
|
||||
)
|
||||
|
||||
// VolumeError reports a store volume this call knows of but could not use:
|
||||
// one a failed prepare left behind, or one that is not encrypted. It is kept.
|
||||
type VolumeError struct {
|
||||
VolumeID string
|
||||
Err error
|
||||
}
|
||||
|
||||
func (e *VolumeError) Error() string {
|
||||
return fmt.Sprintf("SQS store data volume %s is kept: %v", e.VolumeID, e.Err)
|
||||
}
|
||||
|
||||
func (e *VolumeError) Unwrap() error { return e.Err }
|
||||
|
||||
// Launcher realizes the store on NodeID, the node the Region's worker is
|
||||
// bound to. The daemon never holds the worker's private key.
|
||||
type Launcher struct {
|
||||
Records Records
|
||||
Workers workerident.Store
|
||||
FirstStore firststore.Store
|
||||
Volumes Volumes
|
||||
Network NetworkAllocator
|
||||
Host Host
|
||||
VMs VMLauncher
|
||||
NodeID string
|
||||
|
||||
// RetainedVolume extracts a volume ID from a Prepare error, if any.
|
||||
RetainedVolume func(error) string
|
||||
newClaimID func() (string, error)
|
||||
}
|
||||
|
||||
// Result is the Region's bound store. Bound is not ready.
|
||||
type Result struct {
|
||||
InstanceID string
|
||||
Address string
|
||||
InstanceType string
|
||||
}
|
||||
|
||||
// Ensure returns the Region's bound store or launches it. Every step whose
|
||||
// outcome can be lost is marked first, so a retry after a lost reply resumes
|
||||
// or fails closed and never creates a second volume, ENI or VM.
|
||||
func (l *Launcher) Ensure(ctx context.Context, region, instanceType string, sizeGiB int64) (Result, error) {
|
||||
if l.Records.KV == nil || l.Workers.KV == nil || l.FirstStore.KV == nil || l.Volumes == nil ||
|
||||
l.Network == nil || l.Host == nil || l.VMs == nil || l.NodeID == "" || instanceType == "" ||
|
||||
sizeGiB < 1 || sizeGiB > MaxSizeGiB {
|
||||
return Result{}, ErrIncomplete
|
||||
}
|
||||
rec, err := l.Records.Current(ctx, region)
|
||||
switch {
|
||||
case errors.Is(err, jetstream.ErrKeyNotFound):
|
||||
case err != nil:
|
||||
return Result{}, err
|
||||
case rec.NodeID != l.NodeID:
|
||||
return Result{}, ErrWrongNode
|
||||
case rec.InstanceType != instanceType:
|
||||
return Result{}, fmt.Errorf("%w: %s", ErrTypeChanged, rec.InstanceType)
|
||||
case rec.SizeGiB != sizeGiB:
|
||||
return Result{}, fmt.Errorf("%w: %d GiB", ErrSizeChanged, rec.SizeGiB)
|
||||
case rec.State == StateBound:
|
||||
return Result{InstanceID: rec.InstanceID, Address: rec.Address, InstanceType: rec.InstanceType}, nil
|
||||
case rec.Dispatched:
|
||||
return Result{}, ErrOutcomeUnknown
|
||||
}
|
||||
if err := l.Host.Ready(ctx); err != nil {
|
||||
return Result{}, fmt.Errorf("%w: %w", ErrHostNotReady, err)
|
||||
}
|
||||
pub, err := l.recordedKey(ctx, region)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
claim := rec.ClaimID
|
||||
if claim == "" {
|
||||
if claim, err = l.claimID(); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
}
|
||||
if err := l.Records.Begin(ctx, region, claim, l.NodeID, pub, instanceType, sizeGiB); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
if rec, err = l.Records.Current(ctx, region); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
|
||||
volumeID, err := l.volume(ctx, region, rec)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
network, err := l.network(ctx, region, claim)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
if err := l.Records.MarkDispatched(ctx, region, claim); err != nil {
|
||||
if errors.Is(err, ErrMarked) {
|
||||
return Result{}, ErrOutcomeUnknown
|
||||
}
|
||||
return Result{}, err
|
||||
}
|
||||
instanceID, err := l.VMs.LaunchStore(ctx, Spec{InstanceType: instanceType, Network: *network,
|
||||
DataVolumeID: volumeID, ClientNKey: pub})
|
||||
if err != nil {
|
||||
return Result{}, fmt.Errorf("%w: %w", ErrOutcomeUnknown, err)
|
||||
}
|
||||
if err := l.bind(ctx, region, claim, instanceID, network.ENIIP); err != nil {
|
||||
return Result{}, fmt.Errorf("store %s launched but not bound: %w", instanceID, err)
|
||||
}
|
||||
return Result{InstanceID: instanceID, Address: network.ENIIP, InstanceType: instanceType}, nil
|
||||
}
|
||||
|
||||
// recordedKey requires the worker bound on this node and returns the public
|
||||
// key it recorded, after recording the launch intent that confirms it.
|
||||
func (l *Launcher) recordedKey(ctx context.Context, region string) (string, error) {
|
||||
w, err := l.Workers.Current(ctx, region)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return "", ErrWorkerNotBound
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if w.State != workerident.StateBound || w.NodeID != l.NodeID {
|
||||
return "", ErrWorkerNotBound
|
||||
}
|
||||
id := firststore.Identity{InstanceID: w.InstanceID, ClaimID: w.ClaimID, Generation: w.Generation}
|
||||
holder, pub, err := l.FirstStore.RecordedKey(ctx, region)
|
||||
if errors.Is(err, firststore.ErrNotClaimed) || errors.Is(err, firststore.ErrKeyNotRecorded) {
|
||||
return "", ErrKeyNotRecorded
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if holder != id {
|
||||
return "", fmt.Errorf("%w: the key was recorded by another worker identity", ErrKeyNotRecorded)
|
||||
}
|
||||
if err := l.FirstStore.RecordLaunchIntent(ctx, region, id, pub); err != nil {
|
||||
return "", fmt.Errorf("record store launch intent: %w", err)
|
||||
}
|
||||
return pub, nil
|
||||
}
|
||||
|
||||
// volume returns the recorded volume, or marks the record and prepares one.
|
||||
// Its encryption is read from the volume's own record before any use.
|
||||
func (l *Launcher) volume(ctx context.Context, region string, rec Record) (string, error) {
|
||||
volumeID := rec.DataVolumeID
|
||||
if volumeID == "" {
|
||||
if err := l.Records.MarkVolumeAttempted(ctx, region, rec.ClaimID); err != nil {
|
||||
if errors.Is(err, ErrMarked) {
|
||||
return "", ErrVolumeOutcomeUnknown
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
id, err := l.Volumes.Prepare(ctx, rec.SizeGiB)
|
||||
if errors.Is(err, storevolume.ErrNotCreated) {
|
||||
if cerr := l.Records.ClearVolumeAttempt(ctx, region, rec.ClaimID); cerr != nil {
|
||||
return "", fmt.Errorf("%w: no volume was created, but its mark was not cleared: %w", ErrVolumeOutcomeUnknown, cerr)
|
||||
}
|
||||
return "", fmt.Errorf("%w: %w", ErrVolumeNotCreated, err)
|
||||
}
|
||||
if err != nil {
|
||||
err = fmt.Errorf("%w: %w", ErrVolumeOutcomeUnknown, err)
|
||||
if l.RetainedVolume != nil {
|
||||
if kept := l.RetainedVolume(err); kept != "" {
|
||||
return "", &VolumeError{VolumeID: kept, Err: err}
|
||||
}
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
if err := l.Records.RecordVolume(ctx, region, rec.ClaimID, id); err != nil {
|
||||
return "", &VolumeError{VolumeID: id, Err: fmt.Errorf("%w: %w", ErrVolumeOutcomeUnknown, err)}
|
||||
}
|
||||
volumeID = id
|
||||
}
|
||||
encrypted, err := l.Volumes.Encrypted(ctx, volumeID)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read store data volume %s: %w", volumeID, err)
|
||||
}
|
||||
if !encrypted {
|
||||
return "", &VolumeError{VolumeID: volumeID, Err: ErrUnencrypted}
|
||||
}
|
||||
return volumeID, nil
|
||||
}
|
||||
|
||||
// network reuses the claim's visible ENI, or marks the record and sends one
|
||||
// create. Once marked, an ENI that is not visible fails closed.
|
||||
func (l *Launcher) network(ctx context.Context, region, claim string) (*workerlaunch.Network, error) {
|
||||
found, err := l.Network.FindStoreNetwork(ctx, region, claim)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("store network: %w", err)
|
||||
}
|
||||
if found != nil {
|
||||
return found, nil
|
||||
}
|
||||
if err := l.Records.MarkNetworkAttempted(ctx, region, claim); err != nil {
|
||||
if errors.Is(err, ErrMarked) {
|
||||
return nil, ErrNetworkOutcomeUnknown
|
||||
}
|
||||
return nil, fmt.Errorf("store network: %w", err)
|
||||
}
|
||||
created, err := l.Network.CreateStoreNetwork(ctx, region, claim)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrNetworkOutcomeUnknown, err)
|
||||
}
|
||||
return created, nil
|
||||
}
|
||||
|
||||
// bind retries a lost reply with the same arguments, which Bind accepts as
|
||||
// idempotent.
|
||||
func (l *Launcher) bind(ctx context.Context, region, claim, instanceID, address string) error {
|
||||
var err error
|
||||
for range 3 {
|
||||
if err = l.Records.Bind(ctx, region, claim, instanceID, address); err == nil {
|
||||
return nil
|
||||
}
|
||||
if errors.Is(err, ErrSuperseded) || errors.Is(err, ErrConflict) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (l *Launcher) claimID() (string, error) {
|
||||
if l.newClaimID != nil {
|
||||
return l.newClaimID()
|
||||
}
|
||||
b := make([]byte, 12)
|
||||
if _, err := rand.Read(b); err != nil {
|
||||
return "", fmt.Errorf("mint claim ID: %w", err)
|
||||
}
|
||||
return "store-" + hex.EncodeToString(b), nil
|
||||
}
|
||||
@@ -0,0 +1,513 @@
|
||||
package storelaunch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
testutil "github.com/mulgadc/spinifex/internal/testkit"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/firststore"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/storevolume"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/workerident"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/workerlaunch"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/clustersize"
|
||||
"github.com/nats-io/nkeys"
|
||||
)
|
||||
|
||||
const (
|
||||
region = "ap-southeast-2"
|
||||
node = "node-a"
|
||||
itype = "sys.store"
|
||||
sizeGiB = 8
|
||||
workerInst = "i-0000000000000000a"
|
||||
)
|
||||
|
||||
type fakeVolumes struct {
|
||||
mu sync.Mutex
|
||||
sizes []int64
|
||||
encrypted bool
|
||||
prepareErr error
|
||||
}
|
||||
|
||||
func (f *fakeVolumes) Prepare(_ context.Context, size int64) (string, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.sizes = append(f.sizes, size)
|
||||
if f.prepareErr != nil {
|
||||
return "", f.prepareErr
|
||||
}
|
||||
return fmt.Sprintf("vol-%017x", len(f.sizes)), nil
|
||||
}
|
||||
|
||||
func (f *fakeVolumes) Encrypted(context.Context, string) (bool, error) { return f.encrypted, nil }
|
||||
|
||||
func (f *fakeVolumes) count() int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return len(f.sizes)
|
||||
}
|
||||
|
||||
// fakeNetwork holds one ENI per claim. A create with fail set is applied but
|
||||
// reports failure; with hidden set it stays invisible to Find until shown.
|
||||
type fakeNetwork struct {
|
||||
mu sync.Mutex
|
||||
creates int
|
||||
created map[string]bool
|
||||
shown map[string]bool
|
||||
fail error
|
||||
hidden bool
|
||||
}
|
||||
|
||||
func storeNet() *workerlaunch.Network {
|
||||
return &workerlaunch.Network{SubnetID: "subnet-1", ENIID: "eni-2", ENIMac: "02:00:00:00:00:02",
|
||||
ENIIP: "10.240.0.11", CIDR: "10.240.0.11/24", Gateway: "10.240.0.1"}
|
||||
}
|
||||
|
||||
func (f *fakeNetwork) FindStoreNetwork(_ context.Context, _, claimID string) (*workerlaunch.Network, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.created[claimID] && (!f.hidden || f.shown[claimID]) {
|
||||
return storeNet(), nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (f *fakeNetwork) CreateStoreNetwork(_ context.Context, _, claimID string) (*workerlaunch.Network, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.creates++
|
||||
if f.created == nil {
|
||||
f.created = map[string]bool{}
|
||||
}
|
||||
f.created[claimID] = true
|
||||
if f.fail != nil {
|
||||
return nil, f.fail
|
||||
}
|
||||
return storeNet(), nil
|
||||
}
|
||||
|
||||
func (f *fakeNetwork) show(claimID string) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.shown == nil {
|
||||
f.shown = map[string]bool{}
|
||||
}
|
||||
f.shown[claimID] = true
|
||||
}
|
||||
|
||||
func (f *fakeNetwork) count() int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.creates
|
||||
}
|
||||
|
||||
type fakeVMs struct {
|
||||
mu sync.Mutex
|
||||
specs []Spec
|
||||
fail error
|
||||
}
|
||||
|
||||
func (f *fakeVMs) LaunchStore(_ context.Context, spec Spec) (string, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.specs = append(f.specs, spec)
|
||||
if f.fail != nil {
|
||||
return "", f.fail
|
||||
}
|
||||
return fmt.Sprintf("i-%017x", len(f.specs)+0x100), nil
|
||||
}
|
||||
|
||||
func (f *fakeVMs) count() int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return len(f.specs)
|
||||
}
|
||||
|
||||
type fakeHost struct{ err error }
|
||||
|
||||
func (f *fakeHost) Ready(context.Context) error { return f.err }
|
||||
|
||||
type rig struct {
|
||||
l *Launcher
|
||||
vol *fakeVolumes
|
||||
net *fakeNetwork
|
||||
vms *fakeVMs
|
||||
host *fakeHost
|
||||
pub string
|
||||
seed string
|
||||
}
|
||||
|
||||
func newRig(t *testing.T) rig {
|
||||
t.Helper()
|
||||
clustersize.DeclareForTest(t, 1)
|
||||
_, _, js := testutil.StartTestJetStream(t)
|
||||
ctx := context.Background()
|
||||
recs, err := Open(ctx, js)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ids, err := workerident.Open(ctx, js)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fs, err := firststore.Open(ctx, js)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := rig{vol: &fakeVolumes{encrypted: true}, net: &fakeNetwork{}, vms: &fakeVMs{}, host: &fakeHost{}}
|
||||
r.l = &Launcher{Records: recs, Workers: ids, FirstStore: fs, Volumes: r.vol, Network: r.net,
|
||||
Host: r.host, VMs: r.vms, NodeID: node, RetainedVolume: func(err error) string {
|
||||
if v, ok := errors.AsType[*retained](err); ok {
|
||||
return v.id
|
||||
}
|
||||
return ""
|
||||
}}
|
||||
return r
|
||||
}
|
||||
|
||||
type retained struct{ id string }
|
||||
|
||||
func (r *retained) Error() string { return "retained " + r.id }
|
||||
|
||||
// bindWorker binds the Region's worker on nodeID; record also records its
|
||||
// public key in the first-store decision, as the worker's boot does.
|
||||
func (r *rig) bindWorker(t *testing.T, nodeID string, record bool) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
intent, err := r.l.Workers.BeginLaunch(ctx, region, workerident.LaunchRequest{InstanceType: "t3.small", ClaimID: "claim-worker-1",
|
||||
NodeID: nodeID, DataVolumeID: "vol-0000000000000000a", DataVolumeEncrypted: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := r.l.Workers.MarkDispatched(ctx, intent); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := r.l.Workers.BindInstance(ctx, intent, workerInst); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !record {
|
||||
return
|
||||
}
|
||||
kp, err := nkeys.CreateUser()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pub, _ := kp.PublicKey()
|
||||
seed, _ := kp.Seed()
|
||||
r.pub, r.seed = pub, string(seed)
|
||||
id := firststore.Identity{InstanceID: workerInst, ClaimID: "claim-worker-1", Generation: intent.Generation}
|
||||
if d, err := r.l.FirstStore.Claim(ctx, region, id); err != nil || !d.CreateKey {
|
||||
t.Fatalf("claim = %+v, %v", d, err)
|
||||
}
|
||||
if err := r.l.FirstStore.RecordPublicKey(ctx, region, id, pub); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *rig) record(t *testing.T) Record {
|
||||
t.Helper()
|
||||
rec, err := r.l.Records.Current(context.Background(), region)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return rec
|
||||
}
|
||||
|
||||
func (r *rig) nothingAllocated(t *testing.T) {
|
||||
t.Helper()
|
||||
if r.vol.count() != 0 || r.net.count() != 0 || r.vms.count() != 0 {
|
||||
t.Fatalf("allocated: %d volumes, %d ENIs, %d VMs", r.vol.count(), r.net.count(), r.vms.count())
|
||||
}
|
||||
}
|
||||
|
||||
// The store launches once with the worker's recorded public key, a fresh
|
||||
// volume of the operator's size and an ENI in the worker's subnet, and the
|
||||
// record binds it. Ensuring again returns that store with nothing new.
|
||||
func TestEnsure_LaunchesWithTheRecordedPublicKeyAndIsIdempotent(t *testing.T) {
|
||||
r := newRig(t)
|
||||
r.bindWorker(t, node, true)
|
||||
ctx := context.Background()
|
||||
res, err := r.l.Ensure(ctx, region, itype, sizeGiB)
|
||||
if err != nil || res.Address != storeNet().ENIIP {
|
||||
t.Fatalf("ensure = %+v, %v", res, err)
|
||||
}
|
||||
spec := r.vms.specs[0]
|
||||
if spec.ClientNKey != r.pub || !nkeys.IsValidPublicUserKey(spec.ClientNKey) || spec.InstanceType != itype ||
|
||||
spec.Network.ENIID != storeNet().ENIID || spec.DataVolumeID != "vol-00000000000000001" {
|
||||
t.Fatalf("spec = %+v", spec)
|
||||
}
|
||||
if r.vol.sizes[0] != sizeGiB {
|
||||
t.Fatalf("volume size = %d", r.vol.sizes[0])
|
||||
}
|
||||
if rec := r.record(t); rec.State != StateBound || rec.InstanceID != res.InstanceID || rec.PublicKey != r.pub {
|
||||
t.Fatalf("record = %+v", rec)
|
||||
}
|
||||
again, err := r.l.Ensure(ctx, region, itype, sizeGiB)
|
||||
if err != nil || again != res || r.vms.count() != 1 || r.vol.count() != 1 || r.net.count() != 1 {
|
||||
t.Fatalf("again = %+v, %v; %d VMs, %d volumes, %d ENIs", again, err, r.vms.count(), r.vol.count(), r.net.count())
|
||||
}
|
||||
}
|
||||
|
||||
// No store is launched, nor anything allocated, until the worker is bound on
|
||||
// this node and has recorded its public key.
|
||||
func TestEnsure_RequiresTheWorkerBoundHereWithItsKeyRecorded(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
setup func(*testing.T, *rig)
|
||||
wantIs error
|
||||
}{
|
||||
{"no worker", func(*testing.T, *rig) {}, ErrWorkerNotBound},
|
||||
{"worker on another node", func(t *testing.T, r *rig) { r.bindWorker(t, "node-b", true) }, ErrWorkerNotBound},
|
||||
{"key not recorded", func(t *testing.T, r *rig) { r.bindWorker(t, node, false) }, ErrKeyNotRecorded},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
r := newRig(t)
|
||||
tc.setup(t, &r)
|
||||
_, err := r.l.Ensure(context.Background(), region, itype, sizeGiB)
|
||||
if !errors.Is(err, tc.wantIs) {
|
||||
t.Fatalf("ensure = %v, want %v", err, tc.wantIs)
|
||||
}
|
||||
r.nothingAllocated(t)
|
||||
if _, err := r.l.Records.Current(context.Background(), region); err == nil {
|
||||
t.Fatal("a refused ensure must record nothing")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A node that cannot boot the store refuses before any record or allocation,
|
||||
// and a later call succeeds once it can.
|
||||
func TestEnsure_HostNotReadyAllocatesNothing(t *testing.T) {
|
||||
r := newRig(t)
|
||||
r.bindWorker(t, node, true)
|
||||
r.host.err = errors.New("no store image")
|
||||
_, err := r.l.Ensure(context.Background(), region, itype, sizeGiB)
|
||||
if !errors.Is(err, ErrHostNotReady) {
|
||||
t.Fatalf("ensure = %v", err)
|
||||
}
|
||||
r.nothingAllocated(t)
|
||||
r.host.err = nil
|
||||
if _, err := r.l.Ensure(context.Background(), region, itype, sizeGiB); err != nil || r.vms.count() != 1 {
|
||||
t.Fatalf("ensure once ready = %v, %d VMs", err, r.vms.count())
|
||||
}
|
||||
}
|
||||
|
||||
// A dispatched launch whose reply was lost is never sent again.
|
||||
func TestEnsure_AmbiguousLaunchIsNeverResent(t *testing.T) {
|
||||
r := newRig(t)
|
||||
r.bindWorker(t, node, true)
|
||||
r.vms.fail = errors.New("request timed out")
|
||||
for i := range 3 {
|
||||
if _, err := r.l.Ensure(context.Background(), region, itype, sizeGiB); !errors.Is(err, ErrOutcomeUnknown) {
|
||||
t.Fatalf("call %d = %v", i, err)
|
||||
}
|
||||
}
|
||||
if r.vms.count() != 1 || r.vol.count() != 1 || r.net.count() != 1 {
|
||||
t.Fatalf("%d VMs, %d volumes, %d ENIs", r.vms.count(), r.vol.count(), r.net.count())
|
||||
}
|
||||
}
|
||||
|
||||
// A volume create whose outcome is unknown is never sent again, and a volume
|
||||
// it is known to have left behind is reported by ID.
|
||||
func TestEnsure_AmbiguousVolumeCreateIsNeverResent(t *testing.T) {
|
||||
r := newRig(t)
|
||||
r.bindWorker(t, node, true)
|
||||
r.vol.prepareErr = &retained{id: "vol-0000000000000beef"}
|
||||
_, err := r.l.Ensure(context.Background(), region, itype, sizeGiB)
|
||||
v, ok := errors.AsType[*VolumeError](err)
|
||||
if !ok || v.VolumeID != "vol-0000000000000beef" || !errors.Is(err, ErrVolumeOutcomeUnknown) {
|
||||
t.Fatalf("ensure = %v", err)
|
||||
}
|
||||
for range 2 {
|
||||
if _, err := r.l.Ensure(context.Background(), region, itype, sizeGiB); !errors.Is(err, ErrVolumeOutcomeUnknown) {
|
||||
t.Fatalf("retry = %v", err)
|
||||
}
|
||||
}
|
||||
if r.vol.count() != 1 || r.net.count() != 0 || r.vms.count() != 0 {
|
||||
t.Fatalf("%d volumes, %d ENIs, %d VMs", r.vol.count(), r.net.count(), r.vms.count())
|
||||
}
|
||||
}
|
||||
|
||||
// A volume whose own record is not encrypted is kept and reported, never used
|
||||
// or replaced.
|
||||
func TestEnsure_RefusesAnUnencryptedVolume(t *testing.T) {
|
||||
r := newRig(t)
|
||||
r.bindWorker(t, node, true)
|
||||
r.vol.encrypted = false
|
||||
for range 2 {
|
||||
_, err := r.l.Ensure(context.Background(), region, itype, sizeGiB)
|
||||
v, ok := errors.AsType[*VolumeError](err)
|
||||
if !ok || v.VolumeID != "vol-00000000000000001" || !errors.Is(err, ErrUnencrypted) {
|
||||
t.Fatalf("ensure = %v", err)
|
||||
}
|
||||
}
|
||||
if r.vol.count() != 1 || r.net.count() != 0 || r.vms.count() != 0 {
|
||||
t.Fatalf("%d volumes, %d ENIs, %d VMs", r.vol.count(), r.net.count(), r.vms.count())
|
||||
}
|
||||
}
|
||||
|
||||
// An ENI create that timed out before its interface is visible is never sent
|
||||
// again; once it appears, the next call launches exactly once.
|
||||
func TestEnsure_DelayedENICreateIsNeverResent(t *testing.T) {
|
||||
r := newRig(t)
|
||||
r.bindWorker(t, node, true)
|
||||
r.net.hidden, r.net.fail = true, errors.New("timeout")
|
||||
for range 3 {
|
||||
if _, err := r.l.Ensure(context.Background(), region, itype, sizeGiB); !errors.Is(err, ErrNetworkOutcomeUnknown) {
|
||||
t.Fatalf("ensure = %v", err)
|
||||
}
|
||||
}
|
||||
if r.net.count() != 1 || r.vms.count() != 0 {
|
||||
t.Fatalf("%d ENI creates, %d VMs", r.net.count(), r.vms.count())
|
||||
}
|
||||
r.net.show(r.record(t).ClaimID)
|
||||
if _, err := r.l.Ensure(context.Background(), region, itype, sizeGiB); err != nil || r.vms.count() != 1 || r.net.count() != 1 {
|
||||
t.Fatalf("ensure once visible = %v; %d VMs, %d ENI creates", err, r.vms.count(), r.net.count())
|
||||
}
|
||||
}
|
||||
|
||||
// A retry asking for a different volume size is refused rather than applied.
|
||||
func TestEnsure_RefusesADifferentSizeOnRetry(t *testing.T) {
|
||||
r := newRig(t)
|
||||
r.bindWorker(t, node, true)
|
||||
r.vms.fail = errors.New("timeout")
|
||||
_, _ = r.l.Ensure(context.Background(), region, itype, sizeGiB)
|
||||
r.vms.fail = nil
|
||||
_, err := r.l.Ensure(context.Background(), region, itype, sizeGiB+1)
|
||||
if !errors.Is(err, ErrOutcomeUnknown) && !errors.Is(err, ErrSizeChanged) {
|
||||
t.Fatalf("ensure = %v", err)
|
||||
}
|
||||
if r.vol.count() != 1 {
|
||||
t.Fatalf("%d volumes", r.vol.count())
|
||||
}
|
||||
}
|
||||
|
||||
// Once the store is bound, a request for another volume size is refused
|
||||
// rather than silently returning the existing store.
|
||||
func TestEnsure_RefusesADifferentSizeOnceBound(t *testing.T) {
|
||||
r := newRig(t)
|
||||
r.bindWorker(t, node, true)
|
||||
if _, err := r.l.Ensure(context.Background(), region, itype, sizeGiB); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := r.l.Ensure(context.Background(), region, itype, sizeGiB+1); !errors.Is(err, ErrSizeChanged) {
|
||||
t.Fatalf("bound store with another size = %v", err)
|
||||
}
|
||||
if r.vol.count() != 1 || r.vms.count() != 1 {
|
||||
t.Fatalf("%d volumes, %d VMs", r.vol.count(), r.vms.count())
|
||||
}
|
||||
}
|
||||
|
||||
// Before any dispatch, a retry with a different size is refused at Begin.
|
||||
func TestEnsure_RefusesADifferentSizeBeforeDispatch(t *testing.T) {
|
||||
r := newRig(t)
|
||||
r.bindWorker(t, node, true)
|
||||
r.net.fail = errors.New("eni quota")
|
||||
_, _ = r.l.Ensure(context.Background(), region, itype, sizeGiB)
|
||||
_, err := r.l.Ensure(context.Background(), region, itype, sizeGiB+1)
|
||||
if !errors.Is(err, ErrSizeChanged) {
|
||||
t.Fatalf("ensure = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A Prepare that failed before sending any create (a missing tool, scratch or
|
||||
// mkfs failure) leaves the Region retryable, and the retry launches once.
|
||||
func TestEnsure_RetriesAVolumePrepareThatCreatedNothing(t *testing.T) {
|
||||
r := newRig(t)
|
||||
r.bindWorker(t, node, true)
|
||||
r.vol.prepareErr = fmt.Errorf("%w: mkfs.ext4 not found", storevolume.ErrNotCreated)
|
||||
for range 2 {
|
||||
if _, err := r.l.Ensure(context.Background(), region, itype, sizeGiB); !errors.Is(err, ErrVolumeNotCreated) {
|
||||
t.Fatalf("ensure = %v", err)
|
||||
}
|
||||
if rec := r.record(t); rec.VolumeAttempted || rec.DataVolumeID != "" {
|
||||
t.Fatalf("record after an uncreated volume: %+v", rec)
|
||||
}
|
||||
}
|
||||
r.vol.prepareErr = nil
|
||||
res, err := r.l.Ensure(context.Background(), region, itype, sizeGiB)
|
||||
if err != nil || res.InstanceID == "" || r.vms.count() != 1 || r.net.count() != 1 {
|
||||
t.Fatalf("ensure after the fix = %+v, %v; %d VMs, %d ENIs", res, err, r.vms.count(), r.net.count())
|
||||
}
|
||||
}
|
||||
|
||||
// A retry before dispatch asking for a different instance type is refused,
|
||||
// and so is a request for another type once the store is bound.
|
||||
func TestEnsure_RefusesADifferentInstanceType(t *testing.T) {
|
||||
r := newRig(t)
|
||||
r.bindWorker(t, node, true)
|
||||
r.net.fail = errors.New("eni quota")
|
||||
_, _ = r.l.Ensure(context.Background(), region, itype, sizeGiB)
|
||||
if _, err := r.l.Ensure(context.Background(), region, "sys.other", sizeGiB); !errors.Is(err, ErrTypeChanged) {
|
||||
t.Fatalf("retry with another type = %v", err)
|
||||
}
|
||||
r.net.fail = nil
|
||||
res, err := r.l.Ensure(context.Background(), region, itype, sizeGiB)
|
||||
if err != nil || res.InstanceType != itype || r.vms.specs[0].InstanceType != itype {
|
||||
t.Fatalf("ensure = %+v, %v", res, err)
|
||||
}
|
||||
if _, err := r.l.Ensure(context.Background(), region, "sys.other", sizeGiB); !errors.Is(err, ErrTypeChanged) {
|
||||
t.Fatalf("bound store with another type = %v", err)
|
||||
}
|
||||
if r.vms.count() != 1 {
|
||||
t.Fatalf("%d VMs", r.vms.count())
|
||||
}
|
||||
}
|
||||
|
||||
// The record, every error and the launch spec carry only the public key.
|
||||
func TestEnsure_NeverCarriesTheSeed(t *testing.T) {
|
||||
r := newRig(t)
|
||||
r.bindWorker(t, node, true)
|
||||
r.vms.fail = errors.New("timeout")
|
||||
_, err := r.l.Ensure(context.Background(), region, itype, sizeGiB)
|
||||
if err == nil || containsSeed(err.Error(), r.seed) || containsSeed(fmt.Sprintf("%+v", r.record(t)), r.seed) ||
|
||||
containsSeed(fmt.Sprintf("%+v", r.vms.specs), r.seed) {
|
||||
t.Fatalf("seed leaked or no error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func containsSeed(s, seed string) bool { return seed != "" && strings.Contains(s, seed) }
|
||||
|
||||
// After the worker is replaced, the store is not launched until the bound
|
||||
// successor has claimed the recorded key; it then launches with that key.
|
||||
func TestEnsure_WaitsForAReplacedWorkersSuccessorToClaim(t *testing.T) {
|
||||
r := newRig(t)
|
||||
r.bindWorker(t, node, true)
|
||||
ctx := context.Background()
|
||||
old := workerident.Intent{Region: region, Generation: 1, ClaimID: "claim-worker-1"}
|
||||
if err := r.l.Workers.RecordStopped(ctx, old, workerInst); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := r.l.FirstStore.Fence(ctx, region, 2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
next, err := r.l.Workers.BeginLaunch(ctx, region, workerident.LaunchRequest{InstanceType: "t3.small", ClaimID: "claim-worker-2",
|
||||
Supersedes: 1, NodeID: node, DataVolumeID: "vol-0000000000000000a", DataVolumeEncrypted: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := r.l.Workers.MarkDispatched(ctx, next); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const successor = "i-0000000000000000b"
|
||||
if err := r.l.Workers.BindInstance(ctx, next, successor); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if _, err := r.l.Ensure(ctx, region, itype, sizeGiB); !errors.Is(err, ErrKeyNotRecorded) {
|
||||
t.Fatalf("ensure before the successor claims = %v", err)
|
||||
}
|
||||
r.nothingAllocated(t)
|
||||
|
||||
id := firststore.Identity{InstanceID: successor, ClaimID: "claim-worker-2", Generation: next.Generation}
|
||||
if d, err := r.l.FirstStore.Claim(ctx, region, id); err != nil || d.CreateKey || d.PublicKey != r.pub {
|
||||
t.Fatalf("successor claim = %+v, %v", d, err)
|
||||
}
|
||||
if _, err := r.l.Ensure(ctx, region, itype, sizeGiB); err != nil || r.vms.specs[0].ClientNKey != r.pub {
|
||||
t.Fatalf("ensure after the successor claims = %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,342 @@
|
||||
// Package storelaunch realizes the Region's SQS store guest on the shared
|
||||
// worker's node, with the worker's recorded public key. Its record carries a
|
||||
// one-shot marker before every create or dispatch whose outcome can be lost,
|
||||
// so an ambiguous step is never repeated; it waits to be reconciled.
|
||||
package storelaunch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
|
||||
sqsjs "github.com/mulgadc/spinifex/spinifex/domains/sqs/message/jetstream"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/nats-io/nkeys"
|
||||
)
|
||||
|
||||
// Bucket is the SQS domain's KV bucket for store launches.
|
||||
const Bucket = "spinifex-sqs-store-launch"
|
||||
|
||||
// BucketConfig is the bucket's configuration for test fixtures; production
|
||||
// opens it through Open, which takes the replica count from the cluster.
|
||||
func BucketConfig(replicas int) jetstream.KeyValueConfig {
|
||||
return jetstream.KeyValueConfig{
|
||||
Bucket: Bucket,
|
||||
History: 1,
|
||||
Storage: jetstream.FileStorage,
|
||||
Replicas: replicas,
|
||||
}
|
||||
}
|
||||
|
||||
type State string
|
||||
|
||||
const (
|
||||
StateLaunching State = "launching"
|
||||
StateBound State = "bound"
|
||||
)
|
||||
|
||||
// Record is the Region's store launch. It holds the worker's public key only;
|
||||
// no private key material is ever written here.
|
||||
type Record struct {
|
||||
Region string `json:"region"`
|
||||
State State `json:"state"`
|
||||
ClaimID string `json:"claimID"`
|
||||
NodeID string `json:"nodeID"`
|
||||
PublicKey string `json:"publicKey"`
|
||||
SizeGiB int64 `json:"sizeGiB"`
|
||||
// The operator's instance type, fixed at Begin.
|
||||
InstanceType string `json:"instanceType"`
|
||||
// VolumeAttempted is set before the volume create is sent; DataVolumeID
|
||||
// records its result. A set marker without a volume is never retried.
|
||||
VolumeAttempted bool `json:"volumeAttempted,omitempty"`
|
||||
DataVolumeID string `json:"dataVolumeID,omitempty"`
|
||||
NetworkAttempted bool `json:"networkAttempted,omitempty"`
|
||||
Dispatched bool `json:"dispatched,omitempty"`
|
||||
InstanceID string `json:"instanceID,omitempty"`
|
||||
Address string `json:"address,omitempty"`
|
||||
}
|
||||
|
||||
var (
|
||||
ErrUnreadable = errors.New("SQS store launch record is unreadable")
|
||||
ErrContended = errors.New("SQS store launch record changed concurrently; retry")
|
||||
ErrSuperseded = errors.New("SQS store launch record names another claim")
|
||||
ErrConflict = errors.New("SQS store launch record is not launching")
|
||||
ErrMarked = errors.New("SQS store launch step was already sent; its outcome must be reconciled")
|
||||
ErrKeyChanged = errors.New("SQS store launch record holds a different public key")
|
||||
ErrSizeChanged = errors.New("SQS store launch record holds a different volume size")
|
||||
ErrTypeChanged = errors.New("SQS store launch record holds a different instance type")
|
||||
ErrInvalidRecording = errors.New("invalid SQS store launch value")
|
||||
)
|
||||
|
||||
const maxAttempts = 8
|
||||
|
||||
// MaxSizeGiB is the EBS gp3 volume size limit.
|
||||
const MaxSizeGiB = 16384
|
||||
|
||||
var (
|
||||
regionPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,62}$`)
|
||||
claimPattern = regexp.MustCompile(`^[A-Za-z0-9-]{8,64}$`)
|
||||
nodePattern = regexp.MustCompile(`^[A-Za-z0-9_-]{1,64}$`)
|
||||
volumePattern = regexp.MustCompile(`^vol-[0-9a-f]{17}$`)
|
||||
instancePattern = regexp.MustCompile(`^i-[0-9a-f]{8,17}$`)
|
||||
typePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9.-]{0,63}$`)
|
||||
)
|
||||
|
||||
// Records is the store launch state machine. Reads go to the stream leader
|
||||
// and every change is a compare-and-swap at the revision read.
|
||||
type Records struct {
|
||||
JS jetstream.JetStream
|
||||
KV jetstream.KeyValue
|
||||
}
|
||||
|
||||
// Open opens or creates the bucket and refuses one that could lose records.
|
||||
func Open(ctx context.Context, js jetstream.JetStream) (Records, error) {
|
||||
kv, err := sqsjs.OpenDurableKV(ctx, js, Bucket, "SQS store launches: the store guest per Region")
|
||||
if err != nil {
|
||||
return Records{}, err
|
||||
}
|
||||
return Records{JS: js, KV: kv}, nil
|
||||
}
|
||||
|
||||
// Bind binds the existing bucket without creating it, refusing one that could
|
||||
// lose records; a reader that must not provision state uses it.
|
||||
func Bind(ctx context.Context, js jetstream.JetStream) (Records, error) {
|
||||
kv, err := sqsjs.BindDurableKV(ctx, js, Bucket)
|
||||
if err != nil {
|
||||
return Records{}, err
|
||||
}
|
||||
return Records{JS: js, KV: kv}, nil
|
||||
}
|
||||
|
||||
func key(region string) string { return "region." + region }
|
||||
|
||||
// Current is the Region's record, read from the stream leader.
|
||||
func (r Records) Current(ctx context.Context, region string) (Record, error) {
|
||||
rec, _, err := r.read(ctx, region)
|
||||
if err != nil {
|
||||
return Record{}, err
|
||||
}
|
||||
return *rec, nil
|
||||
}
|
||||
|
||||
// Begin creates the Region's launching record. A retry with the same claim,
|
||||
// key and size returns nil; any other existing record is refused.
|
||||
func (r Records) Begin(ctx context.Context, region, claimID, nodeID, publicKey, instanceType string, sizeGiB int64) error {
|
||||
next := Record{Region: region, State: StateLaunching, ClaimID: claimID, NodeID: nodeID, PublicKey: publicKey,
|
||||
SizeGiB: sizeGiB, InstanceType: instanceType}
|
||||
if !wellFormed(&next, region) {
|
||||
return ErrInvalidRecording
|
||||
}
|
||||
for range maxAttempts {
|
||||
rec, rev, err := r.read(ctx, region)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
if r.put(ctx, region, next, 0) == nil {
|
||||
return nil
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if rec.ClaimID != claimID || rec.NodeID != nodeID {
|
||||
return ErrSuperseded
|
||||
}
|
||||
if rec.PublicKey != publicKey {
|
||||
return ErrKeyChanged
|
||||
}
|
||||
if rec.SizeGiB != sizeGiB {
|
||||
return ErrSizeChanged
|
||||
}
|
||||
if rec.InstanceType != instanceType {
|
||||
return ErrTypeChanged
|
||||
}
|
||||
// Unchanged for a retry: the rewrite proves the read was current.
|
||||
if r.put(ctx, region, *rec, rev) == nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return ErrContended
|
||||
}
|
||||
|
||||
// MarkVolumeAttempted records that the claim's one volume create is about to
|
||||
// be sent. A second call gets ErrMarked.
|
||||
func (r Records) MarkVolumeAttempted(ctx context.Context, region, claimID string) error {
|
||||
return r.advance(ctx, region, claimID, func(rec *Record) error {
|
||||
if rec.VolumeAttempted {
|
||||
return ErrMarked
|
||||
}
|
||||
rec.VolumeAttempted = true
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// ClearVolumeAttempt withdraws the volume mark after a Prepare proven to have
|
||||
// sent no create, so a retry may try again. It refuses once a volume is recorded.
|
||||
func (r Records) ClearVolumeAttempt(ctx context.Context, region, claimID string) error {
|
||||
return r.advance(ctx, region, claimID, func(rec *Record) error {
|
||||
if !rec.VolumeAttempted || rec.DataVolumeID != "" {
|
||||
return ErrConflict
|
||||
}
|
||||
rec.VolumeAttempted = false
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// RecordVolume records the volume the attempted create returned. It is
|
||||
// idempotent for the same volume.
|
||||
func (r Records) RecordVolume(ctx context.Context, region, claimID, volumeID string) error {
|
||||
if !volumePattern.MatchString(volumeID) {
|
||||
return ErrInvalidRecording
|
||||
}
|
||||
return r.advance(ctx, region, claimID, func(rec *Record) error {
|
||||
switch {
|
||||
case !rec.VolumeAttempted:
|
||||
return ErrConflict
|
||||
case rec.DataVolumeID == volumeID:
|
||||
return nil
|
||||
case rec.DataVolumeID != "":
|
||||
return ErrConflict
|
||||
}
|
||||
rec.DataVolumeID = volumeID
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// MarkNetworkAttempted records that the claim's one ENI create is about to be
|
||||
// sent. A second call gets ErrMarked.
|
||||
func (r Records) MarkNetworkAttempted(ctx context.Context, region, claimID string) error {
|
||||
return r.advance(ctx, region, claimID, func(rec *Record) error {
|
||||
if rec.NetworkAttempted {
|
||||
return ErrMarked
|
||||
}
|
||||
rec.NetworkAttempted = true
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// MarkDispatched records that the launch is about to be sent. It requires a
|
||||
// recorded volume; a second call gets ErrMarked.
|
||||
func (r Records) MarkDispatched(ctx context.Context, region, claimID string) error {
|
||||
return r.advance(ctx, region, claimID, func(rec *Record) error {
|
||||
switch {
|
||||
case rec.DataVolumeID == "":
|
||||
return ErrConflict
|
||||
case rec.Dispatched:
|
||||
return ErrMarked
|
||||
}
|
||||
rec.Dispatched = true
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// Bind records the launched store instance and its address. It is idempotent
|
||||
// for the same instance and address, so a lost reply can be retried.
|
||||
func (r Records) Bind(ctx context.Context, region, claimID, instanceID, address string) error {
|
||||
if !instancePattern.MatchString(instanceID) || address == "" {
|
||||
return ErrInvalidRecording
|
||||
}
|
||||
for range maxAttempts {
|
||||
rec, rev, err := r.read(ctx, region)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if rec.ClaimID != claimID {
|
||||
return ErrSuperseded
|
||||
}
|
||||
switch rec.State {
|
||||
case StateBound:
|
||||
if rec.InstanceID != instanceID || rec.Address != address {
|
||||
return ErrConflict
|
||||
}
|
||||
case StateLaunching:
|
||||
if !rec.Dispatched {
|
||||
return ErrConflict
|
||||
}
|
||||
rec.State, rec.InstanceID, rec.Address = StateBound, instanceID, address
|
||||
}
|
||||
if r.put(ctx, region, *rec, rev) == nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return ErrContended
|
||||
}
|
||||
|
||||
// advance applies step to the claim's launching record under compare-and-swap.
|
||||
func (r Records) advance(ctx context.Context, region, claimID string, step func(*Record) error) error {
|
||||
for range maxAttempts {
|
||||
rec, rev, err := r.read(ctx, region)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if rec.ClaimID != claimID {
|
||||
return ErrSuperseded
|
||||
}
|
||||
if rec.State != StateLaunching {
|
||||
return ErrConflict
|
||||
}
|
||||
if err := step(rec); err != nil {
|
||||
return err
|
||||
}
|
||||
if r.put(ctx, region, *rec, rev) == nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return ErrContended
|
||||
}
|
||||
|
||||
func (r Records) read(ctx context.Context, region string) (*Record, uint64, error) {
|
||||
if !regionPattern.MatchString(region) {
|
||||
return nil, 0, ErrInvalidRecording
|
||||
}
|
||||
value, rev, err := sqsjs.LeaderKVGet(ctx, r.JS, Bucket, key(region))
|
||||
if err != nil {
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil, 0, err
|
||||
}
|
||||
return nil, 0, fmt.Errorf("read SQS store launch record: %w", err)
|
||||
}
|
||||
var rec Record
|
||||
if err := json.Unmarshal(value, &rec); err != nil || !wellFormed(&rec, region) {
|
||||
return nil, 0, ErrUnreadable
|
||||
}
|
||||
return &rec, rev, nil
|
||||
}
|
||||
|
||||
func wellFormed(r *Record, region string) bool {
|
||||
if r.Region != region || !claimPattern.MatchString(r.ClaimID) || !nodePattern.MatchString(r.NodeID) ||
|
||||
!nkeys.IsValidPublicUserKey(r.PublicKey) || r.SizeGiB < 1 || r.SizeGiB > MaxSizeGiB ||
|
||||
!typePattern.MatchString(r.InstanceType) {
|
||||
return false
|
||||
}
|
||||
if r.DataVolumeID != "" && (!r.VolumeAttempted || !volumePattern.MatchString(r.DataVolumeID)) {
|
||||
return false
|
||||
}
|
||||
if r.Dispatched && r.DataVolumeID == "" {
|
||||
return false
|
||||
}
|
||||
switch r.State {
|
||||
case StateLaunching:
|
||||
return r.InstanceID == "" && r.Address == ""
|
||||
case StateBound:
|
||||
return r.Dispatched && instancePattern.MatchString(r.InstanceID) && r.Address != ""
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// put creates the key when rev is zero and is a revision compare-and-swap
|
||||
// otherwise.
|
||||
func (r Records) put(ctx context.Context, region string, rec Record, rev uint64) error {
|
||||
b, err := json.Marshal(rec)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if rev == 0 {
|
||||
_, err = r.KV.Create(ctx, key(region), b)
|
||||
} else {
|
||||
_, err = r.KV.Update(ctx, key(region), b, rev)
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
package storelaunch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
testutil "github.com/mulgadc/spinifex/internal/testkit"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/clustersize"
|
||||
"github.com/nats-io/nkeys"
|
||||
)
|
||||
|
||||
func newRecords(t *testing.T) (Records, string) {
|
||||
t.Helper()
|
||||
clustersize.DeclareForTest(t, 1)
|
||||
_, _, js := testutil.StartTestJetStream(t)
|
||||
r, err := Open(context.Background(), js)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kp, err := nkeys.CreateUser()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pub, _ := kp.PublicKey()
|
||||
return r, pub
|
||||
}
|
||||
|
||||
// Begin is idempotent for the same claim, node, key and size, and refuses any
|
||||
// other caller, key or size rather than overwriting the record.
|
||||
func TestBegin_IsIdempotentAndRefusesChanges(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
r, pub := newRecords(t)
|
||||
for range 2 {
|
||||
if err := r.Begin(ctx, region, "store-aaaa", node, pub, "m5.large", 8); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
kp, _ := nkeys.CreateUser()
|
||||
pub2, _ := kp.PublicKey()
|
||||
cases := []struct {
|
||||
name string
|
||||
claim, nodeID string
|
||||
key string
|
||||
size int64
|
||||
want error
|
||||
}{
|
||||
{"another claim", "store-bbbb", node, pub, 8, ErrSuperseded},
|
||||
{"another node", "store-aaaa", "node-b", pub, 8, ErrSuperseded},
|
||||
{"another key", "store-aaaa", node, pub2, 8, ErrKeyChanged},
|
||||
{"another size", "store-aaaa", node, pub, 9, ErrSizeChanged},
|
||||
{"a seed", "store-aaaa", node, "SUAEXAMPLE", 8, ErrInvalidRecording},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if err := r.Begin(ctx, region, tc.claim, tc.nodeID, tc.key, "m5.large", tc.size); !errors.Is(err, tc.want) {
|
||||
t.Fatalf("%s: begin = %v, want %v", tc.name, err, tc.want)
|
||||
}
|
||||
}
|
||||
if err := r.Begin(ctx, region, "store-aaaa", node, pub, "m5.xlarge", 8); !errors.Is(err, ErrTypeChanged) {
|
||||
t.Fatalf("another type: begin = %v", err)
|
||||
}
|
||||
rec, err := r.Current(ctx, region)
|
||||
if err != nil || rec.ClaimID != "store-aaaa" || rec.PublicKey != pub || rec.SizeGiB != 8 || rec.InstanceType != "m5.large" {
|
||||
t.Fatalf("record = %+v, %v", rec, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The volume mark may be withdrawn only while it is set and no volume is
|
||||
// recorded; once a volume is recorded it stays one-shot.
|
||||
func TestClearVolumeAttempt_OnlyBeforeAVolumeIsRecorded(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
r, pub := newRecords(t)
|
||||
if err := r.Begin(ctx, region, "store-aaaa", node, pub, "m5.large", 8); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := r.ClearVolumeAttempt(ctx, region, "store-aaaa"); !errors.Is(err, ErrConflict) {
|
||||
t.Fatalf("clear without a mark = %v", err)
|
||||
}
|
||||
if err := r.MarkVolumeAttempted(ctx, region, "store-aaaa"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := r.ClearVolumeAttempt(ctx, region, "store-aaaa"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := r.MarkVolumeAttempted(ctx, region, "store-aaaa"); err != nil {
|
||||
t.Fatalf("mark after clear = %v", err)
|
||||
}
|
||||
if err := r.RecordVolume(ctx, region, "store-aaaa", "vol-00000000000000001"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := r.ClearVolumeAttempt(ctx, region, "store-aaaa"); !errors.Is(err, ErrConflict) {
|
||||
t.Fatalf("clear after a recorded volume = %v", err)
|
||||
}
|
||||
if rec, err := r.Current(ctx, region); err != nil || !rec.VolumeAttempted || rec.DataVolumeID == "" {
|
||||
t.Fatalf("record = %+v, %v", rec, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Each one-shot mark is accepted once; dispatch requires a recorded volume,
|
||||
// and bind requires dispatch and is idempotent for the same instance.
|
||||
func TestMarks_AreOneShotAndOrdered(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
r, pub := newRecords(t)
|
||||
if err := r.Begin(ctx, region, "store-aaaa", node, pub, "m5.large", 8); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := r.RecordVolume(ctx, region, "store-aaaa", "vol-00000000000000001"); !errors.Is(err, ErrConflict) {
|
||||
t.Fatalf("volume before mark = %v", err)
|
||||
}
|
||||
if err := r.MarkDispatched(ctx, region, "store-aaaa"); !errors.Is(err, ErrConflict) {
|
||||
t.Fatalf("dispatch before volume = %v", err)
|
||||
}
|
||||
for _, mark := range []func(context.Context, string, string) error{r.MarkVolumeAttempted, r.MarkNetworkAttempted} {
|
||||
if err := mark(ctx, region, "store-aaaa"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := mark(ctx, region, "store-aaaa"); !errors.Is(err, ErrMarked) {
|
||||
t.Fatalf("second mark = %v", err)
|
||||
}
|
||||
}
|
||||
for range 2 {
|
||||
if err := r.RecordVolume(ctx, region, "store-aaaa", "vol-00000000000000001"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := r.RecordVolume(ctx, region, "store-aaaa", "vol-00000000000000002"); !errors.Is(err, ErrConflict) {
|
||||
t.Fatalf("another volume = %v", err)
|
||||
}
|
||||
if err := r.Bind(ctx, region, "store-aaaa", "i-00000000000000001", "10.0.0.5"); !errors.Is(err, ErrConflict) {
|
||||
t.Fatalf("bind before dispatch = %v", err)
|
||||
}
|
||||
if err := r.MarkDispatched(ctx, region, "store-aaaa"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := r.MarkDispatched(ctx, region, "store-aaaa"); !errors.Is(err, ErrMarked) {
|
||||
t.Fatalf("second dispatch = %v", err)
|
||||
}
|
||||
for range 2 {
|
||||
if err := r.Bind(ctx, region, "store-aaaa", "i-00000000000000001", "10.0.0.5"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := r.Bind(ctx, region, "store-aaaa", "i-00000000000000002", "10.0.0.5"); !errors.Is(err, ErrConflict) {
|
||||
t.Fatalf("bind another instance = %v", err)
|
||||
}
|
||||
if err := r.MarkNetworkAttempted(ctx, region, "store-bbbb"); !errors.Is(err, ErrSuperseded) {
|
||||
t.Fatalf("mark by another claim = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A malformed or private-key-bearing record is unreadable and authorizes
|
||||
// nothing.
|
||||
func TestRead_RefusesMalformedRecords(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
r, _ := newRecords(t)
|
||||
kp, _ := nkeys.CreateUser()
|
||||
seed, _ := kp.Seed()
|
||||
for _, raw := range []string{"{", `{"region":"ap-southeast-2","state":"launching","claimID":"store-aaaa","nodeID":"node-a","publicKey":"` + string(seed) + `","sizeGiB":8}`} {
|
||||
if _, err := r.KV.Put(ctx, key(region), []byte(raw)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := r.Current(ctx, region); !errors.Is(err, ErrUnreadable) {
|
||||
t.Fatalf("read %q = %v", raw[:1], err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package storelaunch
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/workerlaunch"
|
||||
)
|
||||
|
||||
// EnsureSubjectPrefix addresses one node's store launcher. The subject has no
|
||||
// queue group, so only the named node answers and the store stays pinned.
|
||||
const EnsureSubjectPrefix = "sqs.store.Ensure.v1."
|
||||
|
||||
// EnsureRequest asks a node to ensure its Region's SQS store guest.
|
||||
type EnsureRequest struct {
|
||||
NodeID string `json:"NodeId"`
|
||||
InstanceType string `json:"InstanceType"`
|
||||
VolumeSizeGiB int64 `json:"VolumeSizeGiB"`
|
||||
}
|
||||
|
||||
// EnsureResponse names the bound store VM and its address. Bound is not
|
||||
// ready: the worker's own authenticated check decides that.
|
||||
type EnsureResponse struct {
|
||||
NodeID string `json:"NodeId"`
|
||||
Region string `json:"Region"`
|
||||
InstanceID string `json:"InstanceId"`
|
||||
Address string `json:"Address"`
|
||||
// The instance type the store was launched with, fixed by its first request.
|
||||
InstanceType string `json:"InstanceType"`
|
||||
State string `json:"State"`
|
||||
}
|
||||
|
||||
// StateBoundResponse is the only state an EnsureResponse reports.
|
||||
const StateBoundResponse = "bound"
|
||||
|
||||
// EnsureSubject is the subject node's store launcher listens on.
|
||||
func EnsureSubject(node string) (string, error) {
|
||||
if !workerlaunch.ValidNodeID(node) {
|
||||
return "", fmt.Errorf("%w: invalid node ID", ErrIncomplete)
|
||||
}
|
||||
return EnsureSubjectPrefix + node, nil
|
||||
}
|
||||
@@ -87,27 +87,31 @@ var ErrToolMissing = errors.New("required host tool is missing")
|
||||
// may not, since mkfs lives in sbin.
|
||||
var sbinDirs = []string{"/usr/sbin", "/sbin"}
|
||||
|
||||
// ErrNotCreated marks a Prepare failure from before any volume create was
|
||||
// sent, so the caller may safely retry.
|
||||
var ErrNotCreated = errors.New("storevolume: no volume was created")
|
||||
|
||||
// Prepare creates a new volume, writes an empty ext4 filesystem to it and
|
||||
// releases it as available. It never writes any volume it did not just create.
|
||||
func (p *Preparer) Prepare(ctx context.Context, req Request) (string, error) {
|
||||
if p.Volumes == nil || p.Provider == nil || p.WriteImage == nil || p.NodeID == "" || p.ScratchDir == "" {
|
||||
return "", errors.New("storevolume: volume service, provider, image writer, node ID and scratch dir are required")
|
||||
return "", fmt.Errorf("%w: volume service, provider, image writer, node ID and scratch dir are required", ErrNotCreated)
|
||||
}
|
||||
if req.AccountID == "" || req.AvailabilityZone == "" || req.SizeGiB < 1 {
|
||||
return "", fmt.Errorf("storevolume: account, availability zone and a size of at least 1 GiB are required")
|
||||
return "", fmt.Errorf("%w: account, availability zone and a size of at least 1 GiB are required", ErrNotCreated)
|
||||
}
|
||||
mkfs, err := p.findTool("mkfs.ext4")
|
||||
if err != nil {
|
||||
return "", err
|
||||
return "", fmt.Errorf("%w: %w", ErrNotCreated, err)
|
||||
}
|
||||
// The copy runs qemu-img by name, so only a PATH lookup proves it will run.
|
||||
if _, err := p.findOnPath("qemu-img"); err != nil {
|
||||
return "", err
|
||||
return "", fmt.Errorf("%w: %w", ErrNotCreated, err)
|
||||
}
|
||||
|
||||
scratch, err := os.MkdirTemp(p.ScratchDir, "sqs-store-volume-")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("storevolume: scratch dir: %w", err)
|
||||
return "", fmt.Errorf("%w: scratch dir: %w", ErrNotCreated, err)
|
||||
}
|
||||
defer func() {
|
||||
if err := os.RemoveAll(scratch); err != nil {
|
||||
@@ -116,7 +120,7 @@ func (p *Preparer) Prepare(ctx context.Context, req Request) (string, error) {
|
||||
}()
|
||||
image := filepath.Join(scratch, "ext4.raw")
|
||||
if err := buildImage(ctx, p.mkfsRunner(), mkfs, image, req.SizeGiB); err != nil {
|
||||
return "", err
|
||||
return "", fmt.Errorf("%w: %w", ErrNotCreated, err)
|
||||
}
|
||||
|
||||
vol, err := p.Volumes.CreateVolumeForPreparation(ctx, &ec2.CreateVolumeInput{
|
||||
|
||||
@@ -177,6 +177,7 @@ func TestPrepare_MissingToolAllocatesNothing(t *testing.T) {
|
||||
h := &harness{missing: map[string]bool{tool: true}}
|
||||
_, err := prepare(t, h)
|
||||
require.ErrorIs(t, err, ErrToolMissing)
|
||||
require.ErrorIs(t, err, ErrNotCreated)
|
||||
assert.Contains(t, err.Error(), tool)
|
||||
assert.Empty(t, h.calls)
|
||||
})
|
||||
@@ -189,6 +190,7 @@ func TestPrepare_MissingImageWriterAllocatesNothing(t *testing.T) {
|
||||
p.WriteImage = nil
|
||||
_, err := p.Prepare(context.Background(), Request{AccountID: testAccount, AvailabilityZone: "ap-southeast-2a", SizeGiB: 1})
|
||||
require.ErrorContains(t, err, "image writer")
|
||||
require.ErrorIs(t, err, ErrNotCreated)
|
||||
assert.Empty(t, h.calls)
|
||||
}
|
||||
|
||||
@@ -223,6 +225,7 @@ func TestPrepare_FailedFormatAllocatesNothing(t *testing.T) {
|
||||
h := &harness{mkfsErr: errors.New("mkfs exploded")}
|
||||
_, err := prepare(t, h)
|
||||
require.ErrorContains(t, err, "mkfs exploded")
|
||||
require.ErrorIs(t, err, ErrNotCreated)
|
||||
assert.Equal(t, []string{"mkfs ext4.raw"}, h.calls)
|
||||
}
|
||||
|
||||
@@ -230,6 +233,7 @@ func TestPrepare_FailedCreateTouchesNoVolume(t *testing.T) {
|
||||
h := &harness{createErr: errors.New("no capacity")}
|
||||
_, err := prepare(t, h)
|
||||
require.ErrorContains(t, err, "no capacity")
|
||||
assert.NotErrorIs(t, err, ErrNotCreated, "a failed create may still have applied")
|
||||
assert.Equal(t, []string{"mkfs ext4.raw", "create 000000000000 1GiB"}, h.calls)
|
||||
}
|
||||
|
||||
@@ -239,6 +243,7 @@ func TestPrepare_VolumeNotBornCreatingIsNeverWritten(t *testing.T) {
|
||||
h := &harness{createState: "available"}
|
||||
_, err := prepare(t, h)
|
||||
requireRetained(t, err)
|
||||
assert.NotErrorIs(t, err, ErrNotCreated)
|
||||
assert.Equal(t, []string{"mkfs ext4.raw", "create 000000000000 1GiB"}, h.calls)
|
||||
}
|
||||
|
||||
@@ -246,6 +251,7 @@ func TestPrepare_FailedCopyUnpublishesThenDiscards(t *testing.T) {
|
||||
h := &harness{writeErr: errors.New("qemu-img convert: broken pipe")}
|
||||
_, err := prepare(t, h)
|
||||
require.ErrorContains(t, err, "broken pipe")
|
||||
assert.NotErrorIs(t, err, ErrNotCreated)
|
||||
require.ErrorContains(t, err, "discarded")
|
||||
var retained *RetainedVolumeError
|
||||
assert.NotErrorAs(t, err, &retained)
|
||||
@@ -402,7 +408,7 @@ func TestPrepare_RejectsIncompleteRequestBeforeAnything(t *testing.T) {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
h := &harness{}
|
||||
_, err := h.preparer(t).Prepare(context.Background(), req)
|
||||
require.Error(t, err)
|
||||
require.ErrorIs(t, err, ErrNotCreated)
|
||||
assert.Empty(t, h.calls)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -0,0 +1,687 @@
|
||||
// Package workerident is the SQS domain's authority on which VM is the
|
||||
// Region's shared SQS worker. A regional record moves launching -> bound ->
|
||||
// stopped, or launching -> failed, under compare-and-swap; a per-instance
|
||||
// index only finds the record.
|
||||
package workerident
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/firststore"
|
||||
sqsjs "github.com/mulgadc/spinifex/spinifex/domains/sqs/message/jetstream"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// Bucket is the SQS domain's KV bucket for worker identity.
|
||||
const Bucket = "spinifex-sqs-worker-identity"
|
||||
|
||||
// BucketConfig is the bucket's configuration for test fixtures; production
|
||||
// opens it through Open, which takes the replica count from the cluster.
|
||||
func BucketConfig(replicas int) jetstream.KeyValueConfig {
|
||||
return jetstream.KeyValueConfig{
|
||||
Bucket: Bucket,
|
||||
History: 1,
|
||||
Storage: jetstream.FileStorage,
|
||||
Replicas: replicas,
|
||||
}
|
||||
}
|
||||
|
||||
type State string
|
||||
|
||||
const (
|
||||
// A launch with this generation and claim ID is intended; no instance is
|
||||
// the worker yet.
|
||||
StateLaunching State = "launching"
|
||||
// The record names the one instance that is the worker.
|
||||
StateBound State = "bound"
|
||||
// The caller confirmed the launch created no instance holding the volume.
|
||||
StateFailed State = "failed"
|
||||
// The caller confirmed the bound instance stopped and released the volume.
|
||||
StateStopped State = "stopped"
|
||||
)
|
||||
|
||||
// Record is the Region's current worker. Generation only grows; the first cut
|
||||
// pins the worker to NodeID and to one encrypted data volume.
|
||||
type Record struct {
|
||||
Region string `json:"region"`
|
||||
State State `json:"state"`
|
||||
Generation uint64 `json:"generation"`
|
||||
ClaimID string `json:"claimID"`
|
||||
NodeID string `json:"nodeID"`
|
||||
DataVolumeID string `json:"dataVolumeID,omitempty"`
|
||||
InstanceType string `json:"instanceType"`
|
||||
InstanceID string `json:"instanceID,omitempty"`
|
||||
// Dispatched is set once, before the launch is sent. A dispatched launch
|
||||
// is never sent again: its outcome is reconciled, not retried.
|
||||
Dispatched bool `json:"dispatched,omitempty"`
|
||||
// NetworkAttempted is set once, before the claim's ENI create is sent. A
|
||||
// create whose outcome was not seen is never sent again for that claim.
|
||||
NetworkAttempted bool `json:"networkAttempted,omitempty"`
|
||||
// VolumeAttempted is set once, before the first volume create is sent;
|
||||
// DataVolumeID is recorded only after it. A replacement keeps both.
|
||||
VolumeAttempted bool `json:"volumeAttempted,omitempty"`
|
||||
}
|
||||
|
||||
// IndexEntry finds an instance's regional record. It is a lookup, never proof
|
||||
// that the instance is current.
|
||||
type IndexEntry struct {
|
||||
Region string `json:"region"`
|
||||
Generation uint64 `json:"generation"`
|
||||
ClaimID string `json:"claimID"`
|
||||
}
|
||||
|
||||
// Allocated reports whether the launch has sent any create, so its intent
|
||||
// can no longer be re-typed without a replacement.
|
||||
func (r Record) Allocated() bool {
|
||||
return r.VolumeAttempted || r.NetworkAttempted || r.Dispatched
|
||||
}
|
||||
|
||||
// LaunchRequest is one launch attempt. ClaimID is freshly minted per attempt
|
||||
// and reused only to retry it; only the current claim is recorded, so a VM
|
||||
// launch must be idempotent on generation and claim together. Supersedes is the generation the caller
|
||||
// read and is replacing, zero for the Region's first worker, so a replayed
|
||||
// request cannot take a generation it never observed. DataVolumeID is empty
|
||||
// for a first launch whose volume is not yet made.
|
||||
type LaunchRequest struct {
|
||||
ClaimID string
|
||||
Supersedes uint64
|
||||
NodeID string
|
||||
DataVolumeID string
|
||||
DataVolumeEncrypted bool
|
||||
// InstanceType is fixed for a generation; only a replacement may change it.
|
||||
InstanceType string
|
||||
}
|
||||
|
||||
// Intent is a recorded launch the caller may realize.
|
||||
type Intent struct {
|
||||
Region string
|
||||
Generation uint64
|
||||
ClaimID string
|
||||
}
|
||||
|
||||
// Identity is an instance's authorized worker identity, taken from the
|
||||
// current regional record rather than from anything the instance supplies.
|
||||
type Identity struct {
|
||||
Region string
|
||||
InstanceID string
|
||||
Generation uint64
|
||||
ClaimID string
|
||||
NodeID string
|
||||
DataVolumeID string
|
||||
}
|
||||
|
||||
var (
|
||||
ErrUnknownInstance = errors.New("instance is not an SQS worker")
|
||||
ErrNotCurrent = errors.New("instance is not the Region's current SQS worker")
|
||||
ErrSuperseded = errors.New("launch intent has been superseded")
|
||||
ErrInFlight = errors.New("another SQS worker launch is in flight or bound")
|
||||
ErrConflict = errors.New("worker identity conflicts with the recorded state")
|
||||
ErrWrongNode = errors.New("SQS worker is pinned to another node")
|
||||
ErrVolumeMismatch = errors.New("SQS worker is pinned to another data volume")
|
||||
ErrTypeChanged = errors.New("SQS worker launch holds a different instance type")
|
||||
ErrUnencryptedVolume = errors.New("SQS worker data volume must be encrypted")
|
||||
ErrUnreadable = errors.New("worker identity record is unreadable")
|
||||
ErrContended = errors.New("worker identity changed concurrently; retry")
|
||||
ErrNotFenced = errors.New("first-store fence does not yet exclude the replaced generation")
|
||||
ErrDispatched = errors.New("SQS worker launch was already dispatched; its outcome must be reconciled")
|
||||
ErrNetworkAttempted = errors.New("SQS worker network create was already sent; its outcome must be reconciled")
|
||||
ErrVolumeAttempted = errors.New("SQS worker volume create was already sent; its outcome must be reconciled")
|
||||
)
|
||||
|
||||
const maxAttempts = 8
|
||||
|
||||
var (
|
||||
regionPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,62}$`)
|
||||
instancePattern = regexp.MustCompile(`^i-[0-9a-f]{8,17}$`)
|
||||
claimPattern = regexp.MustCompile(`^[A-Za-z0-9-]{8,64}$`)
|
||||
volumePattern = regexp.MustCompile(`^vol-[0-9a-f]{17}$`)
|
||||
nodePattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,62}$`)
|
||||
typePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9.-]{0,63}$`)
|
||||
)
|
||||
|
||||
// Store is the worker identity authority. Reads go to the stream leader and
|
||||
// every decision that grants something is a compare-and-swap at the revision
|
||||
// read, so a stale or ambiguous result grants nothing.
|
||||
type Store struct {
|
||||
JS jetstream.JetStream
|
||||
KV jetstream.KeyValue
|
||||
// Fences reads the first-store fence a replacement launch requires. A nil
|
||||
// reader refuses every replacement.
|
||||
Fences FenceReader
|
||||
}
|
||||
|
||||
// FenceReader reports the lowest worker generation the first-store decision
|
||||
// still accepts in a Region.
|
||||
type FenceReader interface {
|
||||
FencedAt(ctx context.Context, region string) (uint64, error)
|
||||
}
|
||||
|
||||
var _ FenceReader = firststore.Store{}
|
||||
|
||||
// Open opens or creates the bucket and refuses one that could lose records.
|
||||
func Open(ctx context.Context, js jetstream.JetStream) (Store, error) {
|
||||
kv, err := sqsjs.OpenDurableKV(ctx, js, Bucket, "SQS worker identity: the current worker per Region and its instance index")
|
||||
if err != nil {
|
||||
return Store{}, err
|
||||
}
|
||||
fences, err := firststore.Open(ctx, js)
|
||||
if err != nil {
|
||||
return Store{}, err
|
||||
}
|
||||
return Store{JS: js, KV: kv, Fences: fences}, nil
|
||||
}
|
||||
|
||||
// Bind binds the existing bucket without creating it, refusing one that could
|
||||
// lose records; a reader that must not provision state uses it. It reads no
|
||||
// fence, so it refuses every replacement launch.
|
||||
func Bind(ctx context.Context, js jetstream.JetStream) (Store, error) {
|
||||
kv, err := sqsjs.BindDurableKV(ctx, js, Bucket)
|
||||
if err != nil {
|
||||
return Store{}, err
|
||||
}
|
||||
return Store{JS: js, KV: kv}, nil
|
||||
}
|
||||
|
||||
func regionKey(region string) string { return "region." + region }
|
||||
|
||||
func indexKey(instanceID string) string { return "instance." + instanceID }
|
||||
|
||||
// BeginLaunch records a launch intent. A retry of the same attempt returns the
|
||||
// same intent. A new attempt is refused while the previous one is launching or
|
||||
// bound, including one whose outcome is unknown; it takes the next generation
|
||||
// only after FailLaunch or RecordStopped confirmed nothing holds the volume,
|
||||
// and only once the first-store fence admits no older generation.
|
||||
func (s Store) BeginLaunch(ctx context.Context, region string, req LaunchRequest) (Intent, error) {
|
||||
switch {
|
||||
case !regionPattern.MatchString(region):
|
||||
return Intent{}, errors.New("invalid region")
|
||||
case !claimPattern.MatchString(req.ClaimID):
|
||||
return Intent{}, errors.New("invalid claim ID")
|
||||
case !nodePattern.MatchString(req.NodeID):
|
||||
return Intent{}, errors.New("invalid node ID")
|
||||
case req.DataVolumeID != "" && !volumePattern.MatchString(req.DataVolumeID):
|
||||
return Intent{}, errors.New("invalid data volume ID")
|
||||
case req.DataVolumeID != "" && !req.DataVolumeEncrypted:
|
||||
return Intent{}, ErrUnencryptedVolume
|
||||
case !typePattern.MatchString(req.InstanceType):
|
||||
return Intent{}, errors.New("invalid instance type")
|
||||
}
|
||||
for range maxAttempts {
|
||||
rec, rev, err := s.readRecord(ctx, region)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
if req.Supersedes != 0 {
|
||||
return Intent{}, ErrConflict
|
||||
}
|
||||
next := Record{Region: region, State: StateLaunching, Generation: 1,
|
||||
ClaimID: req.ClaimID, NodeID: req.NodeID, DataVolumeID: req.DataVolumeID, InstanceType: req.InstanceType,
|
||||
VolumeAttempted: req.DataVolumeID != ""}
|
||||
// A failed create may still have applied; the re-read decides.
|
||||
if s.put(ctx, regionKey(region), next, 0) == nil {
|
||||
return intentOf(next), nil
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return Intent{}, err
|
||||
}
|
||||
switch {
|
||||
case rec.NodeID != req.NodeID:
|
||||
return Intent{}, ErrWrongNode
|
||||
case rec.DataVolumeID != req.DataVolumeID:
|
||||
return Intent{}, ErrVolumeMismatch
|
||||
}
|
||||
next := *rec
|
||||
switch {
|
||||
case rec.ClaimID == req.ClaimID:
|
||||
if rec.State != StateLaunching || rec.Generation != req.Supersedes+1 {
|
||||
return Intent{}, ErrSuperseded
|
||||
}
|
||||
if rec.InstanceType != req.InstanceType {
|
||||
// An intent that has sent nothing allocated nothing, so it may be re-typed.
|
||||
if rec.Allocated() {
|
||||
return Intent{}, ErrTypeChanged
|
||||
}
|
||||
next.InstanceType = req.InstanceType
|
||||
}
|
||||
case req.Supersedes != rec.Generation:
|
||||
return Intent{}, ErrSuperseded
|
||||
case rec.State == StateLaunching || rec.State == StateBound:
|
||||
return Intent{}, ErrInFlight
|
||||
default:
|
||||
if err := s.requireFence(ctx, region, rec.Generation+1); err != nil {
|
||||
return Intent{}, err
|
||||
}
|
||||
next.State, next.Generation, next.ClaimID, next.InstanceID, next.Dispatched, next.NetworkAttempted =
|
||||
StateLaunching, rec.Generation+1, req.ClaimID, "", false, false
|
||||
next.InstanceType = req.InstanceType
|
||||
}
|
||||
// Unchanged for a retry: the rewrite proves the read was current.
|
||||
if s.put(ctx, regionKey(region), next, rev) == nil {
|
||||
return intentOf(next), nil
|
||||
}
|
||||
}
|
||||
return Intent{}, ErrContended
|
||||
}
|
||||
|
||||
// requireFence refuses a replacement until the first-store fence is at least
|
||||
// gen. The fence never falls, so one leader read that satisfies it stays true.
|
||||
func (s Store) requireFence(ctx context.Context, region string, gen uint64) error {
|
||||
if s.Fences == nil {
|
||||
return ErrNotFenced
|
||||
}
|
||||
fence, err := s.Fences.FencedAt(ctx, region)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if fence < gen {
|
||||
return ErrNotFenced
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// BindInstance makes instanceID the worker for intent. The index entry is
|
||||
// written first, so the record never names an instance the index cannot
|
||||
// find; a lost reply is retried with the same arguments.
|
||||
func (s Store) BindInstance(ctx context.Context, intent Intent, instanceID string) error {
|
||||
if err := validIntent(intent); err != nil {
|
||||
return err
|
||||
}
|
||||
if !instancePattern.MatchString(instanceID) {
|
||||
return errors.New("invalid instance ID")
|
||||
}
|
||||
entry := IndexEntry(intent)
|
||||
if err := s.ensureIndex(ctx, instanceID, entry); err != nil {
|
||||
return err
|
||||
}
|
||||
for range maxAttempts {
|
||||
rec, rev, err := s.readRecord(ctx, intent.Region)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return ErrSuperseded
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if rec.Generation != intent.Generation || rec.ClaimID != intent.ClaimID {
|
||||
return ErrSuperseded
|
||||
}
|
||||
next := *rec
|
||||
switch rec.State {
|
||||
case StateLaunching:
|
||||
next.State, next.InstanceID = StateBound, instanceID
|
||||
case StateBound:
|
||||
if rec.InstanceID != instanceID {
|
||||
return ErrConflict
|
||||
}
|
||||
default:
|
||||
return ErrSuperseded
|
||||
}
|
||||
if s.put(ctx, regionKey(intent.Region), next, rev) == nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return ErrContended
|
||||
}
|
||||
|
||||
// FailLaunch records that intent's launch created no instance holding the data
|
||||
// volume. Call it only once that is confirmed; a launch with an unknown outcome
|
||||
// must stay launching. It refuses once bound or superseded, and is idempotent.
|
||||
func (s Store) FailLaunch(ctx context.Context, intent Intent) error {
|
||||
if err := validIntent(intent); err != nil {
|
||||
return err
|
||||
}
|
||||
for range maxAttempts {
|
||||
rec, rev, err := s.readRecord(ctx, intent.Region)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return ErrSuperseded
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if rec.Generation != intent.Generation || rec.ClaimID != intent.ClaimID {
|
||||
return ErrSuperseded
|
||||
}
|
||||
next := *rec
|
||||
switch rec.State {
|
||||
case StateLaunching:
|
||||
next.State = StateFailed
|
||||
case StateFailed:
|
||||
default:
|
||||
return ErrConflict
|
||||
}
|
||||
if s.put(ctx, regionKey(intent.Region), next, rev) == nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return ErrContended
|
||||
}
|
||||
|
||||
// MarkDispatched records that intent's launch is about to be sent. Exactly one
|
||||
// call wins; a second call, or a retry after a lost reply, gets ErrDispatched,
|
||||
// so an ambiguous dispatch is never repeated.
|
||||
func (s Store) MarkDispatched(ctx context.Context, intent Intent) error {
|
||||
if err := validIntent(intent); err != nil {
|
||||
return err
|
||||
}
|
||||
for range maxAttempts {
|
||||
rec, rev, err := s.readRecord(ctx, intent.Region)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return ErrSuperseded
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch {
|
||||
case rec.Generation != intent.Generation || rec.ClaimID != intent.ClaimID:
|
||||
return ErrSuperseded
|
||||
case rec.State != StateLaunching:
|
||||
return ErrConflict
|
||||
case rec.Dispatched:
|
||||
return ErrDispatched
|
||||
case rec.DataVolumeID == "":
|
||||
return ErrConflict
|
||||
}
|
||||
next := *rec
|
||||
next.Dispatched = true
|
||||
if s.put(ctx, regionKey(intent.Region), next, rev) == nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return ErrContended
|
||||
}
|
||||
|
||||
// MarkNetworkAttempted records that intent's ENI create is about to be sent.
|
||||
// Exactly one call wins; a second call, or a retry after a lost reply, gets
|
||||
// ErrNetworkAttempted, so an uncertain create is never repeated.
|
||||
func (s Store) MarkNetworkAttempted(ctx context.Context, intent Intent) error {
|
||||
if err := validIntent(intent); err != nil {
|
||||
return err
|
||||
}
|
||||
for range maxAttempts {
|
||||
rec, rev, err := s.readRecord(ctx, intent.Region)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return ErrSuperseded
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch {
|
||||
case rec.Generation != intent.Generation || rec.ClaimID != intent.ClaimID:
|
||||
return ErrSuperseded
|
||||
case rec.State != StateLaunching:
|
||||
return ErrConflict
|
||||
case rec.NetworkAttempted:
|
||||
return ErrNetworkAttempted
|
||||
}
|
||||
next := *rec
|
||||
next.NetworkAttempted = true
|
||||
if s.put(ctx, regionKey(intent.Region), next, rev) == nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return ErrContended
|
||||
}
|
||||
|
||||
// MarkVolumeAttempted records that intent's one volume create is about to be
|
||||
// sent. Exactly one call wins; any later call gets ErrVolumeAttempted, so a
|
||||
// create that may have applied is never sent again.
|
||||
func (s Store) MarkVolumeAttempted(ctx context.Context, intent Intent) error {
|
||||
return s.updateLaunching(ctx, intent, func(rec *Record) error {
|
||||
if rec.VolumeAttempted {
|
||||
return ErrVolumeAttempted
|
||||
}
|
||||
rec.VolumeAttempted = true
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// ClearVolumeAttempt withdraws the volume mark after a create proven never to
|
||||
// have been sent. It refuses once a volume is recorded, and is idempotent.
|
||||
func (s Store) ClearVolumeAttempt(ctx context.Context, intent Intent) error {
|
||||
return s.updateLaunching(ctx, intent, func(rec *Record) error {
|
||||
if rec.DataVolumeID != "" {
|
||||
return ErrConflict
|
||||
}
|
||||
rec.VolumeAttempted = false
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// RecordVolume records the volume the attempted create returned. A retry with
|
||||
// the same volume succeeds; a different one is refused.
|
||||
func (s Store) RecordVolume(ctx context.Context, intent Intent, volumeID string) error {
|
||||
if !volumePattern.MatchString(volumeID) {
|
||||
return errors.New("invalid data volume ID")
|
||||
}
|
||||
return s.updateLaunching(ctx, intent, func(rec *Record) error {
|
||||
switch {
|
||||
case !rec.VolumeAttempted:
|
||||
return ErrConflict
|
||||
case rec.DataVolumeID == volumeID:
|
||||
case rec.DataVolumeID != "":
|
||||
return ErrVolumeMismatch
|
||||
}
|
||||
rec.DataVolumeID = volumeID
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// updateLaunching applies change to intent's record while it is launching and
|
||||
// undispatched, under compare-and-swap at the revision read.
|
||||
func (s Store) updateLaunching(ctx context.Context, intent Intent, change func(*Record) error) error {
|
||||
if err := validIntent(intent); err != nil {
|
||||
return err
|
||||
}
|
||||
for range maxAttempts {
|
||||
rec, rev, err := s.readRecord(ctx, intent.Region)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return ErrSuperseded
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch {
|
||||
case rec.Generation != intent.Generation || rec.ClaimID != intent.ClaimID:
|
||||
return ErrSuperseded
|
||||
case rec.State != StateLaunching || rec.Dispatched:
|
||||
return ErrConflict
|
||||
}
|
||||
next := *rec
|
||||
if err := change(&next); err != nil {
|
||||
return err
|
||||
}
|
||||
if s.put(ctx, regionKey(intent.Region), next, rev) == nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return ErrContended
|
||||
}
|
||||
|
||||
// Current is the Region's record, read from the stream leader. It decides
|
||||
// nothing by itself; every transition re-reads it under compare-and-swap.
|
||||
func (s Store) Current(ctx context.Context, region string) (Record, error) {
|
||||
if !regionPattern.MatchString(region) {
|
||||
return Record{}, errors.New("invalid region")
|
||||
}
|
||||
rec, _, err := s.readRecord(ctx, region)
|
||||
if err != nil {
|
||||
return Record{}, err
|
||||
}
|
||||
return *rec, nil
|
||||
}
|
||||
|
||||
// RecordStopped records that intent's bound instance is confirmed stopped and
|
||||
// its data volume detached and no longer exported, which is the only way a
|
||||
// bound worker can be replaced. It is idempotent.
|
||||
func (s Store) RecordStopped(ctx context.Context, intent Intent, instanceID string) error {
|
||||
if err := validIntent(intent); err != nil {
|
||||
return err
|
||||
}
|
||||
for range maxAttempts {
|
||||
rec, rev, err := s.readRecord(ctx, intent.Region)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return ErrSuperseded
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if rec.Generation != intent.Generation || rec.ClaimID != intent.ClaimID {
|
||||
return ErrSuperseded
|
||||
}
|
||||
if rec.InstanceID != instanceID {
|
||||
return ErrConflict
|
||||
}
|
||||
next := *rec
|
||||
switch rec.State {
|
||||
case StateBound:
|
||||
next.State = StateStopped
|
||||
case StateStopped:
|
||||
default:
|
||||
return ErrConflict
|
||||
}
|
||||
if s.put(ctx, regionKey(intent.Region), next, rev) == nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return ErrContended
|
||||
}
|
||||
|
||||
// Authorize returns instanceID's worker identity only if the Region's current
|
||||
// record is bound to exactly that instance, generation and claim. A missing,
|
||||
// stale or partially written index entry authorizes nothing.
|
||||
func (s Store) Authorize(ctx context.Context, instanceID string) (Identity, error) {
|
||||
if !instancePattern.MatchString(instanceID) {
|
||||
return Identity{}, ErrUnknownInstance
|
||||
}
|
||||
var entry IndexEntry
|
||||
if _, err := s.read(ctx, indexKey(instanceID), &entry); err != nil {
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return Identity{}, ErrUnknownInstance
|
||||
}
|
||||
return Identity{}, err
|
||||
}
|
||||
if !regionPattern.MatchString(entry.Region) || entry.Generation == 0 || !claimPattern.MatchString(entry.ClaimID) {
|
||||
return Identity{}, ErrUnreadable
|
||||
}
|
||||
for range maxAttempts {
|
||||
rec, rev, err := s.readRecord(ctx, entry.Region)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return Identity{}, ErrNotCurrent
|
||||
}
|
||||
if err != nil {
|
||||
return Identity{}, err
|
||||
}
|
||||
if rec.State != StateBound || rec.InstanceID != instanceID ||
|
||||
rec.Generation != entry.Generation || rec.ClaimID != entry.ClaimID {
|
||||
return Identity{}, ErrNotCurrent
|
||||
}
|
||||
// The unchanged rewrite proves the read was current, so a lagging
|
||||
// replica cannot keep a replaced worker authorized.
|
||||
if s.put(ctx, regionKey(entry.Region), *rec, rev) == nil {
|
||||
return Identity{Region: rec.Region, InstanceID: instanceID, Generation: rec.Generation,
|
||||
ClaimID: rec.ClaimID, NodeID: rec.NodeID, DataVolumeID: rec.DataVolumeID}, nil
|
||||
}
|
||||
}
|
||||
return Identity{}, ErrContended
|
||||
}
|
||||
|
||||
// ensureIndex creates the entry once; instance IDs are never reused, so an
|
||||
// existing different entry is a conflict rather than something to overwrite.
|
||||
func (s Store) ensureIndex(ctx context.Context, instanceID string, entry IndexEntry) error {
|
||||
for range maxAttempts {
|
||||
var have IndexEntry
|
||||
_, err := s.read(ctx, indexKey(instanceID), &have)
|
||||
switch {
|
||||
case errors.Is(err, jetstream.ErrKeyNotFound):
|
||||
// A failed create may still have applied; the re-read decides.
|
||||
if s.put(ctx, indexKey(instanceID), entry, 0) == nil {
|
||||
return nil
|
||||
}
|
||||
continue
|
||||
case err != nil:
|
||||
return err
|
||||
case have != entry:
|
||||
return ErrConflict
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return ErrContended
|
||||
}
|
||||
|
||||
func validIntent(in Intent) error {
|
||||
if !regionPattern.MatchString(in.Region) || in.Generation == 0 || !claimPattern.MatchString(in.ClaimID) {
|
||||
return errors.New("invalid launch intent")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func intentOf(r Record) Intent {
|
||||
return Intent{Region: r.Region, Generation: r.Generation, ClaimID: r.ClaimID}
|
||||
}
|
||||
|
||||
func (s Store) readRecord(ctx context.Context, region string) (*Record, uint64, error) {
|
||||
var rec Record
|
||||
rev, err := s.read(ctx, regionKey(region), &rec)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
if !wellFormed(&rec, region) {
|
||||
return nil, 0, ErrUnreadable
|
||||
}
|
||||
return &rec, rev, nil
|
||||
}
|
||||
|
||||
func wellFormed(r *Record, region string) bool {
|
||||
if r.Region != region || r.Generation == 0 || !claimPattern.MatchString(r.ClaimID) ||
|
||||
!nodePattern.MatchString(r.NodeID) || !typePattern.MatchString(r.InstanceType) {
|
||||
return false
|
||||
}
|
||||
if r.DataVolumeID == "" {
|
||||
// Only an undispatched launch, or one that failed before it, lacks a volume.
|
||||
if r.Dispatched || (r.State != StateLaunching && r.State != StateFailed) {
|
||||
return false
|
||||
}
|
||||
} else if !r.VolumeAttempted || !volumePattern.MatchString(r.DataVolumeID) {
|
||||
return false
|
||||
}
|
||||
switch r.State {
|
||||
case StateBound, StateStopped:
|
||||
return instancePattern.MatchString(r.InstanceID)
|
||||
case StateLaunching, StateFailed:
|
||||
return r.InstanceID == ""
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (s Store) read(ctx context.Context, key string, into any) (uint64, error) {
|
||||
value, rev, err := sqsjs.LeaderKVGet(ctx, s.JS, Bucket, key)
|
||||
if err != nil {
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return 0, err
|
||||
}
|
||||
return 0, fmt.Errorf("read worker identity %s: %w", key, err)
|
||||
}
|
||||
if err := json.Unmarshal(value, into); err != nil {
|
||||
return 0, ErrUnreadable
|
||||
}
|
||||
return rev, nil
|
||||
}
|
||||
|
||||
// put creates the key when rev is zero and is a revision compare-and-swap
|
||||
// otherwise.
|
||||
func (s Store) put(ctx context.Context, key string, v any, rev uint64) error {
|
||||
b, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if rev == 0 {
|
||||
_, err = s.KV.Create(ctx, key, b)
|
||||
} else {
|
||||
_, err = s.KV.Update(ctx, key, b, rev)
|
||||
}
|
||||
return err
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,44 @@
|
||||
package workerlaunch
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
)
|
||||
|
||||
// EnsureSubjectPrefix addresses one node's worker launcher. The subject has no
|
||||
// queue group, so only the named node answers and the worker stays pinned.
|
||||
const EnsureSubjectPrefix = "sqs.worker.Ensure.v1."
|
||||
|
||||
// EnsureRequest asks a node to ensure its Region's shared SQS worker.
|
||||
type EnsureRequest struct {
|
||||
NodeID string `json:"NodeId"`
|
||||
InstanceType string `json:"InstanceType"`
|
||||
}
|
||||
|
||||
// EnsureResponse names the bound worker VM. Bound is not ready: it says
|
||||
// nothing about the guest's boot, its key or the store.
|
||||
type EnsureResponse struct {
|
||||
NodeID string `json:"NodeId"`
|
||||
Region string `json:"Region"`
|
||||
InstanceID string `json:"InstanceId"`
|
||||
Generation uint64 `json:"Generation"`
|
||||
// The instance type the worker was launched with, fixed for its generation.
|
||||
InstanceType string `json:"InstanceType"`
|
||||
State string `json:"State"`
|
||||
}
|
||||
|
||||
// StateBound is the only state an EnsureResponse reports.
|
||||
const StateBound = "bound"
|
||||
|
||||
var nodeIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{1,64}$`)
|
||||
|
||||
// ValidNodeID reports whether id is safe to use as a single subject token.
|
||||
func ValidNodeID(id string) bool { return nodeIDPattern.MatchString(id) }
|
||||
|
||||
// EnsureSubject is the subject node's launcher listens on.
|
||||
func EnsureSubject(node string) (string, error) {
|
||||
if !ValidNodeID(node) {
|
||||
return "", fmt.Errorf("%w: invalid node ID", ErrIncomplete)
|
||||
}
|
||||
return EnsureSubjectPrefix + node, nil
|
||||
}
|
||||
@@ -0,0 +1,340 @@
|
||||
// Package workerlaunch realizes the Region's shared SQS worker from its
|
||||
// identity record: one intent, one dispatched launch, one bound instance. A
|
||||
// launch whose outcome is unknown is never sent again; it waits to be
|
||||
// reconciled, so an ambiguous reply can never start a second VM.
|
||||
package workerlaunch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/firststore"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/storevolume"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/workerident"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// Volumes prepares the worker's data volume and reads its own metadata.
|
||||
type Volumes interface {
|
||||
// Prepare creates a new, empty, formatted volume and returns its ID. An
|
||||
// error wrapping storevolume.ErrNotCreated proves no create was sent.
|
||||
Prepare(ctx context.Context) (string, error)
|
||||
// Encrypted reports the volume record's own encryption flag.
|
||||
Encrypted(ctx context.Context, volumeID string) (bool, error)
|
||||
}
|
||||
|
||||
// Roles ensures the worker's instance role and profile and returns the
|
||||
// profile ARN.
|
||||
type Roles interface {
|
||||
EnsureWorkerProfile(ctx context.Context) (string, error)
|
||||
}
|
||||
|
||||
// Network is the worker's primary interface in the SQS system VPC.
|
||||
type Network struct {
|
||||
SubnetID string
|
||||
ENIID string
|
||||
ENIMac string
|
||||
ENIIP string
|
||||
// CIDR is the ENI address with the subnet's prefix length.
|
||||
CIDR string
|
||||
Gateway string
|
||||
}
|
||||
|
||||
// NetworkAllocator manages the one ENI for a launch claim. Find only looks
|
||||
// and returns nil when none is visible; Create sends a create, after reusing
|
||||
// one that is already visible.
|
||||
type NetworkAllocator interface {
|
||||
FindWorkerNetwork(ctx context.Context, region, claimID string) (*Network, error)
|
||||
CreateWorkerNetwork(ctx context.Context, region, claimID string) (*Network, error)
|
||||
}
|
||||
|
||||
// Spec is one worker VM launch. It carries no key: the guest takes its
|
||||
// first-store decision from the gateway at boot.
|
||||
type Spec struct {
|
||||
InstanceType string
|
||||
ProfileARN string
|
||||
Network Network
|
||||
DataVolumeID string
|
||||
Region string
|
||||
}
|
||||
|
||||
// Host reports whether this node can boot a worker now: its image is
|
||||
// complete and its boot inputs exist. It allocates and records nothing.
|
||||
type Host interface {
|
||||
Ready(ctx context.Context) error
|
||||
}
|
||||
|
||||
// VMLauncher starts the worker VM on this node. Any error leaves the outcome
|
||||
// unknown to the caller.
|
||||
type VMLauncher interface {
|
||||
LaunchWorker(ctx context.Context, spec Spec) (string, error)
|
||||
}
|
||||
|
||||
var (
|
||||
// ErrOutcomeUnknown is returned when a dispatched launch may or may not
|
||||
// have started a VM. The intent stays recorded until it is reconciled.
|
||||
ErrOutcomeUnknown = errors.New("SQS worker launch outcome is unknown; reconcile before relaunching")
|
||||
ErrUnencrypted = errors.New("SQS worker data volume is not encrypted")
|
||||
ErrIncomplete = errors.New("SQS worker launcher is missing a capability")
|
||||
ErrHostNotReady = errors.New("this node cannot boot an SQS worker yet")
|
||||
// ErrNetworkOutcomeUnknown is returned when the claim's ENI create was
|
||||
// sent but no ENI is visible; the create is not repeated until reconciled.
|
||||
ErrNetworkOutcomeUnknown = errors.New("SQS worker network create outcome is unknown; reconcile before relaunching")
|
||||
// ErrVolumeNotCreated is a volume Prepare that sent no create; a retry may
|
||||
// prepare one.
|
||||
ErrVolumeNotCreated = errors.New("SQS worker volume was not created")
|
||||
// ErrVolumeOutcomeUnknown is returned for a volume create that was sent
|
||||
// but not recorded; the create is not repeated until reconciled.
|
||||
ErrVolumeOutcomeUnknown = errors.New("SQS worker volume create outcome is unknown; reconcile before relaunching")
|
||||
)
|
||||
|
||||
// Launcher realizes the worker on NodeID, the node it runs on and the only
|
||||
// node the worker is pinned to.
|
||||
type Launcher struct {
|
||||
Identities workerident.Store
|
||||
FirstStore firststore.Store
|
||||
Volumes Volumes
|
||||
Roles Roles
|
||||
Network NetworkAllocator
|
||||
Host Host
|
||||
VMs VMLauncher
|
||||
NodeID string
|
||||
// RetainedVolume extracts a volume ID from a Prepare error, if any.
|
||||
RetainedVolume func(error) string
|
||||
|
||||
newClaimID func() (string, error)
|
||||
}
|
||||
|
||||
// Result is the Region's realized worker.
|
||||
type Result struct {
|
||||
InstanceID string
|
||||
Generation uint64
|
||||
InstanceType string
|
||||
}
|
||||
|
||||
// Ensure returns the Region's bound worker, or launches it. A record that is
|
||||
// bound is returned as is; one launching and already dispatched is refused
|
||||
// with ErrOutcomeUnknown; a stopped or failed one is replaced only after the
|
||||
// first-store fence excludes its generation.
|
||||
func (l *Launcher) Ensure(ctx context.Context, region, instanceType string) (Result, error) {
|
||||
if l.Identities.KV == nil || l.FirstStore.KV == nil || l.Volumes == nil || l.Roles == nil ||
|
||||
l.Network == nil || l.Host == nil || l.VMs == nil || l.NodeID == "" || instanceType == "" {
|
||||
return Result{}, ErrIncomplete
|
||||
}
|
||||
cur, err := l.Identities.Current(ctx, region)
|
||||
var req workerident.LaunchRequest
|
||||
switch {
|
||||
case errors.Is(err, jetstream.ErrKeyNotFound):
|
||||
// The first launch records its intent before any volume exists.
|
||||
if err := l.hostReady(ctx); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
claim, err := l.claimID()
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
req = workerident.LaunchRequest{ClaimID: claim, NodeID: l.NodeID}
|
||||
case err != nil:
|
||||
return Result{}, err
|
||||
case cur.NodeID != l.NodeID:
|
||||
return Result{}, workerident.ErrWrongNode
|
||||
case cur.InstanceType != instanceType &&
|
||||
(cur.State == workerident.StateBound || (cur.State == workerident.StateLaunching && cur.Allocated())):
|
||||
// Changing the type needs a replacement, never a silent answer for the old VM.
|
||||
return Result{}, fmt.Errorf("%w: %s", workerident.ErrTypeChanged, cur.InstanceType)
|
||||
case cur.State == workerident.StateBound:
|
||||
return Result{InstanceID: cur.InstanceID, Generation: cur.Generation, InstanceType: cur.InstanceType}, nil
|
||||
case cur.State == workerident.StateLaunching && cur.Dispatched:
|
||||
return Result{}, ErrOutcomeUnknown
|
||||
case cur.State == workerident.StateLaunching:
|
||||
// An undispatched intent started nothing; resume it with its claim.
|
||||
if err := l.hostReady(ctx); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
req = workerident.LaunchRequest{ClaimID: cur.ClaimID, Supersedes: cur.Generation - 1,
|
||||
NodeID: l.NodeID, DataVolumeID: cur.DataVolumeID}
|
||||
default:
|
||||
// Stopped or failed: confirmed to hold nothing. Fence the replaced
|
||||
// generation out of the first-store decision before the next begins.
|
||||
if err := l.hostReady(ctx); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
if err := l.FirstStore.Fence(ctx, region, cur.Generation+1); err != nil {
|
||||
return Result{}, fmt.Errorf("fence replaced worker generation: %w", err)
|
||||
}
|
||||
claim, err := l.claimID()
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
req = workerident.LaunchRequest{ClaimID: claim, Supersedes: cur.Generation,
|
||||
NodeID: l.NodeID, DataVolumeID: cur.DataVolumeID}
|
||||
}
|
||||
|
||||
req.InstanceType = instanceType
|
||||
intent, err := l.begin(ctx, region, req)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
|
||||
// Nothing below is dispatched yet: a failure leaves an undispatched
|
||||
// intent that the next call resumes with the same claim.
|
||||
volumeID, err := l.volume(ctx, intent, req.DataVolumeID)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
profile, err := l.Roles.EnsureWorkerProfile(ctx)
|
||||
if err != nil {
|
||||
return Result{}, fmt.Errorf("ensure worker instance profile: %w", err)
|
||||
}
|
||||
network, err := l.network(ctx, region, intent)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
spec := Spec{InstanceType: instanceType, ProfileARN: profile, Network: *network,
|
||||
DataVolumeID: volumeID, Region: region}
|
||||
|
||||
if err := l.Identities.MarkDispatched(ctx, intent); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
instanceID, err := l.VMs.LaunchWorker(ctx, spec)
|
||||
if err != nil {
|
||||
return Result{}, fmt.Errorf("%w: %w", ErrOutcomeUnknown, err)
|
||||
}
|
||||
if err := l.bind(ctx, intent, instanceID); err != nil {
|
||||
return Result{}, fmt.Errorf("worker %s launched but not bound: %w", instanceID, err)
|
||||
}
|
||||
return Result{InstanceID: instanceID, Generation: intent.Generation, InstanceType: instanceType}, nil
|
||||
}
|
||||
|
||||
// network reuses the claim's visible ENI, or marks the intent and sends one
|
||||
// create. Once marked, an ENI that is not visible fails closed: the create
|
||||
// may still land, so a second one could give the claim two interfaces.
|
||||
func (l *Launcher) network(ctx context.Context, region string, intent workerident.Intent) (*Network, error) {
|
||||
found, err := l.Network.FindWorkerNetwork(ctx, region, intent.ClaimID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("worker network: %w", err)
|
||||
}
|
||||
if found != nil {
|
||||
return found, nil
|
||||
}
|
||||
if err := l.Identities.MarkNetworkAttempted(ctx, intent); err != nil {
|
||||
if errors.Is(err, workerident.ErrNetworkAttempted) {
|
||||
return nil, ErrNetworkOutcomeUnknown
|
||||
}
|
||||
return nil, fmt.Errorf("worker network: %w", err)
|
||||
}
|
||||
created, err := l.Network.CreateWorkerNetwork(ctx, region, intent.ClaimID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrNetworkOutcomeUnknown, err)
|
||||
}
|
||||
return created, nil
|
||||
}
|
||||
|
||||
// hostReady runs before any volume, intent, network or dispatch, so a node
|
||||
// that cannot boot the worker refuses with nothing allocated or recorded.
|
||||
func (l *Launcher) hostReady(ctx context.Context) error {
|
||||
if err := l.Host.Ready(ctx); err != nil {
|
||||
return fmt.Errorf("%w: %w", ErrHostNotReady, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// volume returns the intent's recorded volume, or marks the intent and
|
||||
// prepares one. A volume it knows of but cannot use is reported in a
|
||||
// VolumeError and left in place, never deleted.
|
||||
func (l *Launcher) volume(ctx context.Context, intent workerident.Intent, recorded string) (string, error) {
|
||||
if recorded != "" {
|
||||
return recorded, nil
|
||||
}
|
||||
if err := l.Identities.MarkVolumeAttempted(ctx, intent); err != nil {
|
||||
if errors.Is(err, workerident.ErrVolumeAttempted) {
|
||||
return "", ErrVolumeOutcomeUnknown
|
||||
}
|
||||
return "", fmt.Errorf("worker data volume: %w", err)
|
||||
}
|
||||
id, err := l.Volumes.Prepare(ctx)
|
||||
if errors.Is(err, storevolume.ErrNotCreated) {
|
||||
if cerr := l.Identities.ClearVolumeAttempt(ctx, intent); cerr != nil {
|
||||
return "", fmt.Errorf("%w: no volume was created, but its mark was not cleared: %w", ErrVolumeOutcomeUnknown, cerr)
|
||||
}
|
||||
return "", fmt.Errorf("%w: %w", ErrVolumeNotCreated, err)
|
||||
}
|
||||
if err != nil {
|
||||
err = fmt.Errorf("%w: %w", ErrVolumeOutcomeUnknown, err)
|
||||
if l.RetainedVolume != nil {
|
||||
if kept := l.RetainedVolume(err); kept != "" {
|
||||
return "", &VolumeError{VolumeID: kept, Err: err}
|
||||
}
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
if err := l.Identities.RecordVolume(ctx, intent, id); err != nil {
|
||||
return "", &VolumeError{VolumeID: id, Err: fmt.Errorf("%w: %w", ErrVolumeOutcomeUnknown, err)}
|
||||
}
|
||||
encrypted, err := l.Volumes.Encrypted(ctx, id)
|
||||
if err != nil {
|
||||
return "", &VolumeError{VolumeID: id, Err: fmt.Errorf("read its record: %w", err)}
|
||||
}
|
||||
if !encrypted {
|
||||
return "", &VolumeError{VolumeID: id, Err: ErrUnencrypted}
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// begin checks a recorded volume's own record and records the launch intent.
|
||||
func (l *Launcher) begin(ctx context.Context, region string, req workerident.LaunchRequest) (workerident.Intent, error) {
|
||||
if req.DataVolumeID == "" {
|
||||
return l.Identities.BeginLaunch(ctx, region, req)
|
||||
}
|
||||
encrypted, err := l.Volumes.Encrypted(ctx, req.DataVolumeID)
|
||||
if err != nil {
|
||||
return workerident.Intent{}, fmt.Errorf("read data volume %s: %w", req.DataVolumeID, err)
|
||||
}
|
||||
if !encrypted {
|
||||
return workerident.Intent{}, fmt.Errorf("%w: %s", ErrUnencrypted, req.DataVolumeID)
|
||||
}
|
||||
req.DataVolumeEncrypted = true
|
||||
return l.Identities.BeginLaunch(ctx, region, req)
|
||||
}
|
||||
|
||||
// VolumeError reports a worker volume this call knows of but could not use:
|
||||
// it is kept, never deleted, and its ID stays visible to the operator.
|
||||
type VolumeError struct {
|
||||
VolumeID string
|
||||
Err error
|
||||
}
|
||||
|
||||
func (e *VolumeError) Error() string {
|
||||
return fmt.Sprintf("SQS worker data volume %s is kept: %v", e.VolumeID, e.Err)
|
||||
}
|
||||
|
||||
func (e *VolumeError) Unwrap() error { return e.Err }
|
||||
|
||||
// bind retries a lost reply with the same arguments, which BindInstance
|
||||
// accepts as idempotent.
|
||||
func (l *Launcher) bind(ctx context.Context, intent workerident.Intent, instanceID string) error {
|
||||
var err error
|
||||
for range 3 {
|
||||
if err = l.Identities.BindInstance(ctx, intent, instanceID); err == nil {
|
||||
return nil
|
||||
}
|
||||
if errors.Is(err, workerident.ErrSuperseded) || errors.Is(err, workerident.ErrConflict) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (l *Launcher) claimID() (string, error) {
|
||||
if l.newClaimID != nil {
|
||||
return l.newClaimID()
|
||||
}
|
||||
b := make([]byte, 12)
|
||||
if _, err := rand.Read(b); err != nil {
|
||||
return "", fmt.Errorf("mint claim ID: %w", err)
|
||||
}
|
||||
return "claim-" + hex.EncodeToString(b), nil
|
||||
}
|
||||
@@ -0,0 +1,628 @@
|
||||
package workerlaunch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
testutil "github.com/mulgadc/spinifex/internal/testkit"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/firststore"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/storevolume"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/workerident"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/clustersize"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/nats-io/nkeys"
|
||||
)
|
||||
|
||||
const (
|
||||
region = "ap-southeast-2"
|
||||
node = "node-a"
|
||||
itype = "sys.test"
|
||||
)
|
||||
|
||||
// fakeVolumes creates one volume per Prepare. prepareErr refuses before any
|
||||
// create; appliedErr creates the volume, then reports failure without its ID.
|
||||
type fakeVolumes struct {
|
||||
mu sync.Mutex
|
||||
prepared []string
|
||||
encrypted bool
|
||||
prepareErr error
|
||||
appliedErr error
|
||||
}
|
||||
|
||||
func (f *fakeVolumes) Prepare(context.Context) (string, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.prepareErr != nil {
|
||||
return "", f.prepareErr
|
||||
}
|
||||
id := fmt.Sprintf("vol-%017x", len(f.prepared)+1)
|
||||
f.prepared = append(f.prepared, id)
|
||||
if f.appliedErr != nil {
|
||||
return "", f.appliedErr
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
func (f *fakeVolumes) count() int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return len(f.prepared)
|
||||
}
|
||||
|
||||
// keptVolumeError carries the ID of a volume a failed Prepare left in place.
|
||||
type keptVolumeError struct{ id string }
|
||||
|
||||
func (e keptVolumeError) Error() string { return "fill failed for " + e.id }
|
||||
|
||||
func (f *fakeVolumes) Encrypted(context.Context, string) (bool, error) { return f.encrypted, nil }
|
||||
|
||||
type fakeRoles struct{ fail atomic.Int32 }
|
||||
|
||||
func (f *fakeRoles) EnsureWorkerProfile(context.Context) (string, error) {
|
||||
if f.fail.Add(-1) >= 0 {
|
||||
return "", errors.New("iam unavailable")
|
||||
}
|
||||
return "arn:aws:iam::000000000000:instance-profile/sqsWorkerRole", nil
|
||||
}
|
||||
|
||||
// fakeNetwork holds one ENI per claim. A create with fail set is applied but
|
||||
// reports failure; with hidden set it stays invisible to Find until shown.
|
||||
type fakeNetwork struct {
|
||||
mu sync.Mutex
|
||||
claims []string
|
||||
created map[string]bool
|
||||
fail error
|
||||
hidden bool
|
||||
shown map[string]bool
|
||||
}
|
||||
|
||||
func (f *fakeNetwork) net() *Network {
|
||||
return &Network{SubnetID: "subnet-1", ENIID: "eni-1", ENIMac: "02:00:00:00:00:01",
|
||||
ENIIP: "10.240.0.10", CIDR: "10.240.0.10/24", Gateway: "10.240.0.1"}
|
||||
}
|
||||
|
||||
func (f *fakeNetwork) FindWorkerNetwork(_ context.Context, _, claimID string) (*Network, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.created[claimID] && (!f.hidden || f.shown[claimID]) {
|
||||
return f.net(), nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (f *fakeNetwork) CreateWorkerNetwork(_ context.Context, _, claimID string) (*Network, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.claims = append(f.claims, claimID)
|
||||
if f.created == nil {
|
||||
f.created = map[string]bool{}
|
||||
}
|
||||
f.created[claimID] = true
|
||||
if f.fail != nil {
|
||||
return nil, f.fail
|
||||
}
|
||||
return f.net(), nil
|
||||
}
|
||||
|
||||
func (f *fakeNetwork) show(claimID string) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.shown == nil {
|
||||
f.shown = map[string]bool{}
|
||||
}
|
||||
f.shown[claimID] = true
|
||||
}
|
||||
|
||||
type fakeVMs struct {
|
||||
mu sync.Mutex
|
||||
specs []Spec
|
||||
fail error
|
||||
}
|
||||
|
||||
func (f *fakeVMs) LaunchWorker(_ context.Context, spec Spec) (string, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.specs = append(f.specs, spec)
|
||||
if f.fail != nil {
|
||||
return "", f.fail
|
||||
}
|
||||
return fmt.Sprintf("i-%017x", len(f.specs)), nil
|
||||
}
|
||||
|
||||
func (f *fakeVMs) count() int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return len(f.specs)
|
||||
}
|
||||
|
||||
type fakeHost struct {
|
||||
mu sync.Mutex
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeHost) Ready(context.Context) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.err
|
||||
}
|
||||
|
||||
type rig struct {
|
||||
l *Launcher
|
||||
host *fakeHost
|
||||
vol *fakeVolumes
|
||||
iam *fakeRoles
|
||||
net *fakeNetwork
|
||||
vms *fakeVMs
|
||||
}
|
||||
|
||||
func newRig(t *testing.T) rig {
|
||||
t.Helper()
|
||||
clustersize.DeclareForTest(t, 1)
|
||||
_, _, js := testutil.StartTestJetStream(t)
|
||||
ids, err := workerident.Open(context.Background(), js)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fs, err := firststore.Open(context.Background(), js)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := rig{vol: &fakeVolumes{encrypted: true}, iam: &fakeRoles{}, net: &fakeNetwork{}, vms: &fakeVMs{}, host: &fakeHost{}}
|
||||
r.l = &Launcher{Identities: ids, FirstStore: fs, Volumes: r.vol, Roles: r.iam, Network: r.net, Host: r.host, VMs: r.vms, NodeID: node}
|
||||
return r
|
||||
}
|
||||
|
||||
func (r rig) current(t *testing.T) workerident.Record {
|
||||
t.Helper()
|
||||
rec, err := r.l.Identities.Current(context.Background(), region)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return rec
|
||||
}
|
||||
|
||||
// The first launch prepares one volume, launches once with the real profile,
|
||||
// network and a new-store status, and binds the instance so the gateway's
|
||||
// authorization accepts it. Ensuring again returns that worker unchanged.
|
||||
func TestEnsure_FirstLaunchBindsAndIsIdempotent(t *testing.T) {
|
||||
r := newRig(t)
|
||||
ctx := context.Background()
|
||||
res, err := r.l.Ensure(ctx, region, itype)
|
||||
if err != nil || res.Generation != 1 {
|
||||
t.Fatalf("ensure = %+v, %v", res, err)
|
||||
}
|
||||
spec := r.vms.specs[0]
|
||||
if spec.Region != region || spec.ProfileARN == "" ||
|
||||
spec.Network.ENIID != "eni-1" || spec.DataVolumeID != r.vol.prepared[0] || spec.InstanceType != itype {
|
||||
t.Fatalf("spec = %+v", spec)
|
||||
}
|
||||
id, err := r.l.Identities.Authorize(ctx, res.InstanceID)
|
||||
if err != nil || id.Generation != 1 || id.NodeID != node || id.DataVolumeID != spec.DataVolumeID {
|
||||
t.Fatalf("authorize = %+v, %v", id, err)
|
||||
}
|
||||
again, err := r.l.Ensure(ctx, region, itype)
|
||||
if err != nil || again != res || r.vms.count() != 1 || len(r.vol.prepared) != 1 {
|
||||
t.Fatalf("second ensure = %+v, %v; launched %d, prepared %d", again, err, r.vms.count(), len(r.vol.prepared))
|
||||
}
|
||||
}
|
||||
|
||||
// A volume whose own record is not encrypted is recorded and reported, never
|
||||
// launched on, and later calls refuse it rather than preparing another.
|
||||
func TestEnsure_RefusesAnUnencryptedVolume(t *testing.T) {
|
||||
r := newRig(t)
|
||||
r.vol.encrypted = false
|
||||
for range 3 {
|
||||
_, err := r.l.Ensure(context.Background(), region, itype)
|
||||
if !errors.Is(err, ErrUnencrypted) || !strings.Contains(err.Error(), r.vol.prepared[0]) {
|
||||
t.Fatalf("ensure = %v, want ErrUnencrypted naming %s", err, r.vol.prepared[0])
|
||||
}
|
||||
}
|
||||
if rec := r.current(t); rec.DataVolumeID != r.vol.prepared[0] || rec.Dispatched {
|
||||
t.Fatalf("record = %+v", rec)
|
||||
}
|
||||
if r.vol.count() != 1 || r.vms.count() != 0 || len(r.net.claims) != 0 {
|
||||
t.Fatalf("volumes=%d launches=%d network=%d", r.vol.count(), r.vms.count(), len(r.net.claims))
|
||||
}
|
||||
}
|
||||
|
||||
// A Prepare proven to have sent no create withdraws its mark, so a retry may
|
||||
// prepare the volume; the failed call allocated nothing.
|
||||
func TestEnsure_ProvenPreCreateFailureIsRetried(t *testing.T) {
|
||||
r := newRig(t)
|
||||
ctx := context.Background()
|
||||
r.vol.prepareErr = fmt.Errorf("%w: %w: no storage key", storevolume.ErrNotCreated, ErrUnencrypted)
|
||||
for range 2 {
|
||||
_, err := r.l.Ensure(ctx, region, itype)
|
||||
if !errors.Is(err, ErrVolumeNotCreated) || !errors.Is(err, ErrUnencrypted) {
|
||||
t.Fatalf("ensure = %v, want ErrVolumeNotCreated", err)
|
||||
}
|
||||
if _, ok := errors.AsType[*VolumeError](err); ok {
|
||||
t.Fatalf("ensure = %v reports a volume it never created", err)
|
||||
}
|
||||
}
|
||||
if rec := r.current(t); rec.VolumeAttempted || rec.DataVolumeID != "" || rec.Allocated() {
|
||||
t.Fatalf("record after a proven pre-create failure = %+v", rec)
|
||||
}
|
||||
r.vol.prepareErr = nil
|
||||
if _, err := r.l.Ensure(ctx, region, itype); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r.vol.count() != 1 || r.vms.count() != 1 {
|
||||
t.Fatalf("volumes=%d launches=%d", r.vol.count(), r.vms.count())
|
||||
}
|
||||
}
|
||||
|
||||
// A claim that cannot be minted refuses with nothing recorded or prepared.
|
||||
func TestEnsure_NoClaimRecordsNothing(t *testing.T) {
|
||||
r := newRig(t)
|
||||
r.l.newClaimID = func() (string, error) { return "", errors.New("no entropy") }
|
||||
if _, err := r.l.Ensure(context.Background(), region, itype); err == nil {
|
||||
t.Fatal("ensure succeeded without a claim ID")
|
||||
}
|
||||
if r.vol.count() != 0 {
|
||||
t.Fatalf("prepared %v before a refusal that needs no volume", r.vol.prepared)
|
||||
}
|
||||
if _, err := r.l.Identities.Current(context.Background(), region); !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
t.Fatalf("record after refusal: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A create that applied but returned no ID is never sent again: every later
|
||||
// call reports the unknown outcome, even once the volume exists, and nothing
|
||||
// further is allocated or launched.
|
||||
func TestEnsure_AmbiguousVolumeCreateIsNeverResent(t *testing.T) {
|
||||
r := newRig(t)
|
||||
ctx := context.Background()
|
||||
r.vol.appliedErr = errors.New("nats: timeout")
|
||||
if _, err := r.l.Ensure(ctx, region, itype); !errors.Is(err, ErrVolumeOutcomeUnknown) {
|
||||
t.Fatalf("ensure = %v, want ErrVolumeOutcomeUnknown", err)
|
||||
}
|
||||
r.vol.appliedErr = nil
|
||||
for range 3 {
|
||||
if _, err := r.l.Ensure(ctx, region, itype); !errors.Is(err, ErrVolumeOutcomeUnknown) {
|
||||
t.Fatalf("retry = %v, want ErrVolumeOutcomeUnknown", err)
|
||||
}
|
||||
}
|
||||
rec := r.current(t)
|
||||
if r.vol.count() != 1 || !rec.VolumeAttempted || rec.DataVolumeID != "" || r.vms.count() != 0 || len(r.net.claims) != 0 {
|
||||
t.Fatalf("volumes=%d record=%+v launches=%d network=%d", r.vol.count(), rec, r.vms.count(), len(r.net.claims))
|
||||
}
|
||||
if _, err := r.l.Ensure(ctx, region, "sys.other"); !errors.Is(err, workerident.ErrTypeChanged) {
|
||||
t.Fatalf("re-type after a sent create = %v, want ErrTypeChanged", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A failed Prepare that names the volume it left behind reports that ID, and
|
||||
// the create is still never sent again.
|
||||
func TestEnsure_RetainedVolumeIDStaysVisible(t *testing.T) {
|
||||
r := newRig(t)
|
||||
ctx := context.Background()
|
||||
r.l.RetainedVolume = func(err error) string {
|
||||
if k, ok := errors.AsType[keptVolumeError](err); ok {
|
||||
return k.id
|
||||
}
|
||||
return ""
|
||||
}
|
||||
r.vol.appliedErr = keptVolumeError{id: "vol-00000000000000001"}
|
||||
_, err := r.l.Ensure(ctx, region, itype)
|
||||
kept, ok := errors.AsType[*VolumeError](err)
|
||||
if !ok || kept.VolumeID != "vol-00000000000000001" || !errors.Is(err, ErrVolumeOutcomeUnknown) {
|
||||
t.Fatalf("ensure = %v, want the kept volume reported", err)
|
||||
}
|
||||
r.vol.appliedErr = nil
|
||||
if _, err := r.l.Ensure(ctx, region, itype); !errors.Is(err, ErrVolumeOutcomeUnknown) {
|
||||
t.Fatalf("retry = %v, want ErrVolumeOutcomeUnknown", err)
|
||||
}
|
||||
if r.vol.count() != 1 || r.vms.count() != 0 {
|
||||
t.Fatalf("volumes=%d launches=%d", r.vol.count(), r.vms.count())
|
||||
}
|
||||
}
|
||||
|
||||
// An intent that has sent nothing may be re-typed; one that has may not.
|
||||
func TestEnsure_UnallocatedIntentMayBeRetyped(t *testing.T) {
|
||||
r := newRig(t)
|
||||
ctx := context.Background()
|
||||
r.vol.prepareErr = fmt.Errorf("%w: image writer missing", storevolume.ErrNotCreated)
|
||||
if _, err := r.l.Ensure(ctx, region, itype); !errors.Is(err, ErrVolumeNotCreated) {
|
||||
t.Fatalf("ensure = %v, want ErrVolumeNotCreated", err)
|
||||
}
|
||||
r.vol.prepareErr = nil
|
||||
res, err := r.l.Ensure(ctx, region, "sys.other")
|
||||
if err != nil || res.InstanceType != "sys.other" || res.Generation != 1 || r.vms.specs[0].InstanceType != "sys.other" {
|
||||
t.Fatalf("re-typed ensure = %+v, %v", res, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A launch whose reply is lost is never sent again: every later call reports
|
||||
// the unknown outcome and the record keeps the dispatched intent.
|
||||
func TestEnsure_AmbiguousLaunchIsNeverResent(t *testing.T) {
|
||||
r := newRig(t)
|
||||
r.vms.fail = errors.New("nats: timeout")
|
||||
ctx := context.Background()
|
||||
if _, err := r.l.Ensure(ctx, region, itype); !errors.Is(err, ErrOutcomeUnknown) {
|
||||
t.Fatalf("ensure = %v, want ErrOutcomeUnknown", err)
|
||||
}
|
||||
r.vms.fail = nil
|
||||
for range 3 {
|
||||
if _, err := r.l.Ensure(ctx, region, itype); !errors.Is(err, ErrOutcomeUnknown) {
|
||||
t.Fatalf("retry = %v, want ErrOutcomeUnknown", err)
|
||||
}
|
||||
}
|
||||
if rec := r.current(t); r.vms.count() != 1 || rec.State != workerident.StateLaunching || !rec.Dispatched {
|
||||
t.Fatalf("launched %d, record %+v", r.vms.count(), rec)
|
||||
}
|
||||
}
|
||||
|
||||
// Concurrent first launches prepare one volume and start exactly one VM.
|
||||
func TestEnsure_ConcurrentCallsLaunchOnce(t *testing.T) {
|
||||
r := newRig(t)
|
||||
var wg sync.WaitGroup
|
||||
var ok atomic.Int32
|
||||
for range 8 {
|
||||
wg.Go(func() {
|
||||
if _, err := r.l.Ensure(context.Background(), region, itype); err == nil {
|
||||
ok.Add(1)
|
||||
}
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
if r.vms.count() != 1 || r.vol.count() != 1 || ok.Load() < 1 {
|
||||
t.Fatalf("launched %d VMs on %d volumes, %d calls succeeded", r.vms.count(), r.vol.count(), ok.Load())
|
||||
}
|
||||
if rec := r.current(t); rec.State != workerident.StateBound {
|
||||
t.Fatalf("record = %+v, want bound", rec)
|
||||
}
|
||||
}
|
||||
|
||||
// A failure before dispatch leaves an intent the next call resumes with the
|
||||
// same claim, volume and generation.
|
||||
func TestEnsure_ResumesAnUndispatchedIntent(t *testing.T) {
|
||||
r := newRig(t)
|
||||
r.iam.fail.Store(1)
|
||||
ctx := context.Background()
|
||||
if _, err := r.l.Ensure(ctx, region, itype); err == nil {
|
||||
t.Fatal("ensure succeeded with IAM failing")
|
||||
}
|
||||
if rec := r.current(t); rec.State != workerident.StateLaunching || rec.Dispatched {
|
||||
t.Fatalf("record = %+v, want an undispatched intent", rec)
|
||||
}
|
||||
res, err := r.l.Ensure(ctx, region, itype)
|
||||
if err != nil || res.Generation != 1 || r.vms.count() != 1 || len(r.vol.prepared) != 1 {
|
||||
t.Fatalf("resume = %+v, %v; launched %d, prepared %d", res, err, r.vms.count(), len(r.vol.prepared))
|
||||
}
|
||||
if len(r.net.claims) != 1 || r.current(t).ClaimID != r.net.claims[0] {
|
||||
t.Fatalf("network claims %v, record claim %q", r.net.claims, r.current(t).ClaimID)
|
||||
}
|
||||
}
|
||||
|
||||
// After a confirmed stop the replacement fences the old generation out of
|
||||
// first-store and reuses the pinned volume; the guest asks for its key.
|
||||
func TestEnsure_ReplacementFencesAndReusesTheVolume(t *testing.T) {
|
||||
r := newRig(t)
|
||||
ctx := context.Background()
|
||||
first, err := r.l.Ensure(ctx, region, itype)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
old := workerident.Identity{InstanceID: first.InstanceID, Generation: 1, ClaimID: r.current(t).ClaimID}
|
||||
oldFS := firststore.Identity{InstanceID: old.InstanceID, ClaimID: old.ClaimID, Generation: 1}
|
||||
kp, _ := nkeys.CreateUser()
|
||||
pub, _ := kp.PublicKey()
|
||||
if _, err := r.l.FirstStore.Claim(ctx, region, oldFS); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := r.l.FirstStore.RecordPublicKey(ctx, region, oldFS, pub); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
intent := workerident.Intent{Region: region, Generation: 1, ClaimID: old.ClaimID}
|
||||
if err := r.l.Identities.RecordStopped(ctx, intent, first.InstanceID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
next, err := r.l.Ensure(ctx, region, itype)
|
||||
if err != nil || next.Generation != 2 || next.InstanceID == first.InstanceID {
|
||||
t.Fatalf("replacement = %+v, %v", next, err)
|
||||
}
|
||||
spec := r.vms.specs[1]
|
||||
if spec.DataVolumeID != r.vol.prepared[0] || len(r.vol.prepared) != 1 || spec.Region != region {
|
||||
t.Fatalf("replacement spec = %+v", spec)
|
||||
}
|
||||
if f, _ := r.l.FirstStore.FencedAt(ctx, region); f != 2 {
|
||||
t.Fatalf("fence = %d, want 2", f)
|
||||
}
|
||||
if _, err := r.l.FirstStore.Claim(ctx, region, oldFS); !errors.Is(err, firststore.ErrStaleGeneration) {
|
||||
t.Fatalf("old generation claim = %v, want ErrStaleGeneration", err)
|
||||
}
|
||||
if _, err := r.l.Identities.Authorize(ctx, first.InstanceID); err == nil {
|
||||
t.Fatal("the replaced instance is still authorized")
|
||||
}
|
||||
}
|
||||
|
||||
// The worker is pinned: a launcher on another node launches nothing.
|
||||
func TestEnsure_RefusesAnotherNode(t *testing.T) {
|
||||
r := newRig(t)
|
||||
if _, err := r.l.Ensure(context.Background(), region, itype); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
other := *r.l
|
||||
other.NodeID = "node-b"
|
||||
if _, err := other.Ensure(context.Background(), region, itype); !errors.Is(err, workerident.ErrWrongNode) {
|
||||
t.Fatalf("other node = %v, want ErrWrongNode", err)
|
||||
}
|
||||
if r.vms.count() != 1 {
|
||||
t.Fatalf("launched %d", r.vms.count())
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsure_RefusesWithoutEveryCapability(t *testing.T) {
|
||||
r := newRig(t)
|
||||
for name, mutate := range map[string]func(*Launcher){
|
||||
"volumes": func(l *Launcher) { l.Volumes = nil },
|
||||
"roles": func(l *Launcher) { l.Roles = nil },
|
||||
"network": func(l *Launcher) { l.Network = nil },
|
||||
"host": func(l *Launcher) { l.Host = nil },
|
||||
"vms": func(l *Launcher) { l.VMs = nil },
|
||||
"node": func(l *Launcher) { l.NodeID = "" },
|
||||
} {
|
||||
l := *r.l
|
||||
mutate(&l)
|
||||
if _, err := l.Ensure(context.Background(), region, itype); !errors.Is(err, ErrIncomplete) {
|
||||
t.Errorf("%s: %v, want ErrIncomplete", name, err)
|
||||
}
|
||||
}
|
||||
if r.vms.count() != 0 || len(r.vol.prepared) != 0 {
|
||||
t.Fatal("an incomplete launcher did work")
|
||||
}
|
||||
}
|
||||
|
||||
// A host that cannot boot the worker refuses the first launch before a volume,
|
||||
// intent, network or dispatch exists, and the next ready call launches.
|
||||
func TestEnsure_HostNotReadyRefusesBeforeAllocating(t *testing.T) {
|
||||
r := newRig(t)
|
||||
r.host.err = errors.New("image missing")
|
||||
if _, err := r.l.Ensure(context.Background(), region, itype); !errors.Is(err, ErrHostNotReady) {
|
||||
t.Fatalf("Ensure = %v, want ErrHostNotReady", err)
|
||||
}
|
||||
if len(r.vol.prepared) != 0 || len(r.net.claims) != 0 || r.vms.count() != 0 {
|
||||
t.Fatalf("allocated: volumes=%d claims=%d launches=%d", len(r.vol.prepared), len(r.net.claims), r.vms.count())
|
||||
}
|
||||
if _, err := r.l.Identities.Current(context.Background(), region); !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
t.Fatalf("record after refusal: %v", err)
|
||||
}
|
||||
r.host.err = nil
|
||||
if _, err := r.l.Ensure(context.Background(), region, itype); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r.vms.count() != 1 || len(r.vol.prepared) != 1 {
|
||||
t.Fatalf("ready retry: launches=%d volumes=%d", r.vms.count(), len(r.vol.prepared))
|
||||
}
|
||||
}
|
||||
|
||||
// An undispatched intent is not dispatched while the host is not ready, and
|
||||
// a stopped worker is neither fenced nor replaced.
|
||||
func TestEnsure_HostNotReadyLeavesRecordsUntouched(t *testing.T) {
|
||||
r := newRig(t)
|
||||
r.iam.fail.Store(1)
|
||||
if _, err := r.l.Ensure(context.Background(), region, itype); err == nil {
|
||||
t.Fatal("want IAM failure")
|
||||
}
|
||||
before := r.current(t)
|
||||
r.host.err = errors.New("image missing")
|
||||
if _, err := r.l.Ensure(context.Background(), region, itype); !errors.Is(err, ErrHostNotReady) {
|
||||
t.Fatalf("resume = %v, want ErrHostNotReady", err)
|
||||
}
|
||||
if after := r.current(t); after != before || r.vms.count() != 0 {
|
||||
t.Fatalf("resume changed the record or launched: %+v", after)
|
||||
}
|
||||
|
||||
r.host.err = nil
|
||||
first, err := r.l.Ensure(context.Background(), region, itype)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cur := r.current(t)
|
||||
intent := workerident.Intent{Region: region, Generation: cur.Generation, ClaimID: cur.ClaimID}
|
||||
if err := r.l.Identities.RecordStopped(context.Background(), intent, first.InstanceID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stopped := r.current(t)
|
||||
r.host.err = errors.New("image missing")
|
||||
if _, err := r.l.Ensure(context.Background(), region, itype); !errors.Is(err, ErrHostNotReady) {
|
||||
t.Fatalf("replace = %v, want ErrHostNotReady", err)
|
||||
}
|
||||
if after := r.current(t); after != stopped || r.vms.count() != 1 {
|
||||
t.Fatalf("replacement changed the record or launched: %+v", after)
|
||||
}
|
||||
fence, err := r.l.FirstStore.FencedAt(context.Background(), region)
|
||||
if err != nil || fence > stopped.Generation {
|
||||
t.Fatalf("fence = %d, %v while the host was not ready", fence, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A create whose reply is lost and whose ENI is not yet visible is never sent
|
||||
// again: the retry fails closed and dispatches nothing, and once the ENI
|
||||
// becomes visible the next call reuses it and launches once.
|
||||
func TestEnsure_UncertainNetworkCreateIsNotRepeated(t *testing.T) {
|
||||
r := newRig(t)
|
||||
ctx := context.Background()
|
||||
r.net.fail = errors.New("nats: timeout")
|
||||
r.net.hidden = true
|
||||
if _, err := r.l.Ensure(ctx, region, itype); !errors.Is(err, ErrNetworkOutcomeUnknown) {
|
||||
t.Fatalf("first = %v, want ErrNetworkOutcomeUnknown", err)
|
||||
}
|
||||
r.net.fail = nil
|
||||
for range 3 {
|
||||
if _, err := r.l.Ensure(ctx, region, itype); !errors.Is(err, ErrNetworkOutcomeUnknown) {
|
||||
t.Fatalf("retry = %v, want ErrNetworkOutcomeUnknown", err)
|
||||
}
|
||||
}
|
||||
rec := r.current(t)
|
||||
if len(r.net.claims) != 1 || rec.Dispatched || !rec.NetworkAttempted || r.vms.count() != 0 {
|
||||
t.Fatalf("creates=%d record=%+v launches=%d", len(r.net.claims), rec, r.vms.count())
|
||||
}
|
||||
r.net.show(rec.ClaimID)
|
||||
if _, err := r.l.Ensure(ctx, region, itype); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(r.net.claims) != 1 || r.vms.count() != 1 {
|
||||
t.Fatalf("after visible: creates=%d launches=%d", len(r.net.claims), r.vms.count())
|
||||
}
|
||||
}
|
||||
|
||||
// A failure after the ENI is visible resumes with that ENI, not a new create.
|
||||
func TestEnsure_VisibleNetworkIsReusedOnResume(t *testing.T) {
|
||||
r := newRig(t)
|
||||
ctx := context.Background()
|
||||
r.net.fail = errors.New("nats: timeout")
|
||||
if _, err := r.l.Ensure(ctx, region, itype); !errors.Is(err, ErrNetworkOutcomeUnknown) {
|
||||
t.Fatalf("first = %v", err)
|
||||
}
|
||||
r.net.fail = nil
|
||||
if _, err := r.l.Ensure(ctx, region, itype); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(r.net.claims) != 1 || r.vms.count() != 1 {
|
||||
t.Fatalf("creates=%d launches=%d", len(r.net.claims), r.vms.count())
|
||||
}
|
||||
}
|
||||
|
||||
// A request for another type neither resumes an undispatched intent nor
|
||||
// reports the bound VM as that type; only a replacement may change it.
|
||||
func TestEnsure_InstanceTypeIsFixedUntilReplacement(t *testing.T) {
|
||||
r := newRig(t)
|
||||
ctx := context.Background()
|
||||
r.iam.fail.Store(1)
|
||||
if _, err := r.l.Ensure(ctx, region, itype); err == nil {
|
||||
t.Fatal("ensure succeeded with IAM failing")
|
||||
}
|
||||
if _, err := r.l.Ensure(ctx, region, "sys.other"); !errors.Is(err, workerident.ErrTypeChanged) {
|
||||
t.Fatalf("resume with another type = %v, want ErrTypeChanged", err)
|
||||
}
|
||||
first, err := r.l.Ensure(ctx, region, itype)
|
||||
if err != nil || first.InstanceType != itype || r.vms.count() != 1 {
|
||||
t.Fatalf("resume = %+v, %v; launched %d", first, err, r.vms.count())
|
||||
}
|
||||
if _, err := r.l.Ensure(ctx, region, "sys.other"); !errors.Is(err, workerident.ErrTypeChanged) {
|
||||
t.Fatalf("bound with another type = %v, want ErrTypeChanged", err)
|
||||
}
|
||||
if r.vms.count() != 1 {
|
||||
t.Fatalf("a type mismatch launched: %d", r.vms.count())
|
||||
}
|
||||
|
||||
intent := workerident.Intent{Region: region, Generation: 1, ClaimID: r.current(t).ClaimID}
|
||||
if err := r.l.Identities.RecordStopped(ctx, intent, first.InstanceID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
next, err := r.l.Ensure(ctx, region, "sys.other")
|
||||
if err != nil || next.Generation != 2 || next.InstanceType != "sys.other" || r.vms.specs[1].InstanceType != "sys.other" {
|
||||
t.Fatalf("replacement = %+v, %v", next, err)
|
||||
}
|
||||
if rec := r.current(t); rec.InstanceType != "sys.other" {
|
||||
t.Fatalf("replacement record type = %q", rec.InstanceType)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
// Package workerrole names the shared SQS worker's identity contract: the
|
||||
// signing service its internal actions use, the instance role it runs under
|
||||
// and the exact actions that role is granted. The gateway gate and the role
|
||||
// grant both read it, so they cannot disagree.
|
||||
package workerrole
|
||||
|
||||
// ServiceName is the SigV4 credential-scope service the SQS worker signs for.
|
||||
// No AWS SDK signs for it, so no tenant request reaches these actions.
|
||||
const ServiceName = "spinifex-sqs"
|
||||
|
||||
// RoleName is the system-account instance role the SQS worker VM runs under.
|
||||
// Its trust policy must admit only EC2, so STS names every session after the
|
||||
// instance and no caller can choose the session name the gate reads.
|
||||
const RoleName = "sqsWorkerRole"
|
||||
|
||||
const (
|
||||
ActionClaimFirstStore = "ClaimFirstStore"
|
||||
ActionDescribeStore = "DescribeStore"
|
||||
ActionRecordWorkerPublicKey = "RecordWorkerPublicKey"
|
||||
)
|
||||
|
||||
// Actions lists every internal worker action, sorted.
|
||||
func Actions() []string {
|
||||
return []string{ActionClaimFirstStore, ActionDescribeStore, ActionRecordWorkerPublicKey}
|
||||
}
|
||||
+36
-728
@@ -1,166 +1,30 @@
|
||||
// Package gateway provides the AWS-compatible API gateway for the Spinifex platform.
|
||||
package gateway
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/roles/awsgw/policyauth"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"reflect"
|
||||
"strings"
|
||||
"uuid"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/mulgadc/bluebottle/pkg/iampolicy"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/admission/quota"
|
||||
ec2awsapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi"
|
||||
ec2accountapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/account"
|
||||
ec2capacityreservationapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/capacityreservation"
|
||||
ec2eigwapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/eigw"
|
||||
ec2eipapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/eip"
|
||||
ec2igwapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/igw"
|
||||
ec2imageapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/image"
|
||||
ec2instanceapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/instance"
|
||||
ec2keyapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/key"
|
||||
ec2launchtemplateapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/launchtemplate"
|
||||
ec2natgwapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/natgw"
|
||||
ec2placementgroupapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/placementgroup"
|
||||
ec2routetableapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/routetable"
|
||||
ec2snapshotapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/snapshot"
|
||||
ec2spotinstanceapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/spotinstance"
|
||||
ec2tagsapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/tags"
|
||||
ec2volumeapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/volume"
|
||||
ec2vpcapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/vpc"
|
||||
ec2zoneapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/zone"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
awsxml "github.com/mulgadc/spinifex/spinifex/foundation/aws/xml"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/lifecycle/idempotency"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/query"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// EC2Handler processes parsed query args and returns XML response bytes.
|
||||
// r is included for handlers that enforce iam:PassRole; most ignore it.
|
||||
type EC2Handler func(string, map[string]string, *GatewayConfig, string, *http.Request) ([]byte, error)
|
||||
// ec2Quota adapts the account quota service to the capability the EC2 actions
|
||||
// consume, building the NATS-backed instance-type resolver its retype gate takes.
|
||||
type ec2Quota struct{ *quota.Service }
|
||||
|
||||
// ec2Action parses once so authorization and dispatch share one typed input.
|
||||
type ec2Action struct {
|
||||
parse func(map[string]string) (any, error)
|
||||
dispatch func(string, any, *GatewayConfig, string, *http.Request) ([]byte, error)
|
||||
// idempotent records that this action honours ClientToken, so the set of
|
||||
// actions that do is assertable rather than folklore.
|
||||
idempotent bool
|
||||
var _ ec2awsapi.Quota = ec2Quota{}
|
||||
|
||||
func (q ec2Quota) EnforceRetype(ctx context.Context, nc *nats.Conn, nodeIDs func() []string, accountID, instanceID, newType string) (int, error) {
|
||||
return q.Service.EnforceRetype(ctx, quota.NATSInstanceTypeResolver(nc, nodeIDs), accountID, instanceID, newType)
|
||||
}
|
||||
|
||||
func (h ec2Action) withQueryPreprocessor(preprocess func(map[string]string)) ec2Action {
|
||||
parse := h.parse
|
||||
h.parse = func(q map[string]string) (any, error) {
|
||||
preprocess(q)
|
||||
return parse(q)
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
// requestContext returns r's context, or Background for a nil request
|
||||
// (some callers and tests dispatch without an *http.Request). The SDK's
|
||||
// invocation id rides along: it is repeated across retries of one call, so it
|
||||
// is what lets a mutation recognise its own resend instead of doing the work
|
||||
// twice.
|
||||
func requestContext(r *http.Request) context.Context {
|
||||
if r == nil {
|
||||
return context.Background()
|
||||
}
|
||||
return idempotency.WithKey(r.Context(), r.Header.Get(idempotency.SDKInvocationIDHeader))
|
||||
}
|
||||
|
||||
// parseEC2Input parses query args into an action's input type. Shared by every
|
||||
// ec2Action constructor so they all consume the same parse.
|
||||
func parseEC2Input[In any](q map[string]string) (any, error) {
|
||||
input := new(In)
|
||||
if err := query.QueryParamsToStruct(q, input); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return input, nil
|
||||
}
|
||||
|
||||
// ec2Handler creates a type-safe EC2Handler. Parsing is separate from dispatch
|
||||
// so authorization and execution consume the exact same input.
|
||||
func ec2Handler[In any](handler func(ctx context.Context, input *In, gw *GatewayConfig, accountID string) (any, error)) ec2Action {
|
||||
return ec2Action{
|
||||
parse: parseEC2Input[In],
|
||||
dispatch: func(action string, parsed any, gw *GatewayConfig, accountID string, r *http.Request) ([]byte, error) {
|
||||
input, ok := parsed.(*In)
|
||||
if !ok {
|
||||
return nil, errors.New(awserrors.ErrorInternalError)
|
||||
}
|
||||
output, err := handler(requestContext(r), input, gw, accountID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
xmlOutput, err := marshalEC2Response(action, output)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return xmlOutput, nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// ec2ListsAsSet names the list fields AWS omits when empty instead of
|
||||
// rendering them. Each is rendered as its handler set it, so nil omits it.
|
||||
var ec2ListsAsSet = map[reflect.Type][]string{
|
||||
reflect.TypeFor[ec2.CreateKeyPairOutput](): {"Tags"},
|
||||
reflect.TypeFor[ec2.Image](): {"Tags", "ProductCodes"},
|
||||
reflect.TypeFor[ec2.CreateSecurityGroupOutput](): {"Tags"},
|
||||
reflect.TypeFor[ec2.SecurityGroup](): {"Tags"},
|
||||
reflect.TypeFor[ec2.SecurityGroupRule](): {"Tags"},
|
||||
reflect.TypeFor[ec2.RevokeSecurityGroupIngressOutput](): {"UnknownIpPermissions"},
|
||||
reflect.TypeFor[ec2.RevokeSecurityGroupEgressOutput](): {"UnknownIpPermissions"},
|
||||
reflect.TypeFor[ec2.NetworkInterface](): {"Ipv4Prefixes", "Ipv6Prefixes"},
|
||||
reflect.TypeFor[ec2.Vpc](): {"Ipv6CidrBlockAssociationSet"},
|
||||
reflect.TypeFor[ec2.Subnet](): {"Tags"},
|
||||
}
|
||||
|
||||
// marshalEC2Response renders an EC2 handler's output into the action's XML
|
||||
// envelope. Every EC2 action funnels through here (via ec2Handler and
|
||||
// ec2HandlerWithReq), so wire-format fixes belong here, not in each handler.
|
||||
func marshalEC2Response(action string, output any) ([]byte, error) {
|
||||
// BuildXML omits a nil slice's container element entirely but renders an
|
||||
// empty one for a non-nil empty slice; AWS mostly renders the latter.
|
||||
normalized := awsxml.NormalizeOutput(output, ec2ListsAsSet)
|
||||
// The SDK's generated output structs never carry a RequestId field.
|
||||
withRequestID := awsxml.WithRequestID(normalized, uuid.NewV4().String())
|
||||
payload := awsxml.ResponsePayload(action+"Response", withRequestID)
|
||||
xmlOutput, err := awsxml.Marshal(payload)
|
||||
if err != nil {
|
||||
return nil, errors.New("failed to marshal response to XML")
|
||||
}
|
||||
return xmlOutput, nil
|
||||
}
|
||||
|
||||
// ec2HandlerWithReq is ec2Handler for actions that need the original *http.Request,
|
||||
// e.g. RunInstances which enforces iam:PassRole on the supplied instance profile ARN.
|
||||
func ec2HandlerWithReq[In any](handler func(ctx context.Context, input *In, gw *GatewayConfig, accountID string, r *http.Request) (any, error)) ec2Action {
|
||||
return ec2Action{
|
||||
parse: parseEC2Input[In],
|
||||
dispatch: func(action string, parsed any, gw *GatewayConfig, accountID string, r *http.Request) ([]byte, error) {
|
||||
input, ok := parsed.(*In)
|
||||
if !ok {
|
||||
return nil, errors.New(awserrors.ErrorInternalError)
|
||||
}
|
||||
output, err := handler(requestContext(r), input, gw, accountID, r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
xmlOutput, err := marshalEC2Response(action, output)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return xmlOutput, nil
|
||||
},
|
||||
}
|
||||
func (q ec2Quota) ChargeLaunch(ctx context.Context, accountID string, reservation *ec2.Reservation) error {
|
||||
return q.Service.ChargeLaunch(ctx, accountID, reservation)
|
||||
}
|
||||
|
||||
// advertisedEndpoint returns this gateway's own dialable base URL, from the
|
||||
@@ -181,588 +45,32 @@ func (gw *GatewayConfig) advertisedEndpoint() string {
|
||||
return "https://" + net.JoinHostPort(host, port)
|
||||
}
|
||||
|
||||
var ec2Actions = map[string]ec2Action{
|
||||
"DescribeInstances": ec2Handler(func(ctx context.Context, input *ec2.DescribeInstancesInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
out, err := ec2instanceapi.DescribeInstancesChecked(ctx, input, gw.NATSConn, 0, gw.NodeIDs, accountID)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
ec2instanceapi.EnrichInstanceProfileIDs(out, gw.IAMService, accountID)
|
||||
return out, nil
|
||||
}),
|
||||
"RunInstances": ec2HandlerWithReq(func(ctx context.Context, input *ec2.RunInstancesInput, gw *GatewayConfig, accountID string, r *http.Request) (any, error) {
|
||||
passRoleCheck := func(roleARN string) error {
|
||||
return gw.checkPassRole(r, roleARN, ec2ServicePrincipal)
|
||||
}
|
||||
launchQuotaCheck := func() error {
|
||||
return gw.Quota.EnforceLaunch(ctx, accountID, aws.StringValue(input.InstanceType), int(aws.Int64Value(input.MaxCount)))
|
||||
}
|
||||
reservation, err := ec2instanceapi.RunInstances(ctx, input, gw.NATSConn, gw.IAMService, accountID, passRoleCheck, launchQuotaCheck, gw.ExpectedNodes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Charge the actual launched vCPUs; a counter write failure is drift for
|
||||
// reconcile to correct, so it must not fail the already-successful launch.
|
||||
if err := gw.Quota.ChargeLaunch(ctx, accountID, &reservation); err != nil {
|
||||
slog.WarnContext(ctx, "RunInstances: vcpu quota charge failed, reconcile will correct", "account", accountID, "err", err)
|
||||
}
|
||||
return reservation, nil
|
||||
}),
|
||||
"AssociateIamInstanceProfile": ec2HandlerWithReq(func(ctx context.Context, input *ec2.AssociateIamInstanceProfileInput, gw *GatewayConfig, accountID string, r *http.Request) (any, error) {
|
||||
passRoleCheck := func(roleARN string) error {
|
||||
return gw.checkPassRole(r, roleARN, ec2ServicePrincipal)
|
||||
}
|
||||
return ec2instanceapi.AssociateIamInstanceProfile(ctx, input, gw.NATSConn, gw.IAMService, accountID, passRoleCheck)
|
||||
}),
|
||||
"DisassociateIamInstanceProfile": ec2Handler(func(ctx context.Context, input *ec2.DisassociateIamInstanceProfileInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2instanceapi.DisassociateIamInstanceProfile(ctx, input, gw.NATSConn, gw.DiscoverActiveNodes(ctx), accountID)
|
||||
}),
|
||||
"ReplaceIamInstanceProfileAssociation": ec2HandlerWithReq(func(ctx context.Context, input *ec2.ReplaceIamInstanceProfileAssociationInput, gw *GatewayConfig, accountID string, r *http.Request) (any, error) {
|
||||
passRoleCheck := func(roleARN string) error {
|
||||
return gw.checkPassRole(r, roleARN, ec2ServicePrincipal)
|
||||
}
|
||||
return ec2instanceapi.ReplaceIamInstanceProfileAssociation(ctx, input, gw.NATSConn, gw.IAMService, gw.DiscoverActiveNodes(ctx), accountID, passRoleCheck)
|
||||
}),
|
||||
"DescribeIamInstanceProfileAssociations": ec2Handler(func(ctx context.Context, input *ec2.DescribeIamInstanceProfileAssociationsInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2instanceapi.DescribeIamInstanceProfileAssociations(ctx, input, gw.NATSConn, gw.DiscoverActiveNodes(ctx), accountID)
|
||||
}),
|
||||
"StartInstances": ec2Handler(func(ctx context.Context, input *ec2.StartInstancesInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2instanceapi.StartInstances(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"StopInstances": ec2Handler(func(ctx context.Context, input *ec2.StopInstancesInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2instanceapi.StopInstances(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"RebootInstances": ec2Handler(func(ctx context.Context, input *ec2.RebootInstancesInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2instanceapi.RebootInstances(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"MonitorInstances": ec2Handler(func(ctx context.Context, input *ec2.MonitorInstancesInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2instanceapi.MonitorInstances(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"UnmonitorInstances": ec2Handler(func(ctx context.Context, input *ec2.UnmonitorInstancesInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2instanceapi.UnmonitorInstances(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"TerminateInstances": ec2Handler(func(ctx context.Context, input *ec2.TerminateInstancesInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2instanceapi.TerminateInstances(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DescribeInstanceTypes": ec2Handler(func(ctx context.Context, input *ec2.DescribeInstanceTypesInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2instanceapi.DescribeInstanceTypes(ctx, input, gw.NATSConn, gw.ExpectedNodes, gw.NodeIDs, accountID)
|
||||
}),
|
||||
"DescribeInstanceTypeOfferings": ec2Handler(func(ctx context.Context, input *ec2.DescribeInstanceTypeOfferingsInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2instanceapi.DescribeInstanceTypeOfferings(ctx, input, gw.NATSConn, gw.ExpectedNodes, accountID, gw.Region, gw.AZ)
|
||||
}),
|
||||
"DescribeInstanceStatus": ec2Handler(func(ctx context.Context, input *ec2.DescribeInstanceStatusInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2instanceapi.DescribeInstanceStatus(ctx, input, gw.NATSConn, gw.DiscoverActiveNodes(ctx), accountID, gw.AZ, gw.InstanceStatus)
|
||||
}),
|
||||
"GetConsoleOutput": ec2Handler(func(ctx context.Context, input *ec2.GetConsoleOutputInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2instanceapi.GetConsoleOutput(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"GetPasswordData": ec2Handler(func(ctx context.Context, input *ec2.GetPasswordDataInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2instanceapi.GetPasswordData(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"ModifyInstanceAttribute": ec2Handler(func(ctx context.Context, input *ec2.ModifyInstanceAttributeInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
var delta int
|
||||
if input.InstanceType != nil {
|
||||
resolve := quota.NATSInstanceTypeResolver(gw.NATSConn, func() []string { return gw.NodeIDs })
|
||||
d, err := gw.Quota.EnforceRetype(ctx, resolve, accountID, aws.StringValue(input.InstanceId), aws.StringValue(input.InstanceType.Value))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
delta = d
|
||||
}
|
||||
out, err := ec2instanceapi.ModifyInstanceAttribute(ctx, input, gw.NATSConn, accountID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Charge the retype's vCPU growth; a counter write failure is drift for
|
||||
// reconcile to correct, so it must not fail the applied retype.
|
||||
if err := gw.Quota.AddVCPU(ctx, accountID, delta); err != nil {
|
||||
slog.WarnContext(ctx, "ModifyInstanceAttribute: vcpu quota charge failed, reconcile will correct", "account", accountID, "err", err)
|
||||
}
|
||||
return out, nil
|
||||
}),
|
||||
"DescribeInstanceAttribute": ec2Handler(func(ctx context.Context, input *ec2.DescribeInstanceAttributeInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2instanceapi.DescribeInstanceAttribute(ctx, input, gw.NATSConn, gw.DiscoverActiveNodes(ctx), accountID)
|
||||
}),
|
||||
"ModifyInstanceMetadataOptions": ec2Handler(func(ctx context.Context, input *ec2.ModifyInstanceMetadataOptionsInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2instanceapi.ModifyInstanceMetadataOptions(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DescribeInstanceCreditSpecifications": ec2Handler(func(ctx context.Context, input *ec2.DescribeInstanceCreditSpecificationsInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2instanceapi.DescribeInstanceCreditSpecifications(input)
|
||||
}),
|
||||
"CreateKeyPair": ec2Handler(func(ctx context.Context, input *ec2.CreateKeyPairInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2keyapi.CreateKeyPair(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DeleteKeyPair": ec2Handler(func(ctx context.Context, input *ec2.DeleteKeyPairInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2keyapi.DeleteKeyPair(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DescribeKeyPairs": ec2Handler(func(ctx context.Context, input *ec2.DescribeKeyPairsInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2keyapi.DescribeKeyPairs(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"ImportKeyPair": ec2Handler(func(ctx context.Context, input *ec2.ImportKeyPairInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2keyapi.ImportKeyPair(ctx, input, gw.NATSConn, accountID)
|
||||
}).withQueryPreprocessor(func(q map[string]string) {
|
||||
// Parser leaves Base64 padding URL-encoded; decode it before dispatch.
|
||||
if strings.HasSuffix(q["PublicKeyMaterial"], "%3D%3D") {
|
||||
q["PublicKeyMaterial"] = strings.Replace(q["PublicKeyMaterial"], "%3D%3D", "==", 1)
|
||||
}
|
||||
}),
|
||||
"DescribeImages": ec2Handler(func(ctx context.Context, input *ec2.DescribeImagesInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2imageapi.DescribeImages(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"CreateImage": ec2Handler(func(ctx context.Context, input *ec2.CreateImageInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2imageapi.CreateImage(ctx, input, gw.NATSConn, gw.DiscoverActiveNodes(ctx), accountID)
|
||||
}),
|
||||
"DeregisterImage": ec2Handler(func(ctx context.Context, input *ec2.DeregisterImageInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2imageapi.DeregisterImage(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"RegisterImage": ec2Handler(func(ctx context.Context, input *ec2.RegisterImageInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2imageapi.RegisterImage(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"CopyImage": ec2IdempotentHandler(func(ctx context.Context, input *ec2.CopyImageInput, gw *GatewayConfig, accountID string) (ec2.CopyImageOutput, error) {
|
||||
return ec2imageapi.CopyImage(ctx, input, gw.NATSConn, gw.Region, accountID)
|
||||
}),
|
||||
"DescribeImageAttribute": ec2Handler(func(ctx context.Context, input *ec2.DescribeImageAttributeInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2imageapi.DescribeImageAttribute(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"ModifyImageAttribute": ec2Handler(func(ctx context.Context, input *ec2.ModifyImageAttributeInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2imageapi.ModifyImageAttribute(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"ResetImageAttribute": ec2Handler(func(ctx context.Context, input *ec2.ResetImageAttributeInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2imageapi.ResetImageAttribute(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DescribeRegions": ec2Handler(func(ctx context.Context, input *ec2.DescribeRegionsInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2zoneapi.DescribeRegions(input, gw.Region, gw.advertisedEndpoint())
|
||||
}),
|
||||
"DescribeAvailabilityZones": ec2Handler(func(ctx context.Context, input *ec2.DescribeAvailabilityZonesInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2zoneapi.DescribeAvailabilityZones(input, gw.Region, gw.AZ)
|
||||
}),
|
||||
"DescribeVolumes": ec2Handler(func(ctx context.Context, input *ec2.DescribeVolumesInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2volumeapi.DescribeVolumes(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"ModifyVolume": ec2Handler(func(ctx context.Context, input *ec2.ModifyVolumeInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
if err := gw.Quota.EnforceVolumeModify(ctx, gw.NATSConn, accountID, aws.StringValue(input.VolumeId), int(aws.Int64Value(input.Size))); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ec2volumeapi.ModifyVolume(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"CreateVolume": ec2IdempotentHandler(func(ctx context.Context, input *ec2.CreateVolumeInput, gw *GatewayConfig, accountID string) (ec2.Volume, error) {
|
||||
if err := gw.Quota.EnforceVolumeCreate(ctx, gw.NATSConn, accountID, int(aws.Int64Value(input.Size))); err != nil {
|
||||
return ec2.Volume{}, err
|
||||
}
|
||||
return ec2volumeapi.CreateVolume(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DeleteVolume": ec2Handler(func(ctx context.Context, input *ec2.DeleteVolumeInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2volumeapi.DeleteVolume(ctx, input, gw.NATSConn, gw.NodeIDs, accountID)
|
||||
}),
|
||||
"AttachVolume": ec2Handler(func(ctx context.Context, input *ec2.AttachVolumeInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2volumeapi.AttachVolume(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DescribeVolumeStatus": ec2Handler(func(ctx context.Context, input *ec2.DescribeVolumeStatusInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2volumeapi.DescribeVolumeStatus(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DescribeVolumesModifications": ec2Handler(func(ctx context.Context, input *ec2.DescribeVolumesModificationsInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2volumeapi.DescribeVolumesModifications(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DetachVolume": ec2Handler(func(ctx context.Context, input *ec2.DetachVolumeInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2volumeapi.DetachVolume(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DescribeAccountAttributes": ec2Handler(func(ctx context.Context, input *ec2.DescribeAccountAttributesInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2accountapi.DescribeAccountAttributes(input, func() (string, error) {
|
||||
return defaultVPCID(ctx, gw.NATSConn, accountID)
|
||||
})
|
||||
}),
|
||||
"EnableEbsEncryptionByDefault": ec2Handler(func(ctx context.Context, input *ec2.EnableEbsEncryptionByDefaultInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2accountapi.EnableEbsEncryptionByDefault(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DisableEbsEncryptionByDefault": ec2Handler(func(ctx context.Context, input *ec2.DisableEbsEncryptionByDefaultInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2accountapi.DisableEbsEncryptionByDefault(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"GetEbsEncryptionByDefault": ec2Handler(func(ctx context.Context, input *ec2.GetEbsEncryptionByDefaultInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2accountapi.GetEbsEncryptionByDefault(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"GetSerialConsoleAccessStatus": ec2Handler(func(ctx context.Context, input *ec2.GetSerialConsoleAccessStatusInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2accountapi.GetSerialConsoleAccessStatus(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"EnableSerialConsoleAccess": ec2Handler(func(ctx context.Context, input *ec2.EnableSerialConsoleAccessInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2accountapi.EnableSerialConsoleAccess(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DisableSerialConsoleAccess": ec2Handler(func(ctx context.Context, input *ec2.DisableSerialConsoleAccessInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2accountapi.DisableSerialConsoleAccess(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"CreateTags": ec2Handler(func(ctx context.Context, input *ec2.CreateTagsInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2tagsapi.CreateTags(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DeleteTags": ec2Handler(func(ctx context.Context, input *ec2.DeleteTagsInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2tagsapi.DeleteTags(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DescribeTags": ec2Handler(func(ctx context.Context, input *ec2.DescribeTagsInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2tagsapi.DescribeTags(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"CreateSnapshot": ec2Handler(func(ctx context.Context, input *ec2.CreateSnapshotInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2snapshotapi.CreateSnapshot(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DeleteSnapshot": ec2Handler(func(ctx context.Context, input *ec2.DeleteSnapshotInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2snapshotapi.DeleteSnapshot(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DescribeSnapshots": ec2Handler(func(ctx context.Context, input *ec2.DescribeSnapshotsInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2snapshotapi.DescribeSnapshots(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"CopySnapshot": ec2Handler(func(ctx context.Context, input *ec2.CopySnapshotInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2snapshotapi.CopySnapshot(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"CreateInternetGateway": ec2Handler(func(ctx context.Context, input *ec2.CreateInternetGatewayInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2igwapi.CreateInternetGateway(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DeleteInternetGateway": ec2Handler(func(ctx context.Context, input *ec2.DeleteInternetGatewayInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2igwapi.DeleteInternetGateway(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DescribeInternetGateways": ec2Handler(func(ctx context.Context, input *ec2.DescribeInternetGatewaysInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2igwapi.DescribeInternetGateways(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"AttachInternetGateway": ec2Handler(func(ctx context.Context, input *ec2.AttachInternetGatewayInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2igwapi.AttachInternetGateway(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DetachInternetGateway": ec2Handler(func(ctx context.Context, input *ec2.DetachInternetGatewayInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2igwapi.DetachInternetGateway(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"CreateEgressOnlyInternetGateway": ec2IdempotentHandler(func(ctx context.Context, input *ec2.CreateEgressOnlyInternetGatewayInput, gw *GatewayConfig, accountID string) (ec2.CreateEgressOnlyInternetGatewayOutput, error) {
|
||||
return ec2eigwapi.CreateEgressOnlyInternetGateway(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DeleteEgressOnlyInternetGateway": ec2Handler(func(ctx context.Context, input *ec2.DeleteEgressOnlyInternetGatewayInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2eigwapi.DeleteEgressOnlyInternetGateway(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DescribeEgressOnlyInternetGateways": ec2Handler(func(ctx context.Context, input *ec2.DescribeEgressOnlyInternetGatewaysInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2eigwapi.DescribeEgressOnlyInternetGateways(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"CreatePlacementGroup": ec2Handler(func(ctx context.Context, input *ec2.CreatePlacementGroupInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2placementgroupapi.CreatePlacementGroup(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DeletePlacementGroup": ec2Handler(func(ctx context.Context, input *ec2.DeletePlacementGroupInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2placementgroupapi.DeletePlacementGroup(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DescribePlacementGroups": ec2Handler(func(ctx context.Context, input *ec2.DescribePlacementGroupsInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2placementgroupapi.DescribePlacementGroups(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"CreateCapacityReservation": ec2IdempotentHandler(func(ctx context.Context, input *ec2.CreateCapacityReservationInput, gw *GatewayConfig, accountID string) (ec2.CreateCapacityReservationOutput, error) {
|
||||
return ec2capacityreservationapi.CreateCapacityReservation(ctx, input, gw.NATSConn, gw.ExpectedNodes, accountID)
|
||||
}),
|
||||
"DescribeCapacityReservations": ec2Handler(func(ctx context.Context, input *ec2.DescribeCapacityReservationsInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2capacityreservationapi.DescribeCapacityReservations(ctx, input, gw.NATSConn, gw.ExpectedNodes, accountID)
|
||||
}),
|
||||
"CancelCapacityReservation": ec2Handler(func(ctx context.Context, input *ec2.CancelCapacityReservationInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2capacityreservationapi.CancelCapacityReservation(ctx, input, gw.NATSConn, gw.ExpectedNodes, accountID)
|
||||
}),
|
||||
"CreateLaunchTemplate": ec2Handler(func(ctx context.Context, input *ec2.CreateLaunchTemplateInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2launchtemplateapi.CreateLaunchTemplate(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"CreateLaunchTemplateVersion": ec2IdempotentHandler(func(ctx context.Context, input *ec2.CreateLaunchTemplateVersionInput, gw *GatewayConfig, accountID string) (ec2.CreateLaunchTemplateVersionOutput, error) {
|
||||
return ec2launchtemplateapi.CreateLaunchTemplateVersion(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DeleteLaunchTemplate": ec2Handler(func(ctx context.Context, input *ec2.DeleteLaunchTemplateInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2launchtemplateapi.DeleteLaunchTemplate(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DeleteLaunchTemplateVersions": ec2Handler(func(ctx context.Context, input *ec2.DeleteLaunchTemplateVersionsInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2launchtemplateapi.DeleteLaunchTemplateVersions(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"ModifyLaunchTemplate": ec2Handler(func(ctx context.Context, input *ec2.ModifyLaunchTemplateInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2launchtemplateapi.ModifyLaunchTemplate(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DescribeLaunchTemplates": ec2Handler(func(ctx context.Context, input *ec2.DescribeLaunchTemplatesInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2launchtemplateapi.DescribeLaunchTemplates(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DescribeLaunchTemplateVersions": ec2Handler(func(ctx context.Context, input *ec2.DescribeLaunchTemplateVersionsInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2launchtemplateapi.DescribeLaunchTemplateVersions(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"RequestSpotInstances": ec2HandlerWithReq(func(ctx context.Context, input *ec2.RequestSpotInstancesInput, gw *GatewayConfig, accountID string, r *http.Request) (any, error) {
|
||||
passRoleCheck := func(roleARN string) error {
|
||||
return gw.checkPassRole(r, roleARN, ec2ServicePrincipal)
|
||||
}
|
||||
return ec2spotinstanceapi.RequestSpotInstances(ctx, input, gw.NATSConn, gw.IAMService, accountID, gw.AZ, passRoleCheck, gw.Quota, gw.ExpectedNodes)
|
||||
}),
|
||||
"DescribeSpotInstanceRequests": ec2Handler(func(ctx context.Context, input *ec2.DescribeSpotInstanceRequestsInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2spotinstanceapi.DescribeSpotInstanceRequests(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"CancelSpotInstanceRequests": ec2Handler(func(ctx context.Context, input *ec2.CancelSpotInstanceRequestsInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2spotinstanceapi.CancelSpotInstanceRequests(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"CreateVpc": ec2Handler(func(ctx context.Context, input *ec2.CreateVpcInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
if err := gw.Quota.EnforceVPCs(ctx, gw.NATSConn, accountID, 1); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ec2vpcapi.CreateVpc(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DeleteVpc": ec2Handler(func(ctx context.Context, input *ec2.DeleteVpcInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2vpcapi.DeleteVpc(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DescribeVpcs": ec2Handler(func(ctx context.Context, input *ec2.DescribeVpcsInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2vpcapi.DescribeVpcs(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"ModifyVpcAttribute": ec2Handler(func(ctx context.Context, input *ec2.ModifyVpcAttributeInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2vpcapi.ModifyVpcAttribute(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DescribeVpcAttribute": ec2Handler(func(ctx context.Context, input *ec2.DescribeVpcAttributeInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2vpcapi.DescribeVpcAttribute(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
// No VPC endpoint can exist here, because CreateVpcEndpoint is not served, so
|
||||
// the empty list is the true answer rather than a stand-in for one. It has to
|
||||
// be answered all the same: the Terraform AWS provider calls this on every VPC
|
||||
// delete to sweep GuardDuty-managed endpoints, and an InvalidAction there fails
|
||||
// the destroy and strands the VPC.
|
||||
"DescribeVpcEndpoints": ec2Handler(func(_ context.Context, _ *ec2.DescribeVpcEndpointsInput, _ *GatewayConfig, _ string) (any, error) {
|
||||
return ec2.DescribeVpcEndpointsOutput{VpcEndpoints: []*ec2.VpcEndpoint{}}, nil
|
||||
}),
|
||||
"CreateSubnet": ec2Handler(func(ctx context.Context, input *ec2.CreateSubnetInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
if err := gw.Quota.EnforceSubnets(ctx, gw.NATSConn, accountID, 1); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ec2vpcapi.CreateSubnet(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DeleteSubnet": ec2Handler(func(ctx context.Context, input *ec2.DeleteSubnetInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2vpcapi.DeleteSubnet(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DescribeSubnets": ec2Handler(func(ctx context.Context, input *ec2.DescribeSubnetsInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2vpcapi.DescribeSubnets(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"ModifySubnetAttribute": ec2Handler(func(ctx context.Context, input *ec2.ModifySubnetAttributeInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2vpcapi.ModifySubnetAttribute(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"CreateRouteTable": ec2IdempotentHandler(func(ctx context.Context, input *ec2.CreateRouteTableInput, gw *GatewayConfig, accountID string) (ec2.CreateRouteTableOutput, error) {
|
||||
return ec2routetableapi.CreateRouteTable(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DeleteRouteTable": ec2Handler(func(ctx context.Context, input *ec2.DeleteRouteTableInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2routetableapi.DeleteRouteTable(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DescribeRouteTables": ec2Handler(func(ctx context.Context, input *ec2.DescribeRouteTablesInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2routetableapi.DescribeRouteTables(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"CreateRoute": ec2Handler(func(ctx context.Context, input *ec2.CreateRouteInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2routetableapi.CreateRoute(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DeleteRoute": ec2Handler(func(ctx context.Context, input *ec2.DeleteRouteInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2routetableapi.DeleteRoute(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"ReplaceRoute": ec2Handler(func(ctx context.Context, input *ec2.ReplaceRouteInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2routetableapi.ReplaceRoute(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"AssociateRouteTable": ec2Handler(func(ctx context.Context, input *ec2.AssociateRouteTableInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2routetableapi.AssociateRouteTable(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DisassociateRouteTable": ec2Handler(func(ctx context.Context, input *ec2.DisassociateRouteTableInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2routetableapi.DisassociateRouteTable(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"ReplaceRouteTableAssociation": ec2Handler(func(ctx context.Context, input *ec2.ReplaceRouteTableAssociationInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2routetableapi.ReplaceRouteTableAssociation(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"CreateNetworkInterface": ec2IdempotentHandler(func(ctx context.Context, input *ec2.CreateNetworkInterfaceInput, gw *GatewayConfig, accountID string) (ec2.CreateNetworkInterfaceOutput, error) {
|
||||
return ec2vpcapi.CreateNetworkInterface(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DeleteNetworkInterface": ec2Handler(func(ctx context.Context, input *ec2.DeleteNetworkInterfaceInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2vpcapi.DeleteNetworkInterface(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DescribeNetworkInterfaces": ec2Handler(func(ctx context.Context, input *ec2.DescribeNetworkInterfacesInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2vpcapi.DescribeNetworkInterfaces(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"ModifyNetworkInterfaceAttribute": ec2Handler(func(ctx context.Context, input *ec2.ModifyNetworkInterfaceAttributeInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2vpcapi.ModifyNetworkInterfaceAttribute(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"AttachNetworkInterface": ec2Handler(func(ctx context.Context, input *ec2.AttachNetworkInterfaceInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2vpcapi.AttachNetworkInterface(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DetachNetworkInterface": ec2Handler(func(ctx context.Context, input *ec2.DetachNetworkInterfaceInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2vpcapi.DetachNetworkInterface(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"CreateSecurityGroup": ec2Handler(func(ctx context.Context, input *ec2.CreateSecurityGroupInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2vpcapi.CreateSecurityGroup(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DeleteSecurityGroup": ec2Handler(func(ctx context.Context, input *ec2.DeleteSecurityGroupInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2vpcapi.DeleteSecurityGroup(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DescribeSecurityGroups": ec2Handler(func(ctx context.Context, input *ec2.DescribeSecurityGroupsInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2vpcapi.DescribeSecurityGroups(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DescribeSecurityGroupRules": ec2Handler(func(ctx context.Context, input *ec2.DescribeSecurityGroupRulesInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2vpcapi.DescribeSecurityGroupRules(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"GetSecurityGroupsForVpc": ec2Handler(func(ctx context.Context, input *ec2.GetSecurityGroupsForVpcInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2vpcapi.GetSecurityGroupsForVpc(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"AuthorizeSecurityGroupIngress": ec2Handler(func(ctx context.Context, input *ec2.AuthorizeSecurityGroupIngressInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2vpcapi.AuthorizeSecurityGroupIngress(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"AuthorizeSecurityGroupEgress": ec2Handler(func(ctx context.Context, input *ec2.AuthorizeSecurityGroupEgressInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2vpcapi.AuthorizeSecurityGroupEgress(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"RevokeSecurityGroupIngress": ec2Handler(func(ctx context.Context, input *ec2.RevokeSecurityGroupIngressInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2vpcapi.RevokeSecurityGroupIngress(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"RevokeSecurityGroupEgress": ec2Handler(func(ctx context.Context, input *ec2.RevokeSecurityGroupEgressInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2vpcapi.RevokeSecurityGroupEgress(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"UpdateSecurityGroupRuleDescriptionsIngress": ec2Handler(func(ctx context.Context, input *ec2.UpdateSecurityGroupRuleDescriptionsIngressInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2vpcapi.UpdateSecurityGroupRuleDescriptionsIngress(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"UpdateSecurityGroupRuleDescriptionsEgress": ec2Handler(func(ctx context.Context, input *ec2.UpdateSecurityGroupRuleDescriptionsEgressInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2vpcapi.UpdateSecurityGroupRuleDescriptionsEgress(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"ModifySecurityGroupRules": ec2Handler(func(ctx context.Context, input *ec2.ModifySecurityGroupRulesInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2vpcapi.ModifySecurityGroupRules(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"AllocateAddress": ec2Handler(func(ctx context.Context, input *ec2.AllocateAddressInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
if err := gw.Quota.EnforceEIPs(ctx, gw.NATSConn, accountID, 1); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ec2eipapi.AllocateAddress(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"ReleaseAddress": ec2Handler(func(ctx context.Context, input *ec2.ReleaseAddressInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2eipapi.ReleaseAddress(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"AssociateAddress": ec2Handler(func(ctx context.Context, input *ec2.AssociateAddressInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2eipapi.AssociateAddress(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DisassociateAddress": ec2Handler(func(ctx context.Context, input *ec2.DisassociateAddressInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2eipapi.DisassociateAddress(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DescribeAddresses": ec2Handler(func(ctx context.Context, input *ec2.DescribeAddressesInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2eipapi.DescribeAddresses(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DescribeAddressesAttribute": ec2Handler(func(ctx context.Context, input *ec2.DescribeAddressesAttributeInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2eipapi.DescribeAddressesAttribute(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"CreateNatGateway": ec2IdempotentHandler(func(ctx context.Context, input *ec2.CreateNatGatewayInput, gw *GatewayConfig, accountID string) (ec2.CreateNatGatewayOutput, error) {
|
||||
return ec2natgwapi.CreateNatGateway(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DeleteNatGateway": ec2Handler(func(ctx context.Context, input *ec2.DeleteNatGatewayInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2natgwapi.DeleteNatGateway(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DescribeNatGateways": ec2Handler(func(ctx context.Context, input *ec2.DescribeNatGatewaysInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2natgwapi.DescribeNatGateways(ctx, input, gw.NATSConn, accountID)
|
||||
}),
|
||||
}
|
||||
|
||||
// defaultVPCID returns the ID of accountID's default VPC, or "" when it has
|
||||
// none, as the account's own DescribeVpcs sees it.
|
||||
func defaultVPCID(ctx context.Context, natsConn *nats.Conn, accountID string) (string, error) {
|
||||
if natsConn == nil {
|
||||
return "", errors.New(awserrors.ErrorServerInternal)
|
||||
}
|
||||
out, err := ec2vpcapi.DescribeVpcs(ctx, &ec2.DescribeVpcsInput{
|
||||
Filters: []*ec2.Filter{{Name: aws.String("is-default"), Values: aws.StringSlice([]string{"true"})}},
|
||||
}, natsConn, accountID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(out.Vpcs) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
return aws.StringValue(out.Vpcs[0].VpcId), nil
|
||||
}
|
||||
|
||||
// ec2LocalActions are actions that don't require NATS. DescribeAccountAttributes
|
||||
// needs it only for default-vpc.
|
||||
var ec2LocalActions = map[string]bool{
|
||||
"DescribeRegions": true,
|
||||
"DescribeAvailabilityZones": true,
|
||||
"DescribeAccountAttributes": true,
|
||||
// ec2Deps composes the capabilities the EC2 actions dispatch to from the
|
||||
// gateway's own configuration.
|
||||
func (gw *GatewayConfig) ec2Deps() ec2awsapi.Deps {
|
||||
return ec2awsapi.Deps{
|
||||
NATS: gw.NATSConn,
|
||||
IAM: gw.IAMService,
|
||||
Quota: ec2Quota{gw.Quota},
|
||||
ActiveNodes: gw.DiscoverActiveNodes,
|
||||
ExpectedNodes: gw.ExpectedNodes,
|
||||
NodeIDs: gw.NodeIDs,
|
||||
AZ: gw.AZ,
|
||||
Endpoint: gw.advertisedEndpoint(),
|
||||
InstanceStatus: gw.InstanceStatus,
|
||||
}
|
||||
}
|
||||
|
||||
// EC2_Request serves one authenticated EC2 request.
|
||||
func (gw *GatewayConfig) EC2_Request(w http.ResponseWriter, r *http.Request) error {
|
||||
queryArgs, err := readQueryArgs(r)
|
||||
if err != nil {
|
||||
slog.Debug("EC2: malformed query string", "err", err)
|
||||
return errors.New(awserrors.ErrorMalformedQueryString)
|
||||
}
|
||||
|
||||
action := queryArgs["Action"]
|
||||
if action == "" {
|
||||
return errors.New(awserrors.ErrorMissingAction)
|
||||
}
|
||||
handler, ok := ec2Actions[action]
|
||||
if !ok {
|
||||
return errors.New(awserrors.ErrorInvalidAction)
|
||||
}
|
||||
|
||||
// Hoisted above the policy check because the resolver builds ARNs from it.
|
||||
// gw.Region, never the caller-supplied credential-scope region, which would
|
||||
// let a caller sign for another region and slide out from under a Deny.
|
||||
accountID := callerPrincipalOf(r).accountID
|
||||
if accountID == "" {
|
||||
slog.Error("EC2_Request: no account ID in auth context")
|
||||
// InternalError, not ServerInternal: the policy gate used to reach this
|
||||
// case first and that is the code the caller has always seen.
|
||||
return errors.New(awserrors.ErrorInternalError)
|
||||
}
|
||||
|
||||
input, err := handler.parse(queryArgs)
|
||||
if err != nil {
|
||||
if errors.Is(err, query.ErrSliceTooLarge) {
|
||||
return errors.New(awserrors.ErrorMalformedQueryString)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Launch templates supply effective RunInstances resources. Resolve them
|
||||
// before authorization, and dispatch the same expanded input afterwards.
|
||||
if runInput, ok := input.(*ec2.RunInstancesInput); ok && action == "RunInstances" {
|
||||
if err := ec2instanceapi.ExpandLaunchTemplate(r.Context(), gw.NATSConn, runInput, accountID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
resources, err := ec2awsapi.ResourceARNs(action, gw.Region, accountID, input)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := gw.checkPolicyResources(r, "ec2", action, resources); err != nil {
|
||||
// EC2 answers a policy denial with UnauthorizedOperation, not AccessDenied.
|
||||
if code, _ := awserrors.ResolveErrorCode(err); code == awserrors.ErrorAccessDenied {
|
||||
return errors.New(awserrors.ErrorUnauthorizedOperation)
|
||||
}
|
||||
return err
|
||||
}
|
||||
// AWS refuses a bad tag resource type even on a DryRun request.
|
||||
if err := validateTagSpecificationTypes(action, input); err != nil {
|
||||
return err
|
||||
}
|
||||
if dryRunRequested(input) {
|
||||
return errors.New(awserrors.ErrorDryRunOperation)
|
||||
}
|
||||
if err := validateMaxResults(action, input); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if gw.NATSConn == nil && !ec2LocalActions[action] {
|
||||
return errors.New(awserrors.ErrorServerInternal)
|
||||
}
|
||||
|
||||
xmlOutput, err := handler.dispatch(action, input, gw, accountID, r)
|
||||
if err != nil {
|
||||
// A gateway iam:PassRole denial is EC2's UnauthorizedOperation too; an
|
||||
// AccessDenied a daemon returns is left as it is.
|
||||
if _, denied := errors.AsType[*policyauth.DenialError](err); denied {
|
||||
return errors.New(awserrors.ErrorUnauthorizedOperation)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "text/xml")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
if _, err := w.Write(xmlOutput); err != nil {
|
||||
slog.Error("Failed to write EC2 response", "err", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// dryRunRequested reports whether a parsed EC2 input carries DryRun=true. Only
|
||||
// actions whose SDK input models the parameter can answer DryRunOperation.
|
||||
func dryRunRequested(input any) bool {
|
||||
v := reflect.ValueOf(input)
|
||||
if v.Kind() != reflect.Pointer || v.IsNil() || v.Elem().Kind() != reflect.Struct {
|
||||
return false
|
||||
}
|
||||
field := v.Elem().FieldByName("DryRun")
|
||||
if !field.IsValid() {
|
||||
return false
|
||||
}
|
||||
dryRun, ok := reflect.TypeAssert[*bool](field)
|
||||
return ok && aws.BoolValue(dryRun)
|
||||
gw.ec2Once.Do(func() { gw.ec2Handler = ec2awsapi.NewHandler(gw.ec2Deps()) })
|
||||
return gw.ec2Handler.Dispatch(w, dispatch.Invocation{
|
||||
Request: r,
|
||||
AccountID: callerPrincipalOf(r).accountID,
|
||||
Service: ec2awsapi.ServiceName,
|
||||
Region: gw.Region,
|
||||
Authorize: func(service, action string, resources []string, keys iampolicy.ConditionKeys) error {
|
||||
return gw.checkPolicyResourcesWithKeys(r, service, action, resources, keys)
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/gateway"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestAdvertisedEndpoint_ResolvesFromRegistryHost asserts DescribeRegions'
|
||||
@@ -33,17 +32,3 @@ func TestAdvertisedEndpoint_ResolvesFromRegistryHost(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestDescribeRegions_UsesAdvertisedEndpoint drives the real dispatch entry
|
||||
// EC2_Request uses and asserts the resolved gateway address reaches the wire
|
||||
// response. A caller on a concrete-host deployment must never be told to
|
||||
// dial its own loopback, which is the defect this guards against.
|
||||
func TestDescribeRegions_UsesAdvertisedEndpoint(t *testing.T) {
|
||||
gw := &gateway.GatewayConfig{Region: "ap-southeast-2", RegistryHost: "10.0.0.5", RegistryPort: "9999"}
|
||||
xmlOutput, err := gateway.DispatchEC2Action(gw, "DescribeRegions", map[string]string{}, "acct-123")
|
||||
require.NoError(t, err)
|
||||
|
||||
body := string(xmlOutput)
|
||||
assert.Contains(t, body, "<regionEndpoint>https://10.0.0.5:9999</regionEndpoint>")
|
||||
assert.NotContains(t, body, "localhost", "must not advertise the gateway's own loopback")
|
||||
}
|
||||
|
||||
@@ -1,16 +1,14 @@
|
||||
//test:in-package — walks the unexported ec2Actions route table and drives
|
||||
// EC2_Request through the gateway's unexported test helpers.
|
||||
//test:in-package — drives EC2_Request through the gateway's unexported test
|
||||
// helpers.
|
||||
|
||||
package gateway
|
||||
|
||||
import (
|
||||
"maps"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"reflect"
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
ec2awsapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -18,16 +16,6 @@ import (
|
||||
|
||||
const dryRunMessage = "Request would have succeeded, but DryRun flag is set."
|
||||
|
||||
// modelsDryRun reports whether an action's SDK input carries a DryRun field,
|
||||
// independently of the gateway's own check.
|
||||
func modelsDryRun(t *testing.T, action string) bool {
|
||||
t.Helper()
|
||||
input, err := ec2Actions[action].parse(map[string]string{"Action": action})
|
||||
require.NoError(t, err)
|
||||
_, ok := reflect.TypeOf(input).Elem().FieldByName("DryRun")
|
||||
return ok
|
||||
}
|
||||
|
||||
// serveEC2 runs a request through EC2_Request and, on error, ErrorHandler, so
|
||||
// the assertion sees the status and body a client would.
|
||||
func serveEC2(gw *GatewayConfig, body string) *httptest.ResponseRecorder {
|
||||
@@ -44,12 +32,12 @@ func serveEC2(gw *GatewayConfig, body string) *httptest.ResponseRecorder {
|
||||
func TestEC2Request_DryRunAnswersEveryAction(t *testing.T) {
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, Region: authzRegion, IAMService: allowAllIAMService()})
|
||||
|
||||
actions := slices.Sorted(maps.Keys(ec2Actions))
|
||||
actions := ec2awsapi.ActionNames()
|
||||
|
||||
for _, action := range actions {
|
||||
t.Run(action, func(t *testing.T) {
|
||||
w := serveEC2(gw, "Action="+action+"&DryRun=true")
|
||||
if !modelsDryRun(t, action) {
|
||||
if !ec2awsapi.InputHasField(action, "DryRun") {
|
||||
assert.NotContains(t, w.Body.String(), awserrors.ErrorDryRunOperation)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -0,0 +1,460 @@
|
||||
// Characterization tests for EC2 through the full SigV4 pipeline as the legacy
|
||||
// gateway switch and ec2.go implement them today. They assert current
|
||||
// behaviour, not AWS parity.
|
||||
|
||||
package gateway
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/mulgadc/bluebottle/pkg/iampolicy"
|
||||
ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
|
||||
"github.com/mulgadc/spinifex/internal/testkit"
|
||||
iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
|
||||
awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// ec2Daemon stands in for the daemons behind the ec2.* NATS subjects. It
|
||||
// records what the gateway forwarded and answers every subject with an empty
|
||||
// output unless a case needs a real shape.
|
||||
type ec2Daemon struct {
|
||||
mu sync.Mutex
|
||||
calls map[string]int
|
||||
bodies map[string][]byte
|
||||
eigw int
|
||||
// replies overrides the canned answer for a subject.
|
||||
replies map[string]string
|
||||
}
|
||||
|
||||
func (d *ec2Daemon) reply(subject, body string) {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
d.replies[subject] = body
|
||||
}
|
||||
|
||||
func newEC2Daemon(t *testing.T, nc *nats.Conn) *ec2Daemon {
|
||||
t.Helper()
|
||||
d := &ec2Daemon{calls: map[string]int{}, bodies: map[string][]byte{}, replies: map[string]string{}}
|
||||
sub, err := nc.Subscribe("ec2.>", func(m *nats.Msg) {
|
||||
d.mu.Lock()
|
||||
d.calls[m.Subject]++
|
||||
d.bodies[m.Subject] = append([]byte(nil), m.Data...)
|
||||
reply := `{}`
|
||||
switch m.Subject {
|
||||
case "ec2.CreateEgressOnlyInternetGateway":
|
||||
d.eigw++
|
||||
reply = fmt.Sprintf(`{"EgressOnlyInternetGateway":{"EgressOnlyInternetGatewayId":"eigw-%04d"}}`, d.eigw)
|
||||
case "ec2.DescribeVpcs":
|
||||
reply = `{"Vpcs":[]}`
|
||||
}
|
||||
if r, ok := d.replies[m.Subject]; ok {
|
||||
reply = r
|
||||
}
|
||||
d.mu.Unlock()
|
||||
_ = m.Respond([]byte(reply))
|
||||
})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
require.NoError(t, nc.Flush())
|
||||
return d
|
||||
}
|
||||
|
||||
func (d *ec2Daemon) count(subject string) int {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
return d.calls[subject]
|
||||
}
|
||||
|
||||
func (d *ec2Daemon) body(subject string) []byte {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
return d.bodies[subject]
|
||||
}
|
||||
|
||||
// ec2PipelineEnv is what a case may change before the app is built.
|
||||
type ec2PipelineEnv struct {
|
||||
NoNATS bool
|
||||
ResolveErr error
|
||||
}
|
||||
|
||||
type ec2PipelineApp struct {
|
||||
Handler http.Handler
|
||||
Daemon *ec2Daemon
|
||||
}
|
||||
|
||||
// ec2Pipeline wires SigV4AuthMiddleware -> Request as SetupRoutes does, with
|
||||
// EC2 on the legacy switch over a real JetStream and a stub daemon. Policy
|
||||
// statements are the caller's only IAM policy.
|
||||
func ec2Pipeline(t *testing.T, mutate func(*ec2PipelineEnv), statements ...iamdomain.Statement) ec2PipelineApp {
|
||||
t.Helper()
|
||||
encryptedSecret, err := iamdomain.EncryptSecret(testSecretKey, testMasterKey)
|
||||
require.NoError(t, err)
|
||||
docs := []iamdomain.PolicyDocument{{Version: "2012-10-17", Statement: statements}}
|
||||
env := &ec2PipelineEnv{}
|
||||
if mutate != nil {
|
||||
mutate(env)
|
||||
}
|
||||
svc := &profileIAMService{resolveErr: env.ResolveErr, policyMockIAMService: policyMockIAMService{
|
||||
mockIAMService: mockIAMService{
|
||||
masterKey: testMasterKey,
|
||||
accessKeys: map[string]*iamdomain.AccessKey{
|
||||
testAccessKey: {
|
||||
AccessKeyID: testAccessKey,
|
||||
SecretAccessKey: encryptedSecret,
|
||||
UserName: "alice",
|
||||
AccountID: authzAccountID,
|
||||
Status: "Active",
|
||||
},
|
||||
},
|
||||
},
|
||||
getUserPoliciesFn: func(_, _ string) ([]iamdomain.PolicyDocument, error) { return docs, nil },
|
||||
}}
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
daemon := newEC2Daemon(t, nc)
|
||||
|
||||
cfg := &GatewayConfig{
|
||||
DisableLogging: true,
|
||||
Region: testRegion,
|
||||
AZ: testRegion + "a",
|
||||
RegistryHost: "registry.example",
|
||||
RegistryPort: "443",
|
||||
NATSConn: nc,
|
||||
IAMService: svc,
|
||||
ExpectedNodes: 1,
|
||||
}
|
||||
if env.NoNATS {
|
||||
cfg.NATSConn = nil
|
||||
}
|
||||
gw := withVerifier(cfg)
|
||||
r := chi.NewRouter()
|
||||
r.Use(gw.SigV4AuthMiddleware())
|
||||
r.HandleFunc("/*", gw.Request)
|
||||
return ec2PipelineApp{Handler: r, Daemon: daemon}
|
||||
}
|
||||
|
||||
func signEC2Request(t *testing.T, body string) *http.Request {
|
||||
t.Helper()
|
||||
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body))
|
||||
req.Host = "localhost:9999"
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=utf-8")
|
||||
signTestRequestAs(t, req, []byte(body), testAccessKey, testSecretKey, "ec2", time.Now().UTC())
|
||||
return req
|
||||
}
|
||||
|
||||
type ec2ErrorBody struct {
|
||||
XMLName xml.Name `xml:"Response"`
|
||||
Errors struct {
|
||||
Error struct {
|
||||
Code string `xml:"Code"`
|
||||
Message string `xml:"Message"`
|
||||
} `xml:"Error"`
|
||||
} `xml:"Errors"`
|
||||
RequestID string `xml:"RequestID"`
|
||||
}
|
||||
|
||||
type ec2Outcome struct {
|
||||
Status int
|
||||
ContentType string
|
||||
Body string
|
||||
Code string
|
||||
Message string
|
||||
RequestID string
|
||||
}
|
||||
|
||||
func probeEC2(t *testing.T, app http.Handler, body string) ec2Outcome {
|
||||
t.Helper()
|
||||
resp := doRequest(app, signEC2Request(t, body))
|
||||
raw, err := io.ReadAll(resp.Body)
|
||||
require.NoError(t, err)
|
||||
out := ec2Outcome{Status: resp.StatusCode, ContentType: resp.Header.Get("Content-Type"), Body: string(raw)}
|
||||
if resp.StatusCode >= 400 {
|
||||
var env ec2ErrorBody
|
||||
require.NoError(t, xml.Unmarshal(raw, &env), string(raw))
|
||||
out.Code, out.Message, out.RequestID = env.Errors.Error.Code, env.Errors.Error.Message, env.RequestID
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func allowEC2() iamdomain.Statement { return statement("Allow", "ec2:*", "*") }
|
||||
|
||||
// TestEC2Pipeline_ErrorEnvelopes pins status, code and the EC2 error envelope
|
||||
// for each way a request can be refused before or at dispatch.
|
||||
func TestEC2Pipeline_ErrorEnvelopes(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
noNATS bool
|
||||
deny bool
|
||||
body string
|
||||
status int
|
||||
code string
|
||||
}{
|
||||
{name: "missing action", body: "Version=2016-11-15", status: 400, code: "MissingAction"},
|
||||
{name: "invalid action", body: "Action=NoSuchAction", status: 400, code: "InvalidAction"},
|
||||
{name: "policy denial is UnauthorizedOperation", deny: true, body: "Action=DescribeKeyPairs", status: 403, code: "UnauthorizedOperation"},
|
||||
{name: "dry run", body: "Action=DescribeKeyPairs&DryRun=true", status: 412, code: "DryRunOperation"},
|
||||
{name: "max results out of range", body: "Action=DescribeInstances&MaxResults=1", status: 400, code: "InvalidParameterValue"},
|
||||
{name: "tag resource type not taggable", body: "Action=CreateKeyPair&KeyName=k&TagSpecification.1.ResourceType=instance", status: 400, code: "InvalidParameterValue"},
|
||||
{name: "cluster unavailable, non-local action", noNATS: true, body: "Action=DescribeKeyPairs", status: 503, code: "ServiceUnavailable"},
|
||||
{name: "cluster unavailable, local action", noNATS: true, body: "Action=DescribeRegions", status: 503, code: "ServiceUnavailable"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
var stmts []iamdomain.Statement
|
||||
if !c.deny {
|
||||
stmts = append(stmts, allowEC2())
|
||||
}
|
||||
app := ec2Pipeline(t, func(e *ec2PipelineEnv) { e.NoNATS = c.noNATS }, stmts...)
|
||||
got := probeEC2(t, app.Handler, c.body)
|
||||
require.Equal(t, c.status, got.Status, got.Body)
|
||||
assert.Equal(t, c.code, got.Code, got.Body)
|
||||
assert.NotEmpty(t, got.RequestID, "error body carries a request id: %s", got.Body)
|
||||
assert.True(t, strings.HasPrefix(got.Body, "<?xml"), got.Body)
|
||||
assert.Contains(t, got.ContentType, "xml")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestEC2Pipeline_CheckOrder pins which refusal wins when a request earns
|
||||
// several: policy, then tag types, then DryRun, then MaxResults.
|
||||
func TestEC2Pipeline_CheckOrder(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
deny bool
|
||||
body string
|
||||
code string
|
||||
}{
|
||||
{"policy denial beats every parameter error", true, "Action=CreateKeyPair&KeyName=k&DryRun=true&TagSpecification.1.ResourceType=instance", "UnauthorizedOperation"},
|
||||
{"tag type beats dry run", false, "Action=CreateKeyPair&KeyName=k&DryRun=true&TagSpecification.1.ResourceType=instance", "InvalidParameterValue"},
|
||||
{"dry run beats max results", false, "Action=DescribeInstances&DryRun=true&MaxResults=1", "DryRunOperation"},
|
||||
{"max results alone", false, "Action=DescribeInstances&MaxResults=1", "InvalidParameterValue"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
var stmts []iamdomain.Statement
|
||||
if !c.deny {
|
||||
stmts = append(stmts, allowEC2())
|
||||
}
|
||||
app := ec2Pipeline(t, nil, stmts...)
|
||||
got := probeEC2(t, app.Handler, c.body)
|
||||
assert.Equal(t, c.code, got.Code, got.Body)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestEC2Pipeline_Success pins the success envelope for a daemon-backed list
|
||||
// and for the gateway-local actions.
|
||||
func TestEC2Pipeline_Success(t *testing.T) {
|
||||
app := ec2Pipeline(t, nil, allowEC2())
|
||||
|
||||
t.Run("daemon backed list", func(t *testing.T) {
|
||||
got := probeEC2(t, app.Handler, "Action=DescribeKeyPairs")
|
||||
require.Equal(t, 200, got.Status, got.Body)
|
||||
assert.Equal(t, "text/xml", got.ContentType)
|
||||
assert.Contains(t, got.Body, "<DescribeKeyPairsResponse")
|
||||
assert.Contains(t, got.Body, "<requestId>")
|
||||
assert.Equal(t, 1, app.Daemon.count("ec2.DescribeKeyPairs"))
|
||||
})
|
||||
|
||||
t.Run("DescribeRegions needs no daemon", func(t *testing.T) {
|
||||
before := app.Daemon.count("ec2.DescribeRegions")
|
||||
got := probeEC2(t, app.Handler, "Action=DescribeRegions")
|
||||
require.Equal(t, 200, got.Status, got.Body)
|
||||
assert.Contains(t, got.Body, testRegion)
|
||||
assert.Contains(t, got.Body, "https://registry.example", "endpoint is built from the registry host and port")
|
||||
assert.Equal(t, before, app.Daemon.count("ec2.DescribeRegions"))
|
||||
})
|
||||
|
||||
t.Run("DescribeAvailabilityZones needs no daemon", func(t *testing.T) {
|
||||
got := probeEC2(t, app.Handler, "Action=DescribeAvailabilityZones")
|
||||
require.Equal(t, 200, got.Status, got.Body)
|
||||
assert.Contains(t, got.Body, testRegion+"a")
|
||||
})
|
||||
|
||||
t.Run("DescribeAccountAttributes looks up the default VPC", func(t *testing.T) {
|
||||
got := probeEC2(t, app.Handler, "Action=DescribeAccountAttributes")
|
||||
require.Equal(t, 200, got.Status, got.Body)
|
||||
assert.Contains(t, got.Body, "<DescribeAccountAttributesResponse")
|
||||
assert.Equal(t, 1, app.Daemon.count("ec2.DescribeVpcs"))
|
||||
})
|
||||
}
|
||||
|
||||
// TestEC2Pipeline_IdempotentCreate pins ClientToken handling on a creating
|
||||
// action: a replay returns the first result, changed parameters are refused,
|
||||
// and no token means no deduplication.
|
||||
func TestEC2Pipeline_IdempotentCreate(t *testing.T) {
|
||||
app := ec2Pipeline(t, nil, allowEC2())
|
||||
const subject = "ec2.CreateEgressOnlyInternetGateway"
|
||||
create := func(vpc, token string) ec2Outcome {
|
||||
body := "Action=CreateEgressOnlyInternetGateway&VpcId=" + vpc
|
||||
if token != "" {
|
||||
body += "&ClientToken=" + token
|
||||
}
|
||||
return probeEC2(t, app.Handler, body)
|
||||
}
|
||||
|
||||
first := create("vpc-1", "tok-1")
|
||||
require.Equal(t, 200, first.Status, first.Body)
|
||||
assert.Contains(t, first.Body, "eigw-0001")
|
||||
|
||||
replay := create("vpc-1", "tok-1")
|
||||
require.Equal(t, 200, replay.Status, replay.Body)
|
||||
assert.Contains(t, replay.Body, "eigw-0001", "same token returns the first result")
|
||||
assert.Equal(t, 1, app.Daemon.count(subject), "the replay does not reach the daemon")
|
||||
|
||||
mismatch := create("vpc-2", "tok-1")
|
||||
assert.Equal(t, "IdempotentParameterMismatch", mismatch.Code, mismatch.Body)
|
||||
assert.Equal(t, 400, mismatch.Status)
|
||||
assert.Equal(t, 1, app.Daemon.count(subject))
|
||||
|
||||
for range 2 {
|
||||
require.Equal(t, 200, create("vpc-1", "").Status)
|
||||
}
|
||||
assert.Equal(t, 3, app.Daemon.count(subject), "without a token each call creates")
|
||||
}
|
||||
|
||||
// TestEC2Pipeline_PassRole pins the gateway iam:PassRole gate on an action
|
||||
// that hands an instance profile over: a denied role is UnauthorizedOperation,
|
||||
// a permitted one reaches the daemon.
|
||||
func TestEC2Pipeline_PassRole(t *testing.T) {
|
||||
const body = "Action=AssociateIamInstanceProfile&InstanceId=i-dev&IamInstanceProfile.Name=web"
|
||||
passRole := statement("Allow", "iam:PassRole", "*")
|
||||
passRole.Condition = map[string]map[string]iamdomain.ConditionValue{
|
||||
iampolicy.OpStringEquals: {iampolicy.KeyPassedToService: {"ec2.amazonaws.com"}},
|
||||
}
|
||||
|
||||
t.Run("permitted role reaches the daemon", func(t *testing.T) {
|
||||
app := ec2Pipeline(t, nil, allowEC2(), passRole)
|
||||
got := probeEC2(t, app.Handler, body)
|
||||
assert.Less(t, got.Status, 500, got.Body)
|
||||
assert.NotEqual(t, "UnauthorizedOperation", got.Code, got.Body)
|
||||
assert.Equal(t, 1, app.Daemon.count(ec2v1.InstanceCommandSubject("i-dev")))
|
||||
})
|
||||
|
||||
t.Run("no PassRole", func(t *testing.T) {
|
||||
app := ec2Pipeline(t, nil, allowEC2())
|
||||
got := probeEC2(t, app.Handler, body)
|
||||
assert.Equal(t, 403, got.Status, got.Body)
|
||||
assert.Equal(t, "UnauthorizedOperation", got.Code, got.Body)
|
||||
assert.Zero(t, app.Daemon.count(ec2v1.InstanceCommandSubject("i-dev")))
|
||||
})
|
||||
|
||||
t.Run("PassRole for another service", func(t *testing.T) {
|
||||
other := statement("Allow", "iam:PassRole", "*")
|
||||
other.Condition = map[string]map[string]iamdomain.ConditionValue{
|
||||
iampolicy.OpStringEquals: {iampolicy.KeyPassedToService: {"ecs-tasks.amazonaws.com"}},
|
||||
}
|
||||
app := ec2Pipeline(t, nil, allowEC2(), other)
|
||||
got := probeEC2(t, app.Handler, body)
|
||||
assert.Equal(t, 403, got.Status, got.Body)
|
||||
assert.Equal(t, "UnauthorizedOperation", got.Code, got.Body)
|
||||
})
|
||||
}
|
||||
|
||||
// TestEC2Pipeline_ImportKeyPairPadding pins the base64 padding workaround: a
|
||||
// doubly encoded trailing "==" is decoded before the key reaches the daemon,
|
||||
// and a single trailing "=" is not.
|
||||
func TestEC2Pipeline_ImportKeyPairPadding(t *testing.T) {
|
||||
forwarded := func(t *testing.T, material string) string {
|
||||
t.Helper()
|
||||
app := ec2Pipeline(t, nil, allowEC2())
|
||||
body := "Action=ImportKeyPair&KeyName=k&PublicKeyMaterial=" + url.QueryEscape(material)
|
||||
got := probeEC2(t, app.Handler, body)
|
||||
require.Equal(t, 200, got.Status, got.Body)
|
||||
var in ec2.ImportKeyPairInput
|
||||
require.NoError(t, json.NewDecoder(bytes.NewReader(app.Daemon.body("ec2.ImportKeyPair"))).Decode(&in))
|
||||
return string(in.PublicKeyMaterial)
|
||||
}
|
||||
|
||||
t.Run("double padding", func(t *testing.T) {
|
||||
assert.Equal(t, "ssh-ed25519 AAAA==", forwarded(t, "ssh-ed25519 AAAA%3D%3D"))
|
||||
})
|
||||
t.Run("single padding is left encoded", func(t *testing.T) {
|
||||
assert.Equal(t, "ssh-ed25519 AAAA%3D", forwarded(t, "ssh-ed25519 AAAA%3D"))
|
||||
})
|
||||
t.Run("already decoded", func(t *testing.T) {
|
||||
assert.Equal(t, "ssh-ed25519 AAAA==", forwarded(t, "ssh-ed25519 AAAA=="))
|
||||
})
|
||||
}
|
||||
|
||||
const runInstancesBase = "Action=RunInstances&InstanceType=t3.micro&MinCount=1&MaxCount=1"
|
||||
|
||||
// TestEC2Pipeline_RunInstancesLaunchTemplateAuthorizedAfterExpansion pins that
|
||||
// a launch template is expanded before the policy gate, so a resource the
|
||||
// template supplies is authorized and a Deny on it fires.
|
||||
func TestEC2Pipeline_RunInstancesLaunchTemplateAuthorizedAfterExpansion(t *testing.T) {
|
||||
const subject = "ec2.DescribeLaunchTemplateVersions"
|
||||
template := func(image string) string {
|
||||
return `{"LaunchTemplateVersions":[{"LaunchTemplateData":{"ImageId":"` + image + `"}}]}`
|
||||
}
|
||||
const body = "Action=RunInstances&InstanceType=t3.micro&MinCount=1&MaxCount=1&LaunchTemplate.LaunchTemplateName=lt"
|
||||
deny := statement("Deny", "ec2:RunInstances", "arn:aws:ec2:*:*:image/ami-fenced")
|
||||
|
||||
t.Run("template image is denied", func(t *testing.T) {
|
||||
app := ec2Pipeline(t, nil, allowEC2(), deny)
|
||||
app.Daemon.reply(subject, template("ami-fenced"))
|
||||
got := probeEC2(t, app.Handler, body)
|
||||
assert.Equal(t, 403, got.Status, got.Body)
|
||||
assert.Equal(t, "UnauthorizedOperation", got.Code, got.Body)
|
||||
assert.Equal(t, 1, app.Daemon.count(subject))
|
||||
assert.Zero(t, app.Daemon.count("ec2.RunInstances"))
|
||||
})
|
||||
|
||||
t.Run("template image not fenced passes the gate", func(t *testing.T) {
|
||||
app := ec2Pipeline(t, nil, allowEC2(), deny)
|
||||
app.Daemon.reply(subject, template("ami-ok"))
|
||||
got := probeEC2(t, app.Handler, body)
|
||||
assert.NotEqual(t, "UnauthorizedOperation", got.Code, got.Body)
|
||||
})
|
||||
|
||||
t.Run("unknown template", func(t *testing.T) {
|
||||
app := ec2Pipeline(t, nil, allowEC2())
|
||||
app.Daemon.reply(subject, `{"LaunchTemplateVersions":[]}`)
|
||||
got := probeEC2(t, app.Handler, body)
|
||||
assert.Equal(t, "InvalidLaunchTemplateId.VersionNotFound", got.Code, got.Body)
|
||||
})
|
||||
}
|
||||
|
||||
// TestEC2Pipeline_RunInstancesInstanceProfile pins the current order of the
|
||||
// profile checks on RunInstances: the profile is resolved first, so an unknown
|
||||
// one is refused before iam:PassRole is consulted, and a resolved one is then
|
||||
// subject to PassRole on its role. This is the baseline for any parity change.
|
||||
func TestEC2Pipeline_RunInstancesInstanceProfile(t *testing.T) {
|
||||
const body = runInstancesBase + "&ImageId=ami-1&IamInstanceProfile.Name=web"
|
||||
|
||||
t.Run("unknown profile is refused before PassRole", func(t *testing.T) {
|
||||
app := ec2Pipeline(t, func(e *ec2PipelineEnv) { e.ResolveErr = errors.New(awserrors.ErrorIAMNoSuchEntity) }, allowEC2())
|
||||
got := probeEC2(t, app.Handler, body)
|
||||
assert.Equal(t, "InvalidIamInstanceProfile.NotFound", got.Code, got.Body)
|
||||
assert.Equal(t, 404, got.Status)
|
||||
})
|
||||
|
||||
t.Run("resolved profile without PassRole", func(t *testing.T) {
|
||||
app := ec2Pipeline(t, nil, allowEC2())
|
||||
got := probeEC2(t, app.Handler, body)
|
||||
assert.Equal(t, 403, got.Status, got.Body)
|
||||
assert.Equal(t, "UnauthorizedOperation", got.Code, got.Body)
|
||||
})
|
||||
|
||||
t.Run("other resolution failure is returned as is", func(t *testing.T) {
|
||||
app := ec2Pipeline(t, func(e *ec2PipelineEnv) { e.ResolveErr = errors.New(awserrors.ErrorServerInternal) }, allowEC2())
|
||||
got := probeEC2(t, app.Handler, body)
|
||||
assert.Equal(t, "ServerInternal", got.Code, got.Body)
|
||||
})
|
||||
}
|
||||
@@ -1,12 +1,11 @@
|
||||
//test:in-package — walks the unexported ec2Actions route table and drives
|
||||
// EC2_Request through the gateway's unexported test helpers.
|
||||
//test:in-package — drives EC2_Request through the gateway's unexported test
|
||||
// helpers.
|
||||
|
||||
package gateway
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"net/http"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -15,21 +14,6 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// An action added to the route table with TagSpecifications must say which
|
||||
// resource types it tags, or a wrong type would be silently dropped again.
|
||||
func TestEC2TaggableTypes_CoverEveryAction(t *testing.T) {
|
||||
for action, handler := range ec2Actions {
|
||||
input, err := handler.parse(map[string]string{"Action": action})
|
||||
require.NoError(t, err, action)
|
||||
_, tags := reflect.TypeOf(input).Elem().FieldByName("TagSpecifications")
|
||||
_, listed := ec2TaggableTypes[action]
|
||||
assert.Equal(t, tags, listed, "%s: TagSpecifications modelled=%v, listed in ec2TaggableTypes=%v", action, tags, listed)
|
||||
}
|
||||
for action := range ec2TaggableTypes {
|
||||
assert.Contains(t, ec2Actions, action, "ec2TaggableTypes lists an action the gateway does not route")
|
||||
}
|
||||
}
|
||||
|
||||
func tagSpecBody(action, resourceType string) string {
|
||||
return "Action=" + action + "&TagSpecification.1.ResourceType=" + resourceType +
|
||||
"&TagSpecification.1.Tag.1.Key=k&TagSpecification.1.Tag.1.Value=v"
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package gateway
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
@@ -10,20 +9,6 @@ import (
|
||||
// external test package.
|
||||
func (gw *GatewayConfig) AdvertisedEndpoint() string { return gw.advertisedEndpoint() }
|
||||
|
||||
// DispatchEC2Action runs an action through the parse-then-dispatch entry
|
||||
// EC2_Request uses, without an *http.Request.
|
||||
func DispatchEC2Action(gw *GatewayConfig, action string, query map[string]string, accountID string) ([]byte, error) {
|
||||
h, ok := ec2Actions[action]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("no EC2 action %q", action)
|
||||
}
|
||||
input, err := h.parse(query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return h.dispatch(action, input, gw, accountID, nil)
|
||||
}
|
||||
|
||||
// SetDiscoverActiveNodesTimeoutForTest shortens the discovery fan-out for one
|
||||
// test. Tests that call it must not run in parallel.
|
||||
func SetDiscoverActiveNodesTimeoutForTest(tb testing.TB, d time.Duration) {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
// Package gateway provides the AWS-compatible API gateway for the Spinifex platform.
|
||||
package gateway
|
||||
|
||||
import (
|
||||
@@ -10,7 +11,6 @@ import (
|
||||
authlimit "github.com/mulgadc/spinifex/spinifex/ingress/aws/ratelimit"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/roles/awsgw/accountadmission"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/roles/awsgw/policyauth"
|
||||
"io"
|
||||
"log/slog"
|
||||
"maps"
|
||||
"net/http"
|
||||
@@ -23,9 +23,9 @@ import (
|
||||
"github.com/mulgadc/bluebottle/pkg/iampolicy"
|
||||
bbotel "github.com/mulgadc/bluebottle/pkg/otelsetup"
|
||||
"github.com/mulgadc/bluebottle/pkg/ratelimit"
|
||||
"github.com/mulgadc/bluebottle/pkg/sigv4"
|
||||
clusterv1 "github.com/mulgadc/spinifex/contracts/cluster/v1"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/admission/quota"
|
||||
ec2awsapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi"
|
||||
ec2instanceapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/instance"
|
||||
iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
|
||||
stsdomain "github.com/mulgadc/spinifex/spinifex/domains/sts"
|
||||
@@ -39,7 +39,6 @@ import (
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/envelope"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/query"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
@@ -89,11 +88,10 @@ type GatewayConfig struct {
|
||||
activeNodesMu sync.RWMutex
|
||||
activeNodesCount int
|
||||
activeNodesAt time.Time
|
||||
// The EC2 client-token KV binding, bound on first use and reused after, so
|
||||
// the creates that honour ClientToken do not rebind it per request.
|
||||
ec2TokenOnce sync.Once
|
||||
ec2TokenKV jetstream.KeyValue
|
||||
ec2TokenErr error
|
||||
// The EC2 handler, built on first use so its client-token binding is reused
|
||||
// across requests.
|
||||
ec2Once sync.Once
|
||||
ec2Handler *ec2awsapi.Handler
|
||||
InternalSuffix string // Internal DNS suffix for AWS-parity endpoints (e.g. spinifex.internal)
|
||||
// RegistryPort is the gateway's advertised port, appended to the ECR
|
||||
// registry host so docker login/tag/push dial the right port. Empty or
|
||||
@@ -504,13 +502,6 @@ const (
|
||||
eksServicePrincipal = "eks.amazonaws.com"
|
||||
)
|
||||
|
||||
// checkPassRole enforces iam:PassRole on roleARN. iam:PassedToService is fixed
|
||||
// by the call site handing the role over, never read from the request.
|
||||
func (gw *GatewayConfig) checkPassRole(r *http.Request, roleARN, passedTo string) error {
|
||||
return gw.checkPolicyResourcesWithKeys(r, "iam", "PassRole", []string{roleARN},
|
||||
iampolicy.ConditionKeys{iampolicy.KeyPassedToService: passedTo})
|
||||
}
|
||||
|
||||
// checkPolicyResourcesWithKeys is checkPolicyResources with action-scoped
|
||||
// condition keys layered over the request's own.
|
||||
func (gw *GatewayConfig) checkPolicyResourcesWithKeys(r *http.Request, service, action string, resources []string, actionKeys iampolicy.ConditionKeys) error {
|
||||
@@ -631,24 +622,6 @@ func (gw *GatewayConfig) ErrorHandler(w http.ResponseWriter, r *http.Request, er
|
||||
}
|
||||
}
|
||||
|
||||
// readQueryArgs returns parsed query args from context (set by SigV4) or falls
|
||||
// back to parsing the body (unauthenticated/test paths only).
|
||||
func readQueryArgs(r *http.Request) (map[string]string, error) {
|
||||
if args, ok := r.Context().Value(ctxQueryArgs).(map[string]string); ok {
|
||||
return args, nil
|
||||
}
|
||||
// Same cap as the signed path: this fallback only runs when the context
|
||||
// carries no pre-parsed args, so the body has not been bounded upstream.
|
||||
body, err := io.ReadAll(io.LimitReader(r.Body, sigv4.MaxPayloadLen+1))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read body: %w", err)
|
||||
}
|
||||
if int64(len(body)) > sigv4.MaxPayloadLen {
|
||||
return nil, errors.New(awserrors.ErrorRequestEntityTooLarge)
|
||||
}
|
||||
return ParseAWSQueryArgs(string(body))
|
||||
}
|
||||
|
||||
// ParseAWSQueryArgs parses an AWS query-protocol body. Returns an error on
|
||||
// invalid percent-encoding so callers can surface MalformedQueryString.
|
||||
func ParseAWSQueryArgs(body string) (map[string]string, error) {
|
||||
|
||||
@@ -1271,27 +1271,6 @@ func TestCheckPolicy_NonTransientErrorStillFails(t *testing.T) {
|
||||
assert.Equal(t, awserrors.ErrorInternalError, err.Error())
|
||||
}
|
||||
|
||||
func TestImportKeyPair_Base64PaddingWorkaround(t *testing.T) {
|
||||
// The ImportKeyPair handler decodes URL-encoded Base64 padding (%3D%3D → ==)
|
||||
// before passing to the generic handler.
|
||||
handler := ec2Actions["ImportKeyPair"]
|
||||
require.NotNil(t, handler)
|
||||
|
||||
q := map[string]string{
|
||||
"Action": "ImportKeyPair",
|
||||
"KeyName": "test-key",
|
||||
"PublicKeyMaterial": "c3NoLXJzYSBBQUFBQjNOemFDMXljMkVBQUFBREFRQUJBQUFCZ1FD%3D%3D",
|
||||
}
|
||||
|
||||
_, err := handler.parse(q)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.True(t, strings.HasSuffix(q["PublicKeyMaterial"], "=="),
|
||||
"Expected PublicKeyMaterial to end with == but got: %s", q["PublicKeyMaterial"])
|
||||
assert.NotContains(t, q["PublicKeyMaterial"], "%3D",
|
||||
"Expected no URL-encoded padding remaining")
|
||||
}
|
||||
|
||||
// --- Throttle middleware integration tests ---
|
||||
|
||||
func TestThrottleKeyFuncs_ExtractsAccountAndAction(t *testing.T) {
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
package gateway
|
||||
|
||||
import (
|
||||
"maps"
|
||||
"slices"
|
||||
|
||||
ec2awsapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
|
||||
)
|
||||
|
||||
@@ -23,7 +23,7 @@ type ServiceOperationInventory struct {
|
||||
func AWSOperationInventory(registered map[string]dispatch.Inventory) map[string]ServiceOperationInventory {
|
||||
inventory := map[string]ServiceOperationInventory{
|
||||
"ec2": {
|
||||
Registered: mapKeys(ec2Actions),
|
||||
Registered: ec2awsapi.ActionNames(),
|
||||
},
|
||||
}
|
||||
for name, inv := range registered {
|
||||
@@ -35,7 +35,3 @@ func AWSOperationInventory(registered map[string]dispatch.Inventory) map[string]
|
||||
}
|
||||
return inventory
|
||||
}
|
||||
|
||||
func mapKeys[V any](values map[string]V) []string {
|
||||
return slices.Sorted(maps.Keys(values))
|
||||
}
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
package gateway
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
|
||||
rdsawsapi "github.com/mulgadc/spinifex/spinifex/domains/rds/awsapi"
|
||||
sqsworkerapi "github.com/mulgadc/spinifex/spinifex/domains/sqs/awsapi/worker"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/firststore"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/storelaunch"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/workerident"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const sqsWorkerInstance = "i-0abc12345def67890"
|
||||
|
||||
type sqsWorkerIdentities struct{ asked int }
|
||||
|
||||
func (s *sqsWorkerIdentities) Authorize(_ context.Context, instanceID string) (workerident.Identity, error) {
|
||||
s.asked++
|
||||
return workerident.Identity{Region: "ap-southeast-2", InstanceID: instanceID, Generation: 1, ClaimID: "claim-0001"}, nil
|
||||
}
|
||||
|
||||
type sqsWorkerFirstStore struct{}
|
||||
|
||||
func (sqsWorkerFirstStore) Claim(context.Context, string, firststore.Identity) (firststore.Decision, error) {
|
||||
return firststore.Decision{CreateKey: true}, nil
|
||||
}
|
||||
|
||||
func (sqsWorkerFirstStore) RecordPublicKey(context.Context, string, firststore.Identity, string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
type sqsWorkerStores struct{}
|
||||
|
||||
func (sqsWorkerStores) Current(context.Context, string) (storelaunch.Record, error) {
|
||||
return storelaunch.Record{}, jetstream.ErrKeyNotFound
|
||||
}
|
||||
|
||||
// withSQSInternal registers the internal SQS surface beside RDS, as awsgw
|
||||
// composes them, so routing is selected by signing service.
|
||||
func withSQSInternal(gw *GatewayConfig, deps sqsworkerapi.Deps) *GatewayConfig {
|
||||
b := dispatch.NewBuilder()
|
||||
for _, reg := range []dispatch.Registration{rdsawsapi.NewRegistration(rdsawsapi.Deps{}), sqsworkerapi.NewRegistration(deps)} {
|
||||
if err := b.Register(reg); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
gw.Services = b.Build()
|
||||
if err := gw.ValidateServices(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return gw
|
||||
}
|
||||
|
||||
func sqsWorkerRequest(service, body, session string) *http.Request {
|
||||
req := setupRDSRequest(body)
|
||||
roleARN := "arn:aws:iam::" + awsidentifiers.GlobalAccountID + ":role/" + sqsworkerapi.WorkerRoleName
|
||||
ctx := context.WithValue(req.Context(), ctxService, service)
|
||||
ctx = withPrincipalField(ctx, fieldAccountID, awsidentifiers.GlobalAccountID)
|
||||
ctx = withPrincipalField(ctx, fieldIdentity, session)
|
||||
ctx = withPrincipalField(ctx, fieldPrincipalType, principalTypeAssumedRole)
|
||||
ctx = withPrincipalField(ctx, fieldUnderlyingRoleARN, roleARN)
|
||||
ctx = withPrincipalField(ctx, fieldAssumedRoleARN,
|
||||
"arn:aws:sts::"+awsidentifiers.GlobalAccountID+":assumed-role/"+sqsworkerapi.WorkerRoleName+"/"+session)
|
||||
return req.WithContext(ctx)
|
||||
}
|
||||
|
||||
func serveSigned(gw *GatewayConfig, w http.ResponseWriter, r *http.Request) error {
|
||||
entry, ok := gw.registered(requestService(r))
|
||||
if !ok {
|
||||
panic("no registration serves " + requestService(r))
|
||||
}
|
||||
return gw.dispatchRegistered(entry, w, r)
|
||||
}
|
||||
|
||||
func newSQSInternalGateway(policies []iamdomain.PolicyDocument) (*GatewayConfig, *rdsPolicyProbe, *sqsWorkerIdentities) {
|
||||
gw, probe := newRDSPolicyGateway(policies)
|
||||
ids := &sqsWorkerIdentities{}
|
||||
return withSQSInternal(gw, sqsworkerapi.Deps{Identities: ids, FirstStore: sqsWorkerFirstStore{}, Stores: sqsWorkerStores{}}), probe, ids
|
||||
}
|
||||
|
||||
// The worker's session, signed for spinifex-sqs, reaches the internal surface
|
||||
// and is evaluated against its role policy under the spinifex-sqs prefix.
|
||||
func TestSQSInternal_RoutedBySigningServiceAndRolePolicy(t *testing.T) {
|
||||
gw, probe, ids := newSQSInternalGateway(allowPolicy("spinifex-sqs:ClaimFirstStore"))
|
||||
rec := httptest.NewRecorder()
|
||||
require.NoError(t, serveSigned(gw, rec, sqsWorkerRequest(sqsworkerapi.ServiceName, "Action=ClaimFirstStore", sqsWorkerInstance)))
|
||||
assert.Equal(t, 1, probe.consulted)
|
||||
assert.Equal(t, 1, ids.asked)
|
||||
assert.Contains(t, rec.Body.String(), "<CreateKey>true</CreateKey>")
|
||||
}
|
||||
|
||||
// The role policy is real: a worker session whose policy grants another
|
||||
// service's action is denied before any store is consulted.
|
||||
func TestSQSInternal_PolicyForAnotherServiceDenies(t *testing.T) {
|
||||
gw, probe, ids := newSQSInternalGateway(allowPolicy("sqs:ClaimFirstStore"))
|
||||
err := serveSigned(gw, httptest.NewRecorder(), sqsWorkerRequest(sqsworkerapi.ServiceName, "Action=ClaimFirstStore", sqsWorkerInstance))
|
||||
assertDenied(t, err)
|
||||
assert.Equal(t, 1, probe.consulted)
|
||||
assert.Zero(t, ids.asked)
|
||||
}
|
||||
|
||||
// A customer administrator holding every action is refused by the class gate.
|
||||
func TestSQSInternal_CustomerAdminDenied(t *testing.T) {
|
||||
gw, probe, ids := newSQSInternalGateway(allowPolicy("*"))
|
||||
req := setupRDSUserRequest("Action=ClaimFirstStore", rdsTestAccountID)
|
||||
req = req.WithContext(context.WithValue(req.Context(), ctxService, sqsworkerapi.ServiceName))
|
||||
assertDenied(t, serveSigned(gw, httptest.NewRecorder(), req))
|
||||
assert.Zero(t, probe.consulted)
|
||||
assert.Zero(t, ids.asked)
|
||||
}
|
||||
|
||||
// Public SQS is untouched: nothing serves the sqs signing service, and the
|
||||
// internal actions do not answer under RDS's name either.
|
||||
func TestSQSInternal_PublicSQSUnchanged(t *testing.T) {
|
||||
gw, _, ids := newSQSInternalGateway(allowPolicy("*"))
|
||||
_, ok := gw.registered("sqs")
|
||||
assert.False(t, ok, "the sqs signing service must stay unserved")
|
||||
assert.False(t, legacyServices{}.Serves("sqs"))
|
||||
inv := gw.Services.Inventory()
|
||||
assert.NotContains(t, inv, "sqs")
|
||||
assert.ElementsMatch(t, sqsworkerapi.ActionNames(), inv[sqsworkerapi.ServiceName].Registered)
|
||||
|
||||
err := serveSigned(gw, httptest.NewRecorder(), sqsWorkerRequest(rdsawsapi.ServiceName, "Action=ClaimFirstStore", sqsWorkerInstance))
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorInvalidAction, err.Error())
|
||||
assert.Zero(t, ids.asked)
|
||||
}
|
||||
@@ -70,6 +70,8 @@ var adminActions = map[string]bool{
|
||||
"RevokeModelAccess": true,
|
||||
"ListModelAccess": true,
|
||||
"ListOchreCatalog": true,
|
||||
"EnsureSQSWorker": true,
|
||||
"EnsureSQSStore": true,
|
||||
}
|
||||
|
||||
// ActionNames lists every action the registration dispatches, sorted.
|
||||
@@ -230,6 +232,16 @@ func (d Deps) run(ctx context.Context, action string, args map[string]string, ac
|
||||
return nil, err
|
||||
}
|
||||
return ochre.ModelAccessList{AccountID: targetAccount, ModelIDs: models}, nil
|
||||
case "EnsureSQSWorker":
|
||||
if d.NATS == nil {
|
||||
return nil, errors.New(awserrors.ErrorServerInternal)
|
||||
}
|
||||
return EnsureSQSWorker(ctx, d.NATS, d.NodeIDs, args["NodeId"], args["InstanceType"], accountID, sqsWorkerEnsureWait)
|
||||
case "EnsureSQSStore":
|
||||
if d.NATS == nil {
|
||||
return nil, errors.New(awserrors.ErrorServerInternal)
|
||||
}
|
||||
return EnsureSQSStore(ctx, d.NATS, d.NodeIDs, args["NodeId"], args["InstanceType"], args["VolumeSizeGiB"], accountID, sqsWorkerEnsureWait)
|
||||
case "ListOchreCatalog":
|
||||
entries, err := ochre.AdminCatalog(ctx, accountID, d.credentials(), d.access())
|
||||
if err != nil {
|
||||
|
||||
@@ -245,6 +245,8 @@ func TestNewRegistration_DeclaresTheAdminActionsAsItsInventory(t *testing.T) {
|
||||
assert.ElementsMatch(t, []string{
|
||||
"GetVersion", "GetNodes", "GetVMs", "GetStorageStatus", "PromoteImage",
|
||||
"GrantModelAccess", "RevokeModelAccess", "ListModelAccess", "ListOchreCatalog",
|
||||
"EnsureSQSStore",
|
||||
"EnsureSQSWorker",
|
||||
}, reg.Inventory.Registered)
|
||||
assert.NoError(t, dispatch.NewBuilder().Register(reg))
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user