mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-11-02 09:13:33 +00:00
Compare commits
45 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| e4055d4d89 | |||
| a7753557dc | |||
| fe97c45793 | |||
| 1713c711c4 | |||
| bab4e1e53c | |||
| 1a91d5de9e | |||
| e958e4a57b | |||
| a44fd01b85 | |||
| 72f2786360 | |||
| 8916904285 | |||
| 2630d29e13 | |||
| e7c0f0ebae | |||
| d79e537330 | |||
| fe349fb994 | |||
| 061ea49832 | |||
| f4af10713d | |||
| 0eeed40cb6 | |||
| ad62b56555 | |||
| 6e6690a35a | |||
| 2ad9ab9450 | |||
| 3128713f7b | |||
| 9f74933546 | |||
| 9dc32e3373 | |||
| a6459ba7e4 | |||
| 805a71a63c | |||
| 2d9eadbf23 | |||
| 581be4067a | |||
| 96e62ac213 | |||
| 02bba41669 | |||
| 8fa33f07ce | |||
| 3c192b3558 | |||
| 8cded67776 | |||
| aae7f68097 | |||
| df1873b417 | |||
| bf3c5e3dba | |||
| 9a571a7438 | |||
| 045816a44d | |||
| f6686b5436 | |||
| b034f3b2e9 | |||
| 3d387c839d | |||
| a5c07f37fd | |||
| c50cc2512a | |||
| 5310c82e95 | |||
| 943f191bf3 | |||
| 671a8a9ff5 |
@@ -104,7 +104,7 @@ jobs:
|
||||
|
||||
- name: Build microVM artifacts (lb-agent + kernel + initramfs + qboot + SQS store and worker images)
|
||||
if: steps.version.outputs.exists != 'true'
|
||||
run: cd spinifex && make build-microvm-image build-sqs-store-microvm-image build-sqs-worker-microvm-image
|
||||
run: cd spinifex && make build-microvm-image build-sqs-worker-microvm-image
|
||||
|
||||
- name: Build amd64 Tarball
|
||||
if: steps.version.outputs.exists != 'true'
|
||||
|
||||
@@ -58,7 +58,7 @@ jobs:
|
||||
cache: false
|
||||
|
||||
- name: Build microVM artifacts (lb-agent + kernel + initramfs + qboot + SQS store and worker images)
|
||||
run: cd spinifex && make build-microvm-image build-sqs-store-microvm-image build-sqs-worker-microvm-image
|
||||
run: cd spinifex && make build-microvm-image build-sqs-worker-microvm-image
|
||||
|
||||
- name: Build amd64 Tarball
|
||||
env:
|
||||
|
||||
@@ -21,9 +21,6 @@ build/microvm/vmlinuz
|
||||
build/microvm/initramfs.cpio.gz
|
||||
build/microvm/qboot.rom
|
||||
build/microvm/qboot.LICENSE
|
||||
build/microvm/sqs-store/vmlinuz
|
||||
build/microvm/sqs-store/initramfs.cpio.gz
|
||||
build/microvm/sqs-store/.stage.*/
|
||||
build/microvm/sqs-worker/vmlinuz
|
||||
build/microvm/sqs-worker/initramfs.cpio.gz
|
||||
build/microvm/sqs-worker/.stage.*/
|
||||
|
||||
@@ -153,37 +153,32 @@ $(MICROVM_ARTIFACTS) &: $(MICROVM_INPUTS)
|
||||
bin/lb-agent:
|
||||
$(MAKE) build-lb-agent
|
||||
|
||||
# The SQS store microVM image: a separate artifact from the LB image above, not
|
||||
# installed or released by any target yet.
|
||||
SQS_STORE_MICROVM_OUT_DIR := $(MICROVM_OUT_DIR)/sqs-store
|
||||
SQS_STORE_MICROVM_ARTIFACTS := $(SQS_STORE_MICROVM_OUT_DIR)/vmlinuz $(SQS_STORE_MICROVM_OUT_DIR)/initramfs.cpio.gz
|
||||
SQS_STORE_MICROVM_INPUTS := scripts/build-sqs-store-microvm-image.sh scripts/lib/sqs-store-microvm-publish.sh $(SQS_STORE_MICROVM_OUT_DIR)/init.sh $(SQS_STORE_MICROVM_OUT_DIR)/inittab $(addprefix $(SQS_STORE_MICROVM_OUT_DIR)/,sqs-store-lib.sh sqs-store-prepare sqs-store-run sqs-store-stop) bin/nats-server
|
||||
|
||||
$(SQS_STORE_MICROVM_ARTIFACTS) &: $(SQS_STORE_MICROVM_INPUTS)
|
||||
./scripts/build-sqs-store-microvm-image.sh
|
||||
|
||||
# The dedicated SQS store server, built from the go.mod-pinned module so the
|
||||
# image ships exactly the reviewed version. Static for the musl guest.
|
||||
# The dedicated SQS server, built from the go.mod-pinned module so the image
|
||||
# ships exactly the reviewed version. Static for the musl guest.
|
||||
bin/nats-server: go.mod go.sum
|
||||
@mkdir -p bin
|
||||
GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags '-s -w' -o $@ github.com/nats-io/nats-server/v2
|
||||
|
||||
build-sqs-store-microvm-image: $(SQS_STORE_MICROVM_ARTIFACTS) ## Build the SQS store microVM kernel and initramfs
|
||||
# go_local_srcs lists the main module package dirs and files guest package $(1)
|
||||
# links, scanned with the guest build settings; a dir's mtime catches a deleted
|
||||
# or renamed file. A failed scan names a missing prerequisite, so make stops.
|
||||
go_local_srcs = $(shell GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go list -deps -f '{{if and .Module .Module.Main}}{{.Dir}} {{range .GoFiles}}{{$$.Dir}}/{{.}} {{end}}{{range .EmbedFiles}}{{$$.Dir}}/{{.}} {{end}}{{end}}' $(1) || echo go-list-failed-for-$(1))
|
||||
.SECONDEXPANSION:
|
||||
|
||||
# The shared SQS worker microVM image: the store image's /init and kernel with
|
||||
# the worker binary instead of nats-server. Not installed or released yet.
|
||||
# The SQS microVM image: one guest running its own SQS nats-server and the SQS
|
||||
# worker as separate processes. A separate artifact from the LB image above.
|
||||
SQS_WORKER_MICROVM_OUT_DIR := $(MICROVM_OUT_DIR)/sqs-worker
|
||||
SQS_WORKER_MICROVM_ARTIFACTS := $(SQS_WORKER_MICROVM_OUT_DIR)/vmlinuz $(SQS_WORKER_MICROVM_OUT_DIR)/initramfs.cpio.gz
|
||||
SQS_WORKER_MICROVM_INPUTS := scripts/build-sqs-store-microvm-image.sh scripts/lib/sqs-store-microvm-publish.sh $(SQS_STORE_MICROVM_OUT_DIR)/init.sh $(SQS_WORKER_MICROVM_OUT_DIR)/inittab $(addprefix $(SQS_WORKER_MICROVM_OUT_DIR)/,sqs-worker-lib.sh sqs-worker-prepare sqs-worker-ready sqs-worker-stop) bin/sqs-worker
|
||||
SQS_WORKER_MICROVM_INPUTS := scripts/build-sqs-store-microvm-image.sh scripts/lib/sqs-store-microvm-publish.sh $(SQS_WORKER_MICROVM_OUT_DIR)/init.sh $(SQS_WORKER_MICROVM_OUT_DIR)/inittab $(addprefix $(SQS_WORKER_MICROVM_OUT_DIR)/,sqs-worker-lib.sh sqs-worker-prepare sqs-worker-store sqs-worker-ready sqs-worker-stop) bin/nats-server bin/sqs-worker
|
||||
|
||||
$(SQS_WORKER_MICROVM_ARTIFACTS) &: $(SQS_WORKER_MICROVM_INPUTS)
|
||||
SQS_MICROVM_PROFILE=worker ./scripts/build-sqs-store-microvm-image.sh
|
||||
./scripts/build-sqs-store-microvm-image.sh
|
||||
|
||||
bin/sqs-worker: go.mod go.sum $(shell find cmd/sqs-worker spinifex/agents/sqs -name '*.go' ! -name '*_test.go')
|
||||
bin/sqs-worker: go.mod go.sum $$(call go_local_srcs,./cmd/sqs-worker)
|
||||
@mkdir -p bin
|
||||
GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags '-s -w' -o $@ ./cmd/sqs-worker
|
||||
|
||||
build-sqs-worker-microvm-image: $(SQS_WORKER_MICROVM_ARTIFACTS) ## Build the shared SQS worker microVM kernel and initramfs
|
||||
build-sqs-worker-microvm-image: $(SQS_WORKER_MICROVM_ARTIFACTS) ## Build the SQS microVM kernel and initramfs
|
||||
|
||||
# qboot is the microvm firmware. The source tarball ships with each release
|
||||
# because qboot is GPL-2.0.
|
||||
@@ -196,13 +191,7 @@ $(QBOOT_ARTIFACTS) &: scripts/build-qboot.sh
|
||||
|
||||
build-microvm-image: $(MICROVM_ARTIFACTS) $(QBOOT_ARTIFACTS) ## Build microVM kernel, initramfs and qboot firmware (incremental — skips when up to date)
|
||||
|
||||
install-sqs-store-microvm: $(SQS_STORE_MICROVM_ARTIFACTS) ## Install the SQS store microVM image to /usr/share/spinifex/microvm/sqs-store/
|
||||
sudo install -d /usr/share/spinifex/microvm/sqs-store
|
||||
sudo rm -f /usr/share/spinifex/microvm/sqs-store/initramfs.cpio.gz
|
||||
sudo install -m 0644 $(SQS_STORE_MICROVM_OUT_DIR)/vmlinuz /usr/share/spinifex/microvm/sqs-store/vmlinuz
|
||||
sudo install -m 0644 $(SQS_STORE_MICROVM_OUT_DIR)/initramfs.cpio.gz /usr/share/spinifex/microvm/sqs-store/initramfs.cpio.gz
|
||||
|
||||
install-sqs-worker-microvm: $(SQS_WORKER_MICROVM_ARTIFACTS) ## Install the SQS worker microVM image to /usr/share/spinifex/microvm/sqs-worker/
|
||||
install-sqs-worker-microvm: $(SQS_WORKER_MICROVM_ARTIFACTS) ## Install the SQS microVM image to /usr/share/spinifex/microvm/sqs-worker/
|
||||
sudo install -d /usr/share/spinifex/microvm/sqs-worker
|
||||
sudo rm -f /usr/share/spinifex/microvm/sqs-worker/initramfs.cpio.gz
|
||||
sudo install -m 0644 $(SQS_WORKER_MICROVM_OUT_DIR)/vmlinuz /usr/share/spinifex/microvm/sqs-worker/vmlinuz
|
||||
@@ -322,7 +311,7 @@ test-cover:
|
||||
# Run unit tests with race detector
|
||||
test-race:
|
||||
@echo -e "\n....Running tests with race detector for $(GO_PROJECT_NAME)...."
|
||||
$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -timeout 300s ./spinifex/... ./cmd/... ./internal/... ./contracts/... ./architecture/... $(_RACEQ)
|
||||
$(_Q)LOG_IGNORE=1 SPINIFEX_SQS_R3=1 go test $(GOTESTFLAGS) -race -timeout 300s ./spinifex/... ./cmd/... ./internal/... ./contracts/... ./architecture/... $(_RACEQ)
|
||||
|
||||
# Unit tests for in-repo GitHub Actions (e.g. .github/actions/e2e-analyze).
|
||||
# Kept out of `test-cover` so coverage % isn't diluted by CI-only tooling.
|
||||
@@ -514,7 +503,6 @@ distro-amd64:
|
||||
echo "[distro-amd64] WARNING: microvm artifacts missing — tarball will not include them"; \
|
||||
echo "[distro-amd64] run 'make build-microvm-image' before 'make distro-amd64'"; \
|
||||
fi
|
||||
./scripts/lib/stage-sqs-store-microvm.sh $(SQS_STORE_MICROVM_OUT_DIR) dist/amd64
|
||||
./scripts/lib/stage-sqs-store-microvm.sh $(SQS_WORKER_MICROVM_OUT_DIR) dist/amd64 sqs-worker
|
||||
tar -czf dist/spinifex-$(VERSION)-linux-amd64.tar.gz -C dist/amd64 .
|
||||
sha256sum dist/spinifex-$(VERSION)-linux-amd64.tar.gz > dist/spinifex-$(VERSION)-linux-amd64.tar.gz.sha256
|
||||
@@ -534,7 +522,7 @@ distro-arm64:
|
||||
distro-clean:
|
||||
rm -rf dist/
|
||||
|
||||
.PHONY: test-package-check build build-ui test-ui-e2e ui-e2e-container ui-baselines build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image build-sqs-store-microvm-image build-sqs-worker-microvm-image install-microvm install-sqs-store-microvm install-sqs-worker-microvm go_build preflight test test-cover test-race diff-coverage bench fuzz test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle \
|
||||
.PHONY: test-package-check build build-ui test-ui-e2e ui-e2e-container ui-baselines build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image build-sqs-worker-microvm-image install-microvm install-sqs-worker-microvm go_build preflight test test-cover test-race diff-coverage bench fuzz test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle \
|
||||
deploy reinstall clean \
|
||||
install-system install-go install-aws quickinstall \
|
||||
lint fix govulncheck nilaway \
|
||||
|
||||
@@ -38,6 +38,7 @@ var allowlist = []allowance{
|
||||
{From: "cmd/aws-model-coverage", To: "domains/ecs/awsapi", Debt: "cmd/aws-model-coverage", Reason: "reads declared service inventories; named exception, must not construct a registration"},
|
||||
{From: "cmd/aws-model-coverage", To: "domains/eks/awsapi", Debt: "cmd/aws-model-coverage", Reason: "reads declared service inventories; named exception, must not construct a registration"},
|
||||
{From: "cmd/aws-model-coverage", To: "domains/iam/awsapi", Debt: "cmd/aws-model-coverage", Reason: "reads declared service inventories; named exception, must not construct a registration"},
|
||||
{From: "cmd/aws-model-coverage", To: "domains/sqs/awsapi", Debt: "cmd/aws-model-coverage", Reason: "reads declared service inventories; named exception, must not construct a registration"},
|
||||
{From: "cmd/aws-model-coverage", To: "domains/sts/awsapi", Debt: "cmd/aws-model-coverage", Reason: "reads declared service inventories; named exception, must not construct a registration"},
|
||||
|
||||
{From: "runtime/daemon", To: "domains/ec2/awsapi/instance", Debt: "Instance adapter consumers outside EC2", Reason: "runtime/daemon/eks_cp_control.go and rds_deps.go call the instance adapter; needs an EC2 capability"},
|
||||
|
||||
@@ -1,10 +0,0 @@
|
||||
# The power button loads here, not in /init: busybox init has its SIGUSR2
|
||||
# (poweroff) handler installed before sysinit runs, so a press can never reach
|
||||
# the pre-exec shell PID 1. A press before this line is lost; the host re-sends.
|
||||
::sysinit:/sbin/modprobe tiny_power_button power_signal=12
|
||||
::sysinit:/usr/local/lib/sqs-store/sqs-store-prepare
|
||||
::respawn:/usr/local/lib/sqs-store/sqs-store-run
|
||||
::ctrlaltdel:/sbin/reboot
|
||||
# One action: the store stops and its disk unmounts first, and the generic
|
||||
# umount -a -r runs only if the store is gone.
|
||||
::shutdown:/usr/local/lib/sqs-store/sqs-store-stop
|
||||
@@ -1,224 +0,0 @@
|
||||
# sqs-store-lib.sh — shared by the SQS store guest's prepare, run and stop
|
||||
# steps. Sourced, never executed. Every path and command is overridable so the
|
||||
# host test can drive the same code against a fake fw_cfg, sysfs and tools.
|
||||
#
|
||||
# Guest contract, read from fw_cfg under opt/spinifex/sqs-store/:
|
||||
# data-serial virtio-blk serial of the already-formatted ext4 data disk
|
||||
# listen-addr IPv4 address on a configured guest NIC; the only listener
|
||||
# client-nkey NATS user NKey public key allowed to connect; never a seed
|
||||
# Anything absent, malformed or ambiguous refuses the store. Nothing formats.
|
||||
|
||||
SQS_FWCFG=${SQS_FWCFG:-/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/sqs-store}
|
||||
SQS_SYSBLOCK=${SQS_SYSBLOCK:-/sys/block}
|
||||
SQS_DEVDIR=${SQS_DEVDIR:-/dev}
|
||||
# The host test cannot create block nodes, so it relaxes this to -e.
|
||||
SQS_NODE_TEST=${SQS_NODE_TEST:--b}
|
||||
SQS_RUN=${SQS_RUN:-/run/sqs-store}
|
||||
SQS_DATA=${SQS_DATA:-/data}
|
||||
SQS_CONSOLE=${SQS_CONSOLE:-/dev/console}
|
||||
SQS_MOUNTS=${SQS_MOUNTS:-/proc/mounts}
|
||||
SQS_PORT=${SQS_PORT:-4222}
|
||||
SQS_STOP_BUDGET_TENTHS=${SQS_STOP_BUDGET_TENTHS:-120}
|
||||
NATS_BIN=${NATS_BIN:-/usr/local/bin/nats-server}
|
||||
E2FSCK=${E2FSCK:-e2fsck}
|
||||
MOUNT=${MOUNT:-mount}
|
||||
UMOUNT=${UMOUNT:-umount}
|
||||
IP=${IP:-ip}
|
||||
|
||||
sqs_log() { echo "[sqs-store] $*" >> "$SQS_CONSOLE"; }
|
||||
sqs_fatal() { sqs_log "FATAL: $*"; return 1; }
|
||||
|
||||
# sqs_fwcfg <name>: print one fw_cfg value, refusing a missing, empty, oversized
|
||||
# or multi-token file. The value is not echoed on failure.
|
||||
sqs_fwcfg() {
|
||||
f="$SQS_FWCFG/$1/raw"
|
||||
[ -f "$f" ] || { sqs_fatal "fw_cfg $1 not supplied"; return 1; }
|
||||
[ "$(wc -c < "$f")" -le 128 ] || { sqs_fatal "fw_cfg $1 is oversized"; return 1; }
|
||||
v=$(cat "$f")
|
||||
case "$v" in
|
||||
"" | *[[:space:]]*) sqs_fatal "fw_cfg $1 must be exactly one value"; return 1 ;;
|
||||
esac
|
||||
printf '%s' "$v"
|
||||
}
|
||||
|
||||
# A volume serial is the volume ID with dashes stripped: vol + 17 hex.
|
||||
sqs_valid_serial() {
|
||||
[ "${#1}" -eq 20 ] || return 1
|
||||
case "$1" in vol*) ;; *) return 1 ;; esac
|
||||
case "${1#vol}" in *[!0-9a-f]*) return 1 ;; esac
|
||||
}
|
||||
|
||||
# A NATS user public NKey: "U" then 55 base32 characters. The checksum is
|
||||
# left to the pinned server's own config test.
|
||||
sqs_valid_user_nkey() {
|
||||
[ "${#1}" -eq 56 ] || return 1
|
||||
case "$1" in U*) ;; *) return 1 ;; esac
|
||||
case "$1" in *[!A-Z2-7]*) return 1 ;; esac
|
||||
}
|
||||
|
||||
sqs_valid_ipv4() {
|
||||
case "$1" in *[!0-9.]* | .* | *. | *..*) return 1 ;; esac
|
||||
old_ifs=$IFS; IFS=.
|
||||
# shellcheck disable=SC2086 # split on dots on purpose
|
||||
set -- $1
|
||||
IFS=$old_ifs
|
||||
[ "$#" -eq 4 ] || return 1
|
||||
for o in "$@"; do
|
||||
[ "${#o}" -le 3 ] && [ "$o" -le 255 ] || return 1
|
||||
case "$o" in 0?*) return 1 ;; esac
|
||||
done
|
||||
}
|
||||
|
||||
# sqs_find_disk <serial>: print the /dev node of the one block device whose
|
||||
# sysfs serial matches exactly. Zero or several matches refuse.
|
||||
sqs_find_disk() {
|
||||
found="" n=0
|
||||
for b in "$SQS_SYSBLOCK"/*; do
|
||||
[ -r "$b/serial" ] || continue
|
||||
[ "$(cat "$b/serial")" = "$1" ] || continue
|
||||
found=${b##*/}
|
||||
n=$((n + 1))
|
||||
done
|
||||
[ "$n" -eq 1 ] || { sqs_fatal "expected one disk with serial $1, found $n"; return 1; }
|
||||
test "$SQS_NODE_TEST" "$SQS_DEVDIR/$found" || { sqs_fatal "disk $found has no block device node"; return 1; }
|
||||
printf '%s' "$SQS_DEVDIR/$found"
|
||||
}
|
||||
|
||||
# sqs_has_addr <ipv4>: the address is assigned to a non-loopback interface.
|
||||
sqs_has_addr() {
|
||||
"$IP" -o -4 addr show 2>/dev/null \
|
||||
| awk -v a="$1" '$2 != "lo" { split($4, p, "/"); if (p[1] == a) f = 1 } END { exit !f }'
|
||||
}
|
||||
|
||||
# sqs_alive <pid>: running or stopped, not exited (a zombie counts as exited).
|
||||
sqs_alive() { [ -r "/proc/$1/status" ] && ! grep -q '^State:[[:space:]]*Z' "/proc/$1/status" 2>/dev/null; }
|
||||
|
||||
# sqs_starttime <pid>: kernel start time (stat field 22). It survives exec, and
|
||||
# pairing it with the pid reduces PID-reuse risk; without a pidfd the check and
|
||||
# the signal stay separate steps, so it is not an absolute guarantee.
|
||||
sqs_starttime() { sed 's/.*) //' "/proc/$1/stat" 2>/dev/null | cut -d' ' -f20; }
|
||||
|
||||
# sqs_same <pid> <starttime>: the recorded process is still alive.
|
||||
sqs_same() { [ -n "$2" ] && sqs_alive "$1" && [ "$(sqs_starttime "$1")" = "$2" ]; }
|
||||
|
||||
# sqs_write_config <addr> <nkey> <file>: the dedicated server's config. Its
|
||||
# max_payload admits a 1 MiB SQS message with its headers (ADR-0011:S4); the
|
||||
# 1MB NATS default does not.
|
||||
sqs_write_config() {
|
||||
cat > "$3" <<EOF
|
||||
listen: "$1:$SQS_PORT"
|
||||
server_name: "sqs-store"
|
||||
max_payload: 2MiB
|
||||
jetstream {
|
||||
store_dir: "$SQS_DATA"
|
||||
sync_interval: always
|
||||
}
|
||||
authorization {
|
||||
users = [ { nkey: "$2" } ]
|
||||
}
|
||||
EOF
|
||||
}
|
||||
|
||||
# sqs_prepare: validate the contract, check and mount the data disk, write the
|
||||
# server config and mark the store ready. Returns non-zero, unready, on any refusal.
|
||||
sqs_prepare() {
|
||||
rm -f "$SQS_RUN/ready"
|
||||
mkdir -p "$SQS_RUN"
|
||||
serial=$(sqs_fwcfg data-serial) || return 1
|
||||
sqs_valid_serial "$serial" || { sqs_fatal "data-serial is not a volume serial"; return 1; }
|
||||
addr=$(sqs_fwcfg listen-addr) || return 1
|
||||
sqs_valid_ipv4 "$addr" || { sqs_fatal "listen-addr is not an IPv4 address"; return 1; }
|
||||
nkey=$(sqs_fwcfg client-nkey) || return 1
|
||||
sqs_valid_user_nkey "$nkey" || { sqs_fatal "client-nkey is not a user public NKey"; return 1; }
|
||||
sqs_has_addr "$addr" || { sqs_fatal "listen-addr $addr is not on any guest NIC"; return 1; }
|
||||
|
||||
# The server checks the config, including the NKey checksum, before the
|
||||
# disk is touched.
|
||||
umask 077
|
||||
sqs_write_config "$addr" "$nkey" "$SQS_RUN/nats.conf" || return 1
|
||||
if ! "$NATS_BIN" -t -c "$SQS_RUN/nats.conf" > "$SQS_RUN/config-test.out" 2>&1; then
|
||||
sqs_fatal "server config test failed: $(sed 's/U[A-Z2-7]\{55\}/<nkey>/g' "$SQS_RUN/config-test.out" | tr '\n' ' ')"
|
||||
return 1
|
||||
fi
|
||||
|
||||
dev=$(sqs_find_disk "$serial") || return 1
|
||||
rc=0
|
||||
"$E2FSCK" -p "$dev" > "$SQS_RUN/fsck.out" 2>&1 || rc=$?
|
||||
sqs_log "e2fsck -p $dev rc=$rc"
|
||||
# 0 clean, 1 corrected: proceed. Anything higher, including a disk that is
|
||||
# not ext4 at all, refuses; the guest never formats.
|
||||
[ "$rc" -le 1 ] || { sqs_fatal "e2fsck refused $dev: $(tr '\n' ' ' < "$SQS_RUN/fsck.out")"; return 1; }
|
||||
mkdir -p "$SQS_DATA"
|
||||
"$MOUNT" -t ext4 -o nodev,nosuid,noexec "$dev" "$SQS_DATA" || { sqs_fatal "mount $dev failed"; return 1; }
|
||||
: > "$SQS_RUN/ready"
|
||||
sqs_log "prepared: $dev mounted at $SQS_DATA, config written for $addr:$SQS_PORT; server not started yet"
|
||||
}
|
||||
|
||||
# sqs_run: the respawned supervisor body. Its identity is recorded before the
|
||||
# stopping check: a stop that misses it set stopping first, so this parks; a
|
||||
# stop that reads it signals this process whether or not it has exec'd yet.
|
||||
sqs_run() {
|
||||
mkdir -p "$SQS_RUN"
|
||||
echo "$$ $(sqs_starttime $$)" > "$SQS_RUN/pid"
|
||||
if [ -e "$SQS_RUN/stopping" ]; then
|
||||
sqs_log "run: shutdown in progress, not starting"
|
||||
return 2
|
||||
fi
|
||||
if [ ! -e "$SQS_RUN/ready" ]; then
|
||||
sqs_log "run: store not prepared, not starting"
|
||||
return 2
|
||||
fi
|
||||
sqs_log "run: starting pid=$$"
|
||||
# ADR-0011:S11 bounds the store's Go heap; the guest must be larger.
|
||||
GOMEMLIMIT=512MiB exec "$NATS_BIN" -c "$SQS_RUN/nats.conf"
|
||||
}
|
||||
|
||||
# sqs_stop: first shutdown action. Block any respawn, SIGTERM the recorded run
|
||||
# process (a server, or a start not yet exec'd), SIGKILL only after the budget,
|
||||
# then sync and unmount. Each signal rechecks the identity; a survivor keeps the disk mounted.
|
||||
sqs_stop() {
|
||||
mkdir -p "$SQS_RUN"
|
||||
: > "$SQS_RUN/stopping"
|
||||
sqs_log "stop: begin"
|
||||
rec=$(cat "$SQS_RUN/pid" 2>/dev/null || true)
|
||||
pid=${rec%% *} st=${rec#* }
|
||||
[ "$st" != "$rec" ] || st=""
|
||||
if [ -n "$pid" ] && sqs_same "$pid" "$st"; then
|
||||
kill -TERM "$pid" 2>/dev/null || true
|
||||
sqs_log "stop: SIGTERM sent to run pid=$pid"
|
||||
i=0
|
||||
while sqs_same "$pid" "$st" && [ "$i" -lt "$SQS_STOP_BUDGET_TENTHS" ]; do sleep 0.1; i=$((i + 1)); done
|
||||
if sqs_same "$pid" "$st"; then
|
||||
sqs_log "stop: run pid=$pid still running after $((SQS_STOP_BUDGET_TENTHS / 10))s, SIGKILL"
|
||||
kill -KILL "$pid" 2>/dev/null || true
|
||||
j=0
|
||||
while sqs_same "$pid" "$st" && [ "$j" -lt 30 ]; do sleep 0.1; j=$((j + 1)); done
|
||||
fi
|
||||
if sqs_same "$pid" "$st"; then
|
||||
sqs_log "stop: run pid=$pid STILL ALIVE, leaving $SQS_DATA mounted"
|
||||
return 1
|
||||
fi
|
||||
sqs_log "stop: store exited after $((i / 10)).$((i % 10))s"
|
||||
else
|
||||
sqs_log "stop: no running store"
|
||||
fi
|
||||
sync
|
||||
if grep -q " $SQS_DATA " "$SQS_MOUNTS" 2>/dev/null; then
|
||||
rc=0
|
||||
"$UMOUNT" "$SQS_DATA" || rc=$?
|
||||
sqs_log "stop: umount $SQS_DATA rc=$rc"
|
||||
fi
|
||||
}
|
||||
|
||||
# sqs_shutdown: the only shutdown action. The generic read-only remount runs
|
||||
# only after the store stopped; a surviving writer keeps every mount as it is.
|
||||
sqs_shutdown() {
|
||||
if ! sqs_stop; then
|
||||
sync
|
||||
sqs_log "shutdown: store survived, skipping umount -a -r"
|
||||
return 1
|
||||
fi
|
||||
rc=0
|
||||
"$UMOUNT" -a -r || rc=$?
|
||||
echo "[init] shutdown: umount -a -r rc=$rc" >> "$SQS_CONSOLE"
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
#!/bin/sh
|
||||
# sysinit: load the disk drivers, then validate the guest contract, check and
|
||||
# mount the data disk and write the server config. A refusal leaves the store
|
||||
# unprepared, so the run step never starts it.
|
||||
. /usr/local/lib/sqs-store/sqs-store-lib.sh
|
||||
modprobe virtio_blk 2>/dev/null || true
|
||||
modprobe ext4 2>/dev/null || true
|
||||
sqs_prepare || sqs_log "store refused; it will not start"
|
||||
exit 0
|
||||
@@ -1,6 +0,0 @@
|
||||
#!/bin/sh
|
||||
# Respawned by init. A store that must not run parks here instead of exiting,
|
||||
# so init does not respawn it in a tight loop.
|
||||
. /usr/local/lib/sqs-store/sqs-store-lib.sh
|
||||
sqs_run
|
||||
exec sleep 2147483647
|
||||
@@ -1,5 +0,0 @@
|
||||
#!/bin/sh
|
||||
# The only shutdown action: stop the store and unmount its disk, then run the
|
||||
# generic read-only remount only if the store is gone.
|
||||
. /usr/local/lib/sqs-store/sqs-store-lib.sh
|
||||
sqs_shutdown
|
||||
@@ -33,6 +33,10 @@ load_mod failover || true
|
||||
load_mod net_failover || true
|
||||
load_mod virtio_net || true
|
||||
|
||||
# The guest's SQS store listens on 127.0.0.1 and the worker dials it there;
|
||||
# the kernel leaves lo down, so bring it up before init starts either.
|
||||
ip link set lo up
|
||||
|
||||
NETCFG=/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw
|
||||
i=0
|
||||
while [ "$i" -lt 25 ]; do
|
||||
@@ -87,8 +91,8 @@ for n in 0 1 2 3 4 5; do
|
||||
[ -n "$rdst" ] && [ -n "$rvia" ] && ip route add "$rdst" via "$rvia" dev "$iface"
|
||||
done
|
||||
|
||||
# The store image takes no lb-agent environment. A CA bundle is optional here:
|
||||
# nothing in this image dials out yet.
|
||||
# The image takes no lb-agent environment. The worker reads the cluster CA
|
||||
# from fw_cfg itself; this copy only serves other tools in the guest.
|
||||
CA_CERT=/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/ca-cert/raw
|
||||
mkdir -p /etc/ssl/certs
|
||||
[ -f "$CA_CERT" ] && cp "$CA_CERT" /etc/ssl/certs/ca-certificates.crt
|
||||
@@ -3,6 +3,7 @@
|
||||
# the pre-exec shell PID 1. A press before this line is lost; the host re-sends.
|
||||
::sysinit:/sbin/modprobe tiny_power_button power_signal=12
|
||||
::sysinit:/usr/local/lib/sqs-worker/sqs-worker-prepare
|
||||
::respawn:/usr/local/lib/sqs-worker/sqs-worker-store
|
||||
::respawn:/usr/local/lib/sqs-worker/sqs-worker-ready
|
||||
::ctrlaltdel:/sbin/reboot
|
||||
::shutdown:/usr/local/lib/sqs-worker/sqs-worker-stop
|
||||
|
||||
@@ -1,19 +1,23 @@
|
||||
# sqs-worker-lib.sh — shared by the SQS worker guest's readiness and stop
|
||||
# sqs-worker-lib.sh — shared by the SQS guest's store, readiness and stop
|
||||
# steps. Sourced, never executed. Every path and command is overridable so the
|
||||
# host test can drive the same code against test-owned processes and stub tools.
|
||||
#
|
||||
# store-ready exists only while the running watcher's last check passed. A
|
||||
# check proves the store accepted the worker key, not the store server's
|
||||
# identity, and a store failure is noticed up to one interval plus call time late.
|
||||
# The guest runs its own SQS nats-server on loopback and the worker beside it.
|
||||
# The server starts only after boot settled the receipt key; store-ready exists
|
||||
# only while the running watcher's last check of that server passed.
|
||||
|
||||
SQSW_RUN=${SQSW_RUN:-/run/sqs-worker}
|
||||
SQSW_DATA=${SQSW_DATA:-/data}
|
||||
SQSW_CONSOLE=${SQSW_CONSOLE:-/dev/console}
|
||||
SQSW_MOUNTS=${SQSW_MOUNTS:-/proc/mounts}
|
||||
SQSW_BIN=${SQSW_BIN:-/usr/local/bin/sqs-worker}
|
||||
SQSW_LIB=${SQSW_LIB:-/usr/local/lib/sqs-worker/sqs-worker-lib.sh}
|
||||
SQSW_NATS_BIN=${SQSW_NATS_BIN:-/usr/local/bin/nats-server}
|
||||
SQSW_STORE_PORT=${SQSW_STORE_PORT:-4222}
|
||||
SQSW_STOP_BUDGET_TENTHS=${SQSW_STOP_BUDGET_TENTHS:-100}
|
||||
SQSW_RESPAWN_GAP=${SQSW_RESPAWN_GAP:-5}
|
||||
SQSW_PARK_INTERVAL=${SQSW_PARK_INTERVAL:-30}
|
||||
SQSW_PARK=${SQSW_PARK:-sleep 2147483647}
|
||||
SQSW_BOOT_RETRY=${SQSW_BOOT_RETRY:-30}
|
||||
UMOUNT=${UMOUNT:-umount}
|
||||
|
||||
sqsw_log() { echo "[sqs-worker] $*" >> "$SQSW_CONSOLE"; }
|
||||
@@ -28,28 +32,102 @@ sqsw_starttime() { sed 's/.*) //' "/proc/$1/stat" 2>/dev/null | cut -d' ' -f20;
|
||||
# sqsw_same <pid> <starttime>: the recorded process is still alive.
|
||||
sqsw_same() { [ -n "$2" ] && sqsw_alive "$1" && [ "$(sqsw_starttime "$1")" = "$2" ]; }
|
||||
|
||||
# sqsw_write_config <file>: the guest's own server. It listens only on
|
||||
# loopback, with no monitor, route or cluster listener. Its max_payload admits
|
||||
# a 1 MiB SQS message with its headers (ADR-0011:S4); the 1MB default does not.
|
||||
sqsw_write_config() {
|
||||
cat > "$1" <<CONF
|
||||
listen: "127.0.0.1:$SQSW_STORE_PORT"
|
||||
server_name: "sqs-store"
|
||||
max_payload: 2MiB
|
||||
jetstream {
|
||||
store_dir: "$SQSW_DATA/jetstream"
|
||||
sync_interval: always
|
||||
}
|
||||
CONF
|
||||
}
|
||||
|
||||
# sqsw_prepare_store: write and test the server config once boot has settled
|
||||
# the receipt key. Returns non-zero, leaving the store unstarted, otherwise.
|
||||
sqsw_prepare_store() {
|
||||
rm -f "$SQSW_RUN/store.conf"
|
||||
[ -s "$SQSW_RUN/receipt-ready" ] || { sqsw_log "store: receipt key not settled, store not prepared"; return 1; }
|
||||
umask 077
|
||||
sqsw_write_config "$SQSW_RUN/store.conf.new" || return 1
|
||||
if ! "$SQSW_NATS_BIN" -t -c "$SQSW_RUN/store.conf.new" > "$SQSW_RUN/config-test.out" 2>&1; then
|
||||
sqsw_log "store: FATAL: server config test failed: $(tr '\n' ' ' < "$SQSW_RUN/config-test.out")"
|
||||
return 1
|
||||
fi
|
||||
mv "$SQSW_RUN/store.conf.new" "$SQSW_RUN/store.conf"
|
||||
sqsw_log "store: config written for 127.0.0.1:$SQSW_STORE_PORT; server not started yet"
|
||||
}
|
||||
|
||||
# sqsw_park <why>: init respawns a step that exits, so a step that must not run
|
||||
# logs once and waits for shutdown instead of looping.
|
||||
sqsw_park() {
|
||||
sqsw_log "$1"
|
||||
exec $SQSW_PARK
|
||||
}
|
||||
|
||||
# sqsw_record <name>: record this process's identity before the stopping
|
||||
# check, so a stop that misses the record has already set stopping.
|
||||
sqsw_record() { echo "$$ $(sqsw_starttime $$)" > "$SQSW_RUN/$1.pid"; }
|
||||
|
||||
# sqsw_boot: settle the disk and receipt key once per guest boot. A transient
|
||||
# failure (exit 75) waits and exits so init runs it again; boot never remounts
|
||||
# a disk already mounted. Any other failure is final for this guest boot.
|
||||
sqsw_boot() {
|
||||
[ ! -e "$SQSW_RUN/boot-refused" ] || sqsw_park "store: boot refused, not starting"
|
||||
[ -s "$SQSW_RUN/receipt-ready" ] && return 0
|
||||
# The boot child records itself before its stopping check, as the other
|
||||
# steps do, so shutdown either sees its record or it never starts.
|
||||
SQSW_RUN="$SQSW_RUN" SQSW_BIN="$SQSW_BIN" sh -c '. "$1"; sqsw_record boot; [ ! -e "$SQSW_RUN/stopping" ] || exit 75; exec "$SQSW_BIN" boot' \
|
||||
sh "$SQSW_LIB" >> "$SQSW_CONSOLE" 2>&1 &
|
||||
child=$!
|
||||
rc=0
|
||||
wait "$child" || rc=$?
|
||||
rm -f "$SQSW_RUN/boot.pid"
|
||||
[ ! -e "$SQSW_RUN/stopping" ] || sqsw_park "store: shutdown during boot, not starting"
|
||||
case "$rc" in
|
||||
0) return 0 ;;
|
||||
75) sqsw_log "store: boot not settled, retrying in ${SQSW_BOOT_RETRY}s"
|
||||
sleep "$SQSW_BOOT_RETRY"
|
||||
exit 1 ;;
|
||||
*) : > "$SQSW_RUN/boot-refused"
|
||||
sqsw_park "store: boot refused (rc=$rc), not starting" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# sqsw_store: the respawned server body. It starts only on a prepared config.
|
||||
sqsw_store() {
|
||||
mkdir -p "$SQSW_RUN"
|
||||
sqsw_record store
|
||||
[ ! -e "$SQSW_RUN/stopping" ] || sqsw_park "store: shutdown in progress, not starting"
|
||||
sqsw_boot
|
||||
if [ ! -s "$SQSW_RUN/store.conf" ] && ! sqsw_prepare_store; then
|
||||
: > "$SQSW_RUN/boot-refused"
|
||||
sqsw_park "store: not prepared, not starting"
|
||||
fi
|
||||
sqsw_log "store: starting pid=$$"
|
||||
# ADR-0011:S11 bounds the store's Go heap; the guest must be larger.
|
||||
GOMEMLIMIT=512MiB exec "$SQSW_NATS_BIN" -c "$SQSW_RUN/store.conf"
|
||||
}
|
||||
|
||||
# sqsw_run: the respawned readiness body. A predecessor that was killed or
|
||||
# crashed could not remove its marker, so every start removes it first. Its
|
||||
# identity is recorded before the stopping check, as the store's run step does.
|
||||
# crashed could not remove its marker, so every start removes it first.
|
||||
sqsw_run() {
|
||||
mkdir -p "$SQSW_RUN"
|
||||
echo "$$ $(sqsw_starttime $$)" > "$SQSW_RUN/ready.pid"
|
||||
sqsw_record ready
|
||||
if [ -e "$SQSW_RUN/store-ready" ]; then
|
||||
rm -f "$SQSW_RUN/store-ready"
|
||||
sqsw_log "ready: removed a store-ready marker left by an earlier watcher"
|
||||
fi
|
||||
# Returning here would make init respawn and log at once, forever. Park
|
||||
# instead, logging once: shutdown stops this recorded process, and a key
|
||||
# that appears later lets the watcher start.
|
||||
if [ -e "$SQSW_RUN/stopping" ] || [ ! -s "$SQSW_RUN/public-key" ]; then
|
||||
if [ -e "$SQSW_RUN/stopping" ]; then
|
||||
sqsw_log "ready: shutdown in progress, not starting"
|
||||
else
|
||||
sqsw_log "ready: no worker key; readiness not started, waiting"
|
||||
fi
|
||||
while [ -e "$SQSW_RUN/stopping" ] || [ ! -s "$SQSW_RUN/public-key" ]; do
|
||||
sleep "$SQSW_PARK_INTERVAL"
|
||||
done
|
||||
[ ! -e "$SQSW_RUN/stopping" ] || sqsw_park "ready: shutdown in progress, not starting"
|
||||
[ ! -e "$SQSW_RUN/boot-refused" ] || sqsw_park "ready: boot refused, readiness not started"
|
||||
# Boot may still be retrying; check again after a gap rather than parking.
|
||||
if [ ! -s "$SQSW_RUN/store.conf" ]; then
|
||||
sleep "$SQSW_RESPAWN_GAP"
|
||||
exit 0
|
||||
fi
|
||||
# Init respawns at once; a watcher that keeps exiting restarts at most
|
||||
# once per gap instead of in a tight loop.
|
||||
@@ -62,37 +140,47 @@ sqsw_run() {
|
||||
exec "$SQSW_BIN" ready >> "$SQSW_CONSOLE" 2>&1
|
||||
}
|
||||
|
||||
# sqsw_stop: block any respawn, drop the marker, SIGTERM the recorded watcher,
|
||||
# SIGKILL only after the budget, and unmount the worker disk only once the
|
||||
# watcher is confirmed gone. A survivor keeps the disk mounted.
|
||||
# sqsw_end <name>: SIGTERM the recorded process, SIGKILL only after the
|
||||
# budget. Returns non-zero while it is still alive.
|
||||
sqsw_end() {
|
||||
rec=$(cat "$SQSW_RUN/$1.pid" 2>/dev/null || true)
|
||||
pid=${rec%% *} st=${rec#* }
|
||||
[ "$st" != "$rec" ] || st=""
|
||||
if [ -z "$pid" ] || ! sqsw_same "$pid" "$st"; then
|
||||
sqsw_log "stop: no running $1"
|
||||
return 0
|
||||
fi
|
||||
kill -TERM "$pid" 2>/dev/null || true
|
||||
sqsw_log "stop: SIGTERM sent to $1 pid=$pid"
|
||||
i=0
|
||||
while sqsw_same "$pid" "$st" && [ "$i" -lt "$SQSW_STOP_BUDGET_TENTHS" ]; do sleep 0.1; i=$((i + 1)); done
|
||||
if sqsw_same "$pid" "$st"; then
|
||||
sqsw_log "stop: $1 pid=$pid still running after $((SQSW_STOP_BUDGET_TENTHS / 10))s, SIGKILL"
|
||||
kill -KILL "$pid" 2>/dev/null || true
|
||||
j=0
|
||||
while sqsw_same "$pid" "$st" && [ "$j" -lt 30 ]; do sleep 0.1; j=$((j + 1)); done
|
||||
fi
|
||||
if sqsw_same "$pid" "$st"; then
|
||||
sqsw_log "stop: $1 pid=$pid STILL ALIVE"
|
||||
return 1
|
||||
fi
|
||||
sqsw_log "stop: $1 pid=$pid exited"
|
||||
rm -f "$SQSW_RUN/$1.pid"
|
||||
}
|
||||
|
||||
# sqsw_stop: block any respawn, drop the marker, stop the watcher, the server
|
||||
# and any boot in progress, and unmount the disk only once all are confirmed
|
||||
# gone. A survivor keeps the disk mounted.
|
||||
sqsw_stop() {
|
||||
mkdir -p "$SQSW_RUN"
|
||||
: > "$SQSW_RUN/stopping"
|
||||
rm -f "$SQSW_RUN/store-ready"
|
||||
rec=$(cat "$SQSW_RUN/ready.pid" 2>/dev/null || true)
|
||||
pid=${rec%% *} st=${rec#* }
|
||||
[ "$st" != "$rec" ] || st=""
|
||||
if [ -n "$pid" ] && sqsw_same "$pid" "$st"; then
|
||||
kill -TERM "$pid" 2>/dev/null || true
|
||||
sqsw_log "stop: SIGTERM sent to readiness watcher pid=$pid"
|
||||
i=0
|
||||
while sqsw_same "$pid" "$st" && [ "$i" -lt "$SQSW_STOP_BUDGET_TENTHS" ]; do sleep 0.1; i=$((i + 1)); done
|
||||
if sqsw_same "$pid" "$st"; then
|
||||
sqsw_log "stop: readiness watcher pid=$pid still running after $((SQSW_STOP_BUDGET_TENTHS / 10))s, SIGKILL"
|
||||
kill -KILL "$pid" 2>/dev/null || true
|
||||
j=0
|
||||
while sqsw_same "$pid" "$st" && [ "$j" -lt 30 ]; do sleep 0.1; j=$((j + 1)); done
|
||||
fi
|
||||
if sqsw_same "$pid" "$st"; then
|
||||
rm -f "$SQSW_RUN/store-ready"
|
||||
sqsw_log "stop: readiness watcher pid=$pid STILL ALIVE, leaving $SQSW_DATA mounted"
|
||||
return 1
|
||||
fi
|
||||
sqsw_log "stop: readiness watcher pid=$pid exited"
|
||||
else
|
||||
sqsw_log "stop: no running readiness watcher"
|
||||
if ! sqsw_end ready || ! sqsw_end store || ! sqsw_end boot; then
|
||||
rm -f "$SQSW_RUN/store-ready"
|
||||
sqsw_log "stop: leaving $SQSW_DATA mounted"
|
||||
return 1
|
||||
fi
|
||||
rm -f "$SQSW_RUN/store-ready" "$SQSW_RUN/ready.pid"
|
||||
rm -f "$SQSW_RUN/store-ready"
|
||||
sync
|
||||
if grep -q " $SQSW_DATA " "$SQSW_MOUNTS" 2>/dev/null; then
|
||||
rc=0
|
||||
@@ -102,11 +190,11 @@ sqsw_stop() {
|
||||
}
|
||||
|
||||
# sqsw_shutdown: the only shutdown action. The generic read-only remount runs
|
||||
# only after the watcher stopped; a survivor keeps every mount as it is.
|
||||
# only after both processes stopped; a survivor keeps every mount as it is.
|
||||
sqsw_shutdown() {
|
||||
if ! sqsw_stop; then
|
||||
sync
|
||||
sqsw_log "shutdown: readiness watcher survived, skipping umount -a -r"
|
||||
sqsw_log "shutdown: a guest process survived, skipping umount -a -r"
|
||||
return 1
|
||||
fi
|
||||
rc=0
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
#!/bin/sh
|
||||
# sysinit: load the disk drivers, then select, check and mount the worker disk
|
||||
# and load or create the worker's store key on it. Any refusal is logged; the
|
||||
# key is never read or created on a disk that is not proven mounted.
|
||||
# sysinit: load the disk drivers and clear this boot's markers. The respawned
|
||||
# store step mounts the disk, settles the receipt key and starts the store.
|
||||
. /usr/local/lib/sqs-worker/sqs-worker-lib.sh
|
||||
modprobe virtio_blk 2>/dev/null || true
|
||||
modprobe ext4 2>/dev/null || true
|
||||
/usr/local/bin/sqs-worker boot >> /dev/console 2>&1 || echo "[sqs-worker] boot refused" >> /dev/console
|
||||
mkdir -p "$SQSW_RUN"
|
||||
rm -f "$SQSW_RUN/receipt-ready" "$SQSW_RUN/store.conf" "$SQSW_RUN/boot-refused"
|
||||
exit 0
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
#!/bin/sh
|
||||
# Respawned by init. A watcher that must not run parks here instead of exiting,
|
||||
# so init does not respawn it in a tight loop.
|
||||
# Respawned by init: the worker, reporting ready only while the local store
|
||||
# answers. A watcher that must not run parks instead of exiting.
|
||||
. /usr/local/lib/sqs-worker/sqs-worker-lib.sh
|
||||
sqsw_run
|
||||
exec sleep 2147483647
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#!/bin/sh
|
||||
# The only shutdown action: stop the readiness watcher and unmount the worker
|
||||
# disk, then run the generic read-only remount only if the watcher is gone.
|
||||
# The only shutdown action: stop the worker, then the store, and unmount the
|
||||
# data disk, then run the generic read-only remount only if both are gone.
|
||||
. /usr/local/lib/sqs-worker/sqs-worker-lib.sh
|
||||
sqsw_shutdown
|
||||
|
||||
Executable
+5
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
# Respawned by init: the guest's own SQS nats-server, started only on a
|
||||
# prepared config. A store that must not run parks instead of exiting.
|
||||
. /usr/local/lib/sqs-worker/sqs-worker-lib.sh
|
||||
sqsw_store
|
||||
@@ -1,4 +1,4 @@
|
||||
# spinifex-unit-version: 5
|
||||
# spinifex-unit-version: 6
|
||||
[Unit]
|
||||
Description=Spinifex Daemon (VM Orchestration)
|
||||
PartOf=spinifex.target
|
||||
@@ -15,6 +15,7 @@ Slice=spinifex-system.slice
|
||||
User=spinifex-daemon
|
||||
Group=spinifex
|
||||
ExecStartPre=/usr/local/bin/spx admin hostnet check
|
||||
ExecStartPre=-+/usr/local/bin/spx admin node-facts collect
|
||||
ExecStart=/usr/local/bin/spx service spinifex start
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
iamawsapi "github.com/mulgadc/spinifex/spinifex/domains/iam/awsapi"
|
||||
rdsawsapi "github.com/mulgadc/spinifex/spinifex/domains/rds/awsapi"
|
||||
route53awsapi "github.com/mulgadc/spinifex/spinifex/domains/route53/awsapi"
|
||||
sqsawsapi "github.com/mulgadc/spinifex/spinifex/domains/sqs/awsapi"
|
||||
stsawsapi "github.com/mulgadc/spinifex/spinifex/domains/sts/awsapi"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
|
||||
|
||||
@@ -66,6 +67,7 @@ func compareAll() ([]awsmodel.OperationCoverage, error) {
|
||||
ecsawsapi.ServiceName: ecsawsapi.OperationInventory(),
|
||||
eksawsapi.ServiceName: eksawsapi.OperationInventory(),
|
||||
iamawsapi.ServiceName: iamawsapi.OperationInventory(),
|
||||
sqsawsapi.ServiceName: sqsawsapi.OperationInventory(),
|
||||
stsawsapi.ServiceName: stsawsapi.OperationInventory(),
|
||||
})
|
||||
// The loop below walks the embedded models, so a served service with no
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
// Command eks-pod-identity-webhook is the IRSA mutating admission webhook run
|
||||
// on every EKS control-plane server; the implementation is in
|
||||
// agents/eks/podidentity. It listens on loopback only and is called by the
|
||||
// local kube-apiserver.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/agents/eks/podidentity"
|
||||
)
|
||||
|
||||
func main() {
|
||||
log := slog.New(slog.NewTextHandler(os.Stderr, nil))
|
||||
cfg, err := podidentity.ParseServiceConfig(os.Args[1:], os.Getenv, os.Stderr)
|
||||
if err != nil {
|
||||
log.Error("invalid configuration", "error", err)
|
||||
os.Exit(2)
|
||||
}
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
if err := podidentity.Run(ctx, cfg, log); err != nil {
|
||||
log.Error("pod identity webhook stopped", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
+44
-24
@@ -1,17 +1,22 @@
|
||||
// sqs-worker runs inside the shared SQS worker microVM.
|
||||
// sqs-worker runs inside the SQS microVM, next to its own SQS NATS server.
|
||||
//
|
||||
// Usage:
|
||||
//
|
||||
// sqs-worker boot
|
||||
// sqs-worker ready
|
||||
//
|
||||
// boot selects and mounts the worker's data disk named by fw_cfg, asks the
|
||||
// gateway for its current first-store decision with its instance-role
|
||||
// credentials, then loads or creates the store key on that disk and
|
||||
// publishes only the public key. ready then keeps checking the bound store's
|
||||
// JetStream with that key until it is stopped, keeping a readiness marker only
|
||||
// while the check passes; it proves the store accepted the worker key, not
|
||||
// the store server's identity.
|
||||
// boot selects and mounts the guest's data disk named by fw_cfg, asks the
|
||||
// gateway for its receipt key decision with its instance-role credentials,
|
||||
// then loads the recorded key, or creates one on a new volume and records its
|
||||
// fingerprint, before the guest's SQS store first starts on that disk. ready
|
||||
// then keeps checking the guest's own loopback JetStream, keeping a readiness
|
||||
// marker only while it answers, and serves SQS owner calls against it, pulling
|
||||
// them from the gateway with the same credentials.
|
||||
//
|
||||
// Serving is refused unless the gateway reports a single-node host cluster.
|
||||
// A disk whose receipt key is missing or does not match the recorded one
|
||||
// refuses to serve; recovering it needs a procedure not yet designed, and
|
||||
// nothing deletes or replaces the volume.
|
||||
package main
|
||||
|
||||
import (
|
||||
@@ -23,10 +28,14 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/mulgadc/spinifex/internal/gwsign"
|
||||
"github.com/mulgadc/spinifex/internal/sqsworkergw"
|
||||
"github.com/mulgadc/spinifex/spinifex/agents/sqs/workerboot"
|
||||
"github.com/mulgadc/spinifex/spinifex/agents/sqs/workerkey"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/workerserve"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/clustersize"
|
||||
)
|
||||
|
||||
// The cluster CA the daemon supplies; the only root the client trusts.
|
||||
@@ -36,7 +45,11 @@ const caPath = "/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/ca-cert/raw"
|
||||
// no static-key fallback.
|
||||
func newClaimer() workerboot.NewClaimer {
|
||||
return func(region, gatewayURL string) (workerkey.Claimer, error) {
|
||||
return newClient(region, gatewayURL)
|
||||
c, err := newClient(region, gatewayURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return workerboot.GatewayClaimer{Client: c}, nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -48,13 +61,6 @@ func newClient(region, gatewayURL string) (*sqsworkergw.Client, error) {
|
||||
return sqsworkergw.New(gatewayURL, region, ca, gwsign.NewInstanceRole())
|
||||
}
|
||||
|
||||
type describer struct{ c *sqsworkergw.Client }
|
||||
|
||||
func (d describer) DescribeStore(ctx context.Context) (workerboot.Store, error) {
|
||||
st, err := d.c.DescribeStore(ctx)
|
||||
return workerboot.Store(st), err
|
||||
}
|
||||
|
||||
func main() {
|
||||
if len(os.Args) != 2 || (os.Args[1] != "boot" && os.Args[1] != "ready") {
|
||||
fmt.Fprintln(os.Stderr, "usage: sqs-worker boot|ready")
|
||||
@@ -66,11 +72,18 @@ func main() {
|
||||
run = ready
|
||||
}
|
||||
if err := run(logf); err != nil {
|
||||
if os.Args[1] == "boot" && workerboot.Transient(err) {
|
||||
logf("boot not settled, will retry: " + err.Error())
|
||||
os.Exit(exitTransient)
|
||||
}
|
||||
logf("FATAL: " + err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// exitTransient tells the guest's supervisor that boot may be run again.
|
||||
const exitTransient = 75
|
||||
|
||||
func config(logf func(string)) workerboot.Config {
|
||||
return workerboot.Config{
|
||||
FwCfgDir: "/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/sqs-worker",
|
||||
@@ -99,19 +112,26 @@ func boot(logf func(string)) error {
|
||||
func ready(logf func(string)) error {
|
||||
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT)
|
||||
defer cancel()
|
||||
// The store is one guest, so its streams and buckets are R1.
|
||||
if err := clustersize.Declare(1); err != nil {
|
||||
return err
|
||||
}
|
||||
err := workerboot.Watch(ctx, workerboot.ReadyConfig{
|
||||
Config: config(logf),
|
||||
NewDescriber: func(region, gatewayURL string) (workerboot.StoreDescriber, error) {
|
||||
c, err := newClient(region, gatewayURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return describer{c}, nil
|
||||
},
|
||||
Config: config(logf),
|
||||
Checker: workerboot.NATSChecker{},
|
||||
CallTimeout: 30 * time.Second,
|
||||
Refresh: 30 * time.Second,
|
||||
Refused: 5 * time.Minute,
|
||||
Serve: &workerboot.ServeConfig{
|
||||
Connector: workerboot.NATSConnector{},
|
||||
Run: func(ctx context.Context, region, gatewayURL string, js jetstream.JetStream, receiptKey []byte) error {
|
||||
c, err := newClient(region, gatewayURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return workerserve.Run(ctx, c, js, receiptKey, logf)
|
||||
},
|
||||
},
|
||||
})
|
||||
if errors.Is(err, context.Canceled) {
|
||||
return nil
|
||||
|
||||
@@ -0,0 +1,211 @@
|
||||
package clusterv1
|
||||
|
||||
import "regexp"
|
||||
|
||||
const (
|
||||
// NodeFactsSubject fans out a request for each daemon's hardware and
|
||||
// cabling description. Its reply payload is NodeFacts.
|
||||
NodeFactsSubject = "spinifex.node.facts"
|
||||
|
||||
nodeNetTestPrefix = "spinifex.node.nettest."
|
||||
)
|
||||
|
||||
var nodeIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{1,64}$`)
|
||||
|
||||
// ValidNodeID reports whether a node name is safe to place in a subject: no
|
||||
// NATS token separator or wildcard can get through.
|
||||
func ValidNodeID(node string) bool {
|
||||
return nodeIDPattern.MatchString(node)
|
||||
}
|
||||
|
||||
// NetTestServeSubject asks one node to start a one-shot throughput server.
|
||||
func NetTestServeSubject(node string) string {
|
||||
return nodeNetTestPrefix + node + ".serve"
|
||||
}
|
||||
|
||||
// NetTestRunSubject asks one node to run the client side of a throughput test.
|
||||
func NetTestRunSubject(node string) string {
|
||||
return nodeNetTestPrefix + node + ".run"
|
||||
}
|
||||
|
||||
// Planes a node's NICs can carry, named as the installer names them.
|
||||
const (
|
||||
PlaneWAN = "wan"
|
||||
PlaneLAN = "lan"
|
||||
PlaneVPC = "vpc"
|
||||
)
|
||||
|
||||
// NetTestPort is the TCP port reserved for node-to-node throughput tests.
|
||||
const NetTestPort = 5001
|
||||
|
||||
// NodeFacts describes one node's hardware, OS and cabling. Fields a host
|
||||
// cannot report are left empty, and Sources says why.
|
||||
type NodeFacts struct {
|
||||
Node string `json:"node"`
|
||||
CollectedAt string `json:"collected_at,omitempty"`
|
||||
FormFactor string `json:"form_factor"`
|
||||
Virt string `json:"virt,omitempty"`
|
||||
System SystemFacts `json:"system"`
|
||||
OS OSFacts `json:"os"`
|
||||
CPU CPUFacts `json:"cpu"`
|
||||
Memory MemoryFacts `json:"memory"`
|
||||
NICs []NICFacts `json:"nics"`
|
||||
Planes []PlaneFacts `json:"planes"`
|
||||
Sources map[string]string `json:"sources"`
|
||||
}
|
||||
|
||||
// SystemFacts is the machine's identity from DMI.
|
||||
type SystemFacts struct {
|
||||
Vendor string `json:"vendor,omitempty"`
|
||||
Product string `json:"product,omitempty"`
|
||||
Family string `json:"family,omitempty"`
|
||||
Version string `json:"version,omitempty"`
|
||||
Serial string `json:"serial,omitempty"`
|
||||
UUID string `json:"uuid,omitempty"`
|
||||
ChassisType string `json:"chassis_type,omitempty"`
|
||||
ChassisAsset string `json:"chassis_asset,omitempty"`
|
||||
BoardVendor string `json:"board_vendor,omitempty"`
|
||||
BoardProduct string `json:"board_product,omitempty"`
|
||||
BoardSerial string `json:"board_serial,omitempty"`
|
||||
BIOSVendor string `json:"bios_vendor,omitempty"`
|
||||
BIOSVersion string `json:"bios_version,omitempty"`
|
||||
BIOSDate string `json:"bios_date,omitempty"`
|
||||
DeviceModel string `json:"device_model,omitempty"`
|
||||
}
|
||||
|
||||
// OSFacts names the running OS and Spinifex build.
|
||||
type OSFacts struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
Kernel string `json:"kernel,omitempty"`
|
||||
Arch string `json:"arch,omitempty"`
|
||||
Version string `json:"spx_version,omitempty"`
|
||||
Commit string `json:"spx_commit,omitempty"`
|
||||
UptimeSeconds int64 `json:"uptime_seconds,omitempty"`
|
||||
Hostname string `json:"hostname,omitempty"`
|
||||
}
|
||||
|
||||
// CPUFacts summarises the sockets. Threads and Cores are totals. Features are
|
||||
// the flags worth naming; Flags is every flag the CPU reports.
|
||||
type CPUFacts struct {
|
||||
Model string `json:"model,omitempty"`
|
||||
Vendor string `json:"vendor,omitempty"`
|
||||
Sockets int `json:"sockets"`
|
||||
CoresPerSocket int `json:"cores_per_socket,omitempty"`
|
||||
Cores int `json:"cores"`
|
||||
Threads int `json:"threads"`
|
||||
MaxMHz int `json:"max_mhz,omitempty"`
|
||||
CurrentMHz int `json:"current_mhz,omitempty"`
|
||||
Virtualization string `json:"virtualization,omitempty"`
|
||||
Features []string `json:"features,omitempty"`
|
||||
Flags []string `json:"flags,omitempty"`
|
||||
}
|
||||
|
||||
// MemoryFacts groups identical DIMMs so a 32-slot board reads as one line.
|
||||
type MemoryFacts struct {
|
||||
TotalBytes uint64 `json:"total_bytes"`
|
||||
SlotsUsed int `json:"slots_used"`
|
||||
SlotsTotal int `json:"slots_total"`
|
||||
MaxCapacity string `json:"max_capacity,omitempty"`
|
||||
ECC string `json:"ecc,omitempty"`
|
||||
Groups []MemoryGroup `json:"groups"`
|
||||
OSTotalBytes uint64 `json:"os_total_bytes,omitempty"`
|
||||
OSAvailBytes uint64 `json:"os_available_bytes,omitempty"`
|
||||
HugePageBytes uint64 `json:"huge_page_bytes,omitempty"`
|
||||
}
|
||||
|
||||
// MemoryGroup is a set of DIMMs alike in every field shown. ConfiguredMTs
|
||||
// below SpeedMTs means the board runs them slower than rated.
|
||||
type MemoryGroup struct {
|
||||
Count int `json:"count"`
|
||||
SizeBytes uint64 `json:"size_bytes"`
|
||||
Type string `json:"type,omitempty"`
|
||||
TypeDetail string `json:"type_detail,omitempty"`
|
||||
FormFactor string `json:"form_factor,omitempty"`
|
||||
SpeedMTs int `json:"speed_mts,omitempty"`
|
||||
ConfiguredMTs int `json:"configured_mts,omitempty"`
|
||||
Manufacturer string `json:"manufacturer,omitempty"`
|
||||
PartNumber string `json:"part_number,omitempty"`
|
||||
Rank string `json:"rank,omitempty"`
|
||||
Serials []string `json:"serials,omitempty"`
|
||||
}
|
||||
|
||||
// NICFacts is one physical network interface and, where LLDP reports it, the
|
||||
// switch port it is cabled to.
|
||||
type NICFacts struct {
|
||||
Name string `json:"name"`
|
||||
MAC string `json:"mac,omitempty"`
|
||||
Driver string `json:"driver,omitempty"`
|
||||
Firmware string `json:"firmware,omitempty"`
|
||||
BusInfo string `json:"bus_info,omitempty"`
|
||||
Model string `json:"model,omitempty"`
|
||||
SpeedMbps int `json:"speed_mbps,omitempty"`
|
||||
MaxMbps int `json:"max_mbps,omitempty"`
|
||||
Duplex string `json:"duplex,omitempty"`
|
||||
Port string `json:"port,omitempty"`
|
||||
MTU int `json:"mtu,omitempty"`
|
||||
State string `json:"state"`
|
||||
Master string `json:"master,omitempty"`
|
||||
Planes []string `json:"planes,omitempty"`
|
||||
Neighbor *LLDPNeighbor `json:"neighbor,omitempty"`
|
||||
}
|
||||
|
||||
// LLDPNeighbor is what the switch on the other end of a NIC advertises.
|
||||
type LLDPNeighbor struct {
|
||||
ChassisID string `json:"chassis_id"`
|
||||
SystemName string `json:"system_name,omitempty"`
|
||||
SystemDescription string `json:"system_description,omitempty"`
|
||||
PortID string `json:"port_id,omitempty"`
|
||||
PortDescription string `json:"port_description,omitempty"`
|
||||
Capabilities []string `json:"capabilities,omitempty"`
|
||||
}
|
||||
|
||||
// PlaneFacts is the address a plane uses on this node and the NICs under it.
|
||||
type PlaneFacts struct {
|
||||
Plane string `json:"plane"`
|
||||
Bridge string `json:"bridge,omitempty"`
|
||||
Address string `json:"address,omitempty"`
|
||||
Prefix int `json:"prefix,omitempty"`
|
||||
NICs []string `json:"nics,omitempty"`
|
||||
MTU int `json:"mtu,omitempty"`
|
||||
}
|
||||
|
||||
// NetTestServeRequest starts a server bound to the node's address on Plane.
|
||||
type NetTestServeRequest struct {
|
||||
Plane string `json:"plane"`
|
||||
DurationSeconds int `json:"duration_seconds"`
|
||||
}
|
||||
|
||||
// NetTestServeResponse names the address the server listens on.
|
||||
type NetTestServeResponse struct {
|
||||
Address string `json:"address,omitempty"`
|
||||
Port int `json:"port,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// NetTestRunRequest runs the client towards Address from the node's own
|
||||
// address on Plane.
|
||||
type NetTestRunRequest struct {
|
||||
Plane string `json:"plane"`
|
||||
Address string `json:"address"`
|
||||
Port int `json:"port"`
|
||||
DurationSeconds int `json:"duration_seconds"`
|
||||
Streams int `json:"streams"`
|
||||
}
|
||||
|
||||
// NetTestRunResponse is one direction of a pair's measurement.
|
||||
type NetTestRunResponse struct {
|
||||
SourceAddress string `json:"source_address,omitempty"`
|
||||
BitsPerSecond float64 `json:"bits_per_second"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
Seconds float64 `json:"seconds"`
|
||||
Streams int `json:"streams"`
|
||||
StreamBPS []float64 `json:"stream_bps,omitempty"`
|
||||
RTTMinMillis float64 `json:"rtt_min_ms,omitempty"`
|
||||
RTTAvgMillis float64 `json:"rtt_avg_ms,omitempty"`
|
||||
RTTMaxMillis float64 `json:"rtt_max_ms,omitempty"`
|
||||
PacketLoss float64 `json:"packet_loss_percent,omitempty"`
|
||||
LinkMbps int `json:"link_mbps,omitempty"`
|
||||
MTU int `json:"mtu,omitempty"`
|
||||
Tool string `json:"tool,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package telemetryv1
|
||||
|
||||
import "regexp"
|
||||
|
||||
// Node live subjects mirror the instance ones, keyed by node name. Only the
|
||||
// super-admin account's gateway handler subscribes to them.
|
||||
const (
|
||||
nodeLiveWatchSubjectPrefix = "telemetry.watch.node."
|
||||
nodeLiveSampleSubjectPrefix = "telemetry.live.node."
|
||||
)
|
||||
|
||||
var liveNodePattern = regexp.MustCompile(`^[A-Za-z0-9_-]{1,64}$`)
|
||||
|
||||
// ValidLiveNode reports whether a node name is safe to place in a subject.
|
||||
func ValidLiveNode(node string) bool {
|
||||
return liveNodePattern.MatchString(node)
|
||||
}
|
||||
|
||||
// NodeLiveWatchSubject is where a viewer asks a node to keep sampling itself.
|
||||
// A request on it is answered with a NodeLiveRing.
|
||||
func NodeLiveWatchSubject(node string) string {
|
||||
return nodeLiveWatchSubjectPrefix + node
|
||||
}
|
||||
|
||||
// NodeLiveSampleSubject carries one NodeLiveSample per period while watched.
|
||||
func NodeLiveSampleSubject(node string) string {
|
||||
return nodeLiveSampleSubjectPrefix + node
|
||||
}
|
||||
|
||||
// NodeLiveRing answers a watch with the samples newer than its SinceMillis.
|
||||
type NodeLiveRing struct {
|
||||
Samples []NodeLiveSample `json:"samples"`
|
||||
}
|
||||
|
||||
// NodeLiveSample is one host measurement. Rates are per second over the
|
||||
// interval ending at TimeMillis.
|
||||
type NodeLiveSample struct {
|
||||
TimeMillis int64 `json:"t"`
|
||||
CPUPercent float64 `json:"cpu"`
|
||||
// MemoryUsedPercent is used memory less page cache, as free(1) reports it.
|
||||
MemoryUsedPercent float64 `json:"mem"`
|
||||
Load1 float64 `json:"load1"`
|
||||
// Planes holds the traffic of each plane's physical NICs, keyed wan, lan, vpc.
|
||||
Planes map[string]NodeLivePlane `json:"planes"`
|
||||
}
|
||||
|
||||
// NodeLivePlane is one plane's traffic from the host's side.
|
||||
type NodeLivePlane struct {
|
||||
NetworkIn float64 `json:"netIn"`
|
||||
NetworkOut float64 `json:"netOut"`
|
||||
PacketsIn float64 `json:"pktIn"`
|
||||
PacketsOut float64 `json:"pktOut"`
|
||||
}
|
||||
+19
-5
@@ -194,18 +194,32 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
|
||||
|
||||
### SQS System Instances
|
||||
|
||||
These commands start the first SQS system instances on one node. They do not serve the SQS API, and a bound VM is not a ready service.
|
||||
These commands enable SQS on a single-node cluster and start its one system VM.
|
||||
A bound VM is not a ready service: `spx admin sqs status` reports readiness.
|
||||
|
||||
Both send a signed `spinifex` admin action (`EnsureSQSWorker`, `EnsureSQSStore`) to the AWS gateway using the standard AWS credential chain, for example `AWS_PROFILE=spinifex`. The caller must be in the super-admin account (`000000000001`) and hold `spinifex:<Action>`. The VMs, volumes, private VPC and ENIs belong to the system account (`000000000000`). `--endpoint`, `--region` and `--ca-bundle` default to this node's gateway, region and `config/ca.pem`; off a node, `--endpoint` and `--region` are required. Each call waits up to 11 minutes.
|
||||
The SQS VM runs two processes on one encrypted data volume: the Region's dedicated SQS NATS server, which listens only on the guest's loopback address and is separate from platform NATS, and the worker, which serves SQS owner calls from that local store. The worker starts serving only after the local store answers and the receipt-signing key on the volume matches the fingerprint the platform recorded.
|
||||
|
||||
SQS supports only a fresh single-node install. A worker record without a supported launch contract, or any record of the separate SQS store VM from an earlier dev build, cannot be upgraded: `enable` refuses it, and any existing volume is kept, not inspected, emptied or replaced. An existing separate store VM is refused on relaunch with its volume kept, and an existing worker VM is not restarted into a supported mode; upgrading such an install is unsupported.
|
||||
|
||||
**Do not put tenant production data in SQS.** SQS-managed per-message encryption is not implemented: `GetQueueAttributes` reports `SqsManagedSseEnabled=false`, and a request to enable SSE-SQS or SSE-KMS is rejected.
|
||||
The SQS data volume must be encrypted, which is a separate layer under every queue and not SSE.
|
||||
There is no `DeleteQueue` or `PurgeQueue`, so a queue and its unexpired messages cannot be removed once created.
|
||||
Retention is checked when a message is received, against the queue's retention at that moment, and nothing deletes an expired message: the store drops a body only at its age backstop of 14 days and 1 hour.
|
||||
So a message past a short retention can be delivered again if `MessageRetentionPeriod` is later raised while its body is still stored.
|
||||
Account termination does not yet remove SQS queues or messages, and there is no disable: the VM's volume, receipt key and data are kept.
|
||||
|
||||
Each sends a signed `spinifex` admin action (`EnableSQS`, `DescribeSQS`, `EnsureSQSWorker`) to the AWS gateway using the standard AWS credential chain, for example `AWS_PROFILE=spinifex`. The caller must be in the super-admin account (`000000000001`) and hold `spinifex:<Action>`. The VM, volume, private VPC and ENI belong to the system account (`000000000000`). `--endpoint`, `--region` and `--ca-bundle` default to this node's gateway, region and `config/ca.pem`; off a node, `--endpoint` and `--region` are required. `ensure-worker` and `enable` wait up to 11 minutes, and `status` up to 1 minute.
|
||||
|
||||
| Command | Flags | Description |
|
||||
|---------|-------|-------------|
|
||||
| `spx admin sqs ensure-worker` | `--node` (required), `--instance-type` (required), `--endpoint`, `--region`, `--ca-bundle` | Launches the Region's one SQS worker VM on `--node`, or reports the VM already bound there: node, region, instance, instance type and generation. The worker is pinned to its first node; another node is refused with `InvalidParameterValue`, and there is no takeover. It gets a private system VPC with one /24 subnet and no internet gateway, NAT or Elastic IP, and an encrypted 1 GiB system-account data volume on which the guest generates its own store key. The platform records only the public key. A different `--instance-type` is accepted only while no volume, network interface or launch step has been attempted; once any has, it is `IncorrectState`, whether or not the step completed. |
|
||||
| `spx admin sqs ensure-store` | `--node` (required, the worker's node), `--instance-type` (required, more than 512 MiB memory), `--volume-size-gib` (required, 1–16384), `--endpoint`, `--region`, `--ca-bundle` | Launches the Region's SQS store VM on the worker's node with an encrypted system-account data volume, or reports the VM already bound there: node, region, instance, instance type and address. Needs the worker bound with its public key recorded, else `IncorrectState`. The store accepts only that key. A different size, type or key than the first call is `IncorrectState`. |
|
||||
| `spx admin sqs enable` | `--node` (required), `--instance-type` (required, more than 512 MiB memory), `--volume-size-gib` (required, 1–16384), `--endpoint`, `--region`, `--ca-bundle` | Records SQS as enabled for the Region, pinned to `--node`, then launches or reports the SQS VM there. The desired state is recorded first, so rerunning the command resumes an interrupted enable. More than one node in the cluster is refused before anything is recorded. A worker record with a missing or unsupported launch contract, or a record of the separate store VM, is also refused before anything is recorded or launched; any existing volume is kept, and upgrading it is not supported. |
|
||||
| `spx admin sqs status` | `--endpoint`, `--region`, `--ca-bundle` | Shows the requested state, node, generation, VM state, receipt key state, whether a worker is polling this gateway, whether this gateway serves the SQS API, and `Ready`, which is true only when all of them hold. `Unsupported` names a record whose missing or unsupported launch contract blocks enable. |
|
||||
| `spx admin sqs ensure-worker` | `--node` (required), `--instance-type` (required, more than 512 MiB memory), `--volume-size-gib` (required, 1–16384), `--endpoint`, `--region`, `--ca-bundle` | Launches the Region's one SQS VM on `--node`, or reports the VM already bound there: node, region, instance, instance type and generation. The VM is pinned to its first node; another node is refused with `InvalidParameterValue`, and there is no takeover. It gets a private system VPC with one /24 subnet and no internet gateway, NAT or Elastic IP, and an encrypted system-account data volume of `--volume-size-gib` that holds the queue data and the receipt-signing key the guest creates on first boot. The platform records only the key's fingerprint. A different `--instance-type` is accepted only while no volume, network interface or launch step has been attempted; once any has, it is `IncorrectState`, whether or not the step completed. A different `--volume-size-gib` than the recorded one is `IncorrectState`. |
|
||||
|
||||
`ServiceUnavailable` never proves that no SQS resources exist, because an earlier attempt may already have created some. Its message says what it does establish. When no node serves the subject, this request was not delivered. When a volume create was not sent, that particular create did not happen. When the node cannot boot the guest yet or its launcher lacks a capability, this request stopped before allocating or recording anything. When the gateway times out waiting for the node's reply, nothing is known: the launch may still be running, so never assume it created nothing. In every case, call the command again with the same arguments. A retry first checks the durable record of the attempt and continues only when that is safe; it can proceed to the first launch after a create that was proven unsent, and otherwise reports the recorded outcome as bound or `IncorrectState`, or returns `ServiceUnavailable` again. `IncorrectState` from a volume, network interface or launch step means that step was recorded as sent but its outcome was never observed. It is never resent, the message names the volume when its ID is known, and no tool reconciles it yet: recovery is manual.
|
||||
|
||||
`bound` says only that the VM was launched for the current record. The worker decides store readiness itself, by checking that the store is bound to its own public key and that a NATS connection with that key succeeds; nothing reports that readiness to the operator. The worker-to-store connection does not authenticate the store server. Real VPC routing, IMDS and gateway TLS from the guest, and worker-to-store traffic on a real interface have not been tested.
|
||||
`bound` says only that the VM was launched for the current record. The worker is ready only when the guest's local store answers, and it serves only while its receipt key matches the recorded fingerprint; a missing or different key once one is recorded refuses serving and is never replaced.
|
||||
Real VPC routing, IMDS and gateway TLS from the guest have not been tested on a live host.
|
||||
|
||||
## Private Admin API
|
||||
|
||||
|
||||
+25
-113
@@ -278,8 +278,8 @@ storage-provider contract.
|
||||
### SQS message store (internal)
|
||||
|
||||
`domains/sqs/message` defines the store-neutral interface the SQS domain reaches its message store through (ADR-0011:S6), with a JetStream adapter in `domains/sqs/message/jetstream` and a shared conformance suite in `domains/sqs/message/conformance`.
|
||||
It is an internal primitive: nothing outside the SQS domain uses it, no daemon or service is wired to it, and it does not itself deliver an SQS API.
|
||||
The dedicated JetStream server ADR-0011 describes is not deployed by this code, and Spinifex ships no host service, unit, user or firewall rule for it; the guest that could run it is described under *SQS store system instance (not wired)*.
|
||||
It is used only by the SQS worker inside the SQS system instance and does not itself deliver an SQS API.
|
||||
Spinifex ships no host service, unit, user or firewall rule for the dedicated JetStream server; the system instance that runs it is described under *SQS system instance (operator-launched)*.
|
||||
|
||||
The interface covers durable and idempotent append, body and metadata reads, conditional state writes and deletes, per-namespace counts, namespace enumeration, an ordered change feed, a bounded paginated scan, namespace deletion and a sweep of tombstoned bodies.
|
||||
A namespace carries an immutable message-shard and state-shard allocation chosen when the queue is created, so a later change in shard counts never moves an existing queue.
|
||||
@@ -298,124 +298,36 @@ If that consumer delivers nothing for about two seconds while the stream holds a
|
||||
|
||||
Coverage is unit and conformance tests on in-process R1 and R3 servers; there is no end-to-end or multi-host validation of this code, and earlier multi-host evidence predates the final feed.
|
||||
|
||||
### SQS store system instance (not wired)
|
||||
### SQS system instance (operator-launched)
|
||||
|
||||
Spinifex can build, install, launch, restart and give storage to a dedicated SQS store microVM, but nothing calls any of it.
|
||||
No SQS API, domain service, admin command or daemon start-up path creates a store, so none of this is in production use.
|
||||
One system-owned microVM per Region runs both the SQS store and the SQS worker as separate processes, on one encrypted data volume pinned to one node (ADR-0011 Amendment 5, ADR-0012 Amendment 6).
|
||||
Only an operator activates it with the super-admin `EnsureSQSWorker` or `EnableSQS` actions; nothing starts it at daemon startup, and platform NATS is neither used nor changed for SQS data.
|
||||
This first profile is fresh single-node only: a host cluster of more than one node is refused before anything is recorded or allocated, and there is no disable.
|
||||
|
||||
The guest image is built by `make build-sqs-store-microvm-image`, separate from the load balancer image, and bundles the `nats-server` version pinned in `go.mod` plus the ext4 tools.
|
||||
Release tarballs carry it as `microvm-sqs-store/`, and `setup.sh` installs it to `/usr/share/spinifex/microvm/sqs-store/`.
|
||||
A release without a complete image removes any previously installed one, so a launch is then refused rather than booting a stale image.
|
||||
`make build-sqs-worker-microvm-image` builds the one `sqs-worker` image: the static `sqs-worker` binary, the pinned `nats-server`, the ext4 tools and a network `/init`; releases ship it as `microvm-sqs-worker`, which `setup.sh` installs to `/usr/share/spinifex/microvm/sqs-worker`.
|
||||
At boot the guest reads three firmware config values under `opt/spinifex/sqs-worker/` (the data disk's serial, its Region and the gateway URL on this node's mgmt bridge) and refuses before touching any disk if one is missing or malformed.
|
||||
It selects exactly one disk with that serial, runs `e2fsck -p`, mounts it at `/data` and confirms from `/proc/self/mountinfo` that the whole disk is what is mounted; the guest never formats.
|
||||
|
||||
At boot the guest reads three firmware config values: the data disk's serial, an IPv4 address on one of its interfaces, and one NATS user NKey public key.
|
||||
It mounts exactly one matching disk at `/data` after `e2fsck -p`, refuses a missing, duplicate, unformatted or unrepairable disk, and never formats.
|
||||
JetStream keeps its data on `/data` with `sync_interval: always` (ADR-0011:S3), `max_payload: 2MiB` admits a full 1 MiB SQS message with its headers (ADR-0011:S4), and the server listens only on that address and accepts only that key.
|
||||
On shutdown the store is stopped before `/data` is unmounted; a store that survives SIGKILL leaves every mount in place.
|
||||
The store is a dedicated `nats-server` listening only on `127.0.0.1`, with its JetStream directory at `/data/jetstream` and `GOMEMLIMIT=512MiB` (ADR-0011:S11), so nothing outside the VM can reach it and it needs no network identity of its own.
|
||||
The store starts only after the receipt key is settled, and the worker connects only after the store answers JetStream account info, so an unavailable store never appears ready.
|
||||
|
||||
A launch uses the versioned subject `system.LaunchSQSStore.v1`, which a daemon without this code does not serve, so it fails with no responder instead of booting another guest.
|
||||
The legacy `system.LaunchInstance` path refuses the store kind on both send and receive.
|
||||
Every input check runs before capacity is reserved, and the data volume must be an available system-account volume.
|
||||
It is mounted through the normal EBS path and detached, never deleted, on terminate, which is the protection standard EBS gives today and nothing stronger.
|
||||
CPU mitigations stay enabled for the store, unlike the load balancer.
|
||||
A successful launch means the VM is running, not that NATS is ready.
|
||||
The receipt-signing key lives on the data disk under `/data/sqs-worker-key`, written to a temporary file, synced, renamed and its directory synced before its fingerprint is recorded.
|
||||
`domains/sqs/workerident` holds the platform's receipt-key state on the worker's identity record, beside its generation, claim, pinned node and volume: `pending` is set only when the record first creates its volume, and `recorded` holds the fingerprint.
|
||||
Under `pending` the guest may create a key, or reuse a valid key it already wrote before a crash, only while no `/data/jetstream` directory exists; any store data under `pending` refuses, because it proves the volume is not new.
|
||||
Under `recorded` a missing, unreadable or different key refuses the boot and is never regenerated; a record with no receipt-key state is refused as not proven new.
|
||||
`RecordReceiptKey` writes the fingerprint by compare-and-swap that re-checks the current instance, generation, claim and volume, and a lost reply is answered by reading the record again, never by a new key.
|
||||
A replacement worker reuses the pinned volume and keeps its predecessor's receipt-key state.
|
||||
|
||||
The instance record keeps the store's non-secret guest settings so a restart on the node that launched it rewrites the firmware config files.
|
||||
A relaunch on any other node is refused, and cross-node recovery never claims a store.
|
||||
Both checks exist only in daemons with this code: an older daemon can still claim and relaunch a store elsewhere, and an explicit start served by another node claims the instance before its relaunch is refused.
|
||||
The worker reaches the gateway over TLS 1.3 trusting only the cluster CA, signing with instance-role credentials from the metadata service only.
|
||||
The internal `spinifex-sqs` actions are `ClaimReceiptKey`, `RecordReceiptKey`, `PollWork` and `CompleteWork`; the gateway admits only an assumed-role session of `sqsWorkerRole` in the system account and resolves the instance through `workerident.Authorize`, a point-in-time check against the current bound record.
|
||||
The worker pulls tenant work from its own node's gateway relay and posts results back, so no SQS payload crosses platform NATS; the relay holds a bounded, in-memory set of waiting requests and answers `ServiceUnavailable` when it is full or no worker is polling.
|
||||
|
||||
`domains/sqs/storevolume` prepares a fresh data volume, because the guest never formats.
|
||||
It checks for `mkfs.ext4` and `qemu-img` before allocating, creates the volume in `creating`, writes an empty ext4 image to it over NBD on the local node, unpublishes it, and only then makes it `available`.
|
||||
While a volume is `creating`, attach, delete, modify, force-detach and snapshot are refused.
|
||||
A volume is deleted only after a confirmed unpublish; any publish error, an unconfirmed unpublish or a failed release keeps it and reports its ID, with its state reported as unknown after a failed release.
|
||||
`EnsureSQSWorker` takes a `NodeId`, an `InstanceType` and a `VolumeSizeGiB`; the instance type's memory must exceed the store's 512 MiB limit, and a later request for a different type or size is refused as `IncorrectState`.
|
||||
`EnableSQS` records the Region's desired state, pinned to one node, and then ensures the worker; `status` reports it ready only when the worker is bound, polling and the tenant API is registered.
|
||||
A store launch record or VM from the earlier separate-store design, or a worker record without a supported launch contract, is refused with its volume kept: enable stops before recording anything, and a relaunch of an old store VM is marked failed rather than started.
|
||||
|
||||
Evidence is local only: unit tests, real QEMU boots against disposable scratch disks, and one run of the preparer against the `qemunbdd` provider whose volume kept acknowledged writes across a reboot.
|
||||
Viperblock, volume encryption, the EC2 volume record path, multiple nodes and host reachability of the guest address are not tested.
|
||||
Credential issuance and rotation, store placement and NATS readiness checks do not exist, and there is no failover: a lost node leaves its store unavailable.
|
||||
|
||||
### SQS worker guest (operator-launched)
|
||||
|
||||
The shared SQS worker guest holds the SQS store's client credential, so that no platform component does (ADR-0011:S2).
|
||||
Only an operator activates it, with the super-admin `EnsureSQSWorker` action described below; nothing starts it at daemon startup, and no worker service runs after boot.
|
||||
|
||||
`make build-sqs-worker-microvm-image` builds it with the store image's script and network `/init` under `SQS_MICROVM_PROFILE=worker`; it carries the static `sqs-worker` binary and the ext4 tools, no `nats-server`, and an empty `/data` mountpoint.
|
||||
Releases build it beside the store image and ship it as `microvm-sqs-worker`, which `setup.sh` installs to `/usr/share/spinifex/microvm/sqs-worker` with the store image's rules: a release without it removes an earlier one, and an incomplete one is refused.
|
||||
At boot `sqs-worker boot` reads three firmware config values under `opt/spinifex/sqs-worker/`: the worker disk's serial, its Region and the gateway URL; none is a first-store decision, because a static value survives a guest reset.
|
||||
A missing, empty or malformed value refuses before any disk is touched.
|
||||
|
||||
It selects exactly one disk with that serial, runs `e2fsck -p`, mounts it at `/data` and then confirms from `/proc/self/mountinfo` that this disk, as a whole ext4 filesystem, is what is mounted there; an empty mountpoint, a foreign mount or a failed check or mount refuses, and the guest never formats.
|
||||
Only then does it ask the gateway for the current decision with `ClaimFirstStore`, signed by `internal/sqsworkergw` with credentials from the instance metadata service only (`gwsign.NewInstanceRole`, no environment, profile or static-key fallback), over TLS 1.3 that trusts only the cluster CA from `opt/spinifex/ca-cert` and never follows a redirect.
|
||||
Under that decision `workerkey.Acquire` creates the key on `/data/sqs-worker-key`, owner-only and synced, records only its public key and then uses it, or loads the recorded key; a missing or different key once one is recorded refuses and is never replaced.
|
||||
The VM can start before its launch binds the instance, so its first calls are legitimately `AccessDenied`; that denial creates nothing and is asked again, as are a lost reply and a `ServiceUnavailable` or `Throttling` refusal, all within one bound (60 attempts 5 seconds apart), after which a worker that was never bound, or was replaced, refuses the boot.
|
||||
A signature or other refusal, or an answer that is neither "create" nor one valid public key, refuses the boot at once.
|
||||
|
||||
`domains/sqs/firststore` is the SQS domain's first-store decision: one record per Region in the `spinifex-sqs-first-store` KV bucket moves from `claimed` to `key-recorded` to `launch-intent` under compare-and-swap, bound to the worker's instance ID, claim ID and generation.
|
||||
A claim authorizes key creation only while no public key is recorded; once one is, every claim returns only that key, and a store launch intent needs it recorded first.
|
||||
An older generation, a different identity at the same generation or an unreadable record authorizes nothing, and the record holds no private key material.
|
||||
Reads are stream message gets answered by the stream leader, never direct gets a lagging replica can answer, and every authorizing answer is also confirmed by a compare-and-swap at the revision it read, so a stale or unavailable read authorizes nothing.
|
||||
The authenticated caller that will carry this decision must derive the worker's instance ID and generation from authoritative instance state; a generation supplied in a request cannot itself authorize a takeover.
|
||||
|
||||
`domains/sqs/workerident` is that authoritative instance state: one record per Region in the `spinifex-sqs-worker-identity` KV bucket holds a monotonic generation, claim ID, pinned node and encrypted data volume, and moves from `launching` to `bound` to `stopped`, or from `launching` to `failed`, under compare-and-swap.
|
||||
A launch intent is refused while the previous attempt is `launching` or `bound`, including one whose outcome is unknown, so only one caller ever holds an intent to start a VM on the pinned volume.
|
||||
A replacement takes the next generation only after `FailLaunch` or `RecordStopped` records that nothing holds the volume, and each request names the generation it supersedes, so a replayed request cannot take a generation it never observed; there is no automatic takeover.
|
||||
The launch path then raises the first-store fence to the successor's generation with `firststore.Fence` before `BeginLaunch`, which refuses a replacement until a leader read shows that fence; the fence only rises, and `firststore` refuses every call from a generation below it.
|
||||
Only the current claim ID is recorded, so claim IDs are minted fresh per attempt and a VM launch must be idempotent on generation and claim together.
|
||||
Binding writes the instance's index entry first and then binds the record to that instance.
|
||||
The index is only a lookup: `Authorize` accepts an instance only while the Region's record is `bound` to exactly that instance, generation and claim, confirmed by a compare-and-swap at the leader-read revision, so an old, orphaned, rewritten or unbound entry authorizes nothing.
|
||||
A same-instance reboot keeps its instance ID and identity; a replacement takes a new generation, and the first cut refuses another node or data volume and an unencrypted volume, although the encryption flag is the caller's claim and the launch wiring must verify it from the volume's metadata.
|
||||
Both buckets are opened through `OpenDurableKV`, which refuses an existing bucket whose effective stream configuration could expire, evict, lose, lag or refuse records: non-limits retention, an age, per-message or delete-marker TTL, memory storage, asynchronous flush, a size, count or message-size cap, a discard policy other than discard-new, unacknowledged writes, a sealed stream, a mirror or sources, or fewer replicas than the cluster needs.
|
||||
`workerkey.Acquire` drives that decision: it syncs a new key to disk, records its public key and only then returns it, so a crash before recording reuses the key already on disk.
|
||||
|
||||
`domains/sqs/awsapi/worker` carries the decision to the worker as `ClaimFirstStore` and `RecordWorkerPublicKey`, served under the internal `spinifex-sqs` signing service, which no AWS SDK signs for; the public AmazonSQS action inventory is untouched.
|
||||
The gateway admits only an assumed-role session of `sqsWorkerRole` in the system account, then the role's policy, and only then resolves the session name, the instance ID STS set, through `workerident.Authorize`; the request carries no identity, and any parameter beyond the action's own is refused.
|
||||
The first-store identity is the current record's instance, claim and generation, and a worker bound to another Region is refused.
|
||||
A request that passed `Authorize` just before a confirmed stop and replacement still carries the old generation; the fence was raised before the successor began, so `firststore` refuses it whether or not the successor has claimed yet.
|
||||
The adapter never logs a caller-chosen parameter name, because one could carry a secret.
|
||||
`RecordWorkerPublicKey` accepts only a valid public user key, so a seed is refused without being recorded, logged or echoed.
|
||||
The gateway binds both buckets per request and never creates them, so it provisions nothing and answers `ServiceUnavailable` until a worker launch has created them; the worker launch below ensures the role.
|
||||
The private key stays on the worker disk; the guest logs and publishes only the public key to `/run/sqs-worker/public-key`.
|
||||
|
||||
`domains/sqs/workerlaunch` realizes the worker on the node it runs on, its only node: a bound record is returned as is; otherwise it takes the intent, then on the first launch marks the intent's one volume create, prepares the volume and records its ID, reads the volume's own record and refuses it unless encrypted, ensures the role, finds or creates the network for the claim and reads the recorded store key, and marks the intent dispatched before launching.
|
||||
A dispatched launch is never sent again: a lost reply or error leaves the intent `launching` and dispatched, and every later call returns `ErrOutcomeUnknown` until the outcome is reconciled, since the launch path is not idempotent.
|
||||
A failure before dispatch leaves an undispatched intent that the next call resumes with the same claim; a confirmed stop or failure is replaced only after `firststore.Fence` excludes the old generation, reusing the pinned volume.
|
||||
The launch passes no store decision or key: the guest asks the gateway at boot.
|
||||
The intent exists before any volume, so concurrent first launches allocate nothing for the losers. A Prepare proven to have sent no create withdraws the mark and returns `ServiceUnavailable`, or `UnsupportedOperation` when the cluster storage key is not configured, and a retry may prepare the volume; a create that may have applied is never sent again, and every later call returns `IncorrectState` until it is reconciled. A volume the call knows of but cannot use is reported with its ID in a `VolumeError` and left in place, never deleted. An intent that has sent no create may be re-typed; one that has keeps its type until replaced. The worker role is created with EC2-only trust, and an existing role is checked for that trust before any policy is granted; the instance profile must carry exactly the worker role, and one holding any other role is refused unchanged.
|
||||
The daemon's `LaunchSQSWorkerInstance` boots the `sqs-worker` image through the same direct-boot path as the store, with the data volume attached at boot and kept on terminate, a required instance profile, and a required mgmt NIC; it re-reads the volume's own record and refuses one that is not available and encrypted, before anything is allocated.
|
||||
Its firmware config adds the gateway URL on this node's mgmt bridge and the cluster CA from the daemon's own trust store; the caller cannot supply either, and no value written is secret.
|
||||
A relaunch rebuilds that config from the VM record on the node that launched it and is refused anywhere else, and membership takeover already skips SQS-managed instances.
|
||||
`sqsWorkerRole` is ensured in the system account granting exactly `spinifex-sqs:ClaimFirstStore` and `spinifex-sqs:RecordWorkerPublicKey`, from the `domains/sqs/workerrole` contract the gateway gate also reads; an existing role whose trust admits more than the EC2 service is refused.
|
||||
|
||||
Evidence is local only: unit tests with a fake firmware config, sysfs and mount table, tests running the boot through the real client against the real `spinifex-sqs` registration and records, and QEMU boots of the built image against disposable scratch disks and a host-side test gateway and metadata service, showing the first boot creates the key, reboots reuse it, a key lost after recording is never replaced, and the seed reaches no log or request.
|
||||
The worker's network is a private SQS system VPC per Region, built by `systemvpc.EnsurePrivate` from the `[sqs]` supernet (default `10.240.0.0/14`): one VPC and one /24 subnet, with no internet gateway, NAT gateway, EIP, route table or daemon-to-store node port. Each launch claim gets one ENI in that subnet, in the VPC default security group, found again by its claim on a retry; two ENIs for one claim are refused. The launch intent is marked before the one create is sent, so a create whose ENI is not yet visible is never sent again: the launch fails closed with `IncorrectState` until the ENI appears or the outcome is reconciled. Worker-to-store TCP and the guest's route to the metadata service are not yet proven by any traffic test.
|
||||
The super-admin `EnsureSQSWorker` action on the internal `spinifex` operator service takes a `NodeId`, which must be a node of this cluster and a single subject token, and an `InstanceType`, and sends a request to `sqs.worker.Ensure.v1.<node>` with no queue group, so only that node answers and the worker stays pinned.
|
||||
That node's daemon re-checks the node the request names and the instance type, and refuses as `ServiceUnavailable` before allocating a volume or network or recording an intent unless the worker image's kernel and initramfs are non-empty files, its mgmt bridge exists and the cluster CA is readable; a call after the image is installed then launches.
|
||||
It then runs its one composed launcher for its own Region, one launch at a time, in the daemon's own context with a 30-minute bound, so a caller's timeout does not cancel it.
|
||||
A reply lost to that timeout is reported as `ServiceUnavailable`; calling again reports the bound worker, or `IncorrectState` for a dispatched launch whose outcome is unknown, which is never resent; a Region pinned to another node is refused as `InvalidParameterValue`.
|
||||
The reply names the bound VM's node, Region, instance ID and generation; bound is not ready, and it says nothing about the guest's boot, its key or the store.
|
||||
Stop/detach evidence for `RecordStopped` is not wired, the guest's metadata route and gateway TLS on a real ENI and worker-to-store TCP are unproven, and Viperblock's at-rest key is held by the platform.
|
||||
|
||||
The worker identity record holds the instance type of its generation; a request for another type is refused as `IncorrectState`, before dispatch and after binding, the reply names the recorded type, and only a replacement after a confirmed stop may change it.
|
||||
Operators call both actions with `spx admin sqs ensure-worker --node --instance-type` and `spx admin sqs ensure-store --node --instance-type --volume-size-gib`, signed for the `spinifex` service with the standard AWS credential chain like the other `spx admin` gateway commands.
|
||||
Each command refuses missing inputs before sending anything and reports only the bound VM; a bound VM is not a ready worker or store.
|
||||
The super-admin `EnsureSQSStore` action takes a `NodeId`, an `InstanceType` and a `VolumeSizeGiB` and sends a request to `sqs.store.Ensure.v1.<node>` with no queue group; only the node the Region's worker is bound to can launch the store.
|
||||
That node refuses an instance type whose memory does not exceed 512 MiB, the store's `GOMEMLIMIT` under ADR-0011:S11; this is a floor, not a validated size, and no type is defaulted.
|
||||
It refuses before recording or allocating anything unless the store image and the cluster storage key are present, the worker is bound on this node, and that worker's identity has recorded its public key in the first-store decision, which it then advances to a launch intent.
|
||||
The store's own launch record in `spinifex-sqs-store-launch` holds only that public key, the claim, the node and the operator's instance type and size, and marks each step before its uncertain create: the volume, the ENI in the worker's private subnet, and the launch.
|
||||
A later request for a different instance type or size is refused as `IncorrectState`, before dispatch and after binding, and the reply names the type the store was launched with.
|
||||
A volume preparation that failed before sending any create, such as a missing `mkfs.ext4` or a scratch or format failure, withdraws its mark and returns `ServiceUnavailable`, so a retry may try again once the host is fixed.
|
||||
Any other marked step whose result is not visible is reported as `IncorrectState` and never resent; a volume that is not encrypted by its own record is refused as `UnsupportedOperation` with its ID kept, and the daemon re-checks the volume's encryption and the guest's size at launch.
|
||||
The daemon never receives the worker's seed; the store's NATS server accepts only that public key, and the reply's bound VM and address are not readiness, which only the worker can check.
|
||||
|
||||
After boot, the worker's long-running `sqs-worker ready` step mounts and verifies its disk again and calls the internal `spinifex-sqs` `DescribeStore` action, which admits only the current worker and returns the bound store's address and recorded public key, or pending.
|
||||
It loads its key only if it matches that public key, then connects to the store's NATS on port 4222 with it and asks JetStream for account info, each pass bounded to 30 seconds.
|
||||
The step runs until shutdown with no overall deadline, so a store the operator launches at any time is picked up: it asks again every 5 seconds while the store is pending, not answering or the reply was lost, and re-checks a ready store every 30 seconds.
|
||||
A pass writes only the address and public key to `/run/sqs-worker/store-ready`; any failed pass removes that file, and so does the step's own exit.
|
||||
A start that finds shutdown in progress or no worker key logs once and waits in place rather than exiting, so init does not respawn it in a loop; shutdown stops it like a running watcher.
|
||||
Init respawns the step, at most once every 5 seconds, and every start removes a marker left by a watcher that was killed or crashed before the new watcher runs.
|
||||
A ready store that fails is noticed up to one re-check interval plus one bounded call later, so the marker can be up to about a minute stale.
|
||||
Shutdown blocks further respawns, removes the marker, and stops the watcher by its recorded PID and start time, escalating to SIGKILL after a budget; it unmounts the worker disk only once the watcher is confirmed gone, and a survivor leaves every mount in place.
|
||||
A mismatched key, a store that refuses the key, a key missing from the disk or an unclassified gateway refusal is asked again only every 5 minutes, so it never spins and still recovers if the operator fixes it; a missing key is never replaced.
|
||||
The NKey authenticates the worker to the store, but nothing authenticates the store to the worker: there is no TLS, so a pass means only that something at that address accepted the key.
|
||||
This is store readiness for the worker alone; there is no SQS API, and no production sizing or support is claimed.
|
||||
Evidence is local: unit tests, the guest shell test and an AWS SDK test that creates a queue and sends, receives and deletes a message through the real worker loop against a real local JetStream store and a restart.
|
||||
A real guest boot, its metadata route on a real ENI and the signed path on a live node are unproven, Viperblock's at-rest key is held by the platform, and SQS-managed message encryption is not implemented, so the profile must not hold tenant production data.
|
||||
|
||||
## Configuration
|
||||
|
||||
|
||||
@@ -60,7 +60,7 @@ intact; ADR-0001 explicitly defers that design.
|
||||
|
||||
| Current path | Standing | Target ownership | Rule |
|
||||
|---|---|---|---|
|
||||
| `cmd/aws-model-coverage` | Qualification generator | Thin main; reusable code in `internal/tooling` | Produces the dispatch-versus-model inventory; never imported by product packages. |
|
||||
| `cmd/aws-model-coverage` | Qualification generator | Thin main; reusable code in `internal/tooling` | Produces the dispatch-versus-model inventory; never imported by product packages. Debt: it reads each domain's declared `OperationInventory` from `domains/<service>/awsapi` (ACM, EC2, ECR, ECS, EKS, ELBv2, IAM, RDS, Route 53, SQS, STS) and never constructs a registration. |
|
||||
| `cmd/dhcptest` | Development diagnostic | Thin main; reusable code in `internal/tooling` if extraction is useful | It is a DHCP probe, not a supported `spx admin` command. |
|
||||
| `cmd/spx-loadgen` | Qualification/load tool | Thin main; reusable code in `internal/testkit` | It drives test load and is not shipped to nodes. |
|
||||
| `tests/e2e/*/cmd/*`, `tests/bench/*/cmd/*` | Test and benchmark-private tools | Remain test-private | They are invoked by a test or benchmark harness, not a product process. |
|
||||
@@ -489,8 +489,8 @@ Enforcement: the owner decided the layering test is blocking now, with a ratchet
|
||||
- `domains/rds/instance` → `domains/rds/parametergroup` (legacy edges): `parameterGroupReferences` binds `Dependants` by scanning instance records; this clears when the RDS instance gains its own owner and answers `Dependants` from it. `tags.go` writes the meta record through its generic tag registry; this clears when tagging invokes a parameter-group-owned tag mutation capability, and moving the tagging adapter to `domains/rds/awsapi` alone does not clear it. The Modify handler still calls `propagateParameterGroup` after the owner's write; propagation is instance-side realization and stays legacy until it is designed as asynchronous realization (group generation, instance staleness, retry). `parameterGroupValues` returns the persisted `Record` and overrides to instance code; when the instance owner is extracted, replace them with a consumer-owned resolved-parameters projection.
|
||||
- `domains/ecs` → `domains/ecs/taskdefinition` (owner bypass): `tags.go` reads and writes the revision record through its generic tag registry; this clears when ECS tagging invokes an owner-provided tag mutation capability, and moving `tags.go` to `domains/ecs/awsapi` alone does not clear it. `taskDefinitionResolver` returns the persisted `Record` to task and service launch code; when the task and service owners are extracted, replace the record crossing that boundary with a consumer-owned resolved task-definition (launch-spec) projection. Moving packages has not made the record private.
|
||||
- `domains/ecs/taskdefinition` (deliberate duplication, not debt): private `getJSON`, `putJSON` and `keysWithPrefix` copy the raw-KV helpers in `domains/ecs` so the extraction keeps ECS's get/put/list error and recovery semantics. Revisit only if a later, tested state abstraction proves it preserves those semantics; do not migrate ECS to `kvstore` merely to remove them.
|
||||
- `domains/eks/cluster` → `domains/eks/access` (legacy edges): `ResolveTokenReview` opens the EKS bucket in the gateway process and reads records through the exported `access.Get`; this clears when token review asks the owner across a contract rather than opening the bucket itself. `DeleteClusterPrefix` erases access-entry keys as part of its blanket cluster sweep; this clears when cluster delete invokes an owner-provided removal of the cluster's entries. The owner returns the persisted `Record` to the AWS adapter and token review; replace it with consumer-owned projections when those consumers are extracted.
|
||||
- `domains/eks/cluster` → `domains/eks/addon` (legacy edges): `DeleteClusterPrefix` erases add-on records and manifests in its blanket cluster sweep; this clears when cluster delete invokes an add-on-owned teardown capability. `stageGPUDeviceAddon` is a temporary node-group-to-add-on orchestration edge: best-effort, errors logged and swallowed, never undone on node-group delete; it clears when the node-group owner is extracted and the dependency is designed. `ClusterReconciler` hosts the add-on status subscription; it clears when add-on delivery observation has its own host under the lifecycle correction. Cluster existence is checked in the handler, not through an owner-declared capability; it moves into the owner when cluster admission is defined (today status is ignored). Guest wire types (`AddonStatusReport`, `AddonDeliveryPhase`, `AddonStatusSubject`, `StagedAddonManifest`, `InternalAddonsResponse`) are owned by `contracts/eks/v1`, which `domains/eks/cluster`, `gateway/eks` and `agents/eks/gatewayfetch` import directly.
|
||||
- `domains/eks/cluster` → `domains/eks/access` (legacy edges): `ResolveTokenReview` opens the EKS bucket in the gateway process and reads records through the exported `access.Get`; this clears when token review asks the owner across a contract rather than opening the bucket itself. `DeleteClusterChildren` erases access-entry keys as part of its blanket cluster sweep; this clears when cluster delete invokes an owner-provided removal of the cluster's entries. The owner returns the persisted `Record` to the AWS adapter and token review; replace it with consumer-owned projections when those consumers are extracted.
|
||||
- `domains/eks/cluster` → `domains/eks/addon` (legacy edges): `DeleteClusterChildren` erases add-on records and manifests in its blanket cluster sweep; this clears when cluster delete invokes an add-on-owned teardown capability. `stageGPUDeviceAddon` is a temporary node-group-to-add-on orchestration edge: best-effort, errors logged and swallowed, never undone on node-group delete; it clears when the node-group owner is extracted and the dependency is designed. `ClusterReconciler` hosts the add-on status subscription; it clears when add-on delivery observation has its own host under the lifecycle correction. Cluster existence is checked in the handler, not through an owner-declared capability; it moves into the owner when cluster admission is defined (today status is ignored). Guest wire types (`AddonStatusReport`, `AddonDeliveryPhase`, `AddonStatusSubject`, `StagedAddonManifest`, `InternalAddonsResponse`) are owned by `contracts/eks/v1`, which `domains/eks/cluster`, `gateway/eks` and `agents/eks/gatewayfetch` import directly.
|
||||
- `domains/eks/cluster` and `domains/eks/awsapi` → `domains/eks/addon` (further edges): the AWS adapter reads the persisted `Record` through `addon.Get`/`addon.List`; this clears when the owner returns a view and the add-on adapter moves to `domains/eks/awsapi`. The guest route reaches the owner through the `handlers_eks` NATS RPC (`ListStagedAddonManifests`); this clears with per-resource commands behind EKS registration. The add-on ARN is derived in both the owner and `domains/eks/awsapi/authz.go`; this clears when authorization obtains the ARN from the owner, which an incarnation suffix would require. The owner's `Manifest`/`Report` beside the `contracts/eks/v1` types is deliberate duplication, not debt.
|
||||
- `agents/rds/agent` (dependency, narrowed): engine metadata now arrives through the agent-declared `EngineCatalog`, but the agent still imports `domains/rds/instance` for the command protocol, wire and record types (`Command*`, `Parameter`, `EngineHealth*`, `BootstrapMode*`) and API output types. This clears when those move behind a guest/controller wire contract.
|
||||
- `domains/rds/awsapi` → `domains/rds/instance` (`InstanceSizing`): the gateway obtains its `Sizing` through the RDS handler package's adapter rather than its own composition root. This clears when the gateway's RDS composition moves to `runtime/roles/awsgw` and builds `Sizing` there.
|
||||
|
||||
@@ -96,6 +96,53 @@ Limits:
|
||||
The load balancer VM reports target health through its own host's gateway, so while that gateway is down the unhealthy control plane stays in rotation.
|
||||
- Not measured: more than one control plane per host, a physical-node failure instead of a stopped gateway, and the target health state as `DescribeTargetHealth` reports it, because the load balancer and control-plane VMs belong to the system account.
|
||||
|
||||
### IAM roles for service accounts in your workloads
|
||||
|
||||
A pod gets an IAM role by running under a ServiceAccount that carries the role ARN in an annotation.
|
||||
No pod-spec wiring is needed.
|
||||
|
||||
```yaml
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: app
|
||||
annotations:
|
||||
eks.amazonaws.com/role-arn: arn:aws:iam::123456789012:role/app-role
|
||||
```
|
||||
|
||||
Every control plane runs a mutating webhook that acts on pod creation.
|
||||
When the pod's ServiceAccount has the annotation, the webhook adds these to each container and init container:
|
||||
|
||||
- `AWS_ROLE_ARN` and `AWS_WEB_IDENTITY_TOKEN_FILE`, with a projected token volume named `aws-iam-token` mounted at `/var/run/secrets/eks.amazonaws.com/serviceaccount`.
|
||||
- `AWS_REGION`, `AWS_DEFAULT_REGION` and `AWS_STS_REGIONAL_ENDPOINTS`.
|
||||
- `AWS_ENDPOINT_URL_STS`, so the SDK calls the Spinifex STS endpoint.
|
||||
- `AWS_CA_BUNDLE`, pointing at `/etc/spinifex/ca-bundle/ca.pem`, a ConfigMap named `spinifex-ca-bundle` that is created in every namespace and holds the system roots plus the gateway CA.
|
||||
- `SSL_CERT_FILE` with the same path, only when the container does not set it. Clients such as the AWS Load Balancer Controller read this variable instead of `AWS_CA_BUNDLE`.
|
||||
|
||||
Your workload still sets `AWS_ENDPOINT_URL` (or the per-service variable) for every service other than STS.
|
||||
|
||||
The token audience defaults to `sts.amazonaws.com`.
|
||||
The expiry defaults to a value between 79200 and 86400 seconds and is clamped to 600 to 86400 seconds.
|
||||
Set them with the `eks.amazonaws.com/audience` and `eks.amazonaws.com/token-expiration` annotations on the ServiceAccount.
|
||||
The `token-expiration` annotation on the pod wins over the ServiceAccount.
|
||||
|
||||
A container is left alone when it already has the role, region and regional STS settings, which is how the managed add-ons and your own explicit wiring are skipped.
|
||||
When some are present, the missing ones and the mount are added.
|
||||
Containers named in the pod's `eks.amazonaws.com/skip-containers` annotation (comma separated) are not touched.
|
||||
A pod whose ServiceAccount has no role annotation is not changed.
|
||||
|
||||
The webhook fails open: if it cannot decide, the pod is created without injection.
|
||||
Each injection, skip and failure is logged with the namespace, pod and reason in the control-plane console log.
|
||||
Because it fails open, a pod that needs a role but starts without credentials is the symptom to look for, and the console log says why.
|
||||
|
||||
The webhook listens on `127.0.0.1:8445` of each control plane, and each API server calls its own.
|
||||
Port 8444 is a different listener, the readiness check described above.
|
||||
|
||||
Existing clusters get the webhook only when their control planes are replaced by the current `eks-node` image.
|
||||
No user-facing operation replaces a control plane, and automatic replacement does not currently work when the replacement reuses the failed member's IP.
|
||||
Recreate the cluster to adopt it.
|
||||
User-workload injection through a cluster created with an older image is not measured.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
> [!IMPORTANT]
|
||||
|
||||
@@ -3978,6 +3978,86 @@
|
||||
"extra": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"service": "sqs",
|
||||
"api_version": "2012-11-05",
|
||||
"opaque": false,
|
||||
"note": "",
|
||||
"operations": {
|
||||
"modelled": [
|
||||
"AddPermission",
|
||||
"CancelMessageMoveTask",
|
||||
"ChangeMessageVisibility",
|
||||
"ChangeMessageVisibilityBatch",
|
||||
"CreateQueue",
|
||||
"DeleteMessage",
|
||||
"DeleteMessageBatch",
|
||||
"DeleteQueue",
|
||||
"GetQueueAttributes",
|
||||
"GetQueueUrl",
|
||||
"ListDeadLetterSourceQueues",
|
||||
"ListMessageMoveTasks",
|
||||
"ListQueueTags",
|
||||
"ListQueues",
|
||||
"PurgeQueue",
|
||||
"ReceiveMessage",
|
||||
"RemovePermission",
|
||||
"SendMessage",
|
||||
"SendMessageBatch",
|
||||
"SetQueueAttributes",
|
||||
"StartMessageMoveTask",
|
||||
"TagQueue",
|
||||
"UntagQueue"
|
||||
],
|
||||
"registered": [
|
||||
"ChangeMessageVisibility",
|
||||
"ChangeMessageVisibilityBatch",
|
||||
"CreateQueue",
|
||||
"DeleteMessage",
|
||||
"DeleteMessageBatch",
|
||||
"GetQueueAttributes",
|
||||
"GetQueueUrl",
|
||||
"ListQueueTags",
|
||||
"ListQueues",
|
||||
"ReceiveMessage",
|
||||
"SendMessage",
|
||||
"SendMessageBatch",
|
||||
"SetQueueAttributes",
|
||||
"TagQueue",
|
||||
"UntagQueue"
|
||||
],
|
||||
"implemented": [
|
||||
"ChangeMessageVisibility",
|
||||
"ChangeMessageVisibilityBatch",
|
||||
"CreateQueue",
|
||||
"DeleteMessage",
|
||||
"DeleteMessageBatch",
|
||||
"GetQueueAttributes",
|
||||
"GetQueueUrl",
|
||||
"ListQueueTags",
|
||||
"ListQueues",
|
||||
"ReceiveMessage",
|
||||
"SendMessage",
|
||||
"SendMessageBatch",
|
||||
"SetQueueAttributes",
|
||||
"TagQueue",
|
||||
"UntagQueue"
|
||||
],
|
||||
"stubbed": [],
|
||||
"unsupported": [],
|
||||
"missing": [
|
||||
"AddPermission",
|
||||
"CancelMessageMoveTask",
|
||||
"DeleteQueue",
|
||||
"ListDeadLetterSourceQueues",
|
||||
"ListMessageMoveTasks",
|
||||
"PurgeQueue",
|
||||
"RemovePermission",
|
||||
"StartMessageMoveTask"
|
||||
],
|
||||
"extra": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"service": "sts",
|
||||
"api_version": "2011-06-15",
|
||||
|
||||
@@ -1441,6 +1441,21 @@ request-payment: Requester Pays shifts transfer charges to the caller, which nee
|
||||
- `GetBucketRequestPayment`
|
||||
- `PutBucketRequestPayment`
|
||||
|
||||
## sqs
|
||||
|
||||
`2012-11-05` — 15 of 23 modelled operations implemented (65.2%), 0 stubbed, 0 not applicable, 0 registered outside the pinned model.
|
||||
|
||||
### Not implemented
|
||||
|
||||
- `AddPermission`
|
||||
- `CancelMessageMoveTask`
|
||||
- `DeleteQueue`
|
||||
- `ListDeadLetterSourceQueues`
|
||||
- `ListMessageMoveTasks`
|
||||
- `PurgeQueue`
|
||||
- `RemovePermission`
|
||||
- `StartMessageMoveTask`
|
||||
|
||||
## sts
|
||||
|
||||
`2011-06-15` — 5 of 11 modelled operations implemented (45.5%), 0 stubbed, 0 not applicable, 0 registered outside the pinned model.
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
---
|
||||
title: "Oracle Linux Hosts"
|
||||
seoTitle: "Install Spinifex on Oracle Linux 10 — Spinifex Docs"
|
||||
description: "Prepare Oracle Linux 10 hosts for Spinifex: UEK R8, SELinux, a NetworkManager br-wan bridge, the NFV SIG repository, IPsec on libreswan and host updates."
|
||||
description: "Install Spinifex on Oracle Linux 10: prepare the host with UEK R8, SELinux permissive and a NetworkManager br-wan bridge, then run the single-node install."
|
||||
category: "Install"
|
||||
tags:
|
||||
- install
|
||||
@@ -17,55 +17,42 @@ resources:
|
||||
|
||||
# Oracle Linux Hosts
|
||||
|
||||
> What differs when the Spinifex host runs Oracle Linux 10 instead of Debian or Ubuntu.
|
||||
> Install Spinifex on Oracle Linux 10.
|
||||
|
||||
## Table of Contents
|
||||
|
||||
- [Overview](#overview)
|
||||
- [Prerequisites](#prerequisites)
|
||||
- [Instructions](#instructions)
|
||||
- [What the Installer Changes](#what-the-installer-changes)
|
||||
- [Troubleshooting](#troubleshooting)
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
Spinifex installs on Oracle Linux 10.2 or a later 10.x release, running the UEK R8 kernel, on x86_64.
|
||||
The installer, the release tarball and the install steps are the same as on Debian; this page covers what the host needs first and what the installer changes on it.
|
||||
Follow it alongside [Single-Node Install](/docs/install).
|
||||
Spinifex runs on Oracle Linux 10.2, on the UEK R8 kernel, on x86_64 with an x86-64-v3 CPU (Intel Haswell, AMD Excavator or later).
|
||||
|
||||
**Supported on Oracle Linux:**
|
||||
The host must be a dedicated Spinifex host: Spinifex claims every `tap*` device and owns libreswan's `ipsec.service`.
|
||||
|
||||
- A single node, or a cluster in which every node runs Oracle Linux; servers beyond the third join as compute nodes, as on Debian.
|
||||
- Guests, VPCs, routed NAT, Elastic IPs, and Application and Network Load Balancers.
|
||||
- Whole-GPU passthrough, on x86_64 GPU hosts (see [GPU Passthrough](/docs/gpu-passthrough)).
|
||||
**Not supported yet:** clusters mixing Oracle Linux with Debian or Ubuntu, SELinux enforcing, aarch64, Oracle Linux 9 or earlier, the Red Hat Compatible Kernel, a ZFS root, NVIDIA MIG and air-gapped installs.
|
||||
|
||||
**Not supported on Oracle Linux yet:**
|
||||
## Instructions
|
||||
|
||||
- Clusters that mix Oracle Linux with Debian or Ubuntu nodes, on any release.
|
||||
- SELinux enforcing, aarch64, Oracle Linux 9 or earlier, the Red Hat Compatible Kernel, a ZFS root, and NVIDIA MIG.
|
||||
Work through both tabs in order.
|
||||
|
||||
Open vSwitch and OVN come from the CentOS NFV SIG, because Oracle ships neither for Oracle Linux 10.
|
||||
The NFV SIG is a community repository, and Oracle does not support the packages it publishes.
|
||||
Spinifex installs `openvswitch3.5` and `ovn25.03`, the versions Debian 13 ships, and its repository file lets the SIG supply nothing else.
|
||||
QEMU comes from Oracle's KVM Utilities repository, because only that build has the `microvm` machine type Spinifex load balancers boot on.
|
||||
:::tabs
|
||||
@tab 1. Prepare Oracle Linux
|
||||
|
||||
An Oracle Linux node must be a dedicated Spinifex host.
|
||||
Spinifex claims every `tap*` device and owns libreswan's `ipsec.service`, which it keeps stopped on a single node and runs for its own tunnels on a cluster node, so VPNs, hand-run QEMU guests and libreswan tunnels of your own cannot share the host.
|
||||
Crypto policies other than `DEFAULT` and `FIPS` are untested, and so is Secure Boot.
|
||||
### Update and check the host
|
||||
|
||||
## Prerequisites
|
||||
|
||||
### Operating system and kernel
|
||||
|
||||
Install Oracle Linux 10 with Oracle's ISO or a KVM template, then update it to the current minor release and reboot:
|
||||
Update to the current minor release and reboot:
|
||||
|
||||
```bash
|
||||
sudo dnf update -y
|
||||
sudo reboot
|
||||
```
|
||||
|
||||
After the reboot, check the release, the kernel and the CPU:
|
||||
Then check the release, the running kernel and the CPU:
|
||||
|
||||
```bash
|
||||
cat /etc/oracle-release # Oracle Linux Server release 10.2 or later
|
||||
@@ -73,17 +60,7 @@ uname -r # 6.12.0-...el10uek.x86_64
|
||||
grep -c -w avx2 /proc/cpuinfo # non-zero
|
||||
```
|
||||
|
||||
The kernel must be UEK R8 (`6.12.0-*uek`), and it must be the running kernel: an installed but not booted `kernel-uek` is refused.
|
||||
The CPU must be x86-64-v3 or newer (Intel Haswell, AMD Excavator or later), since the NFV SIG and KVM Utilities builds target it.
|
||||
The installer names any missing CPU flag.
|
||||
|
||||
The installer adds `kernel-uek-modules` for the running kernel, which carries `openvswitch.ko` and pulls in `linux-firmware`, about 800 MB.
|
||||
Oracle publishes modules for each kernel separately, so a host running an old kernel may find its modules gone from the repository; the installer then stops and asks for a kernel update and a reboot.
|
||||
|
||||
### SELinux
|
||||
|
||||
Spinifex ships no SELinux policy yet, so SELinux must be permissive or disabled, both now and at the next boot.
|
||||
Oracle's images boot enforcing.
|
||||
### Set SELinux permissive
|
||||
|
||||
```bash
|
||||
sudo setenforce 0
|
||||
@@ -91,14 +68,9 @@ sudo sed -i 's/^SELINUX=.*/SELINUX=permissive/' /etc/selinux/config
|
||||
getenforce # Permissive
|
||||
```
|
||||
|
||||
The installer and `spx admin preflight` refuse a host that is enforcing in either place.
|
||||
### Create the br-wan bridge
|
||||
|
||||
### WAN bridge with NetworkManager
|
||||
|
||||
As on every OS, the host's uplink must already sit on a Linux bridge named `br-wan`, with the host address and default route on the bridge (see [Single-Node Install](/docs/install#prerequisites)).
|
||||
On Oracle Linux, NetworkManager owns `br-wan`; Spinifex leaves the bridge and its NIC to it and marks the links it creates itself as unmanaged.
|
||||
|
||||
Create the bridge from the console, not over SSH on the NIC being moved, since the host loses its address while the profile changes.
|
||||
Run this from the console, not over SSH on the NIC being moved.
|
||||
Replace `eno1` with the uplink NIC and `"System eno1"` with its current profile (`nmcli connection show`):
|
||||
|
||||
```bash
|
||||
@@ -109,118 +81,88 @@ sudo nmcli connection down "System eno1"
|
||||
sudo nmcli connection up br-wan
|
||||
```
|
||||
|
||||
For a static address, give the bridge `ipv4.method manual` with `ipv4.addresses`, `ipv4.gateway` and `ipv4.dns` instead of `ipv4.method auto`.
|
||||
|
||||
`bridge.stp no` matters: a NetworkManager bridge runs STP by default, which holds the port in its forwarding delay for about 15 seconds after every link up, long enough for DHCP on the bridge to time out.
|
||||
Then verify, as on Debian:
|
||||
For a static address, use `ipv4.method manual` with `ipv4.addresses`, `ipv4.gateway` and `ipv4.dns`.
|
||||
Keep `bridge.stp no`, or DHCP on the bridge can time out.
|
||||
|
||||
```bash
|
||||
ip -br link show br-wan # UP
|
||||
ip route # default via ... dev br-wan
|
||||
```
|
||||
|
||||
### Network access during install
|
||||
@tab 2. Install Single Node
|
||||
|
||||
Besides `install.mulgadc.com` and the AWS CLI download, the install reaches `yum.oracle.com` for the Oracle Linux repositories, `mirror.stream.centos.org` for the NFV SIG packages, and `www.centos.org` for the NFV SIG signing key.
|
||||
An air-gapped install on Oracle Linux is not documented yet.
|
||||
|
||||
## Instructions
|
||||
|
||||
## Step 1. Install Spinifex
|
||||
|
||||
Run the installer as a user with `sudo`, exactly as on Debian:
|
||||
### Install Spinifex
|
||||
|
||||
```bash
|
||||
curl -fsSL https://install.mulgadc.com | bash
|
||||
```
|
||||
|
||||
For a single node, continue with [Single-Node Install](/docs/install) from Step 2: `setup-ovn.sh --management`, `spx admin init` and `systemctl start spinifex.target`.
|
||||
IPsec stays off on a single node, because there is no peer to encrypt traffic to.
|
||||
The installer stops and masks firewalld; see [firewalld](#firewalld) before relying on any rule it held.
|
||||
|
||||
For a cluster, prepare and install every node this way, then continue with [Multi-Node Install](/docs/install-multi-node) from Step 2.
|
||||
|
||||
### Clusters
|
||||
|
||||
Every node in an Oracle Linux cluster runs Oracle Linux.
|
||||
`install-node.sh` refuses a host list that mixes Oracle Linux with Debian or Ubuntu, but `spx admin join` does not check, so keep to one OS yourself when joining by hand.
|
||||
|
||||
IPsec encrypts the Geneve tunnels between nodes and is on by default, as on Debian.
|
||||
On Oracle Linux it runs on libreswan, the IKE daemon Oracle ships, rather than strongSwan.
|
||||
Once the node's IPsec certificate exists, the daemon unmasks and starts `ipsec.service` and `openvswitch-ipsec.service`.
|
||||
`ovs-monitor-ipsec` then loads the node certificate, its key and the cluster CA into libreswan's NSS database at `/var/lib/ipsec/nss`, under nicknames that start with `ovs_cert`.
|
||||
|
||||
The tunnels are plain ESP (IP protocol 50), as on Debian, negotiated over UDP 500.
|
||||
Open UDP 500, UDP 4500 and ESP between nodes: libreswan moves to UDP 4500 if it detects NAT between two nodes.
|
||||
The Spinifex host firewall opens all three to cluster peers; a firewall between the nodes must allow them too.
|
||||
While IKE to a peer fails, the node drops its Geneve traffic to that peer rather than send it in clear.
|
||||
The nft table `inet spinifex_ipsec_guard` drops outbound UDP 6081 from the encap address whose route carries no SA, which also covers the moments pluto flushes the xfrm policy table as it starts and stops.
|
||||
It is not persisted: `ipsec.service` puts it back at boot before ovn-controller builds the tunnels, and the daemon re-syncs it every 60 seconds.
|
||||
`spx admin init --ipsec=false` leaves the tunnels unencrypted, as on any OS.
|
||||
|
||||
## Step 2. Know What the Installer Changed
|
||||
|
||||
On Oracle Linux the installer makes these changes, which Debian hosts do not need:
|
||||
|
||||
- **Repositories.** It enables `ol10_UEKR8` and `ol10_kvm_utils`, and adds `qemu-kvm*` and `qemu-img*` to the `excludepkgs` of `ol10_appstream`, keeping any excludes already set, so a later AppStream build can never replace the KVM Utilities QEMU.
|
||||
- **NFV SIG.** It writes `/etc/yum.repos.d/spinifex-nfv-sig.repo` with an `includepkgs=` list of the OVS and OVN packages, and imports the SIG key to `/etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-NFV` only when its fingerprint is `35154228174901BEFA8E69A621465E289D2A76A7`.
|
||||
- **OVS and OVN user.** Both run as root, as on Debian, through `OVS_USER_ID` and `OVN_USER_ID` in `/etc/sysconfig/openvswitch` and `/etc/sysconfig/ovn`.
|
||||
- **DNS.** Where systemd-resolved is installed but not running, it saves `/etc/resolv.conf` to `/var/lib/spinifex-reset/resolv.conf.orig`, starts resolved behind NetworkManager, tells NetworkManager in `/etc/NetworkManager/conf.d/90-spinifex-dns.conf` to leave `/etc/resolv.conf` alone, and links `/etc/resolv.conf` to the resolved stub. It then checks that names still resolve and stops, printing the commands that put DNS back, if they do not.
|
||||
- **chrony.** It appends `confdir /etc/chrony.d` to `/etc/chrony.conf` once, and writes its step policy to `/etc/chrony.d/spinifex.conf`.
|
||||
- **`spx` under `sudo`.** Oracle Linux's `sudo` path leaves out `/usr/local/bin`, so the installer links `/usr/bin/spx` to `/usr/local/bin/spx`.
|
||||
|
||||
The kernel modules list (`/etc/modules-load.d/spinifex.conf`) and the host firewall below apply on every OS.
|
||||
|
||||
## Step 3. Replace firewalld Rules You Depend On
|
||||
|
||||
The installer stops and masks firewalld, because with it running a packet must pass both firewalld's table and the Spinifex one, which silently breaks routed NAT, Elastic IPs and instance metadata.
|
||||
When firewalld was active, the installer turns on the Spinifex nftables policy in its place; pass `--firewall=on` or `--firewall=off` to choose yourself.
|
||||
|
||||
firewalld's rules are not saved or migrated.
|
||||
The Spinifex policy filters inbound traffic only and opens 53, 443, 3000, 8443 and 9999 to any source.
|
||||
It accepts SSH from every IPv4 source until `$trusted_ssh_peers` in `/etc/spinifex/firewall/custom.nft` narrows it.
|
||||
Restate any other restriction you had, such as SSH limited to a management subnet, in that file; [Host Firewall](/docs/host-firewall) shows how.
|
||||
|
||||
From the moment firewalld stops until the daemon first loads the Spinifex policy, the host accepts every connection.
|
||||
Uninstalling Spinifex leaves firewalld unmasked but disabled; it does not turn it back on.
|
||||
|
||||
## Step 4. Keep the Host Updated
|
||||
|
||||
Do not enable `dnf-automatic`.
|
||||
An Open vSwitch package upgrade restarts OVS, which interrupts guest networking, so update `openvswitch3.5`, `ovn25.03*` and `qemu-kvm*` in a maintenance window.
|
||||
Stay on the current Oracle Linux 10 minor release, which is what Spinifex is tested on.
|
||||
|
||||
Before rebooting into a new kernel, check that its modules are installed, or OVS cannot start after the reboot:
|
||||
### Set up OVN networking
|
||||
|
||||
```bash
|
||||
rpm -q kernel-uek-core kernel-uek-modules
|
||||
sudo /usr/local/share/spinifex/setup-ovn.sh --management
|
||||
```
|
||||
|
||||
Each `kernel-uek-core` version needs a `kernel-uek-modules` of the same version; install a missing one with `sudo dnf install kernel-uek-modules-<version>`.
|
||||
### Initialize
|
||||
|
||||
Track Oracle Linux security advisories (ELSA) for the host, the kernel and QEMU, for example with `dnf updateinfo list --security`.
|
||||
Those advisories do not cover the NFV SIG packages, so follow Open vSwitch and OVN security announcements for those.
|
||||
```bash
|
||||
sudo spx admin init --node node1 --nodes 1
|
||||
```
|
||||
|
||||
## Step 5. Uninstall
|
||||
### Start services
|
||||
|
||||
`/usr/local/share/spinifex/uninstall-spx.sh` removes Spinifex as on Debian.
|
||||
On a cluster node it also stops `openvswitch-ipsec.service` and deletes the `ovs_cert*` entries from libreswan's NSS database, so the node can join a different cluster; other NSS identities and your connection files in `/etc/ipsec.d` stay.
|
||||
It lists what it leaves in place on Oracle Linux: the repository changes and the SIG key, `OVS_USER_ID` and `OVN_USER_ID`, the OVS and OVN logrotate edits, the chrony include, `/etc/modules-load.d/spinifex.conf`, the SELinux mode, a masked `ipsec.service`, firewalld disabled, and any GPU passthrough configuration.
|
||||
`--purge-deps` is refused on Oracle Linux, because dnf cannot tell the packages Spinifex installed from the host's own; remove them with `dnf remove` yourself.
|
||||
```bash
|
||||
sudo systemctl start spinifex.target
|
||||
```
|
||||
|
||||
### Verify
|
||||
|
||||
```bash
|
||||
export AWS_PROFILE=spinifex
|
||||
aws ec2 describe-instance-types
|
||||
```
|
||||
|
||||
If this returns a list of instance types, Spinifex is installed.
|
||||
Continue to [Setting Up Your Cluster](/docs/setting-up-your-cluster) to import an AMI and launch your first instance.
|
||||
:::
|
||||
|
||||
**For a cluster**, prepare and install every node with the steps above, then continue with [Multi-Node Install](/docs/install-multi-node) from Step 2.
|
||||
Every node must run Oracle Linux.
|
||||
IPsec between nodes runs on libreswan, so open UDP 500, UDP 4500 and ESP between nodes on any firewall between them.
|
||||
|
||||
## What the Installer Changes
|
||||
|
||||
On Oracle Linux the installer also:
|
||||
|
||||
- Enables `ol10_UEKR8` and `ol10_kvm_utils`, and excludes `qemu-kvm*` and `qemu-img*` from `ol10_appstream`.
|
||||
- Adds the NFV SIG repository as `/etc/yum.repos.d/spinifex-nfv-sig.repo`, limited to the OVS and OVN packages.
|
||||
- Installs `kernel-uek-modules` for the running kernel, about 800 MB with `linux-firmware`.
|
||||
- Switches DNS to systemd-resolved behind NetworkManager where resolved is installed but not running.
|
||||
- Links `/usr/bin/spx` to `/usr/local/bin/spx`, so `sudo spx` works.
|
||||
|
||||
### firewalld
|
||||
|
||||
The installer stops and masks firewalld and turns on the Spinifex nftables policy in its place; firewalld's rules are not migrated.
|
||||
The Spinifex policy opens 53, 443, 3000, 8443, 9999 and SSH to any source.
|
||||
Restate any restriction you relied on, such as SSH limited to a management subnet, in `/etc/spinifex/firewall/custom.nft`; see [Host Firewall](/docs/host-firewall).
|
||||
|
||||
### Updates
|
||||
|
||||
Do not enable `dnf-automatic`: an Open vSwitch upgrade restarts OVS and interrupts guest networking, so update `openvswitch3.5`, `ovn25.03*` and `qemu-kvm*` in a maintenance window.
|
||||
Before rebooting into a new kernel, check `rpm -q kernel-uek-core kernel-uek-modules` shows matching versions, or OVS cannot start.
|
||||
|
||||
### Uninstall
|
||||
|
||||
`/usr/local/share/spinifex/uninstall-spx.sh` removes Spinifex and lists what it leaves in place.
|
||||
`--purge-deps` is refused on Oracle Linux; remove packages with `dnf remove` yourself.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Installer stops: SELinux is enforcing
|
||||
### Installer stops on SELinux, the kernel or kernel modules
|
||||
|
||||
Set SELinux permissive now and in `/etc/selinux/config`, as in [SELinux](#selinux), then run the installer again.
|
||||
Fixing only one of the two is refused too.
|
||||
|
||||
### Installer stops: the running kernel is not UEK R8
|
||||
|
||||
Install `kernel-uek`, reboot into it, check `uname -r`, then run the installer again.
|
||||
|
||||
### Installer stops: kernel-uek-modules is no longer in the repositories
|
||||
|
||||
The running kernel is older than anything the repository still carries modules for.
|
||||
Set SELinux permissive both now and in `/etc/selinux/config`, or update and boot into the current UEK R8 kernel, then run the installer again:
|
||||
|
||||
```bash
|
||||
sudo dnf update -y kernel-uek
|
||||
@@ -229,41 +171,30 @@ sudo reboot
|
||||
|
||||
### Installer stops: a repository is unreachable
|
||||
|
||||
The message names the repository.
|
||||
Check the host can reach `yum.oracle.com` and `mirror.stream.centos.org` over HTTPS, including through any proxy dnf uses, then run the installer again.
|
||||
Check the host can reach `yum.oracle.com` and `mirror.stream.centos.org` over HTTPS, including through any proxy dnf uses.
|
||||
|
||||
### DHCP on br-wan times out
|
||||
|
||||
Check STP is off on the bridge:
|
||||
STP is on. Turn it off:
|
||||
|
||||
```bash
|
||||
nmcli -g bridge.stp connection show br-wan
|
||||
sudo nmcli connection modify br-wan bridge.stp no
|
||||
sudo nmcli connection up br-wan
|
||||
```
|
||||
|
||||
If it prints `yes`, run `sudo nmcli connection modify br-wan bridge.stp no` and `sudo nmcli connection up br-wan`.
|
||||
|
||||
### `sudo spx` finds no command
|
||||
|
||||
Check that `/usr/bin/spx` links to `/usr/local/bin/spx`; run the installer again if it does not, or call `sudo /usr/local/bin/spx` meanwhile.
|
||||
|
||||
### Cluster nodes form no IPsec tunnels
|
||||
|
||||
Check that both IPsec services run, and that libreswan holds the node's certificate and the cluster CA:
|
||||
|
||||
```bash
|
||||
systemctl is-active ipsec.service openvswitch-ipsec.service
|
||||
sudo certutil -L -d sql:/var/lib/ipsec/nss # ovs_certkey_<node>, ovs_cert_cacert
|
||||
sudo ipsec trafficstatus # one line per peer tunnel
|
||||
```
|
||||
|
||||
If the services are inactive, check the daemon log with `journalctl -u spinifex-daemon` for the IPsec reconcile.
|
||||
If the certificates are present but no tunnel is listed, check that UDP 500 and ESP pass between the nodes in both directions, and look for `SEC_ERROR_UNKNOWN_ISSUER` in `journalctl -u ipsec`, which means a peer's certificate is from another CA.
|
||||
Until the tunnel forms, guest traffic between those two nodes is dropped, which is the guard in [Clusters](#clusters) working.
|
||||
If the services are inactive, check `journalctl -u spinifex-daemon`.
|
||||
If no tunnel is listed, check UDP 500 and ESP pass between the nodes in both directions.
|
||||
Guest traffic between two nodes is dropped until their tunnel forms.
|
||||
|
||||
### CA certificate not trusted
|
||||
|
||||
`sudo spx admin init` installs the CA automatically. To install it again by hand on Oracle Linux:
|
||||
|
||||
```bash
|
||||
sudo cp /etc/spinifex/ca.pem /etc/pki/ca-trust/source/anchors/spinifex-ca.crt
|
||||
sudo update-ca-trust
|
||||
|
||||
@@ -16,16 +16,16 @@ Create, update, delete and policy association complete synchronously within the
|
||||
|---|---|---|
|
||||
| Identity | Cluster name plus principal ARN. Key and ARN suffix use `sha256hex(principalARN)` (`domains/eks/access/store.go` `PrincipalARNHash`). ARN `arn:aws:eks:{region}:{account}:access-entry/{cluster}/{hash}` (`foundation/aws/arn/eks.go`); the AWS ARN shape differs and is not verified. | `handlers/eks/service_impl_test.go` `TestAccessEntry_CreateDescribeListDelete`, `domains/eks/access/store_test.go` `TestKeyPaths`, `gateway/eks/authz_test.go` `TestResourceARNsAccessEntryMatchesHandler` |
|
||||
| Scope | Account and Region, within one cluster: keys live under `clusters/{cluster}/access-entries/` in the `eks-account-{account}` bucket. | `handlers/eks/access_lifecycle_test.go` `TestAccessEntryPersistedLayout_TopLevelAndNestedFieldNames`, `TestAccessEntryRecord_DecodesHandWrittenJSON`, `handlers/eks/create_cluster_test.go` `TestCreateCluster_SeedsCreatorAdminAccessEntry` |
|
||||
| Durable owner and record | `domains/eks/access` (`Owner`) is the only writer of `Record` with nested `AssociatedPolicy` and `Scope`: Create, SeedCreatorAdmin (cluster launch), Update, Associate, Disassociate and Delete. Keys and field names are unchanged by the move. `DeleteClusterPrefix` (`handlers/eks/cluster_state.go`) still erases entries with the cluster, and the gateway process reads records through the exported `access.Get` in `ResolveTokenReview` (`handlers/eks/token_review.go`). | `handlers/eks/access_lifecycle_test.go` `TestAccessEntryPersistedLayout_TopLevelAndNestedFieldNames`, `TestAccessEntryRecord_DecodesHandWrittenJSON`, `domains/eks/access/owner_test.go`, `domains/eks/access/store_test.go`, `handlers/eks/create_cluster_test.go` `TestCreateCluster_SeedsCreatorAdminAccessEntry` |
|
||||
| Durable owner and record | `domains/eks/access` (`Owner`) is the only writer of `Record` with nested `AssociatedPolicy` and `Scope`: Create, SeedCreatorAdmin (cluster launch), Update, Associate, Disassociate and Delete. Keys and field names are unchanged by the move. `DeleteClusterChildren` (`domains/eks/cluster/cluster_state.go`) still erases entries with the cluster, and the gateway process reads records through the exported `access.Get` in `ResolveTokenReview` (`handlers/eks/token_review.go`). | `handlers/eks/access_lifecycle_test.go` `TestAccessEntryPersistedLayout_TopLevelAndNestedFieldNames`, `TestAccessEntryRecord_DecodesHandWrittenJSON`, `domains/eks/access/owner_test.go`, `domains/eks/access/store_test.go`, `handlers/eks/create_cluster_test.go` `TestCreateCluster_SeedsCreatorAdminAccessEntry` |
|
||||
| AWS adapter | `handlers/eks` keeps request validation (entry type, supported policy, scope), the cluster existence check (`acctKVForCluster`), awserrors mapping and the AWS projection. `gateway/eks/authz.go` uses `access.PrincipalARNHash`. | `handlers/eks/service_impl_test.go` access cases, `gateway/eks/authz_test.go` `TestResourceARNsAccessEntryMatchesHandler` |
|
||||
| Desired state | Principal, Kubernetes username and groups, type (`STANDARD` only), tags (set at create only) and cluster-scoped access policy associations. | `handlers/eks/service_impl_test.go` `TestAccessEntry_RejectsNodeType`, `TestAccessPolicy_AssociateRejectsUnsupportedPolicyAndScope`, `TestAccessPolicy_AssociateRejectsNamespaceScope`, `handlers/eks/access_lifecycle_test.go` `TestAccessEntry_UpdateWithNoFieldsKeepsUsernameAndGroups` |
|
||||
| Generation | None. Update, associate and disassociate retry against the KV revision; create and delete do not. | |
|
||||
| Observed state | Not tracked; the record is the only state. | |
|
||||
| Readiness | Usable by token review as soon as the write returns. | `handlers/eks/token_review_test.go` `TestResolveTokenReview_AuthenticatesViaVerifyAndKV`, `TestResolveTokenReview_NoAccessEntryDenies`, `handlers/eks/service_impl_test.go` `TestAccessPolicy_DisassociateRemovesProjectedGroup` |
|
||||
| Idempotency | `clientRequestToken` is ignored. Update always advances `modifiedAt`, even when nothing changes. A duplicate create returns `ResourceInUseException`; a repeated delete returns `ResourceNotFoundException`; disassociating an unbound policy succeeds without change. Internal NATS redelivery of create returns `ResourceInUseException` to the redelivered request. | `handlers/eks/service_impl_test.go` `TestAccessEntry_CreateDescribeListDelete`, `TestAccessEntry_DescribeDeleteMissingIsNotFound`, `handlers/eks/access_lifecycle_test.go` `TestAccessPolicy_DisassociateUnassociatedLeavesRecordUnchanged`, `TestAccessPolicy_AssociateWithNoEntryIsNotFound` |
|
||||
| Dependencies | Every action requires only that the cluster's meta record exists (`acctKVForCluster`); cluster status is not read, so actions succeed on a `CREATING`, `FAILED` or `DELETING` cluster. Token review does not check the cluster at all. The principal is not checked against IAM. | `handlers/eks/service_impl_test.go` `TestAccessEntry_UnknownClusterIsNotFound`, `handlers/eks/access_lifecycle_test.go` `TestAccessEntryActionsIgnoreClusterLifecycleStatus`, `TestResolveTokenReview_AuthenticatesWithoutClusterMeta` |
|
||||
| Dependencies | Every action requires that the cluster's meta record exists (`acctKVForCluster`). `CreateAccessEntry` also refuses a `DELETING` cluster with `ResourceInUseException`, since a recorded delete admits no new dependant; no other status is read, so actions succeed on a `CREATING` or `FAILED` cluster. Token review does not check the cluster at all. The principal is not checked against IAM. | `handlers/eks/service_impl_test.go` `TestAccessEntry_UnknownClusterIsNotFound`, `handlers/eks/access_lifecycle_test.go` `TestAccessEntryActionsIgnoreFailedClusterStatus`, `delete_cluster_async_test.go` `TestS2_EKSDeletingClusterAdmitsNoNewDependants`, `TestResolveTokenReview_AuthenticatesWithoutClusterMeta` |
|
||||
| Authorization | Gateway authz derives the access-entry ARN with `PrincipalARNHash` (`gateway/eks/authz.go`). No `iam:PassRole`. The STS step of token review is the existing `eks.VerifyToken` NATS contract served by awsgw. | `gateway/eks/authz_test.go` `TestResourceARNsAccessEntryMatchesHandler` |
|
||||
| Deletion ordering | Delete is get-then-delete, not conditional. Cluster delete erases entries in the prefix purge only after all infrastructure teardown succeeds; on failure they survive while the cluster stays `DELETING`. | `handlers/eks/access_lifecycle_test.go` `TestDeleteClusterPrefix_SweepsAccessEntriesScopedToCluster` |
|
||||
| Deletion ordering | Delete is get-then-delete, not conditional. Cluster delete erases entries with the cluster's other child records, before the meta, only after all infrastructure teardown succeeds; on failure they survive while the cluster stays `DELETING`. | `handlers/eks/access_lifecycle_test.go` `TestDeleteClusterChildren_SweepsAccessEntriesScopedToCluster` |
|
||||
| Restart and interruption | The prefix purge deletes keys in listing order; an interruption after the meta key is gone leaves entries that a new cluster of the same name inherits. Reclaiming a `FAILED` cluster and a failed launch purge with the meta kept, so the recreated cluster inherits the failed attempt's entries. | none |
|
||||
| End-to-end | `tests/e2e/eks/eks_test.go` `AccessEntry` and `GetToken`; `tests/integration/cross_tenant_test.go`. No Terraform apply/destroy evidence for `aws_eks_access_entry` or `aws_eks_access_policy_association`. | |
|
||||
|
||||
@@ -33,7 +33,7 @@ Create, update, delete and policy association complete synchronously within the
|
||||
|
||||
1. Create is get-then-put with no exclusive create, so concurrent creates for one principal both succeed and the later write wins.
|
||||
2. The creator-admin seed at cluster launch is an unconditional put after the cluster meta is final, so it replaces an entry for the same principal created while the cluster was `CREATING`, including its policy associations.
|
||||
3. Access actions do not check cluster status (see dependencies row), and opening the bucket creates it for an account that has none.
|
||||
3. Access actions other than `CreateAccessEntry` do not check cluster status (see dependencies row), and opening the bucket creates it for an account that has none.
|
||||
4. Entries can outlive their cluster (interrupted prefix purge) or carry over to a recreated cluster (reclaim and failed launch keep the meta and do not purge entries separately).
|
||||
5. Token review authenticates against an entry without checking the cluster exists or its status.
|
||||
6. `ResolveTokenReview` opens the EKS bucket and reads the record in the gateway process through the exported `access.Get`, outside the owner.
|
||||
|
||||
@@ -16,9 +16,9 @@ There is no generation, so the record mixes desired fields with the latest obser
|
||||
| Item | Present behaviour | Evidence |
|
||||
|---|---|---|
|
||||
| Identity | Cluster name plus add-on name; one add-on per name per cluster. ARN `arn:aws:eks:{region}:{account}:addon/{cluster}/{addon}` (`foundation/aws/arn/eks.go` `FormatEKSAddon`), deterministic by name, with no per-incarnation suffix. The gateway authorizer derives the same ARN (`domains/eks/awsapi/authz.go` `addonARN`). | `handlers/eks/addon_lifecycle_test.go` `TestAddonPersistedLayout_RecordAndManifestKeysAndFieldNames`, `domains/eks/awsapi/authz_test.go` `TestResourceARNsFidelity` |
|
||||
| Scope | Account and Region, within one cluster: keys live under `clusters/{cluster}/addons/` in the `eks-account-{account}` bucket. | `TestAddonPersistedLayout_RecordAndManifestKeysAndFieldNames`, `TestDeleteClusterPrefix_SweepsAddonRecordsAndManifestsScopedToCluster` |
|
||||
| Scope | Account and Region, within one cluster: keys live under `clusters/{cluster}/addons/` in the `eks-account-{account}` bucket. | `TestAddonPersistedLayout_RecordAndManifestKeysAndFieldNames`, `TestDeleteClusterChildren_SweepsAddonRecordsAndManifestsScopedToCluster` |
|
||||
| Durable record | `clusters/{cluster}/addons/{addon}` holds `Record` (`domains/eks/addon/record.go`): `addonName`, `addonVersion`, `status`, `serviceAccountRoleArn`, `configurationValues`, `health`, `arn`, `tags`, `createdAt`, `modifiedAt` (the optional ones `omitempty`). `clusters/{cluster}/addons/{addon}/manifest` holds `Manifest` (`domains/eks/addon/record.go`): `addonName`, `addonVersion`, `serviceAccountRoleArn`, `configurationValues`. Listing skips the manifest sub-key by segment count. | `TestAddonPersistedLayout_RecordAndManifestKeysAndFieldNames`, `TestAddonRecord_DecodesHandWrittenJSON`, `domains/eks/addon/store_test.go` `TestRecord_RoundTrip`, `TestList_SkipsManifestSubKeys` |
|
||||
| Writers | Record: the owner in `domains/eks/addon/owner.go`, reached from the `CreateAddon`, `UpdateAddon` and `DeleteAddon` handlers, the cluster reconciler's report relay and the GPU node group launch: `Owner.Create` (unconditional put), `Owner.Update` and its failure marking (revision CAS), `Owner.Delete` (get then delete), `Owner.ApplyReport` (revision CAS), `Owner.EnsureGPUDevicePlugin` through `Owner.Create`; and the cluster prefix purge (`DeleteClusterPrefix`). Manifest: `StagingInstaller.Install` (unconditional put on create and update), `StagingInstaller.Uninstall` (`domains/eks/addon/delivery.go`), `Owner.Delete` and the prefix purge. | `TestAddonPersistedLayout_RecordAndManifestKeysAndFieldNames`, `TestUpdateAddon_ReportsSuccessfulWhileRecordIsUpdating`, `TestDeleteAddon_ReturnsDeletingAfterErasingRecordAndManifest`, `TestDeleteClusterPrefix_SweepsAddonRecordsAndManifestsScopedToCluster`, `domains/eks/addon/store_test.go` `TestCasUpdate`, `TestDeleteRecord_RemovesRecordAndManifest` |
|
||||
| Writers | Record: the owner in `domains/eks/addon/owner.go`, reached from the `CreateAddon`, `UpdateAddon` and `DeleteAddon` handlers, the cluster reconciler's report relay and the GPU node group launch: `Owner.Create` (unconditional put), `Owner.Update` and its failure marking (revision CAS), `Owner.Delete` (get then delete), `Owner.ApplyReport` (revision CAS), `Owner.EnsureGPUDevicePlugin` through `Owner.Create`; and the cluster prefix purge (`DeleteClusterChildren`). Manifest: `StagingInstaller.Install` (unconditional put on create and update), `StagingInstaller.Uninstall` (`domains/eks/addon/delivery.go`), `Owner.Delete` and the prefix purge. | `TestAddonPersistedLayout_RecordAndManifestKeysAndFieldNames`, `TestUpdateAddon_ReportsSuccessfulWhileRecordIsUpdating`, `TestDeleteAddon_ReturnsDeletingAfterErasingRecordAndManifest`, `TestDeleteClusterChildren_SweepsAddonRecordsAndManifestsScopedToCluster`, `domains/eks/addon/store_test.go` `TestCasUpdate`, `TestDeleteRecord_RemovesRecordAndManifest` |
|
||||
| Readers | `ListAddons`, `DescribeAddon`, `CreateAddon` (existence check) and the reconciler read the record. `ListStagedAddonManifests` reads every `/manifest` key for the cluster (`addon.ListManifests`); it is served on `eks.ListStagedAddonManifests` (`daemon/daemon.go`) and reached only through the gateway's internal-addons route. | `handlers/eks/addons_test.go` `TestAddon_DescribeListDelete`, `TestListStagedAddonManifests` |
|
||||
| Catalogue | Static in-binary catalogue (`domains/eks/addon/catalog.go`): `aws-load-balancer-controller`, `argocd`, `aws-ebs-csi-driver`, and hidden-but-creatable `nvidia-device-plugin` and `spinifex-noop`; one version each, the first being the default. Create and update reject unknown names and versions with `InvalidParameterValue`. | `domains/eks/addon/catalog_test.go`, `handlers/eks/addons_test.go` `TestDescribeAddonVersions_ReturnsCatalog`, `TestCreateAddon_UnknownAddonRejected`, `TestCreateAddon_UnknownVersionRejected`, `TestUpdateAddon_UnknownVersionRejected` |
|
||||
| Desired state | Version, `serviceAccountRoleArn`, `configurationValues` and create-time tags. `resolveConflicts`, `podIdentityAssociations` and `clientRequestToken` are accepted and dropped; `preserve` on delete is ignored. `configurationValues` reaches the guest but the agent discards it (`mulga-eks-addon-sync.sh` `sync_once`), so it has no effect. | `TestAddonPersistedLayout_RecordAndManifestKeysAndFieldNames`, `TestDeleteAddon_ReturnsDeletingAfterErasingRecordAndManifest` |
|
||||
@@ -37,9 +37,9 @@ There is no generation, so the record mixes desired fields with the latest obser
|
||||
| Cross-resource dependency | A GPU node group's launch calls `stageGPUDeviceAddon`, which checks the cluster exists and calls `Owner.EnsureGPUDevicePlugin` for `nvidia-device-plugin`; an existing record is a no-op and every other error is logged only. Deleting the node group never unstages it, and an existing record in any status, including `CREATE_FAILED`, blocks re-staging. | `handlers/eks/nodegroup_test.go` `TestCreateNodegroup_GPUNodegroupStagesDevicePluginAddon`, `domains/eks/addon/owner_test.go` `TestOwnerEnsureGPUDevicePlugin_StagesDefaultVersionOnceAndKeepsAnyExistingRecord`, `TestCreateNodegroup_NonGPUNodegroupDoesNotStageDevicePluginAddon`, `tests/e2e/gpu/eks_gpu_test.go` `TestEKSGPUPodExposure` |
|
||||
| Idempotency | `clientRequestToken` is ignored. A duplicate create, including an SDK retry with the same token, returns `ResourceInUseException`. A repeated update re-stages and returns another `Successful` with the same id. A repeated delete returns `ResourceNotFoundException`. | `TestCreateAddon_RepeatedClientRequestTokenIsResourceInUse`, `TestUpdateAddon_ReportsSuccessfulWhileRecordIsUpdating`, `TestDeleteAddon_ReturnsDeletingAfterErasingRecordAndManifest`, `handlers/eks/addons_test.go` `TestCreateAddon_CreatesStagingAndDefaultsVersion` |
|
||||
| Absent resource | An unknown cluster returns `ResourceNotFoundException` for every action, including the internal manifest listing. A missing add-on returns `ResourceNotFoundException` from describe, update and delete. | `handlers/eks/addons_test.go` `TestCreateAddon_UnknownClusterIsNotFound`, `TestDescribeDeleteAddon_MissingIsNotFound`, `TestUpdateAddon_MissingIsNotFound`, `TestListStagedAddonManifests` |
|
||||
| Dependencies | Only the cluster meta's existence is checked; cluster status is not read, so add-ons can be created on a `DELETING` cluster. | `TestAddonActionsIgnoreClusterLifecycleStatus` |
|
||||
| Dependencies | `CreateAddon` checks the cluster meta exists and refuses a `DELETING` cluster with `ResourceInUseException`, since a recorded delete admits no new dependant; other statuses are not read. | `TestCreateAddonRefusesOnlyADeletingCluster` |
|
||||
| Authorization | Gateway policy uses the add-on ARN above. `serviceAccountRoleArn` requires `iam:PassRole` on `CreateAddon` and `UpdateAddon` (`domains/eks/awsapi/passrole.go`), the same check `CreateCluster`/`CreateNodegroup` apply, including that the role exists. Tag actions on an add-on ARN return `NotImplemented`. | `domains/eks/awsapi/gateway_wire_test.go` `TestPassedRoleARNs_AddonServiceAccountRoleIsChecked`, `domains/eks/awsapi/gateway_passrole_test.go` `TestEKSCreateAddon_PassRole`, `TestEKSUpdateAddon_PassRole`, `domains/eks/awsapi/authz_test.go` `TestResourceARNsFidelity` |
|
||||
| Cluster deletion | Add-on records and manifests are erased only by the cluster prefix purge, after the cluster's infrastructure teardown succeeds; there is no per-add-on delete. Reclaiming a failed cluster keeps the meta and does not purge, so the recreated cluster inherits the add-ons. | `TestDeleteClusterPrefix_SweepsAddonRecordsAndManifestsScopedToCluster`, `handlers/eks/cluster_state_test.go` `TestDeleteClusterPrefix_SweepsEveryKey` |
|
||||
| Cluster deletion | Add-on records and manifests are erased only by the cluster prefix purge, after the cluster's infrastructure teardown succeeds; there is no per-add-on delete. Reclaiming a failed cluster keeps the meta and does not purge, so the recreated cluster inherits the add-ons. | `TestDeleteClusterChildren_SweepsAddonRecordsAndManifestsScopedToCluster`, `handlers/eks/cluster_state_test.go` `TestDeleteClusterChildren_SweepsEveryKey` |
|
||||
| Restart and interruption | Create interrupted between the record put and staging leaves a `CREATING` record with no manifest that nothing re-stages and that blocks a retry. Delete interrupted after unstaging leaves a record in its previous status while the guest removes the software. An update interrupted after the CAS leaves `UPDATING` with the previous manifest. A daemon restart loses nothing durable, but reports sent while no reconciler runs are lost until the next tick. | none |
|
||||
| End-to-end | `tests/e2e/eks/eks_test.go` `TestEKS/AddonDelivery` (create to `ACTIVE`, objects present, delete removes the namespace) and `tests/e2e/gpu/eks_gpu_test.go` `TestEKSGPUPodExposure`; `tests/integration/cross_tenant_test.go` `TestCrossTenant_EveryScopedActionIsAccountedFor`. No E2E for update, failure, configuration or the role. No Terraform apply/destroy evidence for `aws_eks_addon`. | |
|
||||
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
# EKS cluster: delete lifecycle contract
|
||||
|
||||
Contract under ADR-0003 S1 for the delete half of an EKS cluster's lifecycle, the half ADR-0003 S8 constrains.
|
||||
Create, update and the control-plane reconciler are out of its scope and keep the observed gaps recorded elsewhere.
|
||||
Paths are relative to `spinifex/` unless they start with `tests/`, which is relative to the spinifex repo root.
|
||||
Test names without a path are in `domains/eks/cluster/`.
|
||||
|
||||
## Scope
|
||||
|
||||
A cluster is a composite resource: its record at `clusters/{name}/meta` in the `eks-account-{account}` bucket names the dependants a delete must remove.
|
||||
In the system account those are the control-plane VMs and their root volumes, the endpoint load balancer with its target groups, the managed control-plane VPC and its network, and the spread placement group.
|
||||
In the customer account they are the egress address, the private-endpoint interface and security group, the cluster security groups, a cluster-owned internet gateway, and load balancers the in-cluster controller created.
|
||||
Node group workers and CSI-provisioned EBS volumes are never removed by a cluster delete.
|
||||
|
||||
## Contract
|
||||
|
||||
| Item | Behaviour | Evidence |
|
||||
|---|---|---|
|
||||
| Request | `DeleteCluster` validates, answers `ResourceNotFoundException` for an absent cluster, answers a `DELETING` cluster from its record, refuses a cluster with node groups with `ResourceInUseException`, records `DELETING` and returns. It waits for no teardown step. | `TestS8_EKSDeleteClusterReturnsBeforeTheTeardownCompletes`, `TestRLC1_EKSDeleteClusterNotFoundOnAbsent`, `TestS8_EKSDeleteClusterRefusesWhileNodegroupsExist`, `tests/e2e/eks` `TestEKS/DeleteCluster` |
|
||||
| Obligation | `DELETING` with `DeletingSince` on the record is the tenant's decision. No status writer moves a record off it, and a create launch still running adds its refs to it instead of overwriting it. | `TestS2_EKSDeletingIsStickyForStatusWriters`, `TestS2_EKSLaunchWriteAfterDeleteNeitherRevertsNorRecreates` |
|
||||
| Retry | A repeat delete answers `DELETING` from the record and starts nothing. | `TestS8_EKSSecondDeleteAnswersFromTheRecordWithoutASecondTeardown` |
|
||||
| Convergence | `convergeClusterDeletion` (`domains/eks/cluster/cluster_teardown.go`), started by the request and backed by `EKSDeletingReaper`, runs under a per-cluster in-process gate and the cross-node teardown lease, renewed on the service context. It acts only on a record that is `DELETING` when re-read. | `TestS8_EKSKickAndSameNodeSweepRunOneTeardown`, `TestS8_EKSTeardownLeaseOutlivesTheRequest`, `TestS7_EKSTeardownActsOnlyOnARecordedDelete` |
|
||||
| Retained cleanup | A failed step keeps the record `DELETING` with its refs and the error. The record is removed last, after its child records, and only if no ref appeared that the pass did not tear down. A record with a node group is never removed; a record that is not `DELETING` is refused before anything else is checked. | `TestRLC2_EKSBillableTeardownBeforeKVSweep`, `TestS2_EKSTeardownRemovesTheMetaLast`, `TestS2_EKSTeardownDoesNotSweepARecordChangedDuringTeardown`, `TestS7_EKSTeardownNeverRemovesAClusterWithNodegroupWorkers` |
|
||||
| Recovery | A teardown interrupted by a crash or a lost lease is finished by the reaper on any node. Failed passes back off from 90 s, doubling to a 30 min cap, and are never abandoned; `DeleteReapExhausted` only flags a long failure for an operator. | `TestS2_EKSTeardownResumesAfterAnInterruptedConverge`, `TestRLC4_DeletingReaperReDrivesWedgedTeardown`, `TestDeletingReaper_KeepsRetryingPastTheOldAttemptLimit`, `TestDeletingReaper_ResumesAnAlreadyExhaustedRecord` |
|
||||
| Visibility | `DescribeCluster` and `ListClusters` show the cluster `DELETING` until the record is removed, then `ResourceNotFoundException`. A failed pass shows one `INTERNAL_FAILURE` health issue naming the step, never the internal error. | `TestDescribeCluster_StuckDeleteShowsTheStepNotTheError`, `TestRLC2_EKSBillableTeardownBeforeKVSweep` |
|
||||
| New dependants | `CreateNodegroup` requires `ACTIVE`; `CreateAddon` and `CreateAccessEntry` refuse a `DELETING` cluster with `ResourceInUseException`. | `TestCreateAddonRefusesOnlyADeletingCluster`, `TestS2_EKSDeletingClusterAdmitsNoNewDependants` |
|
||||
| Subject budgets | Every `eks.*` request subject is classified by its handler's work in `eksSubjects` (`domains/eks/cluster/service_nats.go`) and keeps the 30 s budget; `eks.DeleteCluster` is request-scaled. A subject the gateway sends or the daemon serves that is not classified fails the table. | `TestS8_EKSSubjectBudgets` |
|
||||
| Late child records | No child record outlives the cluster or leaks into a later one of the same name. The teardown sweeps child keys again after removing the meta and stops the cluster's bootstrap subscriber, bootstrap refuses to write into a record that is gone or `DELETING`, the launch re-checks its record around seeding the creator's access entry, and a fresh claim clears whatever an earlier cluster left under the name. | `TestS2_EKSLaunchOvertakenByDeleteLeavesNoChildRecords`, `TestS2_EKSBootstrapWritesNothingIntoADeletingCluster`, `TestS2_EKSFreshClaimInheritsNoStaleChildRecords` |
|
||||
| Launch generation | The create launch's writes are fenced on the record's generation (`CreatedAt` and ARN). An overtaken launch never writes a newer cluster of the same name and releases only the instances it started, by ID. One whose record was swept recreates it as `DELETING` with its refs, so the teardown and reaper own the release. One whose record was marked `FAILED` merges its refs and releases what it built. | `TestS2_EKSOvertakenLaunchNeverWritesANewerClusterOfTheSameName`, `TestS2_EKSOvertakenLaunchHandsItsInfraToTheTeardown`, `TestS2_EKSLaunchMarkedFailedReleasesWhatItBuilt` |
|
||||
| Running launch | A create launch heartbeats its owner and time on the record (`LaunchOwner`, `LaunchHeartbeat`). The teardown keeps the record while that heartbeat is fresh, so the cluster reads `DELETING` throughout and never `ResourceNotFoundException` then `DELETING` again; the launch hands its refs over at its next write and starts the teardown when it returns. A heartbeat older than 90 s is taken as a crashed launch and no longer waited for, and recreating a swept record stays as the backstop. The launch's OIDC keypair writes are fenced on the record's generation. | `TestS2_EKSDeletedClusterNeverReadsMissingWhileItsLaunchRuns`, `TestS2_EKSCrashedLaunchDoesNotPinTheDelete`, `TestS2_EKSOvertakenLaunchCannotReplaceANewerClustersSigningKey` |
|
||||
| Bounded passes | One converger run is bounded at 15 minutes, so a hung step cannot hold the gate and lease; the timed-out pass is recorded and backed off. Waiting on node groups is not counted as a failed pass, and deleting the last node group resumes the delete. | `TestS8_EKSHungTeardownStepReleasesTheGate`, `TestS2_EKSLastNodegroupDeletedResumesTheClusterDelete` |
|
||||
| FAILED reclaim | Re-creating a `FAILED` cluster's name tears the old attempt down under the same gate and lease as a delete, and answers `ResourceInUseException` while another holder has it; the per-node teardown bound does not apply to it. | `TestS2_EKSFailedClusterReclaimRunsUnderTheTeardownLease`, `TestS2_EKSFailedClusterReclaimIsNotRefusedForCapacity` |
|
||||
| Tenant data | The teardown never deletes a CSI-provisioned volume or terminates a node group worker. | `TestS7_EKSTeardownNeverDeletesCSIVolumesOrWorkers`, `TestS7_EKSTeardownNeverRemovesAClusterWithNodegroupWorkers` |
|
||||
|
||||
## Gaps
|
||||
|
||||
1. Four `eks.*` handlers still hold their request for work that grows with the resource, and the subject table names them as not conformant: `CreateCluster` (reclaiming a `FAILED` cluster of the same name tears its infra down inline), `DeleteNodegroup` (one awaited terminate per worker), `UpdateNodegroupConfig` (one awaited launch or terminate per worker added or removed) and `RestoreSnapshot` (launches a control-plane VM and fences the old one inline).
|
||||
2. AWS's answer to a repeat delete on a `DELETING` cluster has not been observed; `DELETING` from the record is the chosen behaviour.
|
||||
3. Control-plane, load balancer and VPC resources in the system account are keyed by the bare cluster name, so a teardown by name can reach another tenant's same-named cluster.
|
||||
4. The teardown deletes load balancers the in-cluster controller created in the customer account, which AWS leaves behind.
|
||||
5. An overtaken launch's late write can still land under a newer cluster of the same name in the instant between that cluster's claim sweep and the old launch's generation check; it is bounded by one write, since the old launch stops at its next fenced write.
|
||||
6. The `FAILED`-name reclaim purge still runs inside the `CreateCluster` request (gap 1).
|
||||
@@ -99,6 +99,7 @@ behavior in other words.
|
||||
| 4432 | Formation server | HTTPS | Cluster (bootstrap only) | Cluster join coordination; active only while a join token is valid. Binds the node's `--bind` (lan) address. See *Formation port lifecycle* below. | Short-lived bearer token + TLS¹ |
|
||||
| 4222 | spinifex-nats (client) | NATS + TLS | Cluster | Internal service bus for EC2/EBS/VPC/S3 handlers | Token + mutual TLS (cluster CA) |
|
||||
| 4248 | spinifex-nats (cluster) | NATS + TLS | Cluster | Inter-node NATS federation | Token + mutual TLS (cluster CA) |
|
||||
| 5001 | iperf2 (admin network test) | TCP | Cluster, any plane | Throughput test between two nodes, started only by the super-admin account. Listens on one plane address for the length of one test, then exits. Runs with no capabilities. | None — peer-scoped by firewall, and the client may only target its own plane subnet |
|
||||
| 5300 | northstar | DNS (UDP + TCP) | Cluster | Forward target for every node's per-instance DNS shim, dialled cross-node. Binds the wildcard by design. | None — restrict by firewall |
|
||||
| 6660 | predastore (blob node) | QUIC / UDP | Cluster | Erasure-coded object shard transport between hosts. Multi-node clusters only — see *Predastore ports* below. | TLS 1.3, server certificate verified against the cluster CA |
|
||||
| 7660 | predastore (meta node) | QUIC / UDP | Cluster | Raft consensus over global state — buckets and the object index — between hosts. Multi-node clusters only. | TLS 1.3, server certificate verified against the cluster CA |
|
||||
@@ -221,7 +222,7 @@ table inet spinifex_filter {
|
||||
|
||||
# Cluster, from peer nodes only. Replace with your nodes' lan-plane
|
||||
# addresses — a CIDR does not generalise to nodes on different subnets.
|
||||
ip saddr { 10.0.1.1, 10.0.1.2, 10.0.1.3 } tcp dport { 4222, 4248, 4432, 5300, 6641, 6642, 6643, 6644, 8660 } accept
|
||||
ip saddr { 10.0.1.1, 10.0.1.2, 10.0.1.3 } tcp dport { 4222, 4248, 4432, 5001, 5300, 6641, 6642, 6643, 6644, 8660 } accept
|
||||
ip saddr { 10.0.1.1, 10.0.1.2, 10.0.1.3 } udp dport { 5300, 6660, 7660 } accept
|
||||
|
||||
# Encap, from peer chassis only. Replace with your nodes' vpc-plane
|
||||
|
||||
@@ -129,7 +129,7 @@ The client reports on its serial console because the runner has no route into th
|
||||
|
||||
### `eks`, `ecs`, `ecr` — container services
|
||||
|
||||
`eks`: cluster create/delete, access entries, addon delivery, `GetToken`, kubeconfig artifacts, the EBS CSI volume path, an IRSA web-identity exchange under the stock AWS trust policy (`StringEquals` on the issuer host-path `:sub`/`:aud`, a mismatched `:sub` denied with `AccessDenied`, a `StringLike` `:sub` granted), and an IRSA pod wired with a projected SA token. Worker IMDS is covered too: a managed nodegroup worker reports IMDSv2 required with hop limit 2, a pod with no IRSA resolves to the node role through IMDS, and a nodegroup built from a launch template takes the template's metadata options. Control-plane isolation is covered as well: no control-plane host registers as a Node, a DaemonSet tolerating every taint runs on workers only, metrics-server is Ready, and logs work for every kube-system pod. `ecs`: clusters, container instances, the three network modes (`awsvpc`, `bridge`, `host`), task-role credentials fetched from the container credentials endpoint, a service behind an ELB, and ENI release on terminate. `ecr`: control-plane CRUD plus a real OCI data plane — docker login/build/push/pull round trips, multi-layer images, and raw blob HTTP.
|
||||
`eks`: cluster create/delete, access entries, addon delivery, `GetToken`, kubeconfig artifacts, the EBS CSI volume path, an IRSA web-identity exchange under the stock AWS trust policy (`StringEquals` on the issuer host-path `:sub`/`:aud`, a mismatched `:sub` denied with `AccessDenied`, a `StringLike` `:sub` granted), and an IRSA pod whose spec names only a ServiceAccount with the role-arn annotation, where the pod-identity webhook injects the token volume, `AWS_*` environment, STS endpoint and CA bundle and the pod assumes the role, while a pod of an unannotated ServiceAccount is left untouched. The unmodified aws-load-balancer-controller chart 1.11.0 installs with the values an AWS user passes plus the gateway endpoint override, the Shield and WAF switches and the single-subnet feature gate, and reconciles an Ingress into an active ALB. Worker IMDS is covered too: a managed nodegroup worker reports IMDSv2 required with hop limit 2, a pod with no IRSA resolves to the node role through IMDS, and a nodegroup built from a launch template takes the template's metadata options. Control-plane isolation is covered as well: no control-plane host registers as a Node, a DaemonSet tolerating every taint runs on workers only, metrics-server is Ready, and logs work for every kube-system pod. `ecs`: clusters, container instances, the three network modes (`awsvpc`, `bridge`, `host`), task-role credentials fetched from the container credentials endpoint, a service behind an ELB, and ENI release on terminate. `ecr`: control-plane CRUD plus a real OCI data plane — docker login/build/push/pull round trips, multi-layer images, and raw blob HTTP.
|
||||
|
||||
### `rds` — managed databases
|
||||
|
||||
|
||||
+79
-22
@@ -24,7 +24,7 @@ All colours are OKLCH and come from mulgadc.com, so product and site match.
|
||||
| Token | Use |
|
||||
|---|---|
|
||||
| `background` | The app page. Warm off-white in light, ink `#14181D` in dark. |
|
||||
| `card`, `popover` | Raised surfaces. White in light, about `#1B2027` in dark. |
|
||||
| `card`, `popover` | Raised surfaces. White in light, about `#1B2027` in dark. `popover` is every floating surface: tooltips, popovers, menus and pickers. |
|
||||
| `muted`, `secondary`, `accent` | Hover rows, secondary buttons, quiet fills. |
|
||||
| `border` | Every divider and card edge. `#DCD8CE` in light, `oklch(1 0 0 / 8%)` in dark. |
|
||||
| `brand` | Brand orange `#EE4B12`, for the logo and decoration only. It fails text contrast, so it never carries text. |
|
||||
@@ -39,15 +39,22 @@ Each status has a solid token plus `-foreground` (text on a tinted background),
|
||||
|
||||
| Status | Hue | AWS states |
|
||||
|---|---|---|
|
||||
| `success` | green | running, available, completed, healthy |
|
||||
| `warning` | brand yellow `#F5B500` | pending, stopping, creating, modifying, deleting, draining |
|
||||
| `danger` | cool red, hue 17 | stopped, error, failed, unhealthy, unavailable |
|
||||
| `success` | green | settled and good: running, available, active, associated, attached, in-use, healthy, ok, main |
|
||||
| `warning` | brand yellow `#F5B500` | changing state and will settle: pending, stopping, creating, modifying, deleting, draining |
|
||||
| `caution` | orange, hue 58 | works but needs attention: degraded, impaired, insufficient-data |
|
||||
| `danger` | cool red, hue 17 | stopped, error, failed, update-failed, unhealthy, unavailable, blackhole |
|
||||
| `info` | blue | informational notes |
|
||||
| `neutral` | grey `#9AA1A8` | terminated (muted text), and any unmapped state |
|
||||
| `neutral` | grey `#9AA1A8` | gone or absent (terminated, deleted, no) in muted text, and any unmapped state |
|
||||
|
||||
`stopped` is danger because EC2 reads it that way, although RDS treats it as a resting state; splitting the map by service is its own change.
|
||||
|
||||
`StateBadge` (`src/components/state-badge.tsx`) is the only mapping from an AWS state to a tone, and it exposes the result as `data-tone` for tests.
|
||||
Its tones are `success`, `progress` (the yellow `warning` token), `caution`, `danger`, `muted` and `neutral`.
|
||||
A `progress` state carries the submit button's spinning loader, so a launching instance reads as work under way; `StatusBadge` does not, because its yellow can mean waiting on the user, as a zone's `pending nameservers` does.
|
||||
It shows every state lowercase with hyphens, as EC2 writes them, so EKS's `ACTIVE` reads `active` and Bedrock's `IN_PROGRESS` reads `in-progress`; the API value, filters and CLI panels keep AWS's case.
|
||||
A good or settled state is always green, never grey, and a status that is not an API state, such as a cluster's health, uses `StatusBadge` with a tone rather than a `Badge`.
|
||||
The orange of `caution` is kept clear of the primary accent's hue, which marks actions and never a status.
|
||||
`Badge`'s default variant is the brand primary and is never used for a status; a label that is not a status, such as `default v3` or `Leader`, uses a `secondary` or `outline` `Badge`.
|
||||
Every status badge measures above 4.8:1 against its tinted background in both themes.
|
||||
|
||||
**Danger is separated from the primary accent by hue alone in dark mode**: the two have the same lightness.
|
||||
@@ -67,18 +74,21 @@ Colour is never the only key: label every series.
|
||||
| UI text, tables and forms | Geist (`font-sans`) |
|
||||
| IDs, ARNs, IPs, CIDRs, code and eyebrow labels | JetBrains Mono (`font-mono`), with `tabular-nums` for figures |
|
||||
|
||||
| Step | Size | Use |
|
||||
|---|---|---|
|
||||
| `2xl` | 26px | page title (`PageHeading`) |
|
||||
| `xl` | 22px | section heading |
|
||||
| `lg` | 17px | |
|
||||
| `base` | 15px | |
|
||||
| `sm` | 14px | body and table default |
|
||||
| `xs` | 13px | badges, secondary text |
|
||||
| `2xs` | 12px | mono eyebrow labels |
|
||||
| Step | rem | At the default 16px browser font | Use |
|
||||
|---|---|---|---|
|
||||
| `2xl` | 1.625 | 28.6px | page title (`PageHeading`) |
|
||||
| `xl` | 1.375 | 24.2px | section heading |
|
||||
| `lg` | 1.0625 | 18.7px | |
|
||||
| `base` | 0.9375 | 16.5px | |
|
||||
| `sm` | 0.875 | 15.4px | body and table default |
|
||||
| `xs` | 0.8125 | 14.3px | badges, secondary text |
|
||||
| `2xs` | 0.75 | 13.2px | mono eyebrow labels |
|
||||
|
||||
The scale sits a step above Tailwind's defaults, as consoles such as Vultr's do, because 13px body text read as cramped on a wide screen.
|
||||
Code blocks are set at 12px (`--text-code-block`) so a long command still fits.
|
||||
The root font size is 110% (`html` in `src/styles.css`), so one rem is 17.6px at the browser default.
|
||||
Everything is sized in rem, so text, spacing and breakpoints scale together, as they would at 110% browser zoom, and a user's own font-size setting still applies.
|
||||
The scale sits above Tailwind's defaults, as consoles such as Vultr's do, because the console read as small on a wide screen at 100%.
|
||||
Code blocks are set at 0.75rem (`--text-code-block`) so a long command still fits.
|
||||
Size new things in rem through the type and spacing scales, never in px, or they will not scale with the rest.
|
||||
|
||||
### Primary and contrast
|
||||
|
||||
@@ -116,7 +126,7 @@ The Go console serves `script-src 'self'; style-src 'self'`.
|
||||
## Primitives
|
||||
|
||||
`src/components/ui/` holds the shadcn primitives on Base UI.
|
||||
Added in Phase 1: breadcrumb, checkbox, command (`cmdk`), dialog, hover-card, popover, resizable (`react-resizable-panels`), scroll-area, slider, switch, table, toggle and toggle-group.
|
||||
Added in Phase 1: breadcrumb, checkbox, command (Base UI Autocomplete), dialog, hover-card, popover, resizable (`react-resizable-panels`), scroll-area, slider, switch, table, toggle and toggle-group.
|
||||
`button-group` joins buttons edge to edge, as the instance power buttons are.
|
||||
`radio-group` is one of a few exclusive choices inside a form, such as a load balancer's type; a raw `<input type="radio">` is not used.
|
||||
|
||||
@@ -169,6 +179,22 @@ Every other list is a `DataTable`, and the test also fails when an allowed file
|
||||
| S13 | a component outside `ui/` renders a raw `<input>`, `<select>` or `<textarea>` |
|
||||
| S15 | a route imports the `ui/alert` primitive instead of using `ErrorBanner` |
|
||||
|
||||
Rules that ADR-0016 does not yet number are named by what they hold until their amendment is accepted:
|
||||
|
||||
| Rule | Fails when |
|
||||
|---|---|
|
||||
| status pills come from StateBadge or StatusBadge | a component outside `ui/` renders `<Badge>` with the default variant |
|
||||
| status pills come from StateBadge or StatusBadge | a file outside `state-badge.tsx` builds a `rounded-full` pill on a status colour |
|
||||
| floating surfaces follow the theme | a file fills with `bg-foreground` or `text-background`, or a tooltip, popover, hover card, menu, select or combobox is not on `bg-popover` and `text-popover-foreground` |
|
||||
| expanding a panel keeps its tab | a `fullPage` link does not pass `search={{ tab }}` |
|
||||
| resource tags are edited in place | `PanelTags` or `TagsEditor` comes back |
|
||||
| lists filter through one FilterBar | a component other than `FilterBar` imports `ui/toggle-group` |
|
||||
| a component handed a form reads errors through `useFormState` | a file typed with `UseFormReturn` reads `form.formState` |
|
||||
|
||||
The last one matters because the console builds with the React compiler and the unit tests run without it.
|
||||
With the compiler on, a child handed a react-hook-form object is cached on that object, whose identity never changes, so errors read from `form.formState` never draw and the submit looks dead.
|
||||
`useFormState({ control })` subscribes the child itself.
|
||||
|
||||
An exception is named in that rule's allowlist with its reason, and the test fails when an allowed file stops needing it.
|
||||
S11, S14 and S16 are held by review, since none has a shape in the source a test can read without false alarms.
|
||||
|
||||
@@ -177,7 +203,7 @@ S11, S14 and S16 are held by review, since none has a shape in the source a test
|
||||
`src/lib/nav.ts` is the one description of the console's navigation.
|
||||
The sidebar, the command palette and the breadcrumbs all read it, and `nav.test.ts` checks every destination against the generated route tree, so a renamed route fails a test rather than leaving a dead link.
|
||||
|
||||
- **Sidebar.** Groups take generic names rather than AWS's service marks: Compute, Databases, Authentication and Object Storage, not EC2, RDS, IAM and S3. Pages and the rest of the console keep the AWS names, and each group's `service` keeps its AWS name findable in the command palette, where typing it exactly ranks that group first. Each service group folds, and a folded group stays folded across reloads (`localStorage["spinifex-ui-nav-collapsed"]`). Navigating into a folded group unfolds it so the active item shows, and it can then be folded again, the current group included. On a phone the sidebar is a sheet, and any navigation closes it.
|
||||
- **Sidebar.** Groups take generic names rather than AWS's service marks: Compute, Databases, Authentication and Object Storage, not EC2, RDS, IAM and S3. Pages and the rest of the console keep the AWS names, and each group's `service` keeps its AWS name findable in the command palette, where typing it exactly ranks that group first. Each service group folds, and a folded group stays folded across reloads (`localStorage["spinifex-ui-nav-collapsed"]`). Navigating into a folded group unfolds it so the active item shows, and it can then be folded again, the current group included. Folding animates through the `nav` variant of `CollapsibleContent`, opening over 200ms and closing over 150ms, and switches at once under reduced motion. On a phone the sidebar is a sheet, and any navigation closes it.
|
||||
- **Command palette.** ⌘K or Ctrl K, or the Search button in the header. It lists every page the user can reach, every create flow (`create` on a nav item), and a theme toggle. It is navigation only, with no resource search yet.
|
||||
- **Breadcrumbs.** Service, then list page, then the resource or the create flow. A resource ID is set in mono. On a phone only the last crumb shows.
|
||||
- **Sign in.** The sign-in card sits over the shell on a light, barely blurred overlay rather than on a page of its own, so the console stays in view. The shell behind it is `inert`, which keeps pointer, keyboard and screen reader out of it until the user signs in.
|
||||
@@ -202,8 +228,13 @@ So columns drop out when a side panel narrows the list as well as on a small scr
|
||||
List pages keep their filter, state and sort in the URL, so a filtered view can be bookmarked, shared and survives a reload.
|
||||
EC2 Instances is the reference page (`ec2/(instances)/-components/instances-page.tsx`):
|
||||
|
||||
- The first column names the resource: the `Name` tag in the primary colour, with the ID in mono beneath it, opening its side panel.
|
||||
- The first column is the instance ID in mono, opening its side panel, and the second its `Name` tag.
|
||||
- A long value ends in an ellipsis and shows in full on hover (`TruncatedText`), so it never widens its column.
|
||||
- A time column shows a relative time, `33 min ago` or `8 months ago`, with the full timestamp on hover (`RelativeTime`).
|
||||
- A Tags column shows one tag key as a pill and `+N` for the rest, and hover lists every tag with its value (`TagPills`); it leaves out `Name`, which has its own column.
|
||||
- Below `md`, only the ID, Name and State columns remain.
|
||||
- Bulk actions appear above the table once a row is selected. Each action counts only the selected instances it applies to, so a mixed selection never sends a call that must fail, and Terminate asks first and lists every ID.
|
||||
- Search and state filters sit in one `FilterBar` (`src/components/filter-bar.tsx`) above the table: a search box, an optional set of choices with counts, and Clear filters while anything is filtered.
|
||||
- An empty account gets a call to action, and an empty filter result says the filters are why.
|
||||
|
||||
VPCs and Internet gateways follow the same pattern.
|
||||
@@ -219,6 +250,12 @@ Every list with a delete action has row checkboxes. Selecting a row shows `Selec
|
||||
- **The bulk delete and the panel's delete are the same function.** Where a single delete is several calls, such as removing dependants first, the sequence lives in one place and both use it.
|
||||
- The verb follows the AWS console: Release for an Elastic IP, Deregister for an image.
|
||||
|
||||
### Lists inside a panel
|
||||
|
||||
A list inside a panel tab, such as a hosted zone's records, is a `DataTable` with row checkboxes, and a click on a row opens it for editing in a `FormPanel`.
|
||||
Rows the user cannot change, such as a zone's own SOA and NS, are muted and cannot be selected (`isRowActionable`).
|
||||
A batch delete uses the service's batch call where it has one, so Route 53 removes the whole selection in one `ChangeResourceRecordSets`.
|
||||
|
||||
### Typed confirmation
|
||||
|
||||
A delete that cannot be walked back asks the user to type a word first, as the AWS console does before deleting a volume.
|
||||
@@ -264,13 +301,24 @@ Modelled on UniFi's console: a click on a row opens the resource in a panel besi
|
||||
- **URL.** The panel is search state on the list route: `selected=<id>`, `tab=overview|monitoring|settings`, and `create=true`. Defaults are stripped, so a closed panel leaves the plain list address. Opening or closing a panel pushes a history entry, so Back closes it; a tab or filter change replaces the entry.
|
||||
- **Layout.** From `lg` (1024px) the panel is fixed to the right edge, full height below the header, flush with no outer margin, and it slides in over the list. The list keeps its width and columns underneath, so nothing reflows when a panel opens, and a soft gradient on the panel's left edge reads as the panel lying on top. Below `lg` there is no room to share, so the panel replaces the list as a full page with a back arrow. `useWideLayout` decides, through `matchMedia`.
|
||||
- **`DetailPanel`.** A header with the resource's own page (left) and close (right), the title and its ID, a summary of state and quick actions, then icon tabs with tooltips. Escape closes it unless a dialog or menu is open over it. Closing slides the panel out over 150ms, or at once under reduced motion. Each tab, and a create form, sits in its own error boundary, so a failed query shows in the panel and the list keeps working.
|
||||
- **Contents.** `PanelSection` is a folding card, `PanelFields` and `PanelField` a label and value list, with `copy` for a one-click copy of IDs and addresses, and `PanelTags` the tag list. Create forms take an `onClose(id?)` prop: embedded, they hide their back link and heading, and a create selects the new resource.
|
||||
- **Contents.** `PanelSection` is a folding card, `PanelFields` and `PanelField` a label and value list, with `copy` for a one-click copy of IDs and addresses, and `InlineTags` the tag list. Create forms take an `onClose(id?)` prop: embedded, they hide their back link and heading, and a create selects the new resource.
|
||||
- **One component is both the panel and the page.** Each resource has a single details component (`InstanceDetails`, `VpcDetails`, `InternetGatewayDetails`) that takes `layout="auto" | "page"`. The list renders it as the panel; `/ec2/describe-instances/$id` and its kind render the same component as a page, as one centred column under a standard left-aligned heading, with state and actions on the right and text tabs. A feature added to the panel is on the page by construction, and the page ignores Escape and has a back arrow in place of close. Expanding the panel to its page keeps the tab, and query results such as fetched console output come along from the cache.
|
||||
- **A click on the list's background closes the panel**, as UniFi's does. A row selects another resource instead, and links, buttons and inputs keep their own behaviour. Selecting rows for a bulk action closes it too, so the bulk bar and the panel never compete.
|
||||
- **Actions.** State actions sit in the summary as a button group, and the rest go in an Actions menu, Unikraft style. Stop, Reboot and every destructive action confirm in a dialog.
|
||||
- **Loading.** A panel or page still loading shows `PanelSpinner`, a centred spinner on the background, not a grey skeleton block.
|
||||
- **`DetailCard`, `DetailRow` and `ListCard` are retired.** Every resource page uses the panel sections instead.
|
||||
|
||||
### Tags
|
||||
|
||||
`InlineTags` (`src/components/inline-tags.tsx`) shows a resource's tags in its panel and edits them in place.
|
||||
Each row has a pencil, which turns the key and value into inputs and becomes a tick to save, and a bin to remove the tag; `+ Add tag` appends a row being edited.
|
||||
Enter saves and Escape cancels the row without closing the panel.
|
||||
Every row saves on its own request through the service's own tagging call, and a failure shows beside its row.
|
||||
AWS has no rename, so changing a key puts the new key and then removes the old one, and a failure between the two leaves both rather than neither.
|
||||
Keys are 1 to 128 characters with no `aws:` prefix and no duplicates, and values up to 256.
|
||||
Without `onPut` and `onRemove` it is read-only, as for a system image or an AWS-managed policy.
|
||||
Panels use the per-service wrappers in `src/components/resource-tags.tsx` and the writers in `src/mutations/tags.ts` rather than wiring the calls themselves.
|
||||
|
||||
### Forms from a resource's panel
|
||||
|
||||
A form that acts on an open resource, such as attaching a volume, scaling a node group or adding a listener, opens in `FormPanel` (`src/components/form-panel.tsx`), never in a dialog.
|
||||
@@ -285,8 +333,16 @@ Every create flow opens in the side panel over its list, through `CreatePanel`,
|
||||
|
||||
- **URL.** `create=true` opens it, plus a list-local field where the flow needs context (`volume` for a snapshot, `instance` for an RDS snapshot, `launch=service|task` on ECS clusters). The old `create-*` addresses redirect to the list with the panel open, so links and bookmarks keep working.
|
||||
- **Form.** The fields come in the order AWS asks for them. Settings that are on or off are `Switch`es, and a multi-select picker (subnets, security groups) is a `SwitchList`, a boxed list with one switch per row. Checkboxes only select table rows.
|
||||
- **CLI.** Every create form ends with a `CliCommandPanel` showing the AWS CLI commands that send the same request, built from the form's live values and only the flags the console actually sends.
|
||||
- **Submit.** `FormActions` turns the submit button into a spinner with its pending label in place, and `FormLock`, a native `<fieldset disabled>`, disables every control above it until the request settles. Nothing above the form appears or moves, and the panel does not scroll.
|
||||
- **CLI.** Every create form ends with a `CliCommandPanel` showing the AWS CLI commands that send the same request, built from the form's live values and only the flags the console actually sends. The commands wrap to the panel's width and never scroll sideways; Copy takes the unwrapped text.
|
||||
- **Sections.** Related fields are grouped in `PanelSection` boxes, as the instance launch form groups Image, Instance, Network and Root volume.
|
||||
- **Nothing to pick, no field.** An optional picker with nothing usable to choose, such as Launch Template or Placement Group with none available, is left out of the form rather than shown holding only `none`.
|
||||
- **Advanced settings.** Settings most users leave at their defaults sit in a collapsed Advanced settings section, which opens by itself when a field inside it fails validation.
|
||||
- **Choices with a mark.** A few named products, such as database engines or distributions, are `ChoiceTile`s with the product's own mark from `src/assets/`, not a select. One that cannot be used stays visible and disabled, saying why.
|
||||
- **Durations and windows.** A duration is a select of presets with the AWS default first and named (`Auto (5 min)` for a record's TTL), keeping a custom value the API returned. A time window, such as an RDS backup or maintenance window, is a day, start time and length picker that refuses an overlap before submitting.
|
||||
- **Labels by type.** Where one field means different things by type, its label follows the type, as a record's value reads `IPv4 address` for A and `Mail server` for MX.
|
||||
- **Long pickers.** A `SwitchList` that can run long, such as security groups, scrolls inside its box rather than pushing the form down.
|
||||
- **Help.** A field's longer explanation goes in an (i) beside its label, through the `info` prop of `TextField` and `SelectField`, rather than in text under the field. Text under a field is for what the user must know before submitting.
|
||||
- **Submit.** `FormActions` cross-fades the submit label into a spinner with its pending label over 150ms, at one width, so the button never changes size, and `FormLock`, a native `<fieldset disabled>`, disables every control above it until the request settles. Nothing above the form appears or moves, and the panel does not scroll.
|
||||
- **Failure.** A refused request shows beside the button through `FormActions`'s `error`, and the form unlocks. The handler wraps only the mutation call in `try`, with an empty `catch` that returns, because react-hook-form rethrows whatever the handler throws and the error is already on screen from the mutation's state.
|
||||
- **Success.** `onClose(id)` closes the panel and selects the new resource, where the list has a panel to select it in.
|
||||
- **Appliance images.** A create that needs an ECS, EKS or RDS appliance image shows the missing-image callout inside its panel, never on the list, and only the step that needs the image is blocked.
|
||||
@@ -306,6 +362,7 @@ A gap of more than three sample periods breaks the line rather than drawing acro
|
||||
## Dates
|
||||
|
||||
`formatDateTime` writes the AWS console's form in the viewer's zone, `2026/06/04 02:00 GMT-5`, with a 24-hour clock and UTC as plain `GMT`.
|
||||
In a table the time is relative (`relativeTime` in `src/lib/relative-time.ts`), rounded down to the largest whole unit, and the full form shows on hover.
|
||||
|
||||
## Testing
|
||||
|
||||
|
||||
@@ -97,16 +97,9 @@ The deploy script, `deploy-vultr.sh`, is in the Spinifex repository:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/mulgadc/spinifex.git
|
||||
cd spinifex
|
||||
git switch dev
|
||||
cd scripts/terraform/vultr-spx
|
||||
cd spinifex/scripts/terraform/vultr-spx
|
||||
```
|
||||
|
||||
> [!NOTE]
|
||||
> Vultr support is on the development branch and not yet in a published release.
|
||||
> `git switch dev` gives you the deploy script for it, and `--channel dev` in [step 3](#3-deploy) installs the matching Spinifex build, the pre-release published from every change to `dev`.
|
||||
> Once Vultr support is in a release, skip `git switch dev` and pass `--channel latest`.
|
||||
|
||||
## 2. Choose a deployment
|
||||
|
||||
| Use | Nodes | Plan | Cost while running |
|
||||
@@ -143,14 +136,14 @@ Set your API key, then run the command for the deployment you chose:
|
||||
export VULTR_API_KEY=<your key>
|
||||
|
||||
# Development and testing: one VM
|
||||
./deploy-vultr.sh up --channel dev --plan vx1-g-16c-64g
|
||||
./deploy-vultr.sh up --plan vx1-g-16c-64g
|
||||
|
||||
# Production: three VMs
|
||||
./deploy-vultr.sh up --channel dev --plan vx1-g-16c-64g --nodes 3
|
||||
./deploy-vultr.sh up --plan vx1-g-16c-64g --nodes 3
|
||||
|
||||
# Production on bare metal: one server, or three
|
||||
./deploy-vultr.sh up --channel dev --plan vbm-8c-64gb-amd
|
||||
./deploy-vultr.sh up --channel dev --plan vbm-8c-64gb-amd --nodes 3
|
||||
./deploy-vultr.sh up --plan vbm-8c-64gb-amd
|
||||
./deploy-vultr.sh up --plan vbm-8c-64gb-amd --nodes 3
|
||||
```
|
||||
|
||||
One VM takes about 4 minutes and three VMs about 16 minutes.
|
||||
|
||||
@@ -19,5 +19,6 @@ var modelFiles = map[Service]string{
|
||||
RDS: "models/rds-2014-10-31.json.gz",
|
||||
Route53: "models/route-53-2013-04-01.json.gz",
|
||||
S3: "models/s3-2006-03-01.json.gz",
|
||||
SQS: "models/sqs-2012-11-05.json.gz",
|
||||
STS: "models/sts-2011-06-15.json.gz",
|
||||
}
|
||||
|
||||
@@ -36,6 +36,7 @@ const (
|
||||
RDS Service = "rds"
|
||||
Route53 Service = "route53"
|
||||
S3 Service = "s3"
|
||||
SQS Service = "sqs"
|
||||
STS Service = "sts"
|
||||
)
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
)
|
||||
|
||||
func TestServices(t *testing.T) {
|
||||
want := []Service{ACM, EC2, ECR, ECS, EKS, ElasticLoadBalancingV2, IAM, RDS, Route53, S3, STS}
|
||||
want := []Service{ACM, EC2, ECR, ECS, EKS, ElasticLoadBalancingV2, IAM, RDS, Route53, S3, SQS, STS}
|
||||
if got := Services(); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("Services() = %v, want %v", got, want)
|
||||
}
|
||||
|
||||
Binary file not shown.
@@ -1,6 +1,6 @@
|
||||
// Package sqsworkergw is the shared SQS worker guest's SigV4 client for the
|
||||
// gateway's internal spinifex-sqs service. It sends only public values: the
|
||||
// worker's identity is the signing session, and the store seed never leaves
|
||||
// worker's identity is the signing session, and the receipt key never leaves
|
||||
// the guest.
|
||||
package sqsworkergw
|
||||
|
||||
@@ -10,20 +10,21 @@ import (
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/bluebottle/pkg/tlsconfig"
|
||||
"github.com/nats-io/nkeys"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/relay"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -33,9 +34,12 @@ const (
|
||||
callTimeout = 30 * time.Second
|
||||
maxResponse = 64 << 10
|
||||
|
||||
actionClaim = "ClaimFirstStore"
|
||||
actionDescribe = "DescribeStore"
|
||||
actionRecord = "RecordWorkerPublicKey"
|
||||
actionClaim = "ClaimReceiptKey"
|
||||
actionRecord = "RecordReceiptKey"
|
||||
actionPoll = "PollWork"
|
||||
actionComplete = "CompleteWork"
|
||||
// maxPollResponse fits the gateway's 16 MiB poll batch as base64 XML.
|
||||
maxPollResponse = 32 << 20
|
||||
)
|
||||
|
||||
// Signer signs a request in place for service and region.
|
||||
@@ -43,7 +47,7 @@ type Signer interface {
|
||||
Sign(r *http.Request, payloadHash, service, region string) error
|
||||
}
|
||||
|
||||
// Client calls the internal first-store actions for this worker.
|
||||
// Client calls the internal SQS worker actions for this worker.
|
||||
type Client struct {
|
||||
baseURL string
|
||||
region string
|
||||
@@ -114,80 +118,72 @@ func (e *TransientError) Error() string { return e.Err.Error() }
|
||||
func (e *TransientError) Unwrap() error { return e.Err }
|
||||
func (e *TransientError) Retryable() bool { return true }
|
||||
|
||||
type claimResult struct {
|
||||
CreateKey bool `xml:"CreateKey"`
|
||||
PublicKey string `xml:"PublicKey"`
|
||||
// ReceiptKey is the worker volume's receipt key initialization as the
|
||||
// gateway reports it: pending authorizes creating or resuming one, otherwise
|
||||
// Fingerprint names the key the disk must already hold.
|
||||
type ReceiptKey struct {
|
||||
Pending bool
|
||||
Fingerprint string
|
||||
// HostNodes is the gateway's declared cluster node count; a gateway
|
||||
// that does not report one reads as zero.
|
||||
HostNodes int
|
||||
}
|
||||
|
||||
// Claim asks for this worker's current first-store decision. A decision
|
||||
// that is neither "create" nor one valid public key is refused here.
|
||||
func (c *Client) Claim(ctx context.Context) (bool, string, error) {
|
||||
type claimResult struct {
|
||||
Pending bool `xml:"Pending"`
|
||||
Fingerprint string `xml:"Fingerprint"`
|
||||
HostNodes int `xml:"HostNodes"`
|
||||
}
|
||||
|
||||
// ClaimReceiptKey asks for the receipt key decision. A reply that is neither
|
||||
// pending nor one valid fingerprint is refused here.
|
||||
func (c *Client) ClaimReceiptKey(ctx context.Context) (ReceiptKey, error) {
|
||||
var res claimResult
|
||||
if err := c.call(ctx, actionClaim, nil, &res); err != nil {
|
||||
return false, "", err
|
||||
return ReceiptKey{}, err
|
||||
}
|
||||
switch {
|
||||
case res.CreateKey && res.PublicKey == "":
|
||||
return true, "", nil
|
||||
case !res.CreateKey && nkeys.IsValidPublicUserKey(res.PublicKey):
|
||||
return false, res.PublicKey, nil
|
||||
if res.Pending == (res.Fingerprint != "") || (res.Fingerprint != "" && !ValidFingerprint(res.Fingerprint)) {
|
||||
return ReceiptKey{}, errors.New("spinifex-sqs ClaimReceiptKey: response is not a valid decision")
|
||||
}
|
||||
return false, "", errors.New("spinifex-sqs ClaimFirstStore: response is not a valid decision")
|
||||
return ReceiptKey(res), nil
|
||||
}
|
||||
|
||||
// Store is the Region's store launch as the gateway reports it. Bound means
|
||||
// a VM was launched for it, not that it is ready.
|
||||
type Store struct {
|
||||
Bound bool
|
||||
Address string
|
||||
PublicKey string
|
||||
}
|
||||
|
||||
type describeResult struct {
|
||||
State string `xml:"State"`
|
||||
Address string `xml:"Address"`
|
||||
PublicKey string `xml:"PublicKey"`
|
||||
}
|
||||
|
||||
// DescribeStore reports this worker's store. A bound reply must carry an
|
||||
// IPv4 address and a valid public key; a pending one must carry neither.
|
||||
func (c *Client) DescribeStore(ctx context.Context) (Store, error) {
|
||||
var res describeResult
|
||||
if err := c.call(ctx, actionDescribe, nil, &res); err != nil {
|
||||
return Store{}, err
|
||||
// RecordReceiptKey records the key's fingerprint. Recording the same one
|
||||
// again succeeds, so a lost reply is safe to repeat.
|
||||
func (c *Client) RecordReceiptKey(ctx context.Context, fingerprint string) error {
|
||||
if !ValidFingerprint(fingerprint) {
|
||||
return errors.New("spinifex-sqs RecordReceiptKey: refusing to send a value that is not a fingerprint")
|
||||
}
|
||||
switch {
|
||||
case res.State == "pending" && res.Address == "" && res.PublicKey == "":
|
||||
return Store{}, nil
|
||||
case res.State == "bound" && isIPv4(res.Address) && nkeys.IsValidPublicUserKey(res.PublicKey):
|
||||
return Store{Bound: true, Address: res.Address, PublicKey: res.PublicKey}, nil
|
||||
}
|
||||
return Store{}, errors.New("spinifex-sqs DescribeStore: response is not a valid store state")
|
||||
return c.call(ctx, actionRecord, url.Values{"Fingerprint": {fingerprint}}, nil)
|
||||
}
|
||||
|
||||
func isIPv4(s string) bool {
|
||||
ip := net.ParseIP(s)
|
||||
return ip != nil && ip.To4() != nil && !strings.Contains(s, ":")
|
||||
}
|
||||
|
||||
// RecordPublicKey records the worker's public key. Anything other than a
|
||||
// user public key is refused before it is sent.
|
||||
func (c *Client) RecordPublicKey(ctx context.Context, publicKey string) error {
|
||||
if !nkeys.IsValidPublicUserKey(publicKey) {
|
||||
return errors.New("spinifex-sqs RecordWorkerPublicKey: refusing to send a value that is not a user public key")
|
||||
// ValidFingerprint reports a lowercase hex SHA-256 fingerprint.
|
||||
func ValidFingerprint(s string) bool {
|
||||
if len(s) != 2*sha256.Size || strings.ToLower(s) != s {
|
||||
return false
|
||||
}
|
||||
return c.call(ctx, actionRecord, url.Values{"PublicKey": {publicKey}}, nil)
|
||||
_, err := hex.DecodeString(s)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func (c *Client) call(ctx context.Context, action string, params url.Values, out any) error {
|
||||
form := url.Values{"Action": {action}, "Version": {apiVersion}}
|
||||
maps.Copy(form, params)
|
||||
body := []byte(form.Encode())
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+"/", bytes.NewReader(body))
|
||||
return c.do(ctx, action, "", []byte(form.Encode()), "application/x-www-form-urlencoded; charset=utf-8", maxResponse, out)
|
||||
}
|
||||
|
||||
// do signs and sends one request. rawQuery, when set, carries the action and
|
||||
// its arguments; the body hash is signed either way.
|
||||
func (c *Client) do(ctx context.Context, action, rawQuery string, body []byte, contentType string, limit int64, out any) error {
|
||||
target := c.baseURL + "/"
|
||||
if rawQuery != "" {
|
||||
target += "?" + rawQuery
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, target, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return fmt.Errorf("spinifex-sqs %s: build request: %w", action, err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=utf-8")
|
||||
req.Header.Set("Content-Type", contentType)
|
||||
sum := sha256.Sum256(body)
|
||||
if err := c.signer.Sign(req, hex.EncodeToString(sum[:]), signingService, c.region); err != nil {
|
||||
return &TransientError{Err: fmt.Errorf("spinifex-sqs %s: sign: %w", action, err)}
|
||||
@@ -197,7 +193,7 @@ func (c *Client) call(ctx context.Context, action string, params url.Values, out
|
||||
return &TransientError{Err: fmt.Errorf("spinifex-sqs %s: send: %w", action, err)}
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
respBody, err := io.ReadAll(io.LimitReader(resp.Body, maxResponse))
|
||||
respBody, err := io.ReadAll(io.LimitReader(resp.Body, limit))
|
||||
if err != nil {
|
||||
return &TransientError{Err: fmt.Errorf("spinifex-sqs %s: read response: %w", action, err)}
|
||||
}
|
||||
@@ -210,6 +206,44 @@ func (c *Client) call(ctx context.Context, action string, params url.Values, out
|
||||
return decodeResult(action, respBody, out)
|
||||
}
|
||||
|
||||
// PollWork long-polls the gateway for owner calls queued for this worker.
|
||||
// An empty list is a poll that found nothing.
|
||||
func (c *Client) PollWork(ctx context.Context) ([]relay.Work, error) {
|
||||
var res struct {
|
||||
Items []string `xml:"Items>member"`
|
||||
}
|
||||
if err := c.do(ctx, actionPoll, "", []byte(url.Values{"Action": {actionPoll}, "Version": {apiVersion}}.Encode()),
|
||||
"application/x-www-form-urlencoded; charset=utf-8", maxPollResponse, &res); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]relay.Work, 0, len(res.Items))
|
||||
for _, item := range res.Items {
|
||||
raw, err := base64.StdEncoding.DecodeString(item)
|
||||
if err != nil {
|
||||
return nil, errors.New("spinifex-sqs PollWork: item is not base64")
|
||||
}
|
||||
var w relay.Work
|
||||
if err := json.Unmarshal(raw, &w); err != nil || w.ID == "" || w.Op == "" {
|
||||
return nil, errors.New("spinifex-sqs PollWork: item is not a work item")
|
||||
}
|
||||
out = append(out, w)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// CompleteWork posts an encoded result for work item id and returns the
|
||||
// gateway's outcome name. Any error leaves delivery unknown.
|
||||
func (c *Client) CompleteWork(ctx context.Context, id string, result []byte) (string, error) {
|
||||
q := url.Values{"Action": {actionComplete}, "Version": {apiVersion}, "WorkId": {id}}
|
||||
var res struct {
|
||||
Outcome string `xml:"Outcome"`
|
||||
}
|
||||
if err := c.do(ctx, actionComplete, q.Encode(), result, "application/json", maxResponse, &res); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return res.Outcome, nil
|
||||
}
|
||||
|
||||
func decodeResult(action string, body []byte, out any) error {
|
||||
want := action + "Result"
|
||||
dec := xml.NewDecoder(bytes.NewReader(body))
|
||||
|
||||
@@ -10,8 +10,6 @@ import (
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/nats-io/nkeys"
|
||||
)
|
||||
|
||||
type nopSigner struct{ err error }
|
||||
@@ -24,15 +22,10 @@ func (s nopSigner) Sign(r *http.Request, _, service, region string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func userKey(t *testing.T) (pub, seed string) {
|
||||
t.Helper()
|
||||
kp, err := nkeys.CreateUser()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p, _ := kp.PublicKey()
|
||||
s, _ := kp.Seed()
|
||||
return p, string(s)
|
||||
var fp = strings.Repeat("ab", 32)
|
||||
|
||||
func claimBody(inner string) string {
|
||||
return `<ClaimReceiptKeyResponse><ClaimReceiptKeyResult>` + inner + `</ClaimReceiptKeyResult></ClaimReceiptKeyResponse>`
|
||||
}
|
||||
|
||||
// server answers with body and status, counting and keeping requests.
|
||||
@@ -73,63 +66,56 @@ func TestNew_RefusesWithoutHTTPSRegionCAOrSigner(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestClaim_SignsForTheInternalServiceAndRegion(t *testing.T) {
|
||||
c, _, seen := server(t, 200, `<ClaimFirstStoreResponse><ClaimFirstStoreResult><CreateKey>true</CreateKey></ClaimFirstStoreResult></ClaimFirstStoreResponse>`)
|
||||
create, pub, err := c.Claim(context.Background())
|
||||
if err != nil || !create || pub != "" {
|
||||
t.Fatalf("claim = %v %q %v", create, pub, err)
|
||||
c, _, seen := server(t, 200, claimBody(`<Pending>true</Pending><HostNodes>1</HostNodes>`))
|
||||
got, err := c.ClaimReceiptKey(context.Background())
|
||||
if err != nil || !got.Pending || got.Fingerprint != "" || got.HostNodes != 1 {
|
||||
t.Fatalf("claim = %+v %v", got, err)
|
||||
}
|
||||
if got := (*seen)[0]; !strings.HasPrefix(got, "test spinifex-sqs/ap-southeast-2|") || !strings.Contains(got, "Action=ClaimFirstStore") {
|
||||
t.Fatalf("request = %q", got)
|
||||
if req := (*seen)[0]; !strings.HasPrefix(req, "test spinifex-sqs/ap-southeast-2|") || !strings.Contains(req, "Action=ClaimReceiptKey") {
|
||||
t.Fatalf("request = %q", req)
|
||||
}
|
||||
}
|
||||
|
||||
// Only "create" or exactly one valid public key is a decision.
|
||||
// Only pending alone, or exactly one valid fingerprint, is a decision.
|
||||
func TestClaim_RefusesAnythingButOneDecision(t *testing.T) {
|
||||
pub, seed := userKey(t)
|
||||
cases := map[string]string{
|
||||
"create with a key": `<CreateKey>true</CreateKey><PublicKey>` + pub + `</PublicKey>`,
|
||||
"neither": `<CreateKey>false</CreateKey>`,
|
||||
"seed as key": `<CreateKey>false</CreateKey><PublicKey>` + seed + `</PublicKey>`,
|
||||
"no result": ``,
|
||||
}
|
||||
for name, inner := range cases {
|
||||
for name, inner := range map[string]string{
|
||||
"pending with a key": `<Pending>true</Pending><Fingerprint>` + fp + `</Fingerprint>`,
|
||||
"neither": `<Pending>false</Pending>`,
|
||||
"uppercase key": `<Pending>false</Pending><Fingerprint>` + strings.ToUpper(fp) + `</Fingerprint>`,
|
||||
"short key": `<Pending>false</Pending><Fingerprint>` + fp[:62] + `</Fingerprint>`,
|
||||
"not the claim reply": "",
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
body := `<ClaimFirstStoreResponse><ClaimFirstStoreResult>` + inner + `</ClaimFirstStoreResult></ClaimFirstStoreResponse>`
|
||||
body := claimBody(inner)
|
||||
if inner == "" {
|
||||
body = `<Other/>`
|
||||
}
|
||||
c, _, _ := server(t, 200, body)
|
||||
_, _, err := c.Claim(context.Background())
|
||||
if err == nil {
|
||||
if _, err := c.ClaimReceiptKey(context.Background()); err == nil {
|
||||
t.Fatal("accepted a non-decision")
|
||||
}
|
||||
if strings.Contains(err.Error(), seed) {
|
||||
t.Fatal("the error echoes a seed")
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("existing", func(t *testing.T) {
|
||||
c, _, _ := server(t, 200, `<ClaimFirstStoreResponse><ClaimFirstStoreResult><CreateKey>false</CreateKey><PublicKey>`+pub+`</PublicKey></ClaimFirstStoreResult></ClaimFirstStoreResponse>`)
|
||||
create, got, err := c.Claim(context.Background())
|
||||
if err != nil || create || got != pub {
|
||||
t.Fatalf("claim = %v %q %v", create, got, err)
|
||||
}
|
||||
})
|
||||
c, _, _ := server(t, 200, claimBody(`<Pending>false</Pending><Fingerprint>`+fp+`</Fingerprint>`))
|
||||
got, err := c.ClaimReceiptKey(context.Background())
|
||||
if err != nil || got.Pending || got.Fingerprint != fp {
|
||||
t.Fatalf("recorded claim = %+v %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecordPublicKey_SendsOnlyAPublicKey(t *testing.T) {
|
||||
pub, seed := userKey(t)
|
||||
c, n, seen := server(t, 200, `<RecordWorkerPublicKeyResponse/>`)
|
||||
if err := c.RecordPublicKey(context.Background(), seed); err == nil || strings.Contains(err.Error(), seed) {
|
||||
t.Fatalf("seed err = %v", err)
|
||||
func TestRecordReceiptKey_SendsOnlyAFingerprint(t *testing.T) {
|
||||
c, n, seen := server(t, 200, `<RecordReceiptKeyResponse/>`)
|
||||
const raw = "not-a-fingerprint-and-maybe-key-material"
|
||||
if err := c.RecordReceiptKey(context.Background(), raw); err == nil || strings.Contains(err.Error(), raw) {
|
||||
t.Fatalf("invalid err = %v", err)
|
||||
}
|
||||
if n.Load() != 0 {
|
||||
t.Fatal("a seed was sent")
|
||||
t.Fatal("a non-fingerprint was sent")
|
||||
}
|
||||
if err := c.RecordPublicKey(context.Background(), pub); err != nil {
|
||||
if err := c.RecordReceiptKey(context.Background(), fp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains((*seen)[0], "PublicKey="+pub) {
|
||||
if !strings.Contains((*seen)[0], "Fingerprint="+fp) {
|
||||
t.Fatalf("request = %q", (*seen)[0])
|
||||
}
|
||||
}
|
||||
@@ -151,7 +137,7 @@ func TestErrors_ClassifiedWithoutEchoingTheBody(t *testing.T) {
|
||||
}
|
||||
for _, tc := range cases {
|
||||
c, _, _ := server(t, tc.status, `<ErrorResponse><Error><Code>`+tc.code+`</Code><Message>secret-ish detail</Message></Error></ErrorResponse>`)
|
||||
_, _, err := c.Claim(context.Background())
|
||||
_, err := c.ClaimReceiptKey(context.Background())
|
||||
api, ok := errors.AsType[*APIError](err)
|
||||
if !ok || api.Code != tc.code || api.Retryable() != tc.retryable || api.Pending() != (tc.code == "AccessDenied") {
|
||||
t.Fatalf("%s: err = %v", tc.code, err)
|
||||
@@ -163,7 +149,7 @@ func TestErrors_ClassifiedWithoutEchoingTheBody(t *testing.T) {
|
||||
|
||||
c, _, _ := server(t, 200, "")
|
||||
c.baseURL = "https://127.0.0.1:1"
|
||||
_, _, err := c.Claim(context.Background())
|
||||
_, err := c.ClaimReceiptKey(context.Background())
|
||||
tr, ok := errors.AsType[*TransientError](err)
|
||||
if !ok || !tr.Retryable() {
|
||||
t.Fatalf("unreachable err = %v, want TransientError", err)
|
||||
@@ -181,55 +167,10 @@ func TestCall_DoesNotFollowRedirects(t *testing.T) {
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
c.baseURL = srv.URL
|
||||
if _, _, err := c.Claim(context.Background()); err == nil {
|
||||
if _, err := c.ClaimReceiptKey(context.Background()); err == nil {
|
||||
t.Fatal("a redirect was accepted as a decision")
|
||||
}
|
||||
if followed.Load() {
|
||||
t.Fatal("the signed request followed a redirect")
|
||||
}
|
||||
}
|
||||
|
||||
// Only "pending" with nothing else, or "bound" with an IPv4 address and one
|
||||
// valid public key, is a store state.
|
||||
func TestDescribeStore_AcceptsOnlyAValidState(t *testing.T) {
|
||||
pub, seed := userKey(t)
|
||||
wrap := func(inner string) string {
|
||||
return `<DescribeStoreResponse><DescribeStoreResult>` + inner + `</DescribeStoreResult></DescribeStoreResponse>`
|
||||
}
|
||||
t.Run("pending", func(t *testing.T) {
|
||||
c, _, seen := server(t, 200, wrap(`<State>pending</State>`))
|
||||
st, err := c.DescribeStore(context.Background())
|
||||
if err != nil || st.Bound {
|
||||
t.Fatalf("store = %+v %v", st, err)
|
||||
}
|
||||
if !strings.Contains((*seen)[0], "Action=DescribeStore") {
|
||||
t.Fatalf("request = %q", (*seen)[0])
|
||||
}
|
||||
})
|
||||
t.Run("bound", func(t *testing.T) {
|
||||
c, _, _ := server(t, 200, wrap(`<State>bound</State><Address>10.240.0.9</Address><PublicKey>`+pub+`</PublicKey>`))
|
||||
st, err := c.DescribeStore(context.Background())
|
||||
if err != nil || !st.Bound || st.Address != "10.240.0.9" || st.PublicKey != pub {
|
||||
t.Fatalf("store = %+v %v", st, err)
|
||||
}
|
||||
})
|
||||
for name, inner := range map[string]string{
|
||||
"pending with address": `<State>pending</State><Address>10.240.0.9</Address>`,
|
||||
"bound without key": `<State>bound</State><Address>10.240.0.9</Address>`,
|
||||
"bound with seed": `<State>bound</State><Address>10.240.0.9</Address><PublicKey>` + seed + `</PublicKey>`,
|
||||
"bound with hostname": `<State>bound</State><Address>store.example</Address><PublicKey>` + pub + `</PublicKey>`,
|
||||
"bound with IPv6": `<State>bound</State><Address>::ffff:10.240.0.9</Address><PublicKey>` + pub + `</PublicKey>`,
|
||||
"unknown state": `<State>ready</State>`,
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
c, _, _ := server(t, 200, wrap(inner))
|
||||
_, err := c.DescribeStore(context.Background())
|
||||
if err == nil {
|
||||
t.Fatal("accepted an invalid store state")
|
||||
}
|
||||
if strings.Contains(err.Error(), seed) {
|
||||
t.Fatal("the error echoes a seed")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,22 +1,18 @@
|
||||
#!/bin/bash
|
||||
# build-sqs-store-microvm-image.sh — build the SQS store microVM kernel +
|
||||
# initramfs: the dedicated nats-server, ext4 support, the store's guest steps
|
||||
# and a clean power-down. The guest never formats a disk; build/microvm/
|
||||
# sqs-store/sqs-store-lib.sh states its fw_cfg contract. The LB image
|
||||
# (build-microvm-image.sh) is separate and unchanged.
|
||||
#
|
||||
# SQS_MICROVM_PROFILE=worker builds the shared SQS worker guest instead: the
|
||||
# same network /init and kernel, build/microvm/sqs-worker's steps and
|
||||
# $REPO_ROOT/bin/sqs-worker, with no nats-server.
|
||||
# build-sqs-store-microvm-image.sh — build the SQS microVM kernel + initramfs:
|
||||
# one guest running the dedicated SQS nats-server and the SQS worker as separate
|
||||
# processes, ext4 support, its guest steps and a clean power-down. The guest
|
||||
# never formats a disk. The LB image (build-microvm-image.sh) is separate.
|
||||
#
|
||||
# Inputs:
|
||||
# $REPO_ROOT/build/microvm/sqs-store/init.sh, inittab
|
||||
# $REPO_ROOT/build/microvm/sqs-store/sqs-store-{lib.sh,prepare,run,stop}
|
||||
# $REPO_ROOT/build/microvm/sqs-worker/init.sh, inittab
|
||||
# $REPO_ROOT/build/microvm/sqs-worker/sqs-worker-{lib.sh,prepare,ready,stop}
|
||||
# $REPO_ROOT/bin/nats-server (make bin/nats-server, the go.mod-pinned version)
|
||||
# $REPO_ROOT/bin/sqs-worker
|
||||
#
|
||||
# Outputs:
|
||||
# $REPO_ROOT/build/microvm/sqs-store/vmlinuz
|
||||
# $REPO_ROOT/build/microvm/sqs-store/initramfs.cpio.gz
|
||||
# $REPO_ROOT/build/microvm/sqs-worker/vmlinuz
|
||||
# $REPO_ROOT/build/microvm/sqs-worker/initramfs.cpio.gz
|
||||
#
|
||||
# Rootfs strategy (first match wins):
|
||||
# 1. apk available on host → install directly into a temp chroot (Alpine CI/CD)
|
||||
@@ -25,20 +21,16 @@
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
BUILD_DIR="$REPO_ROOT/build/microvm/sqs-store"
|
||||
BUILD_DIR="$REPO_ROOT/build/microvm/sqs-worker"
|
||||
INIT_SH="$BUILD_DIR/init.sh"
|
||||
INITTAB="$BUILD_DIR/inittab"
|
||||
NATS_SERVER_BIN="$REPO_ROOT/bin/nats-server"
|
||||
SQS_WORKER_BIN="$REPO_ROOT/bin/sqs-worker"
|
||||
|
||||
PROFILE="${SQS_MICROVM_PROFILE:-store}"
|
||||
PROFILE="${SQS_MICROVM_PROFILE:-worker}"
|
||||
case "$PROFILE" in
|
||||
store) ;;
|
||||
worker)
|
||||
BUILD_DIR="$REPO_ROOT/build/microvm/sqs-worker"
|
||||
INITTAB="$BUILD_DIR/inittab"
|
||||
SQS_WORKER_BIN="$REPO_ROOT/bin/sqs-worker"
|
||||
;;
|
||||
*) echo "ERROR: SQS_MICROVM_PROFILE must be store or worker, not $PROFILE" >&2; exit 1 ;;
|
||||
worker) ;;
|
||||
*) echo "ERROR: SQS_MICROVM_PROFILE must be worker, not $PROFILE" >&2; exit 1 ;;
|
||||
esac
|
||||
TAG="[build-sqs-$PROFILE-microvm-image]"
|
||||
|
||||
@@ -68,16 +60,14 @@ for tool in cpio gzip find fakeroot; do
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$PROFILE" = worker ]; then
|
||||
if [ ! -x "$SQS_WORKER_BIN" ]; then
|
||||
echo "ERROR: $SQS_WORKER_BIN not found — run 'make bin/sqs-worker'" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! file -b "$SQS_WORKER_BIN" | grep -q "statically linked"; then
|
||||
echo "ERROR: $SQS_WORKER_BIN is not statically linked — it cannot run on the musl guest" >&2
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
if [ ! -x "$SQS_WORKER_BIN" ]; then
|
||||
echo "ERROR: $SQS_WORKER_BIN not found — run 'make bin/sqs-worker'" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! file -b "$SQS_WORKER_BIN" | grep -q "statically linked"; then
|
||||
echo "ERROR: $SQS_WORKER_BIN is not statically linked — it cannot run on the musl guest" >&2
|
||||
exit 1
|
||||
fi
|
||||
# --- Verify the dedicated nats-server is the go.mod-pinned build ---
|
||||
# A stale or host-installed binary would ship an unreviewed server version.
|
||||
NATS_SERVER_PIN=$(awk '$1 == "github.com/nats-io/nats-server/v2" { print $2; exit }' "$REPO_ROOT/go.mod")
|
||||
@@ -99,7 +89,6 @@ if [ "$nats_server_version" != "$NATS_SERVER_PIN" ]; then
|
||||
exit 1
|
||||
fi
|
||||
echo "$TAG nats-server: $nats_server_version (matches go.mod)"
|
||||
fi
|
||||
|
||||
# --- Create chroot ---
|
||||
CHROOT_DIR=$(mktemp -d)
|
||||
@@ -213,39 +202,23 @@ fi
|
||||
echo "$TAG installing inittab..."
|
||||
install -m 0644 "$INITTAB" "$CHROOT_DIR/etc/inittab"
|
||||
|
||||
if [ "$PROFILE" = worker ]; then
|
||||
echo "$TAG installing worker guest steps..."
|
||||
for step in sqs-worker-lib.sh sqs-worker-prepare sqs-worker-ready sqs-worker-stop; do
|
||||
if [ ! -f "$BUILD_DIR/$step" ]; then
|
||||
echo "ERROR: $BUILD_DIR/$step not found — cannot build initramfs" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
mkdir -p "$CHROOT_DIR/usr/local/lib/sqs-worker" "$CHROOT_DIR/usr/local/bin"
|
||||
install -m 0755 "$BUILD_DIR/sqs-worker-lib.sh" "$BUILD_DIR/sqs-worker-prepare" "$BUILD_DIR/sqs-worker-ready" "$BUILD_DIR/sqs-worker-stop" \
|
||||
"$CHROOT_DIR/usr/local/lib/sqs-worker/"
|
||||
install -m 0755 "$SQS_WORKER_BIN" "$CHROOT_DIR/usr/local/bin/sqs-worker"
|
||||
# The mountpoint ships empty; the worker proves its disk is mounted here.
|
||||
mkdir -p "$CHROOT_DIR/data"
|
||||
else
|
||||
# --- Place the store's guest steps ---
|
||||
echo "$TAG installing store guest steps..."
|
||||
mkdir -p "$CHROOT_DIR/usr/local/lib/sqs-store"
|
||||
for step in sqs-store-lib.sh sqs-store-prepare sqs-store-run sqs-store-stop; do
|
||||
echo "$TAG installing worker guest steps..."
|
||||
for step in sqs-worker-lib.sh sqs-worker-prepare sqs-worker-store sqs-worker-ready sqs-worker-stop; do
|
||||
if [ ! -f "$BUILD_DIR/$step" ]; then
|
||||
echo "ERROR: $BUILD_DIR/$step not found — cannot build initramfs" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
install -m 0644 "$BUILD_DIR/sqs-store-lib.sh" "$CHROOT_DIR/usr/local/lib/sqs-store/"
|
||||
install -m 0755 "$BUILD_DIR/sqs-store-prepare" "$BUILD_DIR/sqs-store-run" "$BUILD_DIR/sqs-store-stop" \
|
||||
"$CHROOT_DIR/usr/local/lib/sqs-store/"
|
||||
|
||||
mkdir -p "$CHROOT_DIR/usr/local/lib/sqs-worker" "$CHROOT_DIR/usr/local/bin"
|
||||
install -m 0755 "$BUILD_DIR/sqs-worker-lib.sh" "$BUILD_DIR/sqs-worker-prepare" "$BUILD_DIR/sqs-worker-store" "$BUILD_DIR/sqs-worker-ready" "$BUILD_DIR/sqs-worker-stop" \
|
||||
"$CHROOT_DIR/usr/local/lib/sqs-worker/"
|
||||
install -m 0755 "$SQS_WORKER_BIN" "$CHROOT_DIR/usr/local/bin/sqs-worker"
|
||||
# The mountpoint ships empty; the worker proves its disk is mounted here.
|
||||
mkdir -p "$CHROOT_DIR/data"
|
||||
# --- Copy nats-server binary ---
|
||||
echo "$TAG copying nats-server binary..."
|
||||
mkdir -p "$CHROOT_DIR/usr/local/bin"
|
||||
install -m 0755 "$NATS_SERVER_BIN" "$CHROOT_DIR/usr/local/bin/nats-server"
|
||||
fi
|
||||
|
||||
# --- Create required directories for init ---
|
||||
mkdir -p \
|
||||
|
||||
@@ -11,13 +11,14 @@ set -eu
|
||||
# cron can branch without re-parsing, then disables itself so it runs exactly
|
||||
# once.
|
||||
#
|
||||
# server → eks-token-webhook, k3s (server), k3s-first-boot (bootstrap
|
||||
# publish), mulga-eks-state-report, mulga-eks-node-gc
|
||||
# server → eks-token-webhook, k3s (server), eks-pod-identity-webhook,
|
||||
# k3s-first-boot (bootstrap publish), mulga-eks-state-report,
|
||||
# mulga-eks-node-gc
|
||||
# server-join → eks-token-webhook, k3s (server, joins the first server's etcd
|
||||
# quorum), mulga-eks-state-report, mulga-eks-node-gc. NO
|
||||
# k3s-first-boot: the first server already publishes the
|
||||
# cluster-identical bootstrap artifacts; a join re-publish only
|
||||
# races the bootstrap bus.
|
||||
# quorum), eks-pod-identity-webhook, mulga-eks-state-report,
|
||||
# mulga-eks-node-gc. NO k3s-first-boot: the first server
|
||||
# already publishes the cluster-identical bootstrap artifacts;
|
||||
# a join re-publish only races the bootstrap bus.
|
||||
# agent → k3s-agent
|
||||
#
|
||||
# Paths are overridable via env so the selection logic is unit-testable (bats)
|
||||
@@ -81,6 +82,14 @@ svc_start() {
|
||||
esac
|
||||
}
|
||||
|
||||
# IRSA injector: every server runs one, called by its own apiserver. It is
|
||||
# optional (failurePolicy Ignore), so it goes last and a fault only costs
|
||||
# injection, never the control plane.
|
||||
svc_optional() {
|
||||
svc_enable "$1" && svc_start "$1" ||
|
||||
log "WARN: optional service $1 failed to enable or start; continuing without it"
|
||||
}
|
||||
|
||||
svc_disable() {
|
||||
case "${INIT_SYSTEM}" in
|
||||
openrc) rc-update del "$1" default 2>/dev/null || true ;;
|
||||
@@ -150,6 +159,7 @@ case "${ROLE}" in
|
||||
svc_start mulga-eks-state-report
|
||||
svc_start mulga-eks-addon-sync
|
||||
svc_start mulga-eks-node-gc
|
||||
svc_optional eks-pod-identity-webhook
|
||||
;;
|
||||
server-join)
|
||||
log "configuring server-join role"
|
||||
@@ -168,6 +178,7 @@ case "${ROLE}" in
|
||||
svc_start konnectivity-server
|
||||
svc_start mulga-eks-state-report
|
||||
svc_start mulga-eks-node-gc
|
||||
svc_optional eks-pod-identity-webhook
|
||||
;;
|
||||
agent)
|
||||
log "configuring agent role"
|
||||
|
||||
@@ -22,6 +22,7 @@ for tool in rc-update rc-service; do
|
||||
cat > "${STUB_OPENRC}/${tool}" <<EOF
|
||||
#!/bin/sh
|
||||
echo "${tool} \$*" >> "\${CALLS}"
|
||||
case "\$*" in *"\${STUB_FAIL:-@none@}"*) exit 1 ;; esac
|
||||
exit 0
|
||||
EOF
|
||||
chmod +x "${STUB_OPENRC}/${tool}"
|
||||
@@ -32,6 +33,7 @@ mkdir -p "${STUB_SYSTEMD}"
|
||||
cat > "${STUB_SYSTEMD}/systemctl" <<'EOF'
|
||||
#!/bin/sh
|
||||
echo "systemctl $*" >> "${CALLS}"
|
||||
case "$*" in *"${STUB_FAIL:-@none@}"*) exit 1 ;; esac
|
||||
exit 0
|
||||
EOF
|
||||
chmod +x "${STUB_SYSTEMD}/systemctl"
|
||||
@@ -129,6 +131,7 @@ for INIT in openrc systemd; do
|
||||
check "server(${INIT}): enables k3s-first-boot" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" k3s-first-boot)")"
|
||||
check "server(${INIT}): starts k3s" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" k3s)")"
|
||||
check "server(${INIT}): enables konnectivity-server" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" konnectivity-server)")"
|
||||
check "server(${INIT}): enables pod-identity-webhook" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" eks-pod-identity-webhook)")"
|
||||
check "server(${INIT}): enables k3s-recovery" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" mulga-eks-k3s-recovery)")"
|
||||
# Recovery is enabled (so it also runs pre-k3s on later boots) AND started
|
||||
# inline now — a restore-snapshot DR seed's directive must apply on this
|
||||
@@ -145,6 +148,34 @@ for INIT in openrc systemd; do
|
||||
check "server(${INIT}): starts node-gc" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" mulga-eks-node-gc)")"
|
||||
check "server(${INIT}): no k3s-agent" "no" "$(present "${CALLS}" "$(enable_cmd "${INIT}" k3s-agent)")"
|
||||
check "server(${INIT}): self-disables" "yes" "$(present "${CALLS}" "$(disable_cmd "${INIT}" eks-node-role)")"
|
||||
check "server(${INIT}): pod-identity-webhook is the last call" "$(start_cmd "${INIT}" eks-pod-identity-webhook)" "$(grep -v "$(disable_cmd "${INIT}" eks-node-role)" "${CALLS}" | tail -n 1)"
|
||||
check "server(${INIT}): pod-identity-webhook enabled after addon-sync start" "yes" "$(
|
||||
awk -v a="$(start_cmd "${INIT}" mulga-eks-addon-sync)" -v w="$(enable_cmd "${INIT}" eks-pod-identity-webhook)" '
|
||||
index($0, a) == 1 { an = NR }
|
||||
index($0, w) == 1 { wn = NR }
|
||||
END { print (an > 0 && wn > an) ? "yes" : "no" }
|
||||
' "${CALLS}"
|
||||
)"
|
||||
|
||||
# A broken optional webhook must leave the control plane booting.
|
||||
for FAILING in enable start; do
|
||||
run_case "${INIT}" server_${FAILING}_fail
|
||||
printf 'SPINIFEX_K3S_ROLE=server\nEKS_CLUSTER_NAME=alpha\n' > "${EKS_NODE_ENVFILE}"
|
||||
if [ "${FAILING}" = enable ]; then
|
||||
STUB_FAIL="$(enable_cmd "${INIT}" eks-pod-identity-webhook)"
|
||||
export STUB_FAIL
|
||||
STUB_FAIL="${STUB_FAIL#rc-update add }"; STUB_FAIL="${STUB_FAIL#systemctl enable }"
|
||||
else
|
||||
STUB_FAIL="$(start_cmd "${INIT}" eks-pod-identity-webhook)"
|
||||
export STUB_FAIL
|
||||
STUB_FAIL="${STUB_FAIL#rc-service }"; STUB_FAIL="${STUB_FAIL#systemctl start --no-block }"
|
||||
fi
|
||||
invoke
|
||||
unset STUB_FAIL
|
||||
check "server(${INIT}): webhook ${FAILING} failure still exits 0" 0 "${rc}"
|
||||
check "server(${INIT}): webhook ${FAILING} failure still starts mulga-eks-addon-sync" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" mulga-eks-addon-sync)")"
|
||||
check "server(${INIT}): webhook ${FAILING} failure still self-disables" "yes" "$(present "${CALLS}" "$(disable_cmd "${INIT}" eks-node-role)")"
|
||||
done
|
||||
|
||||
# --- server-join ---
|
||||
run_case "${INIT}" server_join
|
||||
@@ -158,11 +189,13 @@ for INIT in openrc systemd; do
|
||||
check "server-join(${INIT}): enables node-gc" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" mulga-eks-node-gc)")"
|
||||
check "server-join(${INIT}): starts node-gc" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" mulga-eks-node-gc)")"
|
||||
check "server-join(${INIT}): enables konnectivity-server" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" konnectivity-server)")"
|
||||
check "server-join(${INIT}): enables pod-identity-webhook" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" eks-pod-identity-webhook)")"
|
||||
check "server-join(${INIT}): enables k3s-recovery" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" mulga-eks-k3s-recovery)")"
|
||||
check "server-join(${INIT}): starts k3s-recovery" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" mulga-eks-k3s-recovery)")"
|
||||
# Join servers must NOT re-publish bootstrap: the first server already
|
||||
# published the cluster-identical artifacts, and a join re-publish would
|
||||
# only race the bootstrap bus. This is deliberate, not an oversight.
|
||||
check "server-join(${INIT}): pod-identity-webhook is the last call" "$(start_cmd "${INIT}" eks-pod-identity-webhook)" "$(grep -v "$(disable_cmd "${INIT}" eks-node-role)" "${CALLS}" | tail -n 1)"
|
||||
check "server-join(${INIT}): no k3s-first-boot" "yes" "$(absent_all "${CALLS}" "k3s-first-boot")"
|
||||
check "server-join(${INIT}): no k3s-agent" "yes" "$(absent_all "${CALLS}" "$(enable_cmd "${INIT}" k3s-agent)")"
|
||||
check "server-join(${INIT}): self-disables" "yes" "$(present "${CALLS}" "$(disable_cmd "${INIT}" eks-node-role)")"
|
||||
@@ -174,7 +207,7 @@ for INIT in openrc systemd; do
|
||||
check "agent(${INIT}): exit 0" 0 "${rc}"
|
||||
check "agent(${INIT}): role file" "agent" "$(cat "${ROLE_FILE}")"
|
||||
check "agent(${INIT}): enables k3s-agent" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" k3s-agent)")"
|
||||
check "agent(${INIT}): no server services" "yes" "$(absent_all "${CALLS}" "eks-token-webhook" "k3s-first-boot" "mulga-eks-node-gc" "$(enable_cmd "${INIT}" k3s)")"
|
||||
check "agent(${INIT}): no server services" "yes" "$(absent_all "${CALLS}" "eks-token-webhook" "eks-pod-identity-webhook" "k3s-first-boot" "mulga-eks-node-gc" "$(enable_cmd "${INIT}" k3s)")"
|
||||
|
||||
# --- agent (inferred from agent.env, no explicit role) ---
|
||||
run_case "${INIT}" agent_inferred
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
#!/sbin/openrc-run
|
||||
# eks-pod-identity-webhook — IRSA mutating admission webhook. Binds 127.0.0.1:8445
|
||||
# only; the local kube-apiserver is the sole client. Supervised and respawned so
|
||||
# it can wait out k3s writing its CAs without ever delaying k3s or the token
|
||||
# webhook: it exits until the CAs exist and supervise-daemon retries.
|
||||
|
||||
description="EKS pod identity webhook (Mulga)"
|
||||
command="/usr/local/bin/eks-pod-identity-webhook"
|
||||
command_args="--addr=127.0.0.1:8445"
|
||||
pidfile="/run/eks-pod-identity-webhook.pid"
|
||||
supervisor="supervise-daemon"
|
||||
respawn_delay=5
|
||||
respawn_max=0
|
||||
output_log="/dev/console"
|
||||
error_log="/dev/console"
|
||||
|
||||
if [ -f /etc/spinifex-eks/first-boot.env ]; then
|
||||
# shellcheck disable=SC1091
|
||||
. /etc/spinifex-eks/first-boot.env
|
||||
export EKS_REGION EKS_ADDON_GATEWAY_URL EKS_GATEWAY_CA
|
||||
fi
|
||||
|
||||
depend() {
|
||||
need net
|
||||
after k3s
|
||||
}
|
||||
@@ -22,9 +22,9 @@ APK_PACKAGES="ca-certificates curl jq iptables ip6tables conntrack-tools cgroup-
|
||||
# at a pinned tag (its GitHub releases ship only container images, and its go.mod
|
||||
# carries replace directives that block `go install pkg@version`). The agent half
|
||||
# is the matching proxy-agent container image, pinned in k3s.initd.
|
||||
BUILD_COMMANDS="CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-token-webhook ./cmd/eks-token-webhook && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-gateway-publish ./cmd/eks-gateway-publish && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-gateway-fetch ./cmd/eks-gateway-fetch && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-webhook-cert ./cmd/eks-webhook-cert && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-konnectivity-cert ./cmd/eks-konnectivity-cert && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-etcd-member ./cmd/eks-etcd-member && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/ecr-credential-provider ./cmd/ecr-credential-provider && rm -rf /tmp/anp-build && git clone --depth 1 --branch v0.30.3 https://github.com/kubernetes-sigs/apiserver-network-proxy /tmp/anp-build && ( cd /tmp/anp-build && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o /tmp/anp-build/konnectivity-server ./cmd/server ) && cp /tmp/anp-build/konnectivity-server bin/konnectivity-server && rm -rf /tmp/anp-build"
|
||||
BUILD_COMMANDS="CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-token-webhook ./cmd/eks-token-webhook && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-gateway-publish ./cmd/eks-gateway-publish && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-gateway-fetch ./cmd/eks-gateway-fetch && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-webhook-cert ./cmd/eks-webhook-cert && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-konnectivity-cert ./cmd/eks-konnectivity-cert && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-etcd-member ./cmd/eks-etcd-member && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-pod-identity-webhook ./cmd/eks-pod-identity-webhook && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/ecr-credential-provider ./cmd/ecr-credential-provider && rm -rf /tmp/anp-build && git clone --depth 1 --branch v0.30.3 https://github.com/kubernetes-sigs/apiserver-network-proxy /tmp/anp-build && ( cd /tmp/anp-build && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o /tmp/anp-build/konnectivity-server ./cmd/server ) && cp /tmp/anp-build/konnectivity-server bin/konnectivity-server && rm -rf /tmp/anp-build"
|
||||
|
||||
INSTALL_BINARIES="bin/eks-token-webhook:/usr/local/bin/eks-token-webhook bin/eks-gateway-publish:/usr/local/bin/eks-gateway-publish bin/eks-gateway-fetch:/usr/local/bin/eks-gateway-fetch bin/eks-webhook-cert:/usr/local/bin/eks-webhook-cert bin/eks-konnectivity-cert:/usr/local/bin/eks-konnectivity-cert bin/eks-etcd-member:/usr/local/bin/eks-etcd-member bin/konnectivity-server:/usr/local/bin/konnectivity-server bin/ecr-credential-provider:/usr/local/bin/ecr-credential-provider"
|
||||
INSTALL_BINARIES="bin/eks-token-webhook:/usr/local/bin/eks-token-webhook bin/eks-gateway-publish:/usr/local/bin/eks-gateway-publish bin/eks-gateway-fetch:/usr/local/bin/eks-gateway-fetch bin/eks-webhook-cert:/usr/local/bin/eks-webhook-cert bin/eks-konnectivity-cert:/usr/local/bin/eks-konnectivity-cert bin/eks-etcd-member:/usr/local/bin/eks-etcd-member bin/eks-pod-identity-webhook:/usr/local/bin/eks-pod-identity-webhook bin/konnectivity-server:/usr/local/bin/konnectivity-server bin/ecr-credential-provider:/usr/local/bin/ecr-credential-provider"
|
||||
|
||||
# All role services ship; none but the selector are enabled at bake time.
|
||||
# eks-node-role picks server vs agent at first boot and enables the rest.
|
||||
@@ -36,6 +36,7 @@ scripts/images/eks-node/k3s-prestart.sh:/usr/local/sbin/k3s-prestart \
|
||||
scripts/images/eks-node/k3s-agent.initd:/etc/init.d/k3s-agent \
|
||||
scripts/images/eks-node/k3s-agent-prestart.sh:/usr/local/sbin/k3s-agent-prestart \
|
||||
scripts/images/eks-node/eks-token-webhook.initd:/etc/init.d/eks-token-webhook \
|
||||
scripts/images/eks-node/eks-pod-identity-webhook.initd:/etc/init.d/eks-pod-identity-webhook \
|
||||
scripts/images/eks-node/k3s-first-boot.initd:/etc/init.d/k3s-first-boot \
|
||||
scripts/images/eks-node/k3s-first-boot.sh:/usr/local/sbin/k3s-first-boot \
|
||||
scripts/images/eks-node/mulga-eks-state-report.initd:/etc/init.d/mulga-eks-state-report \
|
||||
|
||||
@@ -44,7 +44,8 @@ ln -sf /usr/local/bin/k3s /usr/local/bin/ctr
|
||||
# role's services are baked; the selector enables the right ones at first boot.
|
||||
chmod 0755 /etc/init.d/eks-node-role /etc/init.d/k3s /etc/init.d/k3s-agent \
|
||||
/etc/init.d/eks-token-webhook /etc/init.d/k3s-first-boot /etc/init.d/mulga-eks-state-report \
|
||||
/etc/init.d/mulga-eks-addon-sync /etc/init.d/mulga-eks-node-gc /etc/init.d/konnectivity-server
|
||||
/etc/init.d/mulga-eks-addon-sync /etc/init.d/mulga-eks-node-gc /etc/init.d/konnectivity-server \
|
||||
/etc/init.d/eks-pod-identity-webhook
|
||||
chmod 0755 /usr/local/sbin/eks-node-role /usr/local/sbin/k3s-first-boot \
|
||||
/usr/local/sbin/mulga-eks-state-report /usr/local/sbin/mulga-eks-addon-sync \
|
||||
/usr/local/sbin/mulga-eks-node-gc
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
#!/bin/sh
|
||||
# Every /etc/init.d script that INSTALL_FILES ships at 0644 must be chmod'd
|
||||
# 0755 by setup.sh, or OpenRC refuses to enable it ("is not executable") and
|
||||
# eks-node-role, running under set -eu, aborts before k3s starts.
|
||||
#
|
||||
# Run: sh scripts/images/eks-node/setup_initd_test.sh
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
|
||||
MANIFEST="${SCRIPT_DIR}/manifest.conf"
|
||||
SETUP="${SETUP_SH:-${SCRIPT_DIR}/setup.sh}"
|
||||
|
||||
fail=0
|
||||
for dst in $(grep -o ':/etc/init\.d/[A-Za-z0-9._-]*' "${MANIFEST}" | cut -d: -f2); do
|
||||
if ! grep -E '^[[:space:]]*(chmod 0755 )?.*'"${dst}"'([[:space:]\\]|$)' "${SETUP}" \
|
||||
| grep -q . ; then
|
||||
echo "FAIL: ${dst} is installed 0644 and never chmod'd 0755 in setup.sh"
|
||||
fail=1
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "${fail}" -ne 0 ]; then
|
||||
exit 1
|
||||
fi
|
||||
echo "PASS: every shipped init script is chmod'd 0755"
|
||||
@@ -1,293 +0,0 @@
|
||||
#!/bin/sh
|
||||
# sqs-store-guest_test.sh — the SQS store guest steps start the store only on
|
||||
# a valid contract and one checked, mounted disk, never format, and stop the
|
||||
# store before unmounting it.
|
||||
#
|
||||
# Drives build/microvm/sqs-store/sqs-store-lib.sh against a fake fw_cfg, sysfs
|
||||
# and stub tools. The stop cases signal only test-owned children.
|
||||
# The case settings below are read by the sourced library, not by this file.
|
||||
# shellcheck disable=SC2034
|
||||
set -eu
|
||||
|
||||
ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/../.." && pwd)
|
||||
LIB=${SQS_GUEST_LIB:-"$ROOT/build/microvm/sqs-store/sqs-store-lib.sh"}
|
||||
TMP=$(mktemp -d)
|
||||
CHILDREN=""
|
||||
cleanup() {
|
||||
for p in $CHILDREN; do kill -KILL "$p" 2>/dev/null || true; done
|
||||
rm -rf "$TMP"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
fails=0
|
||||
pass() { echo " ok - $1"; }
|
||||
fail() { echo " FAIL - $1"; fails=$((fails + 1)); }
|
||||
check() { if eval "$2"; then pass "$1"; else fail "$1"; fi; }
|
||||
|
||||
echo "-- build/microvm/sqs-store/sqs-store-lib.sh"
|
||||
|
||||
SERIAL=vol0123456789abcdef0
|
||||
NKEY=UAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAB
|
||||
ADDR=10.11.12.13
|
||||
|
||||
# Stub tools record each call to calls.log; their exit codes come from files.
|
||||
mkdir -p "$TMP/bin"
|
||||
cat > "$TMP/bin/e2fsck" <<'EOF'
|
||||
#!/bin/sh
|
||||
echo "e2fsck $*" >> "$T/calls.log"
|
||||
exit "$(cat "$T/fsck.rc")"
|
||||
EOF
|
||||
cat > "$TMP/bin/mount" <<'EOF'
|
||||
#!/bin/sh
|
||||
echo "mount $*" >> "$T/calls.log"
|
||||
exit "$(cat "$T/mount.rc")"
|
||||
EOF
|
||||
cat > "$TMP/bin/umount" <<'EOF'
|
||||
#!/bin/sh
|
||||
echo "umount $*" >> "$T/calls.log"
|
||||
EOF
|
||||
for f in mkfs mkfs.ext4 mke2fs; do
|
||||
printf '#!/bin/sh\necho "mkfs %s $*" >> "$T/calls.log"\n' "$f" > "$TMP/bin/$f"
|
||||
done
|
||||
cat > "$TMP/bin/ip" <<'EOF'
|
||||
#!/bin/sh
|
||||
cat "$T/ip.out"
|
||||
EOF
|
||||
cat > "$TMP/bin/nats-server" <<'EOF'
|
||||
#!/bin/sh
|
||||
echo "nats-server $*" >> "$T/calls.log"
|
||||
[ "$1" = -t ] || echo "GOMEMLIMIT=${GOMEMLIMIT:-unset}" >> "$T/env.log"
|
||||
[ "$1" = -t ] && exit "$(cat "$T/configtest.rc")"
|
||||
exit 0
|
||||
EOF
|
||||
chmod +x "$TMP/bin"/*
|
||||
|
||||
# fresh <name>: a case directory with a valid contract and one matching disk
|
||||
# among others. Callers then break one thing.
|
||||
fresh() {
|
||||
T="$TMP/$1"
|
||||
export T
|
||||
mkdir -p "$T/fwcfg" "$T/sys" "$T/dev" "$T/run" "$T/data"
|
||||
for k in data-serial listen-addr client-nkey; do mkdir -p "$T/fwcfg/$k"; done
|
||||
printf '%s\n' "$SERIAL" > "$T/fwcfg/data-serial/raw"
|
||||
printf '%s\n' "$ADDR" > "$T/fwcfg/listen-addr/raw"
|
||||
printf '%s\n' "$NKEY" > "$T/fwcfg/client-nkey/raw"
|
||||
disk vda vol00000000000000000
|
||||
disk vdb "$SERIAL"
|
||||
echo 0 > "$T/fsck.rc"
|
||||
echo 0 > "$T/mount.rc"
|
||||
echo 0 > "$T/configtest.rc"
|
||||
: > "$T/calls.log"
|
||||
: > "$T/console"
|
||||
printf '1: lo inet 127.0.0.1/8 scope host lo\n2: eth0 inet %s/24 brd 10.11.12.255 scope global eth0\n' "$ADDR" > "$T/ip.out"
|
||||
}
|
||||
disk() { mkdir -p "$T/sys/$1"; printf '%s\n' "$2" > "$T/sys/$1/serial"; : > "$T/dev/$1"; }
|
||||
|
||||
# prepare: run sqs_prepare in a subshell with every path pointed at the case.
|
||||
prepare() {
|
||||
(
|
||||
SQS_FWCFG="$T/fwcfg" SQS_SYSBLOCK="$T/sys" SQS_DEVDIR="$T/dev" SQS_NODE_TEST=-e
|
||||
SQS_RUN="$T/run" SQS_DATA="$T/data" SQS_CONSOLE="$T/console"
|
||||
E2FSCK="$TMP/bin/e2fsck" MOUNT="$TMP/bin/mount" IP="$TMP/bin/ip" NATS_BIN="$TMP/bin/nats-server"
|
||||
PATH="$TMP/bin:$PATH"
|
||||
export PATH
|
||||
# shellcheck source=../../build/microvm/sqs-store/sqs-store-lib.sh
|
||||
. "$LIB"
|
||||
sqs_prepare
|
||||
) > /dev/null 2>&1
|
||||
}
|
||||
refused() { [ ! -e "$T/run/ready" ] && grep -q FATAL "$T/console"; }
|
||||
never_mounted() { ! grep -q '^mount ' "$T/calls.log"; }
|
||||
never_formatted() { ! grep -q '^mkfs ' "$T/calls.log"; }
|
||||
never_checked() { ! grep -q '^e2fsck ' "$T/calls.log"; }
|
||||
|
||||
# --- A valid contract prepares the store ---
|
||||
fresh valid
|
||||
check "valid contract is prepared" 'prepare && [ -e "$T/run/ready" ]'
|
||||
check "the matching disk is checked, then mounted at the data dir" \
|
||||
'grep -qx "e2fsck -p $T/dev/vdb" "$T/calls.log" && grep -qx "mount -t ext4 -o nodev,nosuid,noexec $T/dev/vdb $T/data" "$T/calls.log"'
|
||||
check "config listens only on the supplied address" \
|
||||
'grep -c "listen" "$T/run/nats.conf" | grep -qx 1 && grep -qx "listen: \"$ADDR:4222\"" "$T/run/nats.conf"'
|
||||
check "config admits only the supplied user NKey" 'grep -q "users = \[ { nkey: \"$NKEY\" } \]" "$T/run/nats.conf"'
|
||||
check "config syncs JetStream on the data disk" \
|
||||
'grep -q "store_dir: \"$T/data\"" "$T/run/nats.conf" && grep -q "sync_interval: always" "$T/run/nats.conf"'
|
||||
check "config admits a full SQS message (2MiB max_payload)" \
|
||||
'grep -c "max_payload" "$T/run/nats.conf" | grep -qx 1 && grep -qx "max_payload: 2MiB" "$T/run/nats.conf"'
|
||||
check "config is tested by the server before ready" 'grep -qx "nats-server -t -c $T/run/nats.conf" "$T/calls.log"'
|
||||
check "no auth material on the console" '! grep -q "$NKEY" "$T/console"'
|
||||
check "prepare reports the disk and config, not a listener" 'grep -q "prepared: $T/dev/vdb mounted" "$T/console" && ! grep -qi "listening" "$T/console"'
|
||||
|
||||
fresh fsck-corrected; echo 1 > "$T/fsck.rc"
|
||||
check "e2fsck exit 1 (corrected) proceeds" 'prepare && [ -e "$T/run/ready" ]'
|
||||
|
||||
# --- Disk refusals: nothing mounted, nothing formatted ---
|
||||
fresh no-disk; rm -rf "$T/sys/vdb"
|
||||
check "missing disk is refused before any check or mount" '! prepare && refused && never_checked && never_mounted'
|
||||
|
||||
fresh wrong-disk; printf 'vol0fffffffffffffff0\n' > "$T/fwcfg/data-serial/raw"
|
||||
check "disk with another serial is refused" '! prepare && refused && never_checked && never_mounted'
|
||||
|
||||
fresh two-disks; disk vdc "$SERIAL"
|
||||
check "two disks with the serial are refused" '! prepare && refused && never_checked && never_mounted'
|
||||
|
||||
fresh bad-serial; printf 'vol-0123456789abcdef\n' > "$T/fwcfg/data-serial/raw"
|
||||
check "malformed serial is refused" '! prepare && refused && never_mounted'
|
||||
|
||||
fresh no-node; rm -f "$T/dev/vdb"
|
||||
check "disk without a device node is refused" '! prepare && refused && never_mounted'
|
||||
|
||||
for rc in 2 4 8; do
|
||||
fresh "fsck-$rc"; echo "$rc" > "$T/fsck.rc"
|
||||
check "e2fsck exit $rc is refused without mount or format" '! prepare && refused && never_mounted && never_formatted'
|
||||
done
|
||||
|
||||
fresh mount-fail; echo 32 > "$T/mount.rc"
|
||||
check "failed mount is refused" '! prepare && refused'
|
||||
|
||||
# --- Auth and listener refusals happen before the disk is touched ---
|
||||
fresh no-nkey; rm -rf "$T/fwcfg/client-nkey"
|
||||
check "missing NKey is refused before the disk" '! prepare && refused && never_checked && never_mounted'
|
||||
|
||||
for bad in seed lower short two empty; do
|
||||
fresh "nkey-$bad"
|
||||
case $bad in
|
||||
seed) v=SUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA ;;
|
||||
lower) v=uaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab ;;
|
||||
short) v=UAAAA ;;
|
||||
two) v="$NKEY
|
||||
$NKEY" ;;
|
||||
empty) v="" ;;
|
||||
esac
|
||||
printf '%s\n' "$v" > "$T/fwcfg/client-nkey/raw"
|
||||
check "NKey ($bad) is refused before the disk, without echoing it" \
|
||||
'! prepare && refused && never_checked && { [ -z "$v" ] || ! grep -qF "$v" "$T/console"; }'
|
||||
done
|
||||
|
||||
fresh nkey-checksum; echo 1 > "$T/configtest.rc"
|
||||
check "NKey the server rejects is refused before the disk" '! prepare && refused && never_checked && never_mounted'
|
||||
|
||||
fresh no-addr; rm -rf "$T/fwcfg/listen-addr"
|
||||
check "missing listen address is refused" '! prepare && refused && never_mounted'
|
||||
|
||||
fresh addr-absent; printf '1: lo inet 127.0.0.1/8 scope host lo\n2: eth0 inet 10.11.12.99/24 scope global eth0\n' > "$T/ip.out"
|
||||
check "listen address not on a guest NIC is refused" '! prepare && refused && never_mounted'
|
||||
|
||||
fresh addr-loopback; printf '127.0.0.1\n' > "$T/fwcfg/listen-addr/raw"
|
||||
check "loopback listen address is refused" '! prepare && refused && never_mounted'
|
||||
|
||||
for bad in 0.0.0.0x 1.2.3 1.2.3.256 01.2.3.4; do
|
||||
fresh "addr-$bad"; printf '%s\n' "$bad" > "$T/fwcfg/listen-addr/raw"
|
||||
check "malformed listen address $bad is refused" '! prepare && refused && never_mounted'
|
||||
done
|
||||
|
||||
# --- Run: the store starts only when prepared and not stopping ---
|
||||
run_step() {
|
||||
(
|
||||
SQS_RUN="$T/run" SQS_CONSOLE="$T/console" NATS_BIN="$TMP/bin/nats-server"
|
||||
# shellcheck source=../../build/microvm/sqs-store/sqs-store-lib.sh
|
||||
. "$LIB"
|
||||
sqs_run
|
||||
) > /dev/null 2>&1
|
||||
}
|
||||
fresh run-ready; prepare; : > "$T/calls.log"
|
||||
check "a prepared store starts the server" 'run_step && grep -qx "nats-server -c $T/run/nats.conf" "$T/calls.log"'
|
||||
fresh run-memlimit; prepare; : > "$T/env.log"
|
||||
check "the server runs under the ADR-0011:S11 heap limit" 'run_step && grep -qx "GOMEMLIMIT=512MiB" "$T/env.log"'
|
||||
fresh run-unready
|
||||
check "an unprepared store never starts the server" '! run_step && ! grep -q "^nats-server" "$T/calls.log"'
|
||||
fresh run-stopping; prepare; : > "$T/calls.log"; : > "$T/run/stopping"
|
||||
check "a stopping store never starts the server" '! run_step && ! grep -q "^nats-server" "$T/calls.log"'
|
||||
|
||||
# --- Stop: SIGTERM the verified store, then unmount ---
|
||||
STORE="$TMP/bin/fake-store"
|
||||
cp "$(command -v sleep)" "$STORE"
|
||||
stop_step() {
|
||||
(
|
||||
SQS_RUN="$T/run" SQS_DATA="$T/data" SQS_CONSOLE="$T/console" SQS_MOUNTS="$T/mounts"
|
||||
UMOUNT="$TMP/bin/umount" NATS_BIN="$STORE" SQS_STOP_BUDGET_TENTHS=10
|
||||
# shellcheck source=../../build/microvm/sqs-store/sqs-store-lib.sh
|
||||
. "$LIB"
|
||||
sqs_stop
|
||||
) > /dev/null 2>&1
|
||||
}
|
||||
gone() { [ ! -r "/proc/$1/status" ] || grep -q '^State:[[:space:]]*Z' "/proc/$1/status"; }
|
||||
# record <pid>: write the identity sqs_run would record for that process.
|
||||
# shellcheck source=../../build/microvm/sqs-store/sqs-store-lib.sh
|
||||
record() { echo "$1 $( . "$LIB"; sqs_starttime "$1" )" > "$T/run/pid"; }
|
||||
|
||||
fresh stop-orderly; echo "dev $T/data ext4 rw 0 0" > "$T/mounts"
|
||||
"$STORE" 60 & pid=$!; CHILDREN="$CHILDREN $pid"; record "$pid"
|
||||
stop_step
|
||||
check "stop sends SIGTERM and the store exits" 'gone "$pid" && grep -q "SIGTERM sent" "$T/console" && ! grep -q SIGKILL "$T/console"'
|
||||
check "stop blocks respawn and unmounts after the store exits" \
|
||||
'[ -e "$T/run/stopping" ] && grep -qx "umount $T/data" "$T/calls.log"'
|
||||
|
||||
fresh stop-hung; echo "dev $T/data ext4 rw 0 0" > "$T/mounts"
|
||||
(trap '' TERM; exec "$STORE" 60) & pid=$!; CHILDREN="$CHILDREN $pid"; record "$pid"
|
||||
stop_step
|
||||
check "store ignoring SIGTERM is killed after the budget, then unmounted" \
|
||||
'gone "$pid" && grep -q SIGKILL "$T/console" && grep -qx "umount $T/data" "$T/calls.log"'
|
||||
|
||||
fresh stop-reused; echo "dev $T/data ext4 rw 0 0" > "$T/mounts"
|
||||
"$STORE" 60 & pid=$!; CHILDREN="$CHILDREN $pid"; echo "$pid 1" > "$T/run/pid"
|
||||
stop_step
|
||||
check "a recorded PID with another start time is never signalled" '! gone "$pid" && grep -q "no running store" "$T/console"'
|
||||
kill -KILL "$pid"
|
||||
|
||||
# A start held after its stopping check and before its exec: the run step
|
||||
# blocks writing "run: starting" to a FIFO console until the test reads it.
|
||||
fresh stop-pending-start; echo "dev $T/data ext4 rw 0 0" > "$T/mounts"
|
||||
: > "$T/run/ready"
|
||||
mkfifo "$T/run-console"
|
||||
printf '#!/bin/sh\n: > "%s/started"\nexec "%s" 60\n' "$T" "$STORE" > "$TMP/bin/pending-nats"
|
||||
chmod +x "$TMP/bin/pending-nats"
|
||||
SQS_RUN="$T/run" SQS_CONSOLE="$T/run-console" NATS_BIN="$TMP/bin/pending-nats" \
|
||||
sh -c '. "$1"; sqs_run' sh "$LIB" & pid=$!; CHILDREN="$CHILDREN $pid"
|
||||
i=0
|
||||
while [ "$i" -lt 100 ] && ! grep -q "^$pid " "$T/run/pid" 2>/dev/null; do i=$((i + 1)); sleep 0.05; done
|
||||
check "pending start is held before its exec" 'grep -q "^$pid " "$T/run/pid" && ! gone "$pid" && [ ! -e "$T/started" ]'
|
||||
stop_step
|
||||
# Release the FIFO: a start that survived the stop would exec the server now.
|
||||
cat "$T/run-console" > /dev/null 2>&1 &
|
||||
reader=$!; CHILDREN="$CHILDREN $reader"
|
||||
i=0
|
||||
while [ "$i" -lt 20 ] && ! gone "$pid"; do i=$((i + 1)); sleep 0.05; done
|
||||
check "stop ends a pending start before it can exec the server" 'gone "$pid" && [ ! -e "$T/started" ]'
|
||||
check "stop unmounts only after the pending start is gone" \
|
||||
'grep -q "store exited" "$T/console" && grep -qx "umount $T/data" "$T/calls.log"'
|
||||
kill -KILL "$reader" 2>/dev/null || true
|
||||
|
||||
# --- Shutdown: the generic remount runs only once the store is gone ---
|
||||
# shutdown_step [survivor-pid]: with a pid, that process is reported alive
|
||||
# whatever happens to it, as a writer stuck past SIGKILL would be.
|
||||
shutdown_step() {
|
||||
(
|
||||
SQS_RUN="$T/run" SQS_DATA="$T/data" SQS_CONSOLE="$T/console" SQS_MOUNTS="$T/mounts"
|
||||
UMOUNT="$TMP/bin/umount" SQS_STOP_BUDGET_TENTHS=5
|
||||
# shellcheck source=../../build/microvm/sqs-store/sqs-store-lib.sh
|
||||
. "$LIB"
|
||||
if [ -n "${1:-}" ]; then sqs_same() { [ "$1" = "$survivor" ]; }; survivor=$1; fi
|
||||
sqs_shutdown
|
||||
) > /dev/null 2>&1
|
||||
}
|
||||
|
||||
fresh shutdown-orderly; echo "dev $T/data ext4 rw 0 0" > "$T/mounts"
|
||||
"$STORE" 60 & pid=$!; CHILDREN="$CHILDREN $pid"; record "$pid"
|
||||
check "orderly shutdown succeeds" 'shutdown_step'
|
||||
check "orderly shutdown unmounts the data disk, then runs umount -a -r" \
|
||||
'[ "$(grep "^umount" "$T/calls.log" | tr "\n" "|")" = "umount $T/data|umount -a -r|" ]'
|
||||
|
||||
fresh shutdown-survivor; echo "dev $T/data ext4 rw 0 0" > "$T/mounts"
|
||||
"$STORE" 60 & pid=$!; CHILDREN="$CHILDREN $pid"; record "$pid"
|
||||
check "shutdown with a surviving store reports failure" '! shutdown_step "$pid"'
|
||||
check "a surviving store leaves the data disk and every other mount alone" \
|
||||
'! grep -q "^umount" "$T/calls.log" && grep -q "STILL ALIVE" "$T/console" && grep -q "skipping umount -a -r" "$T/console"'
|
||||
|
||||
fresh stop-unmounted; : > "$T/mounts"
|
||||
stop_step
|
||||
check "stop with no data mount does not unmount" '! grep -q "^umount" "$T/calls.log"'
|
||||
|
||||
if [ "$fails" -ne 0 ]; then
|
||||
echo " $fails failure(s)"
|
||||
exit 1
|
||||
fi
|
||||
@@ -11,13 +11,11 @@ STAGE_DIR="${1:?usage: sqs-store-microvm-publish.sh STAGE_DIR OUT_DIR}"
|
||||
OUT_DIR="${2:?usage: sqs-store-microvm-publish.sh STAGE_DIR OUT_DIR}"
|
||||
VMLINUZ="$STAGE_DIR/vmlinuz"
|
||||
INITRAMFS="$STAGE_DIR/initramfs.cpio.gz"
|
||||
PROFILE="${3:-store}"
|
||||
PROFILE="${3:-worker}"
|
||||
case "$PROFILE" in
|
||||
store)
|
||||
payload="usr/local/bin/nats-server usr/local/lib/sqs-store/sqs-store-lib.sh usr/local/lib/sqs-store/sqs-store-prepare usr/local/lib/sqs-store/sqs-store-run usr/local/lib/sqs-store/sqs-store-stop" ;;
|
||||
worker)
|
||||
payload="usr/local/bin/sqs-worker usr/local/lib/sqs-worker/sqs-worker-lib.sh usr/local/lib/sqs-worker/sqs-worker-prepare usr/local/lib/sqs-worker/sqs-worker-ready usr/local/lib/sqs-worker/sqs-worker-stop data" ;;
|
||||
*) echo "ERROR: profile must be store or worker, not $PROFILE" >&2; exit 1 ;;
|
||||
payload="usr/local/bin/nats-server usr/local/bin/sqs-worker usr/local/lib/sqs-worker/sqs-worker-lib.sh usr/local/lib/sqs-worker/sqs-worker-prepare usr/local/lib/sqs-worker/sqs-worker-store usr/local/lib/sqs-worker/sqs-worker-ready usr/local/lib/sqs-worker/sqs-worker-stop data" ;;
|
||||
*) echo "ERROR: profile must be worker, not $PROFILE" >&2; exit 1 ;;
|
||||
esac
|
||||
tag="[build-sqs-$PROFILE-microvm-image]"
|
||||
|
||||
@@ -56,12 +54,7 @@ for want_ko in ext4 tiny_power_button; do
|
||||
fi
|
||||
done
|
||||
if grep -qE '(^|/)(lb-agent|haproxy|nginx)$' <<< "$archive_list"; then
|
||||
echo "ERROR: LB data-plane binaries found in the SQS store image" >&2
|
||||
exit 1
|
||||
fi
|
||||
# The worker holds the store client key; it must never carry the store server.
|
||||
if [ "$PROFILE" = worker ] && grep -qE '(^|/)nats-server$' <<< "$archive_list"; then
|
||||
echo "ERROR: nats-server found in the SQS worker image" >&2
|
||||
echo "ERROR: LB data-plane binaries found in the SQS image" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "$tag content gate: PASS"
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#!/bin/sh
|
||||
# sqs-store-microvm-publish_test.sh — a staged SQS store microVM pair reaches
|
||||
# sqs-store-microvm-publish_test.sh — a staged SQS microVM pair reaches
|
||||
# the output directory only when every gate passes, and a failed build cannot
|
||||
# leave a pair that make then takes as up to date.
|
||||
#
|
||||
@@ -28,9 +28,10 @@ done
|
||||
stage() {
|
||||
dir=$1 omit=${2:-}
|
||||
root="$TMP/root.$$.$(date +%s%N)"
|
||||
for p in init etc/inittab usr/local/bin/nats-server sbin/mkfs.ext4 sbin/e2fsck \
|
||||
usr/local/lib/sqs-store/sqs-store-lib.sh usr/local/lib/sqs-store/sqs-store-prepare \
|
||||
usr/local/lib/sqs-store/sqs-store-run usr/local/lib/sqs-store/sqs-store-stop \
|
||||
for p in init etc/inittab usr/local/bin/nats-server usr/local/bin/sqs-worker data sbin/mkfs.ext4 sbin/e2fsck \
|
||||
usr/local/lib/sqs-worker/sqs-worker-lib.sh usr/local/lib/sqs-worker/sqs-worker-prepare \
|
||||
usr/local/lib/sqs-worker/sqs-worker-store usr/local/lib/sqs-worker/sqs-worker-ready \
|
||||
usr/local/lib/sqs-worker/sqs-worker-stop \
|
||||
dev/console dev/null lib/modules/6.12-virt/kernel/fs/ext4/ext4.ko \
|
||||
lib/modules/6.12-virt/kernel/drivers/acpi/tiny-power-button.ko; do
|
||||
[ "$p" = "$omit" ] && continue
|
||||
@@ -78,7 +79,7 @@ else
|
||||
fi
|
||||
|
||||
# --- Each failing gate keeps the previous valid pair exactly ---
|
||||
for case in nats-server store-stop ext4 size; do
|
||||
for case in nats-server worker-store ext4 size; do
|
||||
d="$TMP/keep-$case"
|
||||
stage "$d/prev"
|
||||
bash "$PUBLISH" "$d/prev" "$d/out" >/dev/null 2>&1
|
||||
@@ -86,7 +87,7 @@ for case in nats-server store-stop ext4 size; do
|
||||
max=50
|
||||
case $case in
|
||||
nats-server) stage "$d/stage" usr/local/bin/nats-server ;;
|
||||
store-stop) stage "$d/stage" usr/local/lib/sqs-store/sqs-store-stop ;;
|
||||
worker-store) stage "$d/stage" usr/local/lib/sqs-worker/sqs-worker-store ;;
|
||||
ext4) stage "$d/stage" lib/modules/6.12-virt/kernel/fs/ext4/ext4.ko ;;
|
||||
size) stage "$d/stage"; max=0 ;;
|
||||
esac
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
#!/bin/sh
|
||||
# sqs-worker-guest_test.sh — the SQS worker guest's readiness watcher is
|
||||
# supervised by init, never leaves a store-ready marker behind a dead watcher,
|
||||
# and shutdown unmounts the worker disk only once the watcher is gone.
|
||||
# sqs-worker-guest_test.sh — the SQS guest's store starts only on a config
|
||||
# prepared after the receipt key settled, its readiness watcher never leaves a
|
||||
# store-ready marker behind a dead watcher, and shutdown unmounts the data disk
|
||||
# only once both the watcher and the store are gone.
|
||||
#
|
||||
# Drives build/microvm/sqs-worker/sqs-worker-lib.sh with test-owned processes
|
||||
# and a stub umount. Every signal goes only to this test's own children.
|
||||
@@ -12,6 +13,7 @@ set -eu
|
||||
ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/../.." && pwd)
|
||||
LIB=${SQSW_GUEST_LIB:-"$ROOT/build/microvm/sqs-worker/sqs-worker-lib.sh"}
|
||||
INITTAB="$ROOT/build/microvm/sqs-worker/inittab"
|
||||
GUEST_INIT="$ROOT/build/microvm/sqs-worker/init.sh"
|
||||
TMP=$(mktemp -d)
|
||||
CHILDREN=""
|
||||
cleanup() {
|
||||
@@ -41,26 +43,56 @@ echo "$$" >> "$T/watchers"
|
||||
echo "10.0.0.5 UPUB" > "$T/run/store-ready"
|
||||
exec sleep 60
|
||||
EOF
|
||||
# The fake server checks its config, or notes how it was started and stays.
|
||||
cat > "$TMP/bin/nats" <<'EOF'
|
||||
#!/bin/sh
|
||||
if [ "$1" = -t ]; then [ -z "${NATS_BAD:-}" ]; exit; fi
|
||||
echo "$GOMEMLIMIT $*" >> "$T/servers"
|
||||
exec sleep 60
|
||||
EOF
|
||||
# The fake boot takes its exit codes in order from $T/boot-rcs, and like the
|
||||
# real one writes the receipt marker only on success; "hold" blocks like a slow call.
|
||||
cat > "$TMP/bin/boot" <<'EOF'
|
||||
#!/bin/sh
|
||||
echo "$*" >> "$T/boots"
|
||||
rc=$(head -n1 "$T/boot-rcs"); sed -i 1d "$T/boot-rcs"
|
||||
if [ "$rc" = hold ]; then echo "$$" > "$T/boot-child"; exec sleep 60; fi
|
||||
[ "$rc" != 0 ] || echo FP > "$T/run/receipt-ready"
|
||||
exit "$rc"
|
||||
EOF
|
||||
chmod +x "$TMP/bin"/*
|
||||
|
||||
fresh() {
|
||||
T="$TMP/$1"
|
||||
export T
|
||||
mkdir -p "$T/run" "$T/data"
|
||||
echo UPUBLICKEY > "$T/run/public-key"
|
||||
echo FP > "$T/run/receipt-ready"
|
||||
echo "listen: 127.0.0.1:4222" > "$T/run/store.conf"
|
||||
for f in calls.log console mounts; do : > "$T/$f"; done
|
||||
}
|
||||
gone() { [ ! -r "/proc/$1/status" ] || grep -q '^State:[[:space:]]*Z' "/proc/$1/status" 2>/dev/null; }
|
||||
waitfor() { i=0; while [ "$i" -lt 100 ] && ! eval "$1"; do i=$((i + 1)); sleep 0.05; done; eval "$1"; }
|
||||
# start_run: one init respawn of the readiness step, in the background.
|
||||
start_run() {
|
||||
SQSW_RUN="$T/run" SQSW_CONSOLE="$T/console" SQSW_BIN="$TMP/bin/watcher" SQSW_RESPAWN_GAP="${GAP:-0}" SQSW_PARK_INTERVAL="${PARK:-30}" \
|
||||
SQSW_RUN="$T/run" SQSW_CONSOLE="$T/console" SQSW_BIN="$TMP/bin/watcher" SQSW_RESPAWN_GAP="${GAP:-0}" SQSW_PARK="sleep 60" \
|
||||
sh -c '. "$1"; sqsw_run' sh "$LIB" & pid=$!
|
||||
CHILDREN="$CHILDREN $pid"
|
||||
}
|
||||
# start_store: one init respawn of the store step, in the background.
|
||||
start_store() {
|
||||
SQSW_RUN="$T/run" SQSW_DATA="$T/data" SQSW_CONSOLE="$T/console" SQSW_NATS_BIN="$TMP/bin/nats" \
|
||||
SQSW_BIN="$TMP/bin/boot" SQSW_BOOT_RETRY=0 SQSW_PARK="sleep 60" SQSW_LIB="$LIB" \
|
||||
sh -c '. "$1"; sqsw_store' sh "$LIB" & spid=$!
|
||||
CHILDREN="$CHILDREN $spid"
|
||||
}
|
||||
# prepare_store: the sysinit step's store preparation, after boot.
|
||||
prepare_store() {
|
||||
SQSW_RUN="$T/run" SQSW_DATA="$T/data" SQSW_CONSOLE="$T/console" SQSW_NATS_BIN="$TMP/bin/nats" \
|
||||
sh -c '. "$1"; sqsw_prepare_store' sh "$LIB"
|
||||
}
|
||||
# record <pid>: write the identity sqs_run would record for that process.
|
||||
# shellcheck source=../../build/microvm/sqs-worker/sqs-worker-lib.sh
|
||||
record() { echo "$1 $( . "$LIB"; sqsw_starttime "$1" )" > "$T/run/ready.pid"; }
|
||||
record() { echo "$1 $( . "$LIB"; sqsw_starttime "$1" )" > "$T/run/${2:-ready}.pid"; }
|
||||
# shutdown_step [survivor-pid]: with a pid, that process is reported alive
|
||||
# whatever happens to it, as a watcher stuck past SIGKILL would be.
|
||||
shutdown_step() {
|
||||
@@ -74,8 +106,86 @@ shutdown_step() {
|
||||
) > /dev/null 2>&1
|
||||
}
|
||||
|
||||
check "init respawns the readiness step rather than running it once" \
|
||||
'grep -qx "::respawn:/usr/local/lib/sqs-worker/sqs-worker-ready" "$INITTAB" && ! grep -q "::once:" "$INITTAB"'
|
||||
check "init respawns the store and readiness steps rather than running them once" \
|
||||
'grep -qx "::respawn:/usr/local/lib/sqs-worker/sqs-worker-store" "$INITTAB" &&
|
||||
grep -qx "::respawn:/usr/local/lib/sqs-worker/sqs-worker-ready" "$INITTAB" && ! grep -q "::once:" "$INITTAB"'
|
||||
# The store binds 127.0.0.1, which a fresh guest kernel leaves down.
|
||||
lo_at=$(grep -n -x "ip link set lo up" "$GUEST_INIT" | cut -d: -f1 | head -n 1)
|
||||
init_at=$(grep -n -x "exec /sbin/init" "$GUEST_INIT" | cut -d: -f1 | head -n 1)
|
||||
check "guest init brings loopback up before init starts the store" \
|
||||
'[ -n "$lo_at" ] && [ -n "$init_at" ] && [ "$lo_at" -lt "$init_at" ]'
|
||||
|
||||
# --- The store is prepared only after boot settled the receipt key ---
|
||||
fresh prepare-ok; rm -f "$T/run/store.conf"
|
||||
check "a settled receipt key prepares the store" 'prepare_store && [ -s "$T/run/store.conf" ]'
|
||||
check "the store listens only on loopback, with no auth, monitor, route or cluster listener" \
|
||||
'grep -qx "listen: \"127.0.0.1:4222\"" "$T/run/store.conf" && grep -qx " store_dir: \"$T/data/jetstream\"" "$T/run/store.conf" &&
|
||||
! grep -Eq "authorization|http|monitor|cluster|routes|leafnodes|websocket|mqtt" "$T/run/store.conf"'
|
||||
fresh prepare-no-key; rm -f "$T/run/store.conf" "$T/run/receipt-ready"
|
||||
check "a receipt key boot did not settle leaves the store unprepared" '! prepare_store && [ ! -e "$T/run/store.conf" ]'
|
||||
fresh prepare-bad; rm -f "$T/run/store.conf"
|
||||
check "a config the server rejects leaves the store unprepared" '! NATS_BAD=1 prepare_store && [ ! -e "$T/run/store.conf" ]'
|
||||
|
||||
# --- The store step starts the server only on a prepared config ---
|
||||
fresh store-run
|
||||
start_store
|
||||
check "a prepared store starts the dedicated server with its heap bound" \
|
||||
'waitfor "[ -s \"\$T/servers\" ]" && grep -qx "512MiB -c $T/run/store.conf" "$T/servers"'
|
||||
kill -KILL "$spid" 2>/dev/null || true
|
||||
fresh store-refused; : > "$T/run/boot-refused"
|
||||
start_store; waitfor "grep -q 'store: boot refused' \"\$T/console\"" || true
|
||||
check "a store whose boot was refused parks instead of starting or respawning" \
|
||||
'! gone "$spid" && [ ! -e "$T/servers" ] && [ ! -e "$T/boots" ]'
|
||||
kill -KILL "$spid" 2>/dev/null || true
|
||||
|
||||
# --- Boot is supervised: a transient failure is retried, a refusal is final ---
|
||||
fresh boot-transient; rm -f "$T/run/store.conf" "$T/run/receipt-ready"; printf '75\n0\n' > "$T/boot-rcs"
|
||||
start_store
|
||||
check "a transient boot failure exits so init runs the step again, starting nothing" \
|
||||
'waitfor "gone $spid" && [ ! -e "$T/servers" ] && [ ! -e "$T/run/boot-refused" ]'
|
||||
start_store
|
||||
check "the respawn after recovery boots again and starts the store" \
|
||||
'waitfor "[ -s \"\$T/servers\" ]" && [ "$(wc -l < "$T/boots")" -eq 2 ] && [ -s "$T/run/store.conf" ]'
|
||||
kill -KILL "$spid" 2>/dev/null || true
|
||||
fresh boot-final; rm -f "$T/run/store.conf" "$T/run/receipt-ready"; printf '1\n0\n' > "$T/boot-rcs"
|
||||
start_store; waitfor "grep -q 'boot refused (rc=1)' \"\$T/console\"" || true
|
||||
check "a final boot refusal parks the store and records the refusal" \
|
||||
'! gone "$spid" && [ -e "$T/run/boot-refused" ] && [ ! -e "$T/servers" ]'
|
||||
kill -KILL "$spid" 2>/dev/null || true
|
||||
start_store; waitfor "grep -q 'store: boot refused, not starting' \"\$T/console\"" || true
|
||||
check "a later respawn never boots again after a final refusal" '[ "$(wc -l < "$T/boots")" -eq 1 ]'
|
||||
kill -KILL "$spid" 2>/dev/null || true
|
||||
fresh boot-settled; rm -f "$T/run/store.conf"; : > "$T/boot-rcs"
|
||||
start_store
|
||||
check "a respawn after the key settled starts the store without booting again" \
|
||||
'waitfor "[ -s \"\$T/servers\" ]" && [ ! -e "$T/boots" ]'
|
||||
kill -KILL "$spid" 2>/dev/null || true
|
||||
fresh boot-held; rm -f "$T/run/store.conf" "$T/run/receipt-ready"; echo hold > "$T/boot-rcs"
|
||||
echo "/dev/vdb $T/data ext4 rw 0 0" > "$T/mounts"
|
||||
start_store; waitfor '[ -s "$T/boot-child" ]' || true
|
||||
bchild=$(cat "$T/boot-child" 2>/dev/null); CHILDREN="$CHILDREN $bchild"
|
||||
shutdown_step
|
||||
check "shutdown stops a boot still in progress before it unmounts the disk" \
|
||||
'waitfor "gone $bchild" && grep -q "umount $T/data" "$T/calls.log"'
|
||||
kill -KILL "$spid" "$bchild" 2>/dev/null || true
|
||||
fresh boot-held-survivor; rm -f "$T/run/store.conf" "$T/run/receipt-ready"; echo hold > "$T/boot-rcs"
|
||||
echo "/dev/vdb $T/data ext4 rw 0 0" > "$T/mounts"
|
||||
start_store; waitfor '[ -s "$T/boot-child" ]' || true
|
||||
bchild=$(cat "$T/boot-child" 2>/dev/null); CHILDREN="$CHILDREN $bchild"
|
||||
check "a boot that survives SIGKILL fails shutdown and keeps the disk mounted" \
|
||||
'! shutdown_step "$bchild" && ! grep -q umount "$T/calls.log" && grep -q "boot pid=$bchild STILL ALIVE" "$T/console"'
|
||||
kill -KILL "$spid" "$bchild" 2>/dev/null || true
|
||||
# Shutdown sets stopping after the store's own check but before the boot child
|
||||
# runs: the child must still record itself and never start sqs-worker boot.
|
||||
fresh boot-window; rm -f "$T/run/store.conf" "$T/run/receipt-ready"; echo 0 > "$T/boot-rcs"
|
||||
: > "$T/run/stopping"
|
||||
SQSW_RUN="$T/run" SQSW_CONSOLE="$T/console" SQSW_BIN="$TMP/bin/boot" SQSW_BOOT_RETRY=0 SQSW_PARK="sleep 60" SQSW_LIB="$LIB" \
|
||||
sh -c '. "$1"; sqsw_boot' sh "$LIB" & wpid=$!
|
||||
CHILDREN="$CHILDREN $wpid"
|
||||
waitfor "grep -q 'shutdown during boot' \"\$T/console\"" || true
|
||||
check "a boot child started after shutdown began never runs boot" \
|
||||
'grep -q "shutdown during boot" "$T/console" && [ ! -e "$T/boots" ] && [ ! -e "$T/run/boot-refused" ]'
|
||||
kill -KILL "$wpid" 2>/dev/null || true
|
||||
|
||||
# --- A watcher killed after readiness: its marker must not outlive it ---
|
||||
fresh killed-after-ready
|
||||
@@ -103,8 +213,7 @@ kill -KILL "$pid" 2>/dev/null || true
|
||||
# A run that cannot start parks rather than exiting, so init does not respawn
|
||||
# it in a loop; a second is many park intervals, and it must log only once.
|
||||
fresh run-stopping; : > "$T/run/stopping"; echo stale > "$T/run/store-ready"
|
||||
PARK=0.05 start_run; waitfor "grep -q 'shutdown in progress' \"\$T/console\"" || true
|
||||
sleep 1
|
||||
start_run; waitfor "grep -q 'shutdown in progress' \"\$T/console\"" || true
|
||||
check "a start during shutdown parks instead of exiting and respawning" '! gone "$pid"'
|
||||
check "a parked start during shutdown starts no watcher and removes the marker" \
|
||||
'[ ! -e "$T/watchers" ] && [ ! -e "$T/run/store-ready" ]'
|
||||
@@ -112,26 +221,24 @@ check "a parked start during shutdown logs once" '[ "$(grep -c "shutdown in prog
|
||||
shutdown_step
|
||||
check "shutdown stops the parked start" 'waitfor "gone $pid"'
|
||||
|
||||
fresh run-no-key; rm -f "$T/run/public-key"
|
||||
PARK=0.05 start_run; waitfor "grep -q 'no worker key' \"\$T/console\"" || true
|
||||
sleep 1
|
||||
check "a guest without a worker key parks instead of exiting and respawning" '! gone "$pid" && [ ! -e "$T/watchers" ]'
|
||||
check "a parked guest without a worker key logs once" '[ "$(grep -c "no worker key" "$T/console")" -eq 1 ]'
|
||||
echo UPUBLICKEY > "$T/run/public-key"
|
||||
check "a key that appears later starts the watcher" 'waitfor "[ -s \"\$T/watchers\" ]"'
|
||||
kill -KILL "$pid" 2>/dev/null || true
|
||||
|
||||
fresh run-no-key-stop; rm -f "$T/run/public-key"
|
||||
PARK=0.05 start_run; waitfor "grep -q 'no worker key' \"\$T/console\"" || true
|
||||
fresh run-unprepared; rm -f "$T/run/store.conf"
|
||||
start_run
|
||||
check "a watcher whose store is not prepared yet exits to be respawned and never reports ready" \
|
||||
'waitfor "gone $pid" && [ ! -e "$T/watchers" ] && [ ! -e "$T/run/store-ready" ]'
|
||||
fresh run-refused; rm -f "$T/run/store.conf"; : > "$T/run/boot-refused"
|
||||
start_run; waitfor "grep -q 'ready: boot refused' \"\$T/console\"" || true
|
||||
check "a guest whose boot was refused parks its watcher and never reports ready" \
|
||||
'! gone "$pid" && [ ! -e "$T/watchers" ] && [ ! -e "$T/run/store-ready" ]'
|
||||
shutdown_step
|
||||
check "shutdown stops a start parked for a missing key" 'waitfor "gone $pid" && [ ! -e "$T/watchers" ]'
|
||||
check "shutdown stops a watcher parked after a refused boot" 'waitfor "gone $pid" && [ ! -e "$T/watchers" ]'
|
||||
|
||||
# --- Shutdown: unmount only once the watcher is confirmed gone ---
|
||||
# --- Shutdown: unmount only once the watcher and the store are gone ---
|
||||
fresh shutdown-orderly; echo "dev $T/data ext4 rw 0 0" > "$T/mounts"
|
||||
start_run; waitfor "[ -s \"\$T/run/store-ready\" ]"
|
||||
start_store; start_run; waitfor "[ -s \"\$T/run/store-ready\" ]"; waitfor "[ -s \"\$T/servers\" ]"
|
||||
check "orderly shutdown succeeds" 'shutdown_step'
|
||||
check "orderly shutdown confirms the watcher exited, removes the marker, then unmounts" \
|
||||
'gone "$pid" && grep -q "readiness watcher pid=$pid exited" "$T/console" && [ ! -e "$T/run/store-ready" ] &&
|
||||
check "orderly shutdown stops the watcher, then the store, removes the marker, then unmounts" \
|
||||
'gone "$pid" && gone "$spid" && [ ! -e "$T/run/store-ready" ] &&
|
||||
[ "$(grep "exited" "$T/console" | sed "s/.*stop: \([a-z]*\) pid.*/\1/" | tr "\n" "|")" = "ready|store|" ] &&
|
||||
[ "$(grep "^umount" "$T/calls.log" | tr "\n" "|")" = "umount $T/data|umount -a -r|" ]'
|
||||
check "shutdown blocks any later respawn" '[ -e "$T/run/stopping" ]'
|
||||
|
||||
@@ -151,10 +258,16 @@ check "a surviving watcher leaves every mount alone and is never reported stoppe
|
||||
check "a surviving watcher still leaves no readiness marker" '[ ! -e "$T/run/store-ready" ]'
|
||||
kill -KILL "$pid" 2>/dev/null || true
|
||||
|
||||
fresh shutdown-store-survivor; echo "dev $T/data ext4 rw 0 0" > "$T/mounts"
|
||||
(trap '' TERM; exec sleep 60) & pid=$!; CHILDREN="$CHILDREN $pid"; record "$pid" store
|
||||
check "shutdown with a surviving store reports failure and leaves every mount alone" \
|
||||
'! shutdown_step "$pid" && ! grep -q "^umount" "$T/calls.log" && grep -q "store pid=$pid STILL ALIVE" "$T/console"'
|
||||
kill -KILL "$pid" 2>/dev/null || true
|
||||
|
||||
fresh shutdown-reused; echo "dev $T/data ext4 rw 0 0" > "$T/mounts"
|
||||
(exec sleep 60) & pid=$!; CHILDREN="$CHILDREN $pid"; echo "$pid 1" > "$T/run/ready.pid"
|
||||
shutdown_step
|
||||
check "a recorded PID with another start time is never signalled" '! gone "$pid" && grep -q "no running readiness watcher" "$T/console"'
|
||||
check "a recorded PID with another start time is never signalled" '! gone "$pid" && grep -q "no running ready" "$T/console"'
|
||||
kill -KILL "$pid" 2>/dev/null || true
|
||||
|
||||
if [ "$fails" -ne 0 ]; then
|
||||
|
||||
@@ -1,11 +1,10 @@
|
||||
#!/bin/sh
|
||||
# stage-sqs-store-microvm.sh <image-dir> <dist-root> [sqs-store|sqs-worker]:
|
||||
# copy a built SQS microVM image into a release tree as <dist-root>/microvm-<name>.
|
||||
# stage-sqs-store-microvm.sh <image-dir> <dist-root> [sqs-worker]: copy the
|
||||
# built SQS microVM image into a release tree as <dist-root>/microvm-sqs-worker.
|
||||
# Without a complete image it stages nothing and says so, as the LB image does.
|
||||
set -eu
|
||||
src=$1 name=${3:-sqs-store}
|
||||
src=$1 name=${3:-sqs-worker}
|
||||
case "$name" in
|
||||
sqs-store) label="SQS store" ;;
|
||||
sqs-worker) label="SQS worker" ;;
|
||||
*) echo "[distro] unknown SQS microVM image '$name'" >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
@@ -100,7 +100,8 @@ profile_binaries() {
|
||||
./cmd/eks-gateway-fetch:usr/local/bin/eks-gateway-fetch \
|
||||
./cmd/eks-webhook-cert:usr/local/bin/eks-webhook-cert \
|
||||
./cmd/eks-konnectivity-cert:usr/local/bin/eks-konnectivity-cert \
|
||||
./cmd/eks-etcd-member:usr/local/bin/eks-etcd-member"
|
||||
./cmd/eks-etcd-member:usr/local/bin/eks-etcd-member \
|
||||
./cmd/eks-pod-identity-webhook:usr/local/bin/eks-pod-identity-webhook"
|
||||
;;
|
||||
ecs) echo "./cmd/ecs-agent:usr/local/bin/ecs-agent" ;;
|
||||
*) echo "" ;;
|
||||
|
||||
+27
-13
@@ -732,7 +732,8 @@ table inet spinifex_filter {
|
||||
# inventory and running instances to anyone who asks. 8660 is
|
||||
# predastore's unauthenticated /healthz and /readyz, which name the
|
||||
# peers a gate cannot reach, so it belongs here for the same reason.
|
||||
ip saddr $spinifex_peers tcp dport { 4222, 4248, 4432, 5300, 6641, 6642, 6643, 6644, 8660 } accept
|
||||
# 5001 is the admin network test, listening only while one runs.
|
||||
ip saddr $spinifex_peers tcp dport { 4222, 4248, 4432, 5001, 5300, 6641, 6642, 6643, 6644, 8660 } accept
|
||||
ip saddr $spinifex_peers udp dport { 5300, 6660, 7660 } accept
|
||||
|
||||
# Encap plane, peer-scoped: Geneve, IKE, NAT-T and ESP. Geneve is a
|
||||
@@ -1309,6 +1310,7 @@ qemu-utils gdisk ovmf qemu-efi-aarch64 less
|
||||
libvirt-daemon-system libvirt-clients
|
||||
pciutils
|
||||
jq curl iproute2 iputils-arping ethtool netcat-openbsd wget unzip xz-utils file
|
||||
dmidecode iperf
|
||||
ovn-central ovn-host openvswitch-switch openvswitch-ipsec strongswan-charon dhcpcd-base
|
||||
chrony nftables"
|
||||
|
||||
@@ -1318,7 +1320,7 @@ chrony nftables"
|
||||
OL_RUNTIME_PACKAGES="nbdkit nbdkit-basic-filters
|
||||
qemu-kvm qemu-img edk2-ovmf seabios-bin less
|
||||
pciutils
|
||||
jq curl iproute iputils ethtool nmap-ncat wget unzip xz file
|
||||
jq curl iproute iputils ethtool nmap-ncat wget unzip xz file dmidecode
|
||||
ovn25.03-central ovn25.03-host openvswitch3.5 openvswitch3.5-ipsec libreswan dhcpcd
|
||||
chrony nftables iptables-nft systemd-resolved"
|
||||
|
||||
@@ -1585,6 +1587,15 @@ download_spinifex() {
|
||||
EXTRACT_DIR="$SPINIFEX_TMPDIR"
|
||||
}
|
||||
|
||||
# install.mulgadc.com serves the latest release's setup.sh, which cannot know what a
|
||||
# newer --channel or --version tarball needs installed, so it runs the tarball's own.
|
||||
hand_off_to_packaged_setup() {
|
||||
[ -z "${INSTALL_SPINIFEX_TARBALL:-}" ] && [ -f "$EXTRACT_DIR/setup.sh" ] || return 0
|
||||
info "Running the setup.sh shipped in the downloaded release"
|
||||
export INSTALL_SPINIFEX_TARBALL="$TARBALL" SPINIFEX_HANDOFF_TMPDIR="$SPINIFEX_TMPDIR"
|
||||
exec bash "$EXTRACT_DIR/setup.sh" "$@"
|
||||
}
|
||||
|
||||
# OL's sudo secure_path leaves out /usr/local/bin, so `sudo spx` would find
|
||||
# nothing there. A link in /usr/bin changes sudo for nothing else on the host.
|
||||
link_spx_for_sudo() {
|
||||
@@ -1679,13 +1690,12 @@ install_files() {
|
||||
$SUDO cp "$EXTRACT_DIR/microvm/"* /usr/share/spinifex/microvm/
|
||||
info " /usr/share/spinifex/microvm/*"
|
||||
fi
|
||||
install_sqs_store_image
|
||||
install_sqs_worker_image
|
||||
}
|
||||
|
||||
# The SQS images ship beside the load balancer's, not inside it: an older
|
||||
# installer copies microvm/* without -r and would stop at a subdirectory.
|
||||
install_sqs_store_image() { install_sqs_microvm_image sqs-store "SQS store"; }
|
||||
# The SQS image ships beside the load balancer's, not inside it: an older
|
||||
# installer copies microvm/* without -r and would stop at a subdirectory. A
|
||||
# store image left by an earlier install is not touched: nothing boots it.
|
||||
install_sqs_worker_image() { install_sqs_microvm_image sqs-worker "SQS worker"; }
|
||||
|
||||
# install_sqs_microvm_image <name> <label>
|
||||
@@ -2233,6 +2243,7 @@ EOF
|
||||
|
||||
# --- Main ---
|
||||
main() {
|
||||
local args=("$@")
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--firewall=*) INSTALL_SPINIFEX_FIREWALL="${1#*=}" ;;
|
||||
@@ -2262,18 +2273,20 @@ main() {
|
||||
detect_os
|
||||
detect_arch
|
||||
|
||||
# Orchestration (stop running services, fix stale /run/spinifex/nbd perms)
|
||||
# only applies in full-install mode on a live host.
|
||||
if [ "${ISO_BUILD:-0}" != "1" ] && [ -z "${SETUP_STAGES:-}" ]; then
|
||||
handle_upgrade
|
||||
fi
|
||||
|
||||
# Stages that need EXTRACT_DIR: files, directories, systemd, logrotate, udev.
|
||||
# Only download/extract when at least one such stage is enabled.
|
||||
# Only download/extract when at least one such stage is enabled, and before
|
||||
# anything on the host changes, so the hand-off finds it untouched.
|
||||
if stage_enabled files || stage_enabled directories \
|
||||
|| stage_enabled systemd || stage_enabled logrotate \
|
||||
|| stage_enabled udev; then
|
||||
download_spinifex
|
||||
hand_off_to_packaged_setup "${args[@]}"
|
||||
fi
|
||||
|
||||
# Orchestration (stop running services, fix stale /run/spinifex/nbd perms)
|
||||
# only applies in full-install mode on a live host.
|
||||
if [ "${ISO_BUILD:-0}" != "1" ] && [ -z "${SETUP_STAGES:-}" ]; then
|
||||
handle_upgrade
|
||||
fi
|
||||
|
||||
stage_enabled deps && install_deps
|
||||
@@ -2303,6 +2316,7 @@ main() {
|
||||
fi
|
||||
|
||||
[ -n "${SPINIFEX_TMPDIR:-}" ] && rm -rf "$SPINIFEX_TMPDIR"
|
||||
[ -n "${SPINIFEX_HANDOFF_TMPDIR:-}" ] && rm -rf "$SPINIFEX_HANDOFF_TMPDIR"
|
||||
|
||||
# Post-install orchestration (service restart, summary, newgrp) only in
|
||||
# full-install mode on a live host.
|
||||
|
||||
Executable
+88
@@ -0,0 +1,88 @@
|
||||
#!/bin/sh
|
||||
# setup_handoff_test.sh — tests that setup.sh runs the setup.sh shipped in the
|
||||
# tarball it downloaded, before it changes anything on the host.
|
||||
#
|
||||
# setup.sh is sourced with INSTALL_SPINIFEX_LIB_ONLY=1 and main runs with the
|
||||
# host-touching functions stubbed; the packaged setup.sh is a fixture that
|
||||
# prints what it was given.
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
|
||||
SETUP="$SCRIPT_DIR/setup.sh"
|
||||
TMP=$(mktemp -d)
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
|
||||
fails=0
|
||||
pass() { echo " ok - $1"; }
|
||||
fail() { echo " FAIL - $1"; fails=$((fails + 1)); }
|
||||
|
||||
echo "-- setup.sh hand-off to the packaged setup.sh"
|
||||
|
||||
mkdir -p "$TMP/extract"
|
||||
cat > "$TMP/packaged-setup.sh" <<'EOF'
|
||||
echo "PACKAGED args=[$*] tarball=${INSTALL_SPINIFEX_TARBALL:-} handoff=${SPINIFEX_HANDOFF_TMPDIR:-}"
|
||||
EOF
|
||||
|
||||
# run <packaged: yes|no> [main args...] — runs main with every step that would
|
||||
# touch the host stubbed to print its name, and sets OUT.
|
||||
run() {
|
||||
packaged="$1"
|
||||
shift
|
||||
rm -f "$TMP/extract/setup.sh"
|
||||
[ "$packaged" = yes ] && cp "$TMP/packaged-setup.sh" "$TMP/extract/setup.sh"
|
||||
OUT=$(INSTALL_SPINIFEX_LIB_ONLY=1 TMPROOT="$TMP" bash -c '
|
||||
. "$1"; shift
|
||||
info() { :; }
|
||||
setup_sudo() { :; }
|
||||
resolve_firewall_mode() { :; }
|
||||
detect_os() { :; }
|
||||
detect_arch() { :; }
|
||||
download_spinifex() {
|
||||
SPINIFEX_TMPDIR="$TMPROOT/extract"
|
||||
TARBALL="$SPINIFEX_TMPDIR/spinifex.tar.gz"
|
||||
EXTRACT_DIR="$SPINIFEX_TMPDIR"
|
||||
}
|
||||
handle_upgrade() { echo HANDLE_UPGRADE; exit 0; }
|
||||
main "$@"
|
||||
' _ "$SETUP" "$@" 2>&1) || true
|
||||
OUT=$(printf '%s\n' "$OUT" | sed '/^$/d')
|
||||
}
|
||||
|
||||
run yes --channel dev --firewall off
|
||||
want="PACKAGED args=[--channel dev --firewall off] tarball=$TMP/extract/spinifex.tar.gz handoff=$TMP/extract"
|
||||
if [ "$OUT" = "$want" ]; then
|
||||
pass "a downloaded tarball's setup.sh runs with the same arguments, before services are stopped"
|
||||
else
|
||||
fail "downloaded tarball: got [$OUT], want [$want]"
|
||||
fi
|
||||
|
||||
run no --channel dev
|
||||
if [ "$OUT" = HANDLE_UPGRADE ]; then
|
||||
pass "a tarball without setup.sh is installed by the running script"
|
||||
else
|
||||
fail "no packaged setup.sh: got [$OUT]"
|
||||
fi
|
||||
|
||||
run_local() {
|
||||
rm -f "$TMP/extract/setup.sh"
|
||||
cp "$TMP/packaged-setup.sh" "$TMP/extract/setup.sh"
|
||||
OUT=$(INSTALL_SPINIFEX_LIB_ONLY=1 INSTALL_SPINIFEX_TARBALL=/somewhere/spinifex.tar.gz bash -c '
|
||||
. "$1"
|
||||
info() { :; }
|
||||
EXTRACT_DIR="$2"; TARBALL="$2/spinifex.tar.gz"; SPINIFEX_TMPDIR="$2"
|
||||
hand_off_to_packaged_setup --channel dev
|
||||
echo STAYED
|
||||
' _ "$SETUP" "$TMP/extract" 2>&1) || true
|
||||
}
|
||||
run_local
|
||||
if [ "$OUT" = STAYED ]; then
|
||||
pass "a local tarball, including the one a hand-off passes on, is not handed off again"
|
||||
else
|
||||
fail "local tarball: got [$OUT]"
|
||||
fi
|
||||
|
||||
if [ "$fails" -gt 0 ]; then
|
||||
echo "setup_handoff_test.sh: $fails failure(s)"
|
||||
exit 1
|
||||
fi
|
||||
echo "setup_handoff_test.sh: ok"
|
||||
Regular → Executable
+48
-88
@@ -1,6 +1,6 @@
|
||||
#!/bin/sh
|
||||
# setup_sqs_store_image_test.sh — tests for shipping the SQS store and worker
|
||||
# microVM images: staging them into a release tree and installing with setup.sh.
|
||||
# setup_sqs_worker_image_test.sh — tests for shipping the SQS microVM
|
||||
# image: staging it into a release tree and installing with setup.sh.
|
||||
#
|
||||
# setup.sh is sourced with INSTALL_SPINIFEX_LIB_ONLY=1 so main never runs, and
|
||||
# installs into a temporary SPINIFEX_MICROVM_DIR with SUDO cleared.
|
||||
@@ -16,7 +16,7 @@ fails=0
|
||||
pass() { echo " ok - $1"; }
|
||||
fail() { echo " FAIL - $1"; fails=$((fails + 1)); }
|
||||
|
||||
echo "-- SQS store and worker microVM image release and install"
|
||||
echo "-- SQS microVM image release and install"
|
||||
|
||||
# image <dir> <tag>: a fake built image whose bytes name its tag.
|
||||
image() {
|
||||
@@ -35,11 +35,11 @@ release() {
|
||||
}
|
||||
|
||||
# install_image <extract-dir> <microvm-dir> [path-prefix]: runs
|
||||
# install_sqs_store_image and sets RC and OUT.
|
||||
# install_sqs_worker_image and sets RC and OUT.
|
||||
install_image() {
|
||||
RC=0
|
||||
OUT=$(PATH="${3:-}$PATH" EXTRACT_DIR="$1" SPINIFEX_MICROVM_DIR="$2" INSTALL_SPINIFEX_LIB_ONLY=1 \
|
||||
bash -c '. "$1"; SUDO=""; install_sqs_store_image' _ "$SETUP" 2>&1) || RC=$?
|
||||
bash -c '. "$1"; SUDO=""; install_sqs_worker_image' _ "$SETUP" 2>&1) || RC=$?
|
||||
}
|
||||
|
||||
same() { cmp -s "$1" "$2"; }
|
||||
@@ -48,13 +48,13 @@ same() { cmp -s "$1" "$2"; }
|
||||
image "$TMP/built" v1
|
||||
release "$TMP/rel"
|
||||
sh "$STAGE" "$TMP/built" "$TMP/rel" > "$TMP/stage.out"
|
||||
if same "$TMP/built/vmlinuz" "$TMP/rel/microvm-sqs-store/vmlinuz" \
|
||||
&& same "$TMP/built/initramfs.cpio.gz" "$TMP/rel/microvm-sqs-store/initramfs.cpio.gz"; then
|
||||
if same "$TMP/built/vmlinuz" "$TMP/rel/microvm-sqs-worker/vmlinuz" \
|
||||
&& same "$TMP/built/initramfs.cpio.gz" "$TMP/rel/microvm-sqs-worker/initramfs.cpio.gz"; then
|
||||
pass "stages the built image beside the load balancer's"
|
||||
else
|
||||
fail "staged image differs from the build"
|
||||
fi
|
||||
if [ ! -e "$TMP/rel/microvm/sqs-store" ] && [ "$(ls "$TMP/rel/microvm" | wc -l)" -eq 3 ]; then
|
||||
if [ ! -e "$TMP/rel/microvm/sqs-worker" ] && [ "$(ls "$TMP/rel/microvm" | wc -l)" -eq 3 ]; then
|
||||
pass "leaves the load balancer's microvm directory as it was"
|
||||
else
|
||||
fail "load balancer directory changed: $(ls "$TMP/rel/microvm")"
|
||||
@@ -63,21 +63,21 @@ fi
|
||||
release "$TMP/rel-none"
|
||||
mkdir -p "$TMP/half"; printf 'k\n' > "$TMP/half/vmlinuz"
|
||||
OUT=$(sh "$STAGE" "$TMP/half" "$TMP/rel-none")
|
||||
if [ ! -e "$TMP/rel-none/microvm-sqs-store" ] && echo "$OUT" | grep -q 'WARNING: SQS store microVM image missing'; then
|
||||
if [ ! -e "$TMP/rel-none/microvm-sqs-worker" ] && echo "$OUT" | grep -q 'WARNING: SQS worker microVM image missing'; then
|
||||
pass "stages nothing from an incomplete build, and says so"
|
||||
else
|
||||
fail "incomplete build staged or unannounced: $OUT"
|
||||
fi
|
||||
|
||||
# --- an older installer meets a release that carries the store image ---
|
||||
# --- an older installer meets a release that carries the SQS image ---
|
||||
# It copies microvm/* without -r; the sibling directory must not reach it.
|
||||
mkdir -p "$TMP/old-dest"
|
||||
if cp "$TMP/rel/microvm/"* "$TMP/old-dest/" 2> "$TMP/old.err" && [ ! -e "$TMP/old-dest/sqs-store" ]; then
|
||||
pass "an older installer's copy still succeeds and ignores the store image"
|
||||
if cp "$TMP/rel/microvm/"* "$TMP/old-dest/" 2> "$TMP/old.err" && [ ! -e "$TMP/old-dest/sqs-worker" ]; then
|
||||
pass "an older installer's copy still succeeds and ignores the SQS image"
|
||||
else
|
||||
fail "older installer copy failed: $(cat "$TMP/old.err")"
|
||||
fi
|
||||
mkdir -p "$TMP/nested/microvm/sqs-store" "$TMP/nested-dest"
|
||||
mkdir -p "$TMP/nested/microvm/sqs-worker" "$TMP/nested-dest"
|
||||
printf 'x\n' > "$TMP/nested/microvm/vmlinuz"
|
||||
if cp "$TMP/nested/microvm/"* "$TMP/nested-dest/" 2> /dev/null; then
|
||||
fail "control: a nested directory did not break the older copy, so the test proves nothing"
|
||||
@@ -87,58 +87,58 @@ fi
|
||||
|
||||
# --- install ---
|
||||
install_image "$TMP/rel" "$TMP/mv"
|
||||
if [ "$RC" = 0 ] && same "$TMP/built/vmlinuz" "$TMP/mv/sqs-store/vmlinuz" \
|
||||
&& same "$TMP/built/initramfs.cpio.gz" "$TMP/mv/sqs-store/initramfs.cpio.gz"; then
|
||||
if [ "$RC" = 0 ] && same "$TMP/built/vmlinuz" "$TMP/mv/sqs-worker/vmlinuz" \
|
||||
&& same "$TMP/built/initramfs.cpio.gz" "$TMP/mv/sqs-worker/initramfs.cpio.gz"; then
|
||||
pass "installs the staged image where the daemon looks"
|
||||
else
|
||||
fail "install rc=$RC: $OUT"
|
||||
fi
|
||||
mode=$(stat -c '%a' "$TMP/mv/sqs-store/initramfs.cpio.gz" 2> /dev/null || echo none)
|
||||
mode=$(stat -c '%a' "$TMP/mv/sqs-worker/initramfs.cpio.gz" 2> /dev/null || echo none)
|
||||
if [ "$mode" = 644 ]; then pass "installs the image 0644"; else fail "initramfs mode $mode"; fi
|
||||
|
||||
image "$TMP/built2" v2
|
||||
release "$TMP/rel2"
|
||||
sh "$STAGE" "$TMP/built2" "$TMP/rel2" > /dev/null
|
||||
install_image "$TMP/rel2" "$TMP/mv"
|
||||
if [ "$RC" = 0 ] && same "$TMP/built2/vmlinuz" "$TMP/mv/sqs-store/vmlinuz" \
|
||||
&& same "$TMP/built2/initramfs.cpio.gz" "$TMP/mv/sqs-store/initramfs.cpio.gz"; then
|
||||
if [ "$RC" = 0 ] && same "$TMP/built2/vmlinuz" "$TMP/mv/sqs-worker/vmlinuz" \
|
||||
&& same "$TMP/built2/initramfs.cpio.gz" "$TMP/mv/sqs-worker/initramfs.cpio.gz"; then
|
||||
pass "an upgrade replaces both files"
|
||||
else
|
||||
fail "upgrade rc=$RC: $OUT"
|
||||
fi
|
||||
|
||||
# A release without the store image: nothing installed, and a clear warning.
|
||||
# A release without the SQS image: nothing installed, and a clear warning.
|
||||
install_image "$TMP/rel-none" "$TMP/mv-none"
|
||||
if [ "$RC" = 0 ] && [ ! -e "$TMP/mv-none/sqs-store" ] && echo "$OUT" | grep -q 'no SQS store microVM image; SQS store launches will be refused'; then
|
||||
if [ "$RC" = 0 ] && [ ! -e "$TMP/mv-none/sqs-worker" ] && echo "$OUT" | grep -q 'no SQS worker microVM image; SQS worker launches will be refused'; then
|
||||
pass "a release without the image installs nothing and warns"
|
||||
else
|
||||
fail "release without image: rc=$RC exists=$([ -e "$TMP/mv-none/sqs-store" ] && echo yes || echo no) $OUT"
|
||||
fail "release without image: rc=$RC exists=$([ -e "$TMP/mv-none/sqs-worker" ] && echo yes || echo no) $OUT"
|
||||
fi
|
||||
|
||||
# installed <dir>: a store image from an earlier install, plus an unrelated file.
|
||||
# installed <dir>: an SQS image from an earlier install, plus an unrelated file.
|
||||
installed() {
|
||||
image "$1/sqs-store" old
|
||||
printf 'keep\n' > "$1/sqs-store/operator-note"
|
||||
image "$1/sqs-worker" old
|
||||
printf 'keep\n' > "$1/sqs-worker/operator-note"
|
||||
}
|
||||
gone() { [ ! -e "$1/sqs-store/vmlinuz" ] && [ ! -e "$1/sqs-store/initramfs.cpio.gz" ]; }
|
||||
kept() { [ "$(cat "$1/sqs-store/operator-note" 2> /dev/null)" = keep ]; }
|
||||
gone() { [ ! -e "$1/sqs-worker/vmlinuz" ] && [ ! -e "$1/sqs-worker/initramfs.cpio.gz" ]; }
|
||||
kept() { [ "$(cat "$1/sqs-worker/operator-note" 2> /dev/null)" = keep ]; }
|
||||
|
||||
# Upgrading to a release without the image must not leave the old one usable.
|
||||
installed "$TMP/mv-up"
|
||||
install_image "$TMP/rel-none" "$TMP/mv-up"
|
||||
if [ "$RC" = 0 ] && gone "$TMP/mv-up" && kept "$TMP/mv-up" \
|
||||
&& echo "$OUT" | grep -q 'Removed the SQS store microVM image left by an earlier install' \
|
||||
&& echo "$OUT" | grep -q 'SQS store launches will be refused'; then
|
||||
&& echo "$OUT" | grep -q 'Removed the SQS worker microVM image left by an earlier install' \
|
||||
&& echo "$OUT" | grep -q 'SQS worker launches will be refused'; then
|
||||
pass "a release without the image removes the earlier one, keeps other files, and warns"
|
||||
else
|
||||
fail "release without image over an installed one: rc=$RC $OUT"
|
||||
fi
|
||||
|
||||
# An incomplete store image refuses and removes the earlier pair, because the
|
||||
# An incomplete SQS image refuses and removes the earlier pair, because the
|
||||
# binaries this release installed before it would otherwise boot the old one.
|
||||
installed "$TMP/mv-bad"
|
||||
cp -r "$TMP/rel2" "$TMP/rel-bad"
|
||||
rm "$TMP/rel-bad/microvm-sqs-store/initramfs.cpio.gz"
|
||||
rm "$TMP/rel-bad/microvm-sqs-worker/initramfs.cpio.gz"
|
||||
install_image "$TMP/rel-bad" "$TMP/mv-bad"
|
||||
if [ "$RC" != 0 ] && echo "$OUT" | grep -q 'incomplete: initramfs.cpio.gz is missing' \
|
||||
&& gone "$TMP/mv-bad" && kept "$TMP/mv-bad"; then
|
||||
@@ -154,17 +154,17 @@ for case in missing incomplete; do
|
||||
sh "$STAGE" "$TMP/built" "$TMP/rel-$case" > /dev/null
|
||||
if [ "$case" = missing ]; then rm -r "$TMP/built-$case"; else rm "$TMP/built-$case/initramfs.cpio.gz"; fi
|
||||
OUT=$(sh "$STAGE" "$TMP/built-$case" "$TMP/rel-$case")
|
||||
if [ ! -e "$TMP/rel-$case/microvm-sqs-store" ] && echo "$OUT" | grep -q 'tarball will not include it'; then
|
||||
if [ ! -e "$TMP/rel-$case/microvm-sqs-worker" ] && echo "$OUT" | grep -q 'tarball will not include it'; then
|
||||
pass "restaging from a $case build ships no earlier image"
|
||||
else
|
||||
fail "restaging from a $case build left: $(ls -R "$TMP/rel-$case/microvm-sqs-store" 2> /dev/null)"
|
||||
fail "restaging from a $case build left: $(ls -R "$TMP/rel-$case/microvm-sqs-worker" 2> /dev/null)"
|
||||
fi
|
||||
done
|
||||
mkdir -p "$TMP/rel-other/microvm-sqs-store"
|
||||
image "$TMP/rel-other/microvm-sqs-store" old
|
||||
printf 'keep\n' > "$TMP/rel-other/microvm-sqs-store/note"
|
||||
mkdir -p "$TMP/rel-other/microvm-sqs-worker"
|
||||
image "$TMP/rel-other/microvm-sqs-worker" old
|
||||
printf 'keep\n' > "$TMP/rel-other/microvm-sqs-worker/note"
|
||||
sh "$STAGE" "$TMP/no-such-build" "$TMP/rel-other" > /dev/null
|
||||
if [ ! -e "$TMP/rel-other/microvm-sqs-store/vmlinuz" ] && [ "$(cat "$TMP/rel-other/microvm-sqs-store/note")" = keep ]; then
|
||||
if [ ! -e "$TMP/rel-other/microvm-sqs-worker/vmlinuz" ] && [ "$(cat "$TMP/rel-other/microvm-sqs-worker/note")" = keep ]; then
|
||||
pass "restaging removes only the image files"
|
||||
else
|
||||
fail "restaging touched more than the image files"
|
||||
@@ -180,65 +180,25 @@ exec /usr/bin/install "$@"
|
||||
EOF
|
||||
chmod +x "$TMP/bin/install"
|
||||
install_image "$TMP/rel" "$TMP/mv" "$TMP/bin:"
|
||||
if [ "$RC" != 0 ] && same "$TMP/built/vmlinuz" "$TMP/mv/sqs-store/vmlinuz" && [ ! -e "$TMP/mv/sqs-store/initramfs.cpio.gz" ]; then
|
||||
if [ "$RC" != 0 ] && same "$TMP/built/vmlinuz" "$TMP/mv/sqs-worker/vmlinuz" && [ ! -e "$TMP/mv/sqs-worker/initramfs.cpio.gz" ]; then
|
||||
pass "an interrupted install leaves the initramfs missing, not mismatched"
|
||||
else
|
||||
fail "interrupted install: rc=$RC initramfs=$([ -e "$TMP/mv/sqs-store/initramfs.cpio.gz" ] && echo present || echo missing)"
|
||||
fail "interrupted install: rc=$RC initramfs=$([ -e "$TMP/mv/sqs-worker/initramfs.cpio.gz" ] && echo present || echo missing)"
|
||||
fi
|
||||
|
||||
# --- SQS worker image: same staging and install path, its own directory ---
|
||||
install_worker() {
|
||||
RC=0
|
||||
OUT=$(EXTRACT_DIR="$1" SPINIFEX_MICROVM_DIR="$2" INSTALL_SPINIFEX_LIB_ONLY=1 \
|
||||
bash -c '. "$1"; SUDO=""; install_sqs_worker_image' _ "$SETUP" 2>&1) || RC=$?
|
||||
}
|
||||
image "$TMP/wbuilt" w1
|
||||
release "$TMP/wrel"
|
||||
sh "$STAGE" "$TMP/built" "$TMP/wrel" > /dev/null
|
||||
sh "$STAGE" "$TMP/wbuilt" "$TMP/wrel" sqs-worker > /dev/null
|
||||
if same "$TMP/wbuilt/vmlinuz" "$TMP/wrel/microvm-sqs-worker/vmlinuz" \
|
||||
&& same "$TMP/wbuilt/initramfs.cpio.gz" "$TMP/wrel/microvm-sqs-worker/initramfs.cpio.gz" \
|
||||
&& same "$TMP/built/vmlinuz" "$TMP/wrel/microvm-sqs-store/vmlinuz"; then
|
||||
pass "the worker image stages beside the store image"
|
||||
image "$TMP/keep/sqs-store" old-store
|
||||
install_image "$TMP/rel" "$TMP/keep"
|
||||
if [ "$RC" = 0 ] && [ "$(cat "$TMP/keep/sqs-store/vmlinuz")" = kernel-old-store ] \
|
||||
&& [ "$(cat "$TMP/keep/sqs-store/initramfs.cpio.gz")" = initramfs-old-store ]; then
|
||||
pass "an install leaves a separate store image from an earlier install untouched"
|
||||
else
|
||||
fail "worker staging: $(ls -R "$TMP/wrel")"
|
||||
fail "earlier store image: rc=$RC $(ls -R "$TMP/keep")"
|
||||
fi
|
||||
install_worker "$TMP/wrel" "$TMP/wmv"
|
||||
if [ "$RC" = 0 ] && same "$TMP/wbuilt/initramfs.cpio.gz" "$TMP/wmv/sqs-worker/initramfs.cpio.gz" \
|
||||
&& [ ! -e "$TMP/wmv/sqs-store" ]; then
|
||||
pass "the worker image installs into its own directory only"
|
||||
if OUT=$(sh "$STAGE" "$TMP/built" "$TMP/rel" sqs-store 2>&1); then
|
||||
fail "the separate store image was staged"
|
||||
else
|
||||
fail "worker install: rc=$RC $OUT"
|
||||
fi
|
||||
release "$TMP/wrel-none"
|
||||
OUT=$(sh "$STAGE" "$TMP/no-such-build" "$TMP/wrel-none" sqs-worker)
|
||||
if [ ! -e "$TMP/wrel-none/microvm-sqs-worker" ] && echo "$OUT" | grep -q 'SQS worker microVM image missing'; then
|
||||
pass "a missing worker build stages nothing and says so"
|
||||
else
|
||||
fail "missing worker build: $OUT"
|
||||
fi
|
||||
install_worker "$TMP/wrel-none" "$TMP/wmv"
|
||||
if [ "$RC" = 0 ] && [ ! -e "$TMP/wmv/sqs-worker/vmlinuz" ] && [ ! -e "$TMP/wmv/sqs-worker/initramfs.cpio.gz" ] \
|
||||
&& echo "$OUT" | grep -q 'no SQS worker microVM image; SQS worker launches will be refused'; then
|
||||
pass "a release without the worker image removes the earlier one and warns"
|
||||
else
|
||||
fail "release without worker image: rc=$RC $OUT"
|
||||
fi
|
||||
cp -r "$TMP/wrel" "$TMP/wrel-bad"
|
||||
rm "$TMP/wrel-bad/microvm-sqs-worker/vmlinuz"
|
||||
install_worker "$TMP/wrel" "$TMP/wmv"
|
||||
install_worker "$TMP/wrel-bad" "$TMP/wmv"
|
||||
if [ "$RC" != 0 ] && echo "$OUT" | grep -q 'SQS worker microVM image in this release is incomplete: vmlinuz is missing' \
|
||||
&& [ ! -e "$TMP/wmv/sqs-worker/initramfs.cpio.gz" ]; then
|
||||
pass "an incomplete worker image is refused and the earlier pair is removed"
|
||||
else
|
||||
fail "incomplete worker image: rc=$RC $OUT"
|
||||
fi
|
||||
if OUT=$(sh "$STAGE" "$TMP/wbuilt" "$TMP/wrel" lb 2>&1); then
|
||||
fail "an unknown image name was staged"
|
||||
else
|
||||
pass "an unknown image name is refused"
|
||||
pass "the separate store image is no longer staged"
|
||||
fi
|
||||
|
||||
[ "$fails" = 0 ] || { echo "FAIL: $fails"; exit 1; }
|
||||
echo " setup_sqs_store_image_test ok"
|
||||
echo " setup_sqs_worker_image_test ok"
|
||||
@@ -29,6 +29,7 @@ MODELS=(
|
||||
rds/2014-10-31
|
||||
route-53/2013-04-01
|
||||
s3/2006-03-01
|
||||
sqs/2012-11-05
|
||||
sts/2011-06-15
|
||||
)
|
||||
|
||||
|
||||
@@ -4,8 +4,9 @@
|
||||
|
||||
```bash
|
||||
export VULTR_API_KEY=... # the key's IP allowlist must admit this host and every node
|
||||
./deploy-vultr.sh up --channel dev # one VX VM
|
||||
./deploy-vultr.sh up --channel dev --nodes 3 --plan vbm-8c-64gb-amd # three bare-metal servers
|
||||
./deploy-vultr.sh up # one VX VM, latest release
|
||||
./deploy-vultr.sh up --nodes 3 --plan vbm-8c-64gb-amd # three bare-metal servers
|
||||
./deploy-vultr.sh up --channel dev # the newest dev build
|
||||
./deploy-vultr.sh up --name v1 --distro spinifex-<version>.tar.gz # a named deployment from a local tarball
|
||||
./deploy-vultr.sh status
|
||||
./deploy-vultr.sh credentials
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
#!/usr/bin/env bash
|
||||
# deploy-vultr.sh — build a one- or three-node Spinifex deployment on Vultr, or tear it down.
|
||||
#
|
||||
# deploy-vultr.sh up --channel dev one VX VM
|
||||
# deploy-vultr.sh up --channel dev --nodes 3 --plan vbm-8c-64gb-amd three bare-metal servers
|
||||
# deploy-vultr.sh up one VX VM
|
||||
# deploy-vultr.sh up --nodes 3 --plan vbm-8c-64gb-amd three bare-metal servers
|
||||
# deploy-vultr.sh status nodes, addresses and what the deployment holds
|
||||
# deploy-vultr.sh credentials print the AWS access key and secret
|
||||
# deploy-vultr.sh ssh [nodeN] [command] SSH to a node, node1 by default
|
||||
@@ -38,7 +38,7 @@ NAME="default"
|
||||
REGION="ord"
|
||||
PLAN="vx1-g-16c-64g"
|
||||
DISTRO=""
|
||||
CHANNEL=""
|
||||
CHANNEL="latest"
|
||||
SPX_REGION="us-east-1"
|
||||
SPX_AZ="us-east-1a"
|
||||
SSH_KEY="$HOME/.ssh/id_ed25519"
|
||||
@@ -57,7 +57,7 @@ Options:
|
||||
--region ID Vultr region (default $REGION)
|
||||
--plan ID Vultr plan (default $PLAN); vbm-8c-64gb-amd for bare metal
|
||||
--distro TARBALL install this local release tarball
|
||||
--channel CHANNEL install from a release channel (latest, dev)
|
||||
--channel CHANNEL install from a release channel: latest (default) or dev
|
||||
--spx-region NAME Spinifex region (default $SPX_REGION)
|
||||
--spx-az NAME Spinifex availability zone (default $SPX_AZ)
|
||||
--ssh-key PATH private key; PATH.pub is installed (default $SSH_KEY)
|
||||
@@ -168,8 +168,6 @@ preflight() {
|
||||
[ -r "$SSH_KEY" ] && [ -r "$SSH_KEY.pub" ] || die "SSH key $SSH_KEY (and .pub) not readable"
|
||||
if [ -n "$DISTRO" ]; then
|
||||
[ -r "$DISTRO" ] || die "--distro $DISTRO not readable"
|
||||
elif [ -z "$CHANNEL" ]; then
|
||||
die "pass --distro TARBALL or --channel latest|dev"
|
||||
fi
|
||||
vultr GET /account >/dev/null || die "the API key was refused; check its IP allowlist"
|
||||
local type=vx1
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
------------------------------------------------- MODULE Capabilities -------------------------------------------------
|
||||
(*********************************************************************************************************************)
|
||||
(* The optional behaviour that an implementation advertises. *)
|
||||
(*********************************************************************************************************************)
|
||||
EXTENDS EBSProvider
|
||||
|
||||
-----------------------------------------------------------------------------------------------------------------------
|
||||
\* Interface methods
|
||||
|
||||
GetCapabilitiesRequests == {<<>>}
|
||||
|
||||
GetCapabilities(req, val, err) ==
|
||||
(*******************************************************************************************************************)
|
||||
(* The implementation reports which optional behaviour it supports. *)
|
||||
(*******************************************************************************************************************)
|
||||
\* The provider failed to answer.
|
||||
\/ /\ val = None
|
||||
/\ err \in ProviderErrors
|
||||
/\ UNCHANGED vars
|
||||
|
||||
\* The request was accepted.
|
||||
\* Every answer is the same, so an earlier one may be cached.
|
||||
\/ /\ val = Capabilities
|
||||
/\ err = None
|
||||
/\ UNCHANGED vars
|
||||
|
||||
=======================================================================================================================
|
||||
@@ -1,163 +0,0 @@
|
||||
------------------------------------------------- MODULE EBSProvider --------------------------------------------------
|
||||
(*********************************************************************************************************************)
|
||||
(* The state that an implementation exposes, and the types and constants that every other module shares. *)
|
||||
(*********************************************************************************************************************)
|
||||
EXTENDS
|
||||
Naturals,
|
||||
Sequences,
|
||||
FiniteSets,
|
||||
Functions,
|
||||
TLC
|
||||
|
||||
-----------------------------------------------------------------------------------------------------------------------
|
||||
\* Constants
|
||||
|
||||
CONSTANTS
|
||||
MaxBytes,
|
||||
VolumeIDs, VolumeHandles,
|
||||
NodeIDs, NBDURIs, DevicePaths,
|
||||
SnapshotIDs, SnapshotHandles,
|
||||
AvailabilityZones, Timestamps,
|
||||
Parameters, ProviderData,
|
||||
ListTokens, MaxListResults,
|
||||
\* The optional behaviour that the implementation advertises, which never changes.
|
||||
Capabilities,
|
||||
\* Whether the provider may fail to answer at any point in a behaviour.
|
||||
ProviderMayFail,
|
||||
\* Stands in for an optional field that a request or response omitted.
|
||||
None
|
||||
|
||||
ASSUME MaxBytes \in Nat \ {0}
|
||||
ASSUME VolumeIDs /= {} /\ VolumeHandles /= {}
|
||||
ASSUME NodeIDs /= {} => NBDURIs /= {} \/ DevicePaths /= {}
|
||||
ASSUME SnapshotIDs /= {} => SnapshotHandles /= {} /\ Timestamps /= {}
|
||||
ASSUME ListTokens /= {}
|
||||
ASSUME MaxListResults \in Nat \ {0}
|
||||
ASSUME None \notin UNION {VolumeIDs, VolumeHandles, NodeIDs, SnapshotIDs, SnapshotHandles,
|
||||
AvailabilityZones, Timestamps, Parameters, ProviderData, ListTokens}
|
||||
ASSUME Capabilities \in [online_expansion : BOOLEAN,
|
||||
sparse_extent_reporting : BOOLEAN,
|
||||
crash_consistent_snapshot : BOOLEAN,
|
||||
volume_seeding : BOOLEAN,
|
||||
read_only_publish : BOOLEAN,
|
||||
volume_enumeration : BOOLEAN,
|
||||
snapshot_enumeration : BOOLEAN,
|
||||
owner_routing : BOOLEAN,
|
||||
exclusion : [scope : {"none", "node", "cluster"},
|
||||
claim_ttl_seconds : Nat,
|
||||
fences_lost_claim : BOOLEAN]]
|
||||
ASSUME ProviderMayFail \in BOOLEAN
|
||||
|
||||
-----------------------------------------------------------------------------------------------------------------------
|
||||
\* Variables
|
||||
|
||||
VARIABLES
|
||||
\* Every volume that the implementation exposes, keyed by volume ID.
|
||||
volumes,
|
||||
\* Every publication that the implementation exposes, keyed by the volume and the node it is published to.
|
||||
publications,
|
||||
\* Every snapshot that the implementation exposes, keyed by snapshot ID.
|
||||
snapshots
|
||||
|
||||
vars == <<volumes, publications, snapshots>>
|
||||
|
||||
-----------------------------------------------------------------------------------------------------------------------
|
||||
\* Miscellaneous
|
||||
|
||||
IsPartialFn(f, D, R) == DOMAIN f \subseteq D /\ \A x \in DOMAIN f : f[x] \in R
|
||||
|
||||
Optional(S) == S \union {None}
|
||||
|
||||
InsertKey(f, x, y) == f @@ (x :> y)
|
||||
|
||||
RemoveKey(f, x) == RestrictDomain(f, LAMBDA x_2 : x_2 /= x)
|
||||
|
||||
KeyExists(f, x) == x \in DOMAIN f
|
||||
|
||||
-----------------------------------------------------------------------------------------------------------------------
|
||||
\* Types
|
||||
|
||||
Bytes == 0..MaxBytes
|
||||
|
||||
\* The errors that an implementation may return when its provider fails to answer.
|
||||
ProviderErrors == IF ProviderMayFail
|
||||
THEN {"internal", "unavailable"}
|
||||
\* Otherwise an implementation could conform by refusing to answer.
|
||||
ELSE {}
|
||||
|
||||
CapacityRanges == [required_bytes : Bytes, limit_bytes : Bytes]
|
||||
|
||||
VolumeStates == {"available", "in-use"}
|
||||
|
||||
Volumes == [capacity_bytes : 1..MaxBytes,
|
||||
state : VolumeStates,
|
||||
handle : VolumeHandles,
|
||||
availability_zone : Optional(AvailabilityZones),
|
||||
|
||||
\* Never returned, but kept to tell a retry from a conflicting request.
|
||||
source_snapshot_id : Optional(SnapshotIDs),
|
||||
parameters : Optional(Parameters),
|
||||
provider_data : Optional(ProviderData)]
|
||||
|
||||
\* How a published volume is reached on its node, either over NBD or as a host block device.
|
||||
Exports == [kind : {"nbd"},
|
||||
nbd_uri : NBDURIs]
|
||||
\union
|
||||
[kind : {"block"},
|
||||
device_path : DevicePaths,
|
||||
size_bytes : 1..MaxBytes]
|
||||
|
||||
\* A volume published to a node, and whether it was published read-only.
|
||||
Publications == [export : Exports,
|
||||
read_only : BOOLEAN]
|
||||
|
||||
SnapshotStates == {"pending", "completed", "error"}
|
||||
|
||||
Snapshots == [size_bytes : 1..MaxBytes,
|
||||
created_at : Timestamps,
|
||||
state : SnapshotStates,
|
||||
handle : SnapshotHandles,
|
||||
source_volume_id : VolumeIDs]
|
||||
|
||||
-----------------------------------------------------------------------------------------------------------------------
|
||||
\* State helpers
|
||||
|
||||
\* The part of a volume that the implementation returns.
|
||||
VolumeView(volID) ==
|
||||
("id" :> volID) @@ Restrict(volumes[volID], {"capacity_bytes",
|
||||
"state",
|
||||
"handle",
|
||||
"availability_zone",
|
||||
"provider_data"})
|
||||
|
||||
\* The nodes that a volume is published to.
|
||||
PublishedNodes(volID) == {n \in NodeIDs : KeyExists(publications, <<volID, n>>)}
|
||||
|
||||
\* The part of a publication that the implementation returns.
|
||||
PublicationView(volID, nodeID) == [volume_id |-> volID,
|
||||
node_id |-> nodeID,
|
||||
export |-> publications[volID, nodeID].export]
|
||||
|
||||
\* The part of a snapshot that the implementation returns.
|
||||
SnapshotView(snapID) == ("id" :> snapID) @@ snapshots[snapID]
|
||||
|
||||
\* The number of refs that a page may hold, with an omitted or oversized request clamped to the maximum.
|
||||
PageSize(m) == IF m <= 0 \/ m > MaxListResults THEN MaxListResults ELSE m
|
||||
|
||||
-----------------------------------------------------------------------------------------------------------------------
|
||||
\* Invariants
|
||||
|
||||
TypeOK ==
|
||||
/\ IsPartialFn(volumes, VolumeIDs, Volumes)
|
||||
/\ IsPartialFn(publications, VolumeIDs \X NodeIDs, Publications)
|
||||
/\ IsPartialFn(snapshots, SnapshotIDs, Snapshots)
|
||||
|
||||
-----------------------------------------------------------------------------------------------------------------------
|
||||
\* Initial state
|
||||
|
||||
Init ==
|
||||
/\ volumes = <<>>
|
||||
/\ publications = <<>>
|
||||
/\ snapshots = <<>>
|
||||
|
||||
=======================================================================================================================
|
||||
@@ -1,140 +0,0 @@
|
||||
# EBSProvider specification
|
||||
|
||||
The contract that every implementation of the `EBSProvider` interface must meet.
|
||||
|
||||
## Terminology
|
||||
|
||||
- **Provider:** the service that backs an implementation's operations, such as Viperblock.
|
||||
The memory implementation has none.
|
||||
- **Implementation:** a Go implementation of the `EBSProvider` interface, which adapts between the shapes of the contract and the shapes that its provider expects.
|
||||
- **Contract:** the behaviour that this specification requires of every implementation, as observed through the interface.
|
||||
|
||||
## Open questions
|
||||
|
||||
### EBSProvider
|
||||
|
||||
- What may a failed call change?
|
||||
A failed call leaves the state unchanged, but a timeout can arrive after the provider acted, and a crash can leave work half done.
|
||||
- Is the schema version part of the contract?
|
||||
Every method refuses an unsupported version with `unsupported_version`, and the conformance suite tests it, but the specification does not model versions.
|
||||
- Are `internal` and `unavailable` the only errors that a failed provider may cause?
|
||||
- Must a present but malformed ID be refused with `invalid_argument`?
|
||||
IDs are opaque here, so only an omitted one is refused.
|
||||
|
||||
### Capabilities
|
||||
|
||||
#### GetCapabilities
|
||||
|
||||
- Is it a lasting promise that capabilities never change, even when an implementation or its provider is upgraded or reconfigured after an earlier answer was cached?
|
||||
|
||||
### Volumes
|
||||
|
||||
- Who decides whether a volume is `in-use`?
|
||||
An implementation may track attachments itself, or derive the state from whether a publication claim is held.
|
||||
|
||||
#### CreateVolume
|
||||
|
||||
- May the capacity be anywhere in the requested range, as CSI allows?
|
||||
An implementation that rounds up would then fail the comparison that tells a retry from a conflicting request.
|
||||
- May seed data be combined with a source snapshot?
|
||||
The Viperblock implementation refuses, the memory implementation accepts.
|
||||
- Must the source snapshot be completed?
|
||||
AWS refuses one that is not.
|
||||
- Must an implementation store the availability zone, parameters and source snapshot, so that it can detect a conflicting retry?
|
||||
The Viperblock implementation does not.
|
||||
- Should a retry be compared against the original request rather than the current volume?
|
||||
A retry after ExpandVolume conflicts on capacity.
|
||||
- Should a retry with a different source snapshot volume ID conflict?
|
||||
It is not compared.
|
||||
- Is byte equality the meaning of equal parameters, or may an implementation compare them semantically?
|
||||
- Should `MaxSeedBytes` be modelled?
|
||||
Seed data larger than it is refused with `invalid_argument`.
|
||||
|
||||
#### GetVolume
|
||||
|
||||
- Must an implementation compare the handle?
|
||||
The Viperblock implementation ignores it.
|
||||
- Is `not_found` the right error for a volume that was deleted and recreated under the same ID?
|
||||
|
||||
#### ListVolumes
|
||||
|
||||
- Must following next tokens list every volume that exists throughout the walk exactly once?
|
||||
The conformance suite tests it, but the specification describes each call alone, so it does not say how one page relates to the next.
|
||||
- What should a starting token that the implementation never issued do?
|
||||
Every implementation treats any token as a position to resume after, while CSI refuses an invalid one with `ABORTED`.
|
||||
- May a page hold fewer refs than requested yet carry a next token?
|
||||
AWS and CSI allow it, but here a next token comes only with a full page, so that a walk cannot stall on empty pages.
|
||||
|
||||
#### ExpandVolume
|
||||
|
||||
- Should a request below the current capacity succeed without change, as CSI recommends, rather than be refused with `invalid_argument`?
|
||||
It is refused here, as AWS refuses `ModifyVolume` to a smaller size and the conformance suite tests, while CSI says the plugin should reply OK when the volume is already at least the requested size.
|
||||
- Must a block export's size follow an online expansion?
|
||||
A published volume keeps its export's `size_bytes`, so after it grows the export reports the old size.
|
||||
|
||||
#### PublishVolume
|
||||
|
||||
- Should `volume_in_use` depend on the exclusion scope?
|
||||
An implementation whose scope is `none` lets two writers in, and one whose scope is `node` cannot see another node, yet both are refused here.
|
||||
- Must a republish to the same node with a different `read_only` be refused?
|
||||
The Viperblock implementation refuses it with `volume_in_use`, while the memory implementation ignores the flag.
|
||||
- May two volumes on one node share an NBD URI or a device path?
|
||||
- Must a block export's size equal the volume's capacity?
|
||||
|
||||
#### UnpublishVolume
|
||||
|
||||
- Must an implementation compare the handle?
|
||||
Both the Viperblock and memory implementations ignore it.
|
||||
- Is a provider that fails to seal the volume a failed provider?
|
||||
The Viperblock implementation answers `internal` and leaves the volume published, which the specification allows only when the provider may fail.
|
||||
- Should a claim that lapses through its TTL be modelled, as a step that happens without a call?
|
||||
- Should a volume that does not exist be refused with `not_found`?
|
||||
It succeeds here, as CSI requires and the conformance suite tests, while AWS refuses `DetachVolume` on a missing volume with `InvalidVolume.NotFound`.
|
||||
|
||||
#### DeleteVolume
|
||||
|
||||
- Must an implementation compare the handle before refusing a published volume?
|
||||
The memory implementation does, and succeeds without deleting it, while the Viperblock implementation ignores the handle and refuses with `volume_in_use`.
|
||||
- Should a volume that does not exist be refused with `not_found`?
|
||||
It succeeds here, as CSI requires and the conformance suite tests, while AWS refuses `DeleteVolume` on a missing volume with `InvalidVolume.NotFound`.
|
||||
|
||||
### Snapshots
|
||||
|
||||
- Should a pending snapshot's move to `completed` or `error` be modelled, as a step that happens without a call?
|
||||
Without it a pending snapshot never completes, and no snapshot is ever in `error`.
|
||||
|
||||
#### CreateSnapshot
|
||||
|
||||
- May success return a `pending` snapshot?
|
||||
AWS starts a snapshot as `pending`, while the memory implementation returns `completed` and the NATS implementation waits for completion and refuses anything else.
|
||||
- Must a retry compare the volume handle?
|
||||
It is ignored, as in the memory implementation, so a volume deleted and recreated under the same ID returns the old snapshot.
|
||||
- Should a copy's ID count as a retry?
|
||||
A copy keeps its source's volume ID, so creating a snapshot of that volume under the copy's ID returns the copy.
|
||||
- Should `crash_consistent_snapshot` be modelled?
|
||||
The specification does not model volume data, so it cannot say what a snapshot of a published volume holds.
|
||||
|
||||
#### CopySnapshot
|
||||
|
||||
- Should a retried copy return the existing copy, as a retried CreateVolume does, rather than `already_exists`?
|
||||
- May a source that is not completed be copied?
|
||||
AWS refuses until it is.
|
||||
- What state does a copy start in?
|
||||
The memory implementation always returns `completed`, while AWS starts a copy as `pending`.
|
||||
|
||||
#### ListSnapshots
|
||||
|
||||
- The questions for ListVolumes apply here too.
|
||||
- Must a ref carry the source volume ID?
|
||||
The Viperblock implementation leaves it empty, since reading it costs a fetch per snapshot, yet reconciliation needs it to act on a snapshot whose metadata is lost.
|
||||
|
||||
#### DeleteSnapshot
|
||||
|
||||
- Is the handle check part of the contract?
|
||||
The Viperblock implementation ignores the handle and deletes anyway.
|
||||
- May a pending snapshot be deleted?
|
||||
AWS allows it.
|
||||
- What does success promise?
|
||||
The Go docs say the snapshot is gone, but a snapshot whose handle did not match is left in place.
|
||||
- Should a snapshot that does not exist be refused with `not_found`?
|
||||
It succeeds here, as CSI requires and the conformance suite tests, while AWS refuses `DeleteSnapshot` on a missing snapshot with `InvalidSnapshot.NotFound`.
|
||||
@@ -1,159 +0,0 @@
|
||||
-------------------------------------------------- MODULE Snapshots ---------------------------------------------------
|
||||
(*********************************************************************************************************************)
|
||||
(* The lifecycle of a snapshot, from its creation to its deletion. *)
|
||||
(*********************************************************************************************************************)
|
||||
EXTENDS EBSProvider
|
||||
|
||||
-----------------------------------------------------------------------------------------------------------------------
|
||||
\* Interface methods
|
||||
|
||||
CreateSnapshotRequests == [snapshot_id : Optional(SnapshotIDs),
|
||||
volume_id : Optional(VolumeIDs),
|
||||
volume_handle : Optional(VolumeHandles)]
|
||||
|
||||
CreateSnapshot(req, val, err) ==
|
||||
(*******************************************************************************************************************)
|
||||
(* A volume's data is frozen as a snapshot, from which new volumes can be restored. A retry of a call that *)
|
||||
(* already succeeded returns the same snapshot. *)
|
||||
(*******************************************************************************************************************)
|
||||
LET snapID == req.snapshot_id
|
||||
volID == req.volume_id
|
||||
|
||||
Errors ==
|
||||
LET checks == [invalid_argument |-> None \in {snapID, volID},
|
||||
already_exists |-> /\ KeyExists(snapshots, snapID)
|
||||
/\ snapshots[snapID].source_volume_id /= volID,
|
||||
\* Only checked for a new snapshot, so a retry outlives the volume.
|
||||
not_found |-> /\ ~KeyExists(snapshots, snapID)
|
||||
/\ \/ ~KeyExists(volumes, volID)
|
||||
\/ req.volume_handle \notin {None, volumes[volID].handle}]
|
||||
IN {e \in DOMAIN checks : checks[e]}
|
||||
|
||||
\* The request was rejected, or the provider failed to answer.
|
||||
IN \/ /\ val = None
|
||||
/\ err \in Errors \union ProviderErrors
|
||||
/\ UNCHANGED vars
|
||||
|
||||
\* The request was accepted.
|
||||
\/ /\ Errors = {}
|
||||
\* A retry returns the existing snapshot.
|
||||
/\ \/ /\ KeyExists(snapshots, snapID)
|
||||
/\ UNCHANGED snapshots
|
||||
|
||||
\/ /\ ~KeyExists(snapshots, snapID)
|
||||
\* The handle, creation time and state are the implementation's choice.
|
||||
/\ \E h \in SnapshotHandles \ {snapshots[x].handle : x \in DOMAIN snapshots},
|
||||
t \in Timestamps,
|
||||
s \in {"pending", "completed"} :
|
||||
snapshots' = InsertKey(snapshots, snapID, [size_bytes |-> volumes[volID].capacity_bytes,
|
||||
created_at |-> t,
|
||||
state |-> s,
|
||||
handle |-> h,
|
||||
source_volume_id |-> volID])
|
||||
/\ val = SnapshotView(snapID)'
|
||||
/\ err = None
|
||||
/\ UNCHANGED <<volumes, publications>>
|
||||
|
||||
|
||||
CopySnapshotRequests == [source_snapshot_id : Optional(SnapshotIDs),
|
||||
destination_snapshot_id : Optional(SnapshotIDs),
|
||||
volume_id : Optional(VolumeIDs)]
|
||||
|
||||
CopySnapshot(req, val, err) ==
|
||||
(*******************************************************************************************************************)
|
||||
(* A snapshot is copied under a new ID, sharing its frozen data, and the copy can be deleted independently *)
|
||||
(* of it. *)
|
||||
(*******************************************************************************************************************)
|
||||
LET srcSnapID == req.source_snapshot_id
|
||||
dstSnapID == req.destination_snapshot_id
|
||||
volID == req.volume_id
|
||||
|
||||
Errors ==
|
||||
LET checks == [invalid_argument |-> \/ None \in {srcSnapID, dstSnapID, volID}
|
||||
\/ srcSnapID = dstSnapID
|
||||
\* Only compared, not looked up, since a snapshot outlives its volume.
|
||||
\/ /\ KeyExists(snapshots, srcSnapID)
|
||||
/\ snapshots[srcSnapID].source_volume_id /= volID,
|
||||
already_exists |-> KeyExists(snapshots, dstSnapID),
|
||||
not_found |-> srcSnapID \notin Optional(DOMAIN snapshots)]
|
||||
IN {e \in DOMAIN checks : checks[e]}
|
||||
|
||||
\* The request was rejected, or the provider failed to answer.
|
||||
IN \/ /\ val = None
|
||||
/\ err \in Errors \union ProviderErrors
|
||||
/\ UNCHANGED vars
|
||||
|
||||
\* The request was accepted.
|
||||
\/ /\ Errors = {}
|
||||
\* The handle and creation time are the implementation's choice.
|
||||
/\ \E h \in SnapshotHandles \ {snapshots[x].handle : x \in DOMAIN snapshots},
|
||||
t \in Timestamps :
|
||||
snapshots' = InsertKey(snapshots, dstSnapID, [created_at |-> t,
|
||||
handle |-> h]
|
||||
@@ snapshots[srcSnapID])
|
||||
/\ val = SnapshotView(dstSnapID)'
|
||||
/\ err = None
|
||||
/\ UNCHANGED <<volumes, publications>>
|
||||
|
||||
|
||||
ListSnapshotsRequests == [max_results : -1..(MaxListResults + 1),
|
||||
starting_token : Optional(ListTokens)]
|
||||
|
||||
ListSnapshots(req, val, err) ==
|
||||
(*******************************************************************************************************************)
|
||||
(* Some of the snapshots are listed, a page at a time, and nothing changes. A call with no starting token that *)
|
||||
(* returns no next token lists every snapshot. *)
|
||||
(*******************************************************************************************************************)
|
||||
LET Errors ==
|
||||
LET checks == [unsupported_capability |-> ~Capabilities.snapshot_enumeration]
|
||||
IN {e \in DOMAIN checks : checks[e]}
|
||||
|
||||
\* The request was rejected, or the provider failed to answer.
|
||||
IN \/ /\ val = None
|
||||
/\ err \in Errors \union ProviderErrors
|
||||
/\ UNCHANGED vars
|
||||
|
||||
\* The request was accepted.
|
||||
\/ /\ Errors = {}
|
||||
/\ LET refs == {[id |-> id,
|
||||
source_volume_id |-> snapshots[id].source_volume_id,
|
||||
handle |-> snapshots[id].handle] : id \in DOMAIN snapshots}
|
||||
\* The page and the next token are the implementation's choice.
|
||||
IN \E page \in SUBSET refs, next \in Optional(ListTokens) :
|
||||
/\ Cardinality(page) <= PageSize(req.max_results)
|
||||
/\ next /= None => Cardinality(page) = PageSize(req.max_results)
|
||||
/\ req.starting_token = None /\ next = None => page = refs
|
||||
/\ val = [snapshots |-> page, next_token |-> next]
|
||||
/\ err = None
|
||||
/\ UNCHANGED vars
|
||||
|
||||
|
||||
DeleteSnapshotRequests == [snapshot_id : Optional(SnapshotIDs),
|
||||
handle : Optional(SnapshotHandles)]
|
||||
|
||||
DeleteSnapshot(req, err) ==
|
||||
(*******************************************************************************************************************)
|
||||
(* A snapshot is deleted, so that no new volume can be restored from it. Volumes already restored from it, *)
|
||||
(* and copies of it, are unaffected. *)
|
||||
(*******************************************************************************************************************)
|
||||
LET snapID == req.snapshot_id
|
||||
|
||||
Errors ==
|
||||
LET checks == [invalid_argument |-> snapID = None]
|
||||
IN {e \in DOMAIN checks : checks[e]}
|
||||
|
||||
\* The request was rejected, or the provider failed to answer.
|
||||
IN \/ /\ err \in Errors \union ProviderErrors
|
||||
/\ UNCHANGED vars
|
||||
|
||||
\* The request was accepted.
|
||||
\/ /\ Errors = {}
|
||||
/\ snapshots' = IF /\ KeyExists(snapshots, snapID)
|
||||
/\ req.handle \notin {None, snapshots[snapID].handle}
|
||||
THEN snapshots
|
||||
\* Record removal is idempotent so that retries succeed.
|
||||
ELSE RemoveKey(snapshots, snapID)
|
||||
/\ err = None
|
||||
/\ UNCHANGED <<volumes, publications>>
|
||||
|
||||
=======================================================================================================================
|
||||
@@ -1,286 +0,0 @@
|
||||
--------------------------------------------------- MODULE Volumes ----------------------------------------------------
|
||||
(*********************************************************************************************************************)
|
||||
(* The lifecycle of a volume, from its creation to its deletion. *)
|
||||
(*********************************************************************************************************************)
|
||||
EXTENDS EBSProvider
|
||||
|
||||
-----------------------------------------------------------------------------------------------------------------------
|
||||
\* Interface methods
|
||||
|
||||
CreateVolumeRequests == [volume_id : Optional(VolumeIDs),
|
||||
capacity_range : CapacityRanges,
|
||||
availability_zone : Optional(AvailabilityZones),
|
||||
source_snapshot_id : Optional(SnapshotIDs),
|
||||
source_snapshot_volume_id : Optional(VolumeIDs),
|
||||
parameters : Optional(Parameters),
|
||||
seed_data : Bytes]
|
||||
|
||||
CreateVolume(req, val, err) ==
|
||||
(*******************************************************************************************************************)
|
||||
(* A volume of the requested capacity is created, either blank or holding a snapshot's data, optionally *)
|
||||
(* with seed data written at its start. A retry of a call that already succeeded returns the same volume. *)
|
||||
(*******************************************************************************************************************)
|
||||
LET volID == req.volume_id
|
||||
snapID == req.source_snapshot_id
|
||||
snapVolID == req.source_snapshot_volume_id
|
||||
capacity == req.capacity_range
|
||||
|
||||
\* Any warranted error may be returned, since the order of checks is not part of the contract.
|
||||
Errors ==
|
||||
LET checks == [invalid_argument |-> \/ volID = None
|
||||
\/ capacity.required_bytes <= 0
|
||||
\/ /\ capacity.limit_bytes > 0
|
||||
/\ capacity.required_bytes > capacity.limit_bytes
|
||||
\/ req.seed_data > capacity.required_bytes
|
||||
\/ snapID /= None /\ snapVolID = None
|
||||
\* Only checked for a new volume, so a retry outlives the snapshot.
|
||||
\/ /\ ~KeyExists(volumes, volID)
|
||||
/\ KeyExists(snapshots, snapID)
|
||||
/\ \/ capacity.required_bytes < snapshots[snapID].size_bytes
|
||||
\/ snapshots[snapID].source_volume_id /= snapVolID,
|
||||
\* Seed data is not compared, since it only applies on creation.
|
||||
already_exists |-> /\ KeyExists(volumes, volID)
|
||||
/\ \/ volumes[volID].capacity_bytes /= capacity.required_bytes
|
||||
\/ volumes[volID].availability_zone /= req.availability_zone
|
||||
\/ volumes[volID].source_snapshot_id /= snapID
|
||||
\/ volumes[volID].parameters /= req.parameters,
|
||||
not_found |-> /\ ~KeyExists(volumes, volID)
|
||||
/\ snapID \notin Optional(DOMAIN snapshots),
|
||||
unsupported_capability |-> /\ req.seed_data > 0
|
||||
/\ ~Capabilities.volume_seeding]
|
||||
IN {e \in DOMAIN checks : checks[e]}
|
||||
|
||||
\* The request was rejected, or the provider failed to answer.
|
||||
IN \/ /\ val = None
|
||||
/\ err \in Errors \union ProviderErrors
|
||||
/\ UNCHANGED vars
|
||||
|
||||
\* The request was accepted.
|
||||
\/ /\ Errors = {}
|
||||
\* A retry returns the existing volume.
|
||||
/\ \/ /\ KeyExists(volumes, volID)
|
||||
/\ UNCHANGED volumes
|
||||
|
||||
\/ /\ ~KeyExists(volumes, volID)
|
||||
\* The handle and provider data are the implementation's choice.
|
||||
/\ \E h \in VolumeHandles \ {volumes[x].handle : x \in DOMAIN volumes},
|
||||
pd \in Optional(ProviderData) :
|
||||
volumes' = InsertKey(volumes, volID, [capacity_bytes |-> capacity.required_bytes,
|
||||
state |-> "available",
|
||||
handle |-> h,
|
||||
availability_zone |-> req.availability_zone,
|
||||
source_snapshot_id |-> snapID,
|
||||
parameters |-> req.parameters,
|
||||
provider_data |-> pd])
|
||||
/\ val = VolumeView(volID)'
|
||||
/\ err = None
|
||||
/\ UNCHANGED <<publications, snapshots>>
|
||||
|
||||
|
||||
GetVolumeRequests == [volume_id : Optional(VolumeIDs),
|
||||
handle : Optional(VolumeHandles)]
|
||||
|
||||
GetVolume(req, val, err) ==
|
||||
(*******************************************************************************************************************)
|
||||
(* A volume's current capacity, state and handle are read, and nothing changes. *)
|
||||
(*******************************************************************************************************************)
|
||||
LET volID == req.volume_id
|
||||
|
||||
Errors ==
|
||||
LET checks == [invalid_argument |-> volID = None,
|
||||
not_found |-> \/ ~KeyExists(volumes, volID)
|
||||
\/ req.handle \notin {None, volumes[volID].handle}]
|
||||
IN {e \in DOMAIN checks : checks[e]}
|
||||
|
||||
\* The request was rejected, or the provider failed to answer.
|
||||
IN \/ /\ val = None
|
||||
/\ err \in Errors \union ProviderErrors
|
||||
/\ UNCHANGED vars
|
||||
|
||||
\* The request was accepted.
|
||||
\/ /\ Errors = {}
|
||||
/\ val = VolumeView(volID)
|
||||
/\ err = None
|
||||
/\ UNCHANGED vars
|
||||
|
||||
|
||||
ListVolumesRequests == [max_results : -1..(MaxListResults + 1),
|
||||
starting_token : Optional(ListTokens)]
|
||||
|
||||
ListVolumes(req, val, err) ==
|
||||
(*******************************************************************************************************************)
|
||||
(* Some of the volumes are listed, a page at a time, and nothing changes. A call with no starting token that *)
|
||||
(* returns no next token lists every volume. *)
|
||||
(*******************************************************************************************************************)
|
||||
LET Errors ==
|
||||
LET checks == [unsupported_capability |-> ~Capabilities.volume_enumeration]
|
||||
IN {e \in DOMAIN checks : checks[e]}
|
||||
|
||||
\* The request was rejected, or the provider failed to answer.
|
||||
IN \/ /\ val = None
|
||||
/\ err \in Errors \union ProviderErrors
|
||||
/\ UNCHANGED vars
|
||||
|
||||
\* The request was accepted.
|
||||
\/ /\ Errors = {}
|
||||
/\ LET refs == {[id |-> id, handle |-> volumes[id].handle] : id \in DOMAIN volumes}
|
||||
\* The page and the next token are the implementation's choice.
|
||||
IN \E page \in SUBSET refs, next \in Optional(ListTokens) :
|
||||
/\ Cardinality(page) <= PageSize(req.max_results)
|
||||
/\ next /= None => Cardinality(page) = PageSize(req.max_results)
|
||||
/\ req.starting_token = None /\ next = None => page = refs
|
||||
/\ val = [volumes |-> page, next_token |-> next]
|
||||
/\ err = None
|
||||
/\ UNCHANGED vars
|
||||
|
||||
|
||||
ExpandVolumeRequests == [volume_id : Optional(VolumeIDs),
|
||||
handle : Optional(VolumeHandles),
|
||||
capacity_range : CapacityRanges]
|
||||
|
||||
ExpandVolume(req, val, err) ==
|
||||
(*******************************************************************************************************************)
|
||||
(* A volume grows to the requested capacity, keeping its data. A published volume grows only if the *)
|
||||
(* implementation supports online expansion, and a request for the current capacity succeeds without change. *)
|
||||
(*******************************************************************************************************************)
|
||||
LET volID == req.volume_id
|
||||
capacity == req.capacity_range
|
||||
|
||||
Errors ==
|
||||
LET checks == [invalid_argument |-> \/ volID = None
|
||||
\/ capacity.required_bytes = 0
|
||||
\/ /\ capacity.limit_bytes > 0
|
||||
/\ capacity.required_bytes > capacity.limit_bytes
|
||||
\/ /\ KeyExists(volumes, volID)
|
||||
/\ capacity.required_bytes < volumes[volID].capacity_bytes,
|
||||
not_found |-> \/ ~KeyExists(volumes, volID)
|
||||
\/ req.handle \notin {None, volumes[volID].handle},
|
||||
volume_in_use |-> /\ PublishedNodes(volID) /= {}
|
||||
/\ ~Capabilities.online_expansion
|
||||
/\ capacity.required_bytes > volumes[volID].capacity_bytes]
|
||||
IN {e \in DOMAIN checks : checks[e]}
|
||||
|
||||
\* The request was rejected, or the provider failed to answer.
|
||||
IN \/ /\ val = None
|
||||
/\ err \in Errors \union ProviderErrors
|
||||
/\ UNCHANGED vars
|
||||
|
||||
\* The request was accepted.
|
||||
\/ /\ Errors = {}
|
||||
/\ volumes' = [volumes EXCEPT ![volID].capacity_bytes = capacity.required_bytes]
|
||||
/\ val = VolumeView(volID)'
|
||||
/\ err = None
|
||||
/\ UNCHANGED <<publications, snapshots>>
|
||||
|
||||
|
||||
PublishVolumeRequests == [volume_id : Optional(VolumeIDs),
|
||||
handle : Optional(VolumeHandles),
|
||||
node_id : Optional(NodeIDs),
|
||||
read_only : BOOLEAN]
|
||||
|
||||
PublishVolume(req, val, err) ==
|
||||
(*******************************************************************************************************************)
|
||||
(* A volume's data becomes reachable on a node, through an export that a VM there can attach, either an NBD *)
|
||||
(* URI or a host block device. Reads and writes through the export reach the volume, and a read-only export *)
|
||||
(* refuses writes. Only one node at a time may reach a volume. *)
|
||||
(*******************************************************************************************************************)
|
||||
LET volID == req.volume_id
|
||||
nodeID == req.node_id
|
||||
|
||||
Errors ==
|
||||
LET checks == [invalid_argument |-> None \in {volID, nodeID},
|
||||
not_found |-> \/ ~KeyExists(volumes, volID)
|
||||
\/ req.handle \notin {None, volumes[volID].handle},
|
||||
unsupported_capability |-> /\ req.read_only
|
||||
/\ ~Capabilities.read_only_publish,
|
||||
volume_in_use |-> PublishedNodes(volID) \ {nodeID} /= {}]
|
||||
IN {e \in DOMAIN checks : checks[e]}
|
||||
|
||||
\* The request was rejected, or the provider failed to answer.
|
||||
IN \/ /\ val = None
|
||||
/\ err \in Errors \union ProviderErrors
|
||||
/\ UNCHANGED vars
|
||||
|
||||
\* The request was accepted.
|
||||
\/ /\ Errors = {}
|
||||
\* A retry returns the existing publication.
|
||||
/\ \/ /\ KeyExists(publications, <<volID, nodeID>>)
|
||||
/\ UNCHANGED publications
|
||||
|
||||
\/ /\ ~KeyExists(publications, <<volID, nodeID>>)
|
||||
\* The type of export is the implementation's choice.
|
||||
/\ \E x \in Exports :
|
||||
publications' = InsertKey(publications, <<volID, nodeID>>, [export |-> x,
|
||||
read_only |-> req.read_only])
|
||||
/\ volumes' = [volumes EXCEPT ![volID].state = "in-use"]
|
||||
/\ val = PublicationView(volID, nodeID)'
|
||||
/\ err = None
|
||||
/\ UNCHANGED snapshots
|
||||
|
||||
|
||||
UnpublishVolumeRequests == [volume_id : Optional(VolumeIDs),
|
||||
handle : Optional(VolumeHandles),
|
||||
node_id : Optional(NodeIDs)]
|
||||
|
||||
UnpublishVolume(req, err) ==
|
||||
(*******************************************************************************************************************)
|
||||
(* A volume's export on a node is torn down, so that its data is no longer reachable there. Once no node *)
|
||||
(* can reach the volume, it is available again. *)
|
||||
(*******************************************************************************************************************)
|
||||
LET volID == req.volume_id
|
||||
nodeID == req.node_id
|
||||
|
||||
Errors ==
|
||||
LET checks == [invalid_argument |-> None \in {volID, nodeID}]
|
||||
IN {e \in DOMAIN checks : checks[e]}
|
||||
|
||||
\* The request was rejected, or the provider failed to answer.
|
||||
IN \/ /\ err \in Errors \union ProviderErrors
|
||||
/\ UNCHANGED vars
|
||||
|
||||
\* The request was accepted.
|
||||
\/ /\ Errors = {}
|
||||
\* Record removal is idempotent so that retries succeed.
|
||||
/\ publications' = RemoveKey(publications, <<volID, nodeID>>)
|
||||
\* The volume is available again once no node holds it.
|
||||
/\ volumes' = IF \/ ~KeyExists(volumes, volID)
|
||||
\/ PublishedNodes(volID)' /= {}
|
||||
THEN volumes
|
||||
ELSE [volumes EXCEPT ![volID].state = "available"]
|
||||
/\ err = None
|
||||
/\ UNCHANGED snapshots
|
||||
|
||||
|
||||
DeleteVolumeRequests == [volume_id : Optional(VolumeIDs),
|
||||
handle : Optional(VolumeHandles)]
|
||||
|
||||
DeleteVolume(req, err) ==
|
||||
(*******************************************************************************************************************)
|
||||
(* A volume is deleted, so that its data is gone. Snapshots taken from it are unaffected. *)
|
||||
(*******************************************************************************************************************)
|
||||
LET volID == req.volume_id
|
||||
|
||||
Errors ==
|
||||
LET checks == [invalid_argument |-> volID = None,
|
||||
volume_in_use |-> /\ PublishedNodes(volID) /= {}
|
||||
/\ req.handle \in {None, volumes[volID].handle}]
|
||||
IN {e \in DOMAIN checks : checks[e]}
|
||||
|
||||
\* The request was rejected, or the provider failed to answer.
|
||||
IN \/ /\ err \in Errors \union ProviderErrors
|
||||
/\ UNCHANGED vars
|
||||
|
||||
\* The request was accepted.
|
||||
\/ /\ Errors = {}
|
||||
\* Record removal is idempotent so that retries succeed.
|
||||
/\ volumes' = IF /\ KeyExists(volumes, volID)
|
||||
/\ req.handle \notin {None, volumes[volID].handle}
|
||||
THEN volumes
|
||||
ELSE RemoveKey(volumes, volID)
|
||||
/\ err = None
|
||||
/\ UNCHANGED <<publications, snapshots>>
|
||||
|
||||
-----------------------------------------------------------------------------------------------------------------------
|
||||
\* Specification
|
||||
|
||||
=======================================================================================================================
|
||||
@@ -76,7 +76,7 @@ func ensureCert(dir, caCertPath, caKeyPath, cn string, dns []string, ips []net.I
|
||||
return certPath, keyPath, nil
|
||||
}
|
||||
|
||||
caCert, caKey, err := loadCA(caCertPath, caKeyPath)
|
||||
caCert, caKey, err := LoadCA(caCertPath, caKeyPath)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
@@ -135,8 +135,8 @@ func certCoversSANs(certPath, keyPath string, dns []string, ips []net.IP) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// loadCA parses the PEM CA cert + EC/PKCS8 private key used to sign the leaf.
|
||||
func loadCA(certPath, keyPath string) (*x509.Certificate, *ecdsa.PrivateKey, error) {
|
||||
// LoadCA parses the PEM CA cert + EC/PKCS8 private key used to sign the leaf.
|
||||
func LoadCA(certPath, keyPath string) (*x509.Certificate, *ecdsa.PrivateKey, error) {
|
||||
certPEM, err := os.ReadFile(certPath)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("read ca cert %s: %w", certPath, err)
|
||||
|
||||
@@ -0,0 +1,196 @@
|
||||
package podidentity
|
||||
|
||||
import (
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/pem"
|
||||
"fmt"
|
||||
"math/big"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/agents/eks/konnectivitycert"
|
||||
)
|
||||
|
||||
const (
|
||||
// ClientCN is the identity the webhook presents to the API server; the
|
||||
// ClusterRoleBinding names this user.
|
||||
ClientCN = "spinifex:eks-pod-identity-webhook"
|
||||
|
||||
servingCertFile = "serving.crt"
|
||||
servingKeyFile = "serving.key"
|
||||
clientCertFile = "client.crt"
|
||||
clientKeyFile = "client.key"
|
||||
certRenewWindow = 30 * 24 * time.Hour
|
||||
certValidity = 365 * 24 * time.Hour
|
||||
)
|
||||
|
||||
// CertConfig names the k3s CAs that sign the webhook's two certificates and
|
||||
// the directory they are cached in.
|
||||
type CertConfig struct {
|
||||
Dir string
|
||||
ServerCACert string
|
||||
ServerCAKey string
|
||||
ClientCACert string
|
||||
ClientCAKey string
|
||||
}
|
||||
|
||||
// Certs are the paths of the cached certificate and key pairs.
|
||||
type Certs struct {
|
||||
ServingCert, ServingKey string
|
||||
ClientCert, ClientKey string
|
||||
}
|
||||
|
||||
// EnsureCerts mints, or reuses while still valid and signed by the current CA,
|
||||
// a serving certificate for 127.0.0.1 signed by the k3s server CA and a client
|
||||
// certificate signed by the k3s client CA. Keys are written 0600.
|
||||
func EnsureCerts(c CertConfig) (Certs, error) {
|
||||
out := Certs{
|
||||
ServingCert: filepath.Join(c.Dir, servingCertFile), ServingKey: filepath.Join(c.Dir, servingKeyFile),
|
||||
ClientCert: filepath.Join(c.Dir, clientCertFile), ClientKey: filepath.Join(c.Dir, clientKeyFile),
|
||||
}
|
||||
if err := os.MkdirAll(c.Dir, 0o700); err != nil {
|
||||
return Certs{}, fmt.Errorf("mkdir %s: %w", c.Dir, err)
|
||||
}
|
||||
serving := leafSpec{cn: "pod-identity-webhook", usage: x509.ExtKeyUsageServerAuth, ips: []net.IP{net.IPv4(127, 0, 0, 1)}}
|
||||
if err := ensureLeaf(out.ServingCert, out.ServingKey, c.ServerCACert, c.ServerCAKey, serving); err != nil {
|
||||
return Certs{}, fmt.Errorf("serving certificate: %w", err)
|
||||
}
|
||||
client := leafSpec{cn: ClientCN, usage: x509.ExtKeyUsageClientAuth}
|
||||
if err := ensureLeaf(out.ClientCert, out.ClientKey, c.ClientCACert, c.ClientCAKey, client); err != nil {
|
||||
return Certs{}, fmt.Errorf("client certificate: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
type leafSpec struct {
|
||||
cn string
|
||||
usage x509.ExtKeyUsage
|
||||
ips []net.IP
|
||||
}
|
||||
|
||||
func ensureLeaf(certPath, keyPath, caCertPath, caKeyPath string, spec leafSpec) error {
|
||||
ca, caKey, err := konnectivitycert.LoadCA(caCertPath, caKeyPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if leafCurrent(certPath, keyPath, ca, spec) {
|
||||
return nil
|
||||
}
|
||||
priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
return fmt.Errorf("generate key: %w", err)
|
||||
}
|
||||
serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
|
||||
if err != nil {
|
||||
return fmt.Errorf("serial: %w", err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
tmpl := x509.Certificate{
|
||||
SerialNumber: serial,
|
||||
Subject: pkix.Name{CommonName: spec.cn},
|
||||
NotBefore: now.Add(-time.Hour),
|
||||
NotAfter: now.Add(certValidity),
|
||||
KeyUsage: x509.KeyUsageDigitalSignature,
|
||||
ExtKeyUsage: []x509.ExtKeyUsage{spec.usage},
|
||||
BasicConstraintsValid: true,
|
||||
IPAddresses: spec.ips,
|
||||
}
|
||||
der, err := x509.CreateCertificate(rand.Reader, &tmpl, ca, &priv.PublicKey, caKey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create certificate: %w", err)
|
||||
}
|
||||
keyDER, err := x509.MarshalECPrivateKey(priv)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal key: %w", err)
|
||||
}
|
||||
// Key first: a cert without its key is never published.
|
||||
if err := writeAtomic(keyPath, pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}), 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
return writeAtomic(certPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o644)
|
||||
}
|
||||
|
||||
// leafCurrent reports whether the cached pair loads, matches, was signed by ca,
|
||||
// has the wanted usage and SANs, and is not near expiry.
|
||||
func leafCurrent(certPath, keyPath string, ca *x509.Certificate, spec leafSpec) bool {
|
||||
pair, err := tls.LoadX509KeyPair(certPath, keyPath)
|
||||
if err != nil || len(pair.Certificate) == 0 {
|
||||
return false
|
||||
}
|
||||
crt, err := x509.ParseCertificate(pair.Certificate[0])
|
||||
if err != nil || crt.CheckSignatureFrom(ca) != nil {
|
||||
return false
|
||||
}
|
||||
if inRenewWindow(crt, time.Now()) || crt.Subject.CommonName != spec.cn {
|
||||
return false
|
||||
}
|
||||
if len(crt.ExtKeyUsage) != 1 || crt.ExtKeyUsage[0] != spec.usage {
|
||||
return false
|
||||
}
|
||||
for _, ip := range spec.ips {
|
||||
found := false
|
||||
for _, have := range crt.IPAddresses {
|
||||
found = found || have.Equal(ip)
|
||||
}
|
||||
if !found {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func writeAtomic(path string, data []byte, mode os.FileMode) error {
|
||||
tmp, err := os.CreateTemp(filepath.Dir(path), ".tmp-*")
|
||||
if err != nil {
|
||||
return fmt.Errorf("create temp for %s: %w", path, err)
|
||||
}
|
||||
defer func() { _ = os.Remove(tmp.Name()) }()
|
||||
if err := tmp.Chmod(mode); err != nil {
|
||||
_ = tmp.Close()
|
||||
return fmt.Errorf("chmod %s: %w", path, err)
|
||||
}
|
||||
if _, err := tmp.Write(data); err != nil {
|
||||
_ = tmp.Close()
|
||||
return fmt.Errorf("write %s: %w", path, err)
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
return fmt.Errorf("close %s: %w", path, err)
|
||||
}
|
||||
if err := os.Rename(tmp.Name(), path); err != nil {
|
||||
return fmt.Errorf("rename %s: %w", path, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// inRenewWindow reports whether crt expires within the renew window at now.
|
||||
func inRenewWindow(crt *x509.Certificate, now time.Time) bool {
|
||||
return crt.NotAfter.Sub(now) < certRenewWindow
|
||||
}
|
||||
|
||||
// NeedsRenewal reports whether either cached certificate is inside its renew
|
||||
// window at now. A running process exits when this is true so the supervisor
|
||||
// restarts it and EnsureCerts reissues; an unreadable certificate also counts.
|
||||
func NeedsRenewal(c Certs, now time.Time) bool {
|
||||
for _, path := range []string{c.ServingCert, c.ClientCert} {
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return true
|
||||
}
|
||||
blk, _ := pem.Decode(b)
|
||||
if blk == nil {
|
||||
return true
|
||||
}
|
||||
crt, err := x509.ParseCertificate(blk.Bytes)
|
||||
if err != nil || inRenewWindow(crt, now) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,263 @@
|
||||
package podidentity
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
kubeCallTimeout = 15 * time.Second
|
||||
watchRetryDelay = 2 * time.Second
|
||||
maxKubeBody = 4 << 20
|
||||
)
|
||||
|
||||
// KubeClient reaches the Kubernetes API with plain REST. It implements
|
||||
// ServiceAccountGetter, ConfigMapStore and NamespaceSource using only the
|
||||
// calls the webhook's ClusterRole grants.
|
||||
type KubeClient struct {
|
||||
base string
|
||||
http *http.Client
|
||||
}
|
||||
|
||||
var (
|
||||
_ ServiceAccountGetter = (*KubeClient)(nil)
|
||||
_ ConfigMapStore = (*KubeClient)(nil)
|
||||
_ NamespaceSource = (*KubeClient)(nil)
|
||||
)
|
||||
|
||||
// NewKubeClient returns a client for apiServer that trusts caFile and presents
|
||||
// the client certificate in certFile and keyFile. The pair is re-read on every
|
||||
// handshake, so a re-issued certificate is used without a restart.
|
||||
func NewKubeClient(apiServer, caFile, certFile, keyFile string) (*KubeClient, error) {
|
||||
ca, err := os.ReadFile(caFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read api server ca: %w", err)
|
||||
}
|
||||
pool := x509.NewCertPool()
|
||||
if !pool.AppendCertsFromPEM(ca) {
|
||||
return nil, fmt.Errorf("api server ca %s holds no certificate", caFile)
|
||||
}
|
||||
if _, err := tls.LoadX509KeyPair(certFile, keyFile); err != nil {
|
||||
return nil, fmt.Errorf("load client certificate: %w", err)
|
||||
}
|
||||
tr := &http.Transport{TLSClientConfig: &tls.Config{
|
||||
RootCAs: pool,
|
||||
MinVersion: tls.VersionTLS12,
|
||||
GetClientCertificate: func(*tls.CertificateRequestInfo) (*tls.Certificate, error) {
|
||||
c, err := tls.LoadX509KeyPair(certFile, keyFile)
|
||||
return &c, err
|
||||
},
|
||||
}}
|
||||
return &KubeClient{base: strings.TrimSuffix(apiServer, "/"), http: &http.Client{Transport: tr}}, nil
|
||||
}
|
||||
|
||||
type kubeObject struct {
|
||||
APIVersion string `json:"apiVersion,omitempty"`
|
||||
Kind string `json:"kind,omitempty"`
|
||||
Metadata struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
Namespace string `json:"namespace,omitempty"`
|
||||
ResourceVersion string `json:"resourceVersion,omitempty"`
|
||||
Annotations map[string]string `json:"annotations,omitempty"`
|
||||
} `json:"metadata"`
|
||||
Data map[string]string `json:"data,omitempty"`
|
||||
}
|
||||
|
||||
// do sends one request and returns the status. A non-2xx status is returned
|
||||
// without an error so callers can map the codes they expect.
|
||||
func (k *KubeClient) do(ctx context.Context, method, path string, body, out any) (int, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, kubeCallTimeout)
|
||||
defer cancel()
|
||||
var rd io.Reader
|
||||
if body != nil {
|
||||
b, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
rd = bytes.NewReader(b)
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, method, k.base+path, rd)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
req.Header.Set("Accept", "application/json")
|
||||
if body != nil {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
resp, err := k.http.Do(req)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
raw, err := io.ReadAll(io.LimitReader(resp.Body, maxKubeBody))
|
||||
if err != nil {
|
||||
return resp.StatusCode, err
|
||||
}
|
||||
if resp.StatusCode/100 == 2 && out != nil {
|
||||
if err := json.Unmarshal(raw, out); err != nil {
|
||||
return resp.StatusCode, fmt.Errorf("decode %s %s: %w", method, path, err)
|
||||
}
|
||||
}
|
||||
return resp.StatusCode, nil
|
||||
}
|
||||
|
||||
func statusErr(method, path string, code int) error {
|
||||
return fmt.Errorf("%s %s: unexpected status %d", method, path, code)
|
||||
}
|
||||
|
||||
// GetServiceAccount implements ServiceAccountGetter.
|
||||
func (k *KubeClient) GetServiceAccount(ctx context.Context, namespace, name string) (*ServiceAccount, error) {
|
||||
path := "/api/v1/namespaces/" + url.PathEscape(namespace) + "/serviceaccounts/" + url.PathEscape(name)
|
||||
var o kubeObject
|
||||
switch code, err := k.do(ctx, http.MethodGet, path, nil, &o); {
|
||||
case err != nil:
|
||||
return nil, err
|
||||
case code == http.StatusNotFound:
|
||||
return nil, ErrServiceAccountNotFound
|
||||
case code != http.StatusOK:
|
||||
return nil, statusErr(http.MethodGet, path, code)
|
||||
}
|
||||
return &ServiceAccount{Annotations: o.Metadata.Annotations}, nil
|
||||
}
|
||||
|
||||
func cmPath(namespace, name string) string {
|
||||
p := "/api/v1/namespaces/" + url.PathEscape(namespace) + "/configmaps"
|
||||
if name != "" {
|
||||
p += "/" + url.PathEscape(name)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func cmBody(cm ConfigMap) kubeObject {
|
||||
o := kubeObject{APIVersion: "v1", Kind: "ConfigMap", Data: cm.Data}
|
||||
o.Metadata.Name, o.Metadata.Namespace, o.Metadata.ResourceVersion = cm.Name, cm.Namespace, cm.ResourceVersion
|
||||
return o
|
||||
}
|
||||
|
||||
// Get implements ConfigMapStore.
|
||||
func (k *KubeClient) Get(ctx context.Context, namespace, name string) (*ConfigMap, error) {
|
||||
path := cmPath(namespace, name)
|
||||
var o kubeObject
|
||||
switch code, err := k.do(ctx, http.MethodGet, path, nil, &o); {
|
||||
case err != nil:
|
||||
return nil, err
|
||||
case code == http.StatusNotFound:
|
||||
return nil, ErrNotFound
|
||||
case code != http.StatusOK:
|
||||
return nil, statusErr(http.MethodGet, path, code)
|
||||
}
|
||||
return &ConfigMap{Namespace: namespace, Name: name, Data: o.Data, ResourceVersion: o.Metadata.ResourceVersion}, nil
|
||||
}
|
||||
|
||||
// Create implements ConfigMapStore.
|
||||
func (k *KubeClient) Create(ctx context.Context, cm ConfigMap) error {
|
||||
path := cmPath(cm.Namespace, "")
|
||||
switch code, err := k.do(ctx, http.MethodPost, path, cmBody(cm), nil); {
|
||||
case err != nil:
|
||||
return err
|
||||
case code == http.StatusConflict:
|
||||
return ErrAlreadyExists
|
||||
case code/100 != 2:
|
||||
return statusErr(http.MethodPost, path, code)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Update implements ConfigMapStore.
|
||||
func (k *KubeClient) Update(ctx context.Context, cm ConfigMap) error {
|
||||
path := cmPath(cm.Namespace, cm.Name)
|
||||
switch code, err := k.do(ctx, http.MethodPut, path, cmBody(cm), nil); {
|
||||
case err != nil:
|
||||
return err
|
||||
case code == http.StatusConflict:
|
||||
return ErrConflict
|
||||
case code == http.StatusNotFound:
|
||||
return ErrNotFound
|
||||
case code/100 != 2:
|
||||
return statusErr(http.MethodPut, path, code)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// List implements NamespaceSource.
|
||||
func (k *KubeClient) List(ctx context.Context) ([]string, error) {
|
||||
const path = "/api/v1/namespaces"
|
||||
var l struct {
|
||||
Items []kubeObject `json:"items"`
|
||||
}
|
||||
if code, err := k.do(ctx, http.MethodGet, path, nil, &l); err != nil {
|
||||
return nil, err
|
||||
} else if code != http.StatusOK {
|
||||
return nil, statusErr(http.MethodGet, path, code)
|
||||
}
|
||||
names := make([]string, 0, len(l.Items))
|
||||
for _, it := range l.Items {
|
||||
names = append(names, it.Metadata.Name)
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
|
||||
// Watch implements NamespaceSource. The stream replays existing namespaces as
|
||||
// ADDED first, which is harmless, and is re-opened when the server ends it.
|
||||
// The channel closes only when ctx ends.
|
||||
func (k *KubeClient) Watch(ctx context.Context) (<-chan string, error) {
|
||||
out := make(chan string, 16)
|
||||
go func() {
|
||||
defer close(out)
|
||||
for ctx.Err() == nil {
|
||||
k.watchOnce(ctx, out)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(watchRetryDelay):
|
||||
}
|
||||
}
|
||||
}()
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (k *KubeClient) watchOnce(ctx context.Context, out chan<- string) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, k.base+"/api/v1/namespaces?watch=true&timeoutSeconds=300", nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
resp, err := k.http.Do(req)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return
|
||||
}
|
||||
dec := json.NewDecoder(resp.Body)
|
||||
for {
|
||||
var ev struct {
|
||||
Type string `json:"type"`
|
||||
Object kubeObject `json:"object"`
|
||||
}
|
||||
if err := dec.Decode(&ev); err != nil {
|
||||
if !errors.Is(err, io.EOF) && ctx.Err() == nil {
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
if ev.Type != "ADDED" || ev.Object.Metadata.Name == "" {
|
||||
continue
|
||||
}
|
||||
select {
|
||||
case out <- ev.Object.Metadata.Name:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
package podidentity
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// fakeAPI is a TLS API server that requires the webhook's client certificate.
|
||||
type fakeAPI struct {
|
||||
srv *httptest.Server
|
||||
handler http.HandlerFunc
|
||||
seenCN string
|
||||
}
|
||||
|
||||
func newFakeAPI(t *testing.T) (*fakeAPI, *KubeClient) {
|
||||
t.Helper()
|
||||
f := &fakeAPI{}
|
||||
f.srv = httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
f.seenCN = r.TLS.PeerCertificates[0].Subject.CommonName
|
||||
f.handler(w, r)
|
||||
}))
|
||||
f.srv.TLS = &tls.Config{ClientAuth: tls.RequireAnyClientCert, MinVersion: tls.VersionTLS12}
|
||||
f.srv.StartTLS()
|
||||
t.Cleanup(f.srv.Close)
|
||||
|
||||
cfg, _ := certTestConfig(t)
|
||||
certs, err := EnsureCerts(cfg)
|
||||
require.NoError(t, err)
|
||||
ca := filepath.Join(t.TempDir(), "ca.crt")
|
||||
require.NoError(t, os.WriteFile(ca, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: f.srv.Certificate().Raw}), 0o644))
|
||||
k, err := NewKubeClient(f.srv.URL+"/", ca, certs.ClientCert, certs.ClientKey)
|
||||
require.NoError(t, err)
|
||||
return f, k
|
||||
}
|
||||
|
||||
func reply(w http.ResponseWriter, code int, body any) {
|
||||
w.WriteHeader(code)
|
||||
if body != nil {
|
||||
_ = json.NewEncoder(w).Encode(body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKubeClient_PresentsTheClientCertificate(t *testing.T) {
|
||||
f, k := newFakeAPI(t)
|
||||
f.handler = func(w http.ResponseWriter, _ *http.Request) {
|
||||
reply(w, 200, map[string]any{"metadata": map[string]any{}})
|
||||
}
|
||||
_, err := k.GetServiceAccount(context.Background(), "app", "sa")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, ClientCN, f.seenCN)
|
||||
}
|
||||
|
||||
func TestKubeClient_GetServiceAccount(t *testing.T) {
|
||||
f, k := newFakeAPI(t)
|
||||
f.handler = func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/api/v1/namespaces/app/serviceaccounts/sa":
|
||||
reply(w, 200, map[string]any{"metadata": map[string]any{"annotations": map[string]string{"a": "b"}}})
|
||||
case "/api/v1/namespaces/app/serviceaccounts/gone":
|
||||
reply(w, 404, nil)
|
||||
default:
|
||||
reply(w, 500, nil)
|
||||
}
|
||||
}
|
||||
sa, err := k.GetServiceAccount(context.Background(), "app", "sa")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, map[string]string{"a": "b"}, sa.Annotations)
|
||||
|
||||
_, err = k.GetServiceAccount(context.Background(), "app", "gone")
|
||||
assert.ErrorIs(t, err, ErrServiceAccountNotFound)
|
||||
_, err = k.GetServiceAccount(context.Background(), "app", "boom")
|
||||
assert.Error(t, err)
|
||||
assert.NotErrorIs(t, err, ErrServiceAccountNotFound)
|
||||
}
|
||||
|
||||
func TestKubeClient_ConfigMapCalls(t *testing.T) {
|
||||
f, k := newFakeAPI(t)
|
||||
var gotBody kubeObject
|
||||
f.handler = func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/api/v1/namespaces/app/configmaps/ca":
|
||||
reply(w, 200, map[string]any{"metadata": map[string]any{"resourceVersion": "7"}, "data": map[string]string{"k": "v"}})
|
||||
case r.Method == http.MethodGet:
|
||||
reply(w, 404, nil)
|
||||
case r.Method == http.MethodPost && r.URL.Path == "/api/v1/namespaces/dup/configmaps":
|
||||
reply(w, 409, nil)
|
||||
case r.Method == http.MethodPost:
|
||||
_ = json.NewDecoder(r.Body).Decode(&gotBody)
|
||||
reply(w, 201, nil)
|
||||
case r.Method == http.MethodPut && r.URL.Path == "/api/v1/namespaces/stale/configmaps/ca":
|
||||
reply(w, 409, nil)
|
||||
case r.Method == http.MethodPut && r.URL.Path == "/api/v1/namespaces/gone/configmaps/ca":
|
||||
reply(w, 404, nil)
|
||||
case r.Method == http.MethodPut && r.URL.Path == "/api/v1/namespaces/bad/configmaps/ca":
|
||||
reply(w, 500, nil)
|
||||
default:
|
||||
reply(w, 200, nil)
|
||||
}
|
||||
}
|
||||
ctx := context.Background()
|
||||
|
||||
cm, err := k.Get(ctx, "app", "ca")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "7", cm.ResourceVersion)
|
||||
assert.Equal(t, "v", cm.Data["k"])
|
||||
_, err = k.Get(ctx, "app", "missing")
|
||||
assert.ErrorIs(t, err, ErrNotFound)
|
||||
|
||||
require.NoError(t, k.Create(ctx, ConfigMap{Namespace: "app", Name: "ca", Data: map[string]string{"x": "y"}}))
|
||||
assert.Equal(t, "ca", gotBody.Metadata.Name)
|
||||
assert.Equal(t, "y", gotBody.Data["x"])
|
||||
assert.ErrorIs(t, k.Create(ctx, ConfigMap{Namespace: "dup", Name: "ca"}), ErrAlreadyExists)
|
||||
|
||||
require.NoError(t, k.Update(ctx, ConfigMap{Namespace: "app", Name: "ca"}))
|
||||
assert.ErrorIs(t, k.Update(ctx, ConfigMap{Namespace: "stale", Name: "ca"}), ErrConflict)
|
||||
assert.ErrorIs(t, k.Update(ctx, ConfigMap{Namespace: "gone", Name: "ca"}), ErrNotFound)
|
||||
err = k.Update(ctx, ConfigMap{Namespace: "bad", Name: "ca"})
|
||||
assert.Error(t, err)
|
||||
assert.NotErrorIs(t, err, ErrConflict)
|
||||
}
|
||||
|
||||
func TestKubeClient_ListNamespaces(t *testing.T) {
|
||||
f, k := newFakeAPI(t)
|
||||
ok := true
|
||||
f.handler = func(w http.ResponseWriter, _ *http.Request) {
|
||||
if !ok {
|
||||
reply(w, 500, nil)
|
||||
return
|
||||
}
|
||||
reply(w, 200, map[string]any{"items": []map[string]any{{"metadata": map[string]string{"name": "a"}}, {"metadata": map[string]string{"name": "b"}}}})
|
||||
}
|
||||
names, err := k.List(context.Background())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"a", "b"}, names)
|
||||
ok = false
|
||||
_, err = k.List(context.Background())
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func TestKubeClient_WatchEmitsAddedNamespacesOnly(t *testing.T) {
|
||||
f, k := newFakeAPI(t)
|
||||
f.handler = func(w http.ResponseWriter, _ *http.Request) {
|
||||
enc := json.NewEncoder(w)
|
||||
for _, ev := range []struct{ t, n string }{{"ADDED", "one"}, {"MODIFIED", "two"}, {"DELETED", "three"}, {"ADDED", "four"}} {
|
||||
_ = enc.Encode(map[string]any{"type": ev.t, "object": map[string]any{"metadata": map[string]string{"name": ev.n}}})
|
||||
}
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
ch, err := k.Watch(ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
got := []string{<-ch, <-ch}
|
||||
assert.Equal(t, []string{"one", "four"}, got)
|
||||
cancel()
|
||||
require.Eventually(t, func() bool {
|
||||
select {
|
||||
case _, open := <-ch:
|
||||
return !open
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}, 5*time.Second, 10*time.Millisecond, "the channel closes when the context ends")
|
||||
}
|
||||
|
||||
func TestNewKubeClient_RejectsBadInputs(t *testing.T) {
|
||||
cfg, _ := certTestConfig(t)
|
||||
certs, err := EnsureCerts(cfg)
|
||||
require.NoError(t, err)
|
||||
dir := t.TempDir()
|
||||
empty := filepath.Join(dir, "empty.crt")
|
||||
require.NoError(t, os.WriteFile(empty, []byte("not pem"), 0o644))
|
||||
|
||||
_, err = NewKubeClient("https://x", filepath.Join(dir, "absent.crt"), certs.ClientCert, certs.ClientKey)
|
||||
assert.Error(t, err)
|
||||
_, err = NewKubeClient("https://x", empty, certs.ClientCert, certs.ClientKey)
|
||||
assert.Error(t, err)
|
||||
_, err = NewKubeClient("https://x", cfg.ServerCACert, filepath.Join(dir, "no.crt"), certs.ClientKey)
|
||||
assert.Error(t, err)
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package podidentity
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
)
|
||||
|
||||
// ClusterRoleName names the ClusterRole and ClusterRoleBinding the webhook runs as.
|
||||
const ClusterRoleName = "spinifex:eks-pod-identity-webhook"
|
||||
|
||||
// RenderManifests returns the cluster objects the webhook needs, as JSON
|
||||
// documents separated by "---" (valid YAML for the k3s manifest deployer): its
|
||||
// least-privilege ClusterRole, the binding to its client-certificate user, and
|
||||
// the MutatingWebhookConfiguration. The output is deterministic so every server
|
||||
// writes the same bytes.
|
||||
func RenderManifests(webhook WebhookConfig) ([]byte, error) {
|
||||
cfg, err := RenderWebhookConfiguration(webhook)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
role := map[string]any{
|
||||
"apiVersion": "rbac.authorization.k8s.io/v1",
|
||||
"kind": "ClusterRole",
|
||||
"metadata": map[string]any{"name": ClusterRoleName},
|
||||
"rules": []map[string]any{
|
||||
{"apiGroups": []string{""}, "resources": []string{"serviceaccounts"}, "verbs": []string{"get"}},
|
||||
{"apiGroups": []string{""}, "resources": []string{"namespaces"}, "verbs": []string{"list", "watch"}},
|
||||
{"apiGroups": []string{""}, "resources": []string{"configmaps"}, "verbs": []string{"create"}},
|
||||
{"apiGroups": []string{""}, "resources": []string{"configmaps"}, "resourceNames": []string{CAConfigMapName}, "verbs": []string{"get", "update"}},
|
||||
},
|
||||
}
|
||||
binding := map[string]any{
|
||||
"apiVersion": "rbac.authorization.k8s.io/v1",
|
||||
"kind": "ClusterRoleBinding",
|
||||
"metadata": map[string]any{"name": ClusterRoleName},
|
||||
"roleRef": map[string]any{"apiGroup": "rbac.authorization.k8s.io", "kind": "ClusterRole", "name": ClusterRoleName},
|
||||
"subjects": []map[string]any{{"apiGroup": "rbac.authorization.k8s.io", "kind": "User", "name": ClientCN}},
|
||||
}
|
||||
var out bytes.Buffer
|
||||
for i, doc := range []any{role, binding, json.RawMessage(cfg)} {
|
||||
b, err := json.MarshalIndent(doc, "", " ")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if i > 0 {
|
||||
out.WriteString("---\n")
|
||||
}
|
||||
out.Write(b)
|
||||
out.WriteByte('\n')
|
||||
}
|
||||
return out.Bytes(), nil
|
||||
}
|
||||
@@ -0,0 +1,307 @@
|
||||
package podidentity
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"math/rand/v2"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
envRoleARN = "AWS_ROLE_ARN"
|
||||
envTokenFile = "AWS_WEB_IDENTITY_TOKEN_FILE" //nolint:gosec // variable name, not a credential
|
||||
envRegion = "AWS_REGION"
|
||||
envDefRegion = "AWS_DEFAULT_REGION"
|
||||
envRegionalS = "AWS_STS_REGIONAL_ENDPOINTS"
|
||||
envSTSURL = "AWS_ENDPOINT_URL_STS"
|
||||
envCABundle = "AWS_CA_BUNDLE"
|
||||
envSSLCert = "SSL_CERT_FILE"
|
||||
maxReviewSize = 8 << 20
|
||||
)
|
||||
|
||||
type podObject struct {
|
||||
Metadata struct {
|
||||
Name string `json:"name"`
|
||||
GenerateName string `json:"generateName"`
|
||||
Namespace string `json:"namespace"`
|
||||
Annotations map[string]string `json:"annotations"`
|
||||
} `json:"metadata"`
|
||||
Spec struct {
|
||||
ServiceAccountName string `json:"serviceAccountName"`
|
||||
ServiceAccount string `json:"serviceAccount"`
|
||||
Containers []containerObject `json:"containers"`
|
||||
InitContainers []containerObject `json:"initContainers"`
|
||||
Volumes []nameOnly `json:"volumes"`
|
||||
} `json:"spec"`
|
||||
}
|
||||
|
||||
type nameOnly struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
type containerObject struct {
|
||||
Name string `json:"name"`
|
||||
Env []nameOnly `json:"env"`
|
||||
VolumeMounts []nameOnly `json:"volumeMounts"`
|
||||
}
|
||||
|
||||
type envVar struct {
|
||||
Name string `json:"name"`
|
||||
Value string `json:"value"`
|
||||
}
|
||||
|
||||
type volumeMount struct {
|
||||
Name string `json:"name"`
|
||||
MountPath string `json:"mountPath"`
|
||||
ReadOnly bool `json:"readOnly"`
|
||||
}
|
||||
|
||||
// injection is what one pod gets, computed from its ServiceAccount and annotations.
|
||||
type injection struct {
|
||||
roleARN string
|
||||
regional bool
|
||||
skip map[string]bool
|
||||
}
|
||||
|
||||
// Mutate returns the patch for one pod creation request. It never denies a pod.
|
||||
func (m *Mutator) Mutate(ctx context.Context, req AdmissionRequest) AdmissionResponse {
|
||||
resp := AdmissionResponse{UID: req.UID, Allowed: true}
|
||||
if req.Operation != "CREATE" {
|
||||
return resp
|
||||
}
|
||||
var pod podObject
|
||||
if err := json.Unmarshal(req.Object, &pod); err != nil {
|
||||
m.note(ctx, slog.LevelWarn, req.Namespace, "", ReasonDecode, "error", err)
|
||||
return resp
|
||||
}
|
||||
ns := cmp.Or(req.Namespace, pod.Metadata.Namespace)
|
||||
name := cmp.Or(pod.Metadata.Name, pod.Metadata.GenerateName)
|
||||
|
||||
saName := cmp.Or(pod.Spec.ServiceAccountName, pod.Spec.ServiceAccount, "default")
|
||||
sa, err := m.sas.GetServiceAccount(ctx, ns, saName)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrServiceAccountNotFound) {
|
||||
m.note(ctx, slog.LevelInfo, ns, name, ReasonServiceAccountMissing, "serviceaccount", saName)
|
||||
} else {
|
||||
m.note(ctx, slog.LevelWarn, ns, name, ReasonServiceAccountLookup, "serviceaccount", saName, "error", err)
|
||||
}
|
||||
return resp
|
||||
}
|
||||
dom := cmp.Or(m.cfg.AnnotationDomain, DefaultAnnotationDomain)
|
||||
roleARN := sa.Annotations[dom+"/role-arn"]
|
||||
if roleARN == "" {
|
||||
return resp
|
||||
}
|
||||
|
||||
in := injection{
|
||||
roleARN: roleARN,
|
||||
regional: sa.Annotations[dom+"/sts-regional-endpoints"] == "true",
|
||||
skip: map[string]bool{},
|
||||
}
|
||||
for c := range strings.SplitSeq(pod.Metadata.Annotations[dom+"/skip-containers"], ",") {
|
||||
if c = strings.TrimSpace(c); c != "" {
|
||||
in.skip[c] = true
|
||||
}
|
||||
}
|
||||
|
||||
var ops []PatchOp
|
||||
patched := 0
|
||||
for _, set := range []struct {
|
||||
field string
|
||||
list []containerObject
|
||||
}{{"containers", pod.Spec.Containers}, {"initContainers", pod.Spec.InitContainers}} {
|
||||
for i, c := range set.list {
|
||||
if in.skip[c.Name] {
|
||||
m.note(ctx, slog.LevelInfo, ns, name, ReasonSkipContainer, "container", c.Name)
|
||||
continue
|
||||
}
|
||||
cops := m.containerOps(fmt.Sprintf("/spec/%s/%d", set.field, i), c, in)
|
||||
if cops == nil {
|
||||
m.note(ctx, slog.LevelInfo, ns, name, ReasonAlreadyConfigured, "container", c.Name)
|
||||
continue
|
||||
}
|
||||
ops = append(ops, cops...)
|
||||
patched++
|
||||
}
|
||||
}
|
||||
if patched == 0 {
|
||||
return resp
|
||||
}
|
||||
|
||||
ops = append(ops, volumeOps(pod, tokenExpiry(sa, pod.Metadata.Annotations, dom), cmp.Or(sa.Annotations[dom+"/audience"], DefaultAudience))...)
|
||||
resp.Patch = ops
|
||||
m.log.InfoContext(ctx, "pod identity injected", "namespace", ns, "pod", name, "serviceaccount", saName, "containers", patched)
|
||||
return resp
|
||||
}
|
||||
|
||||
// note logs a skip or failure with every attribute inline, not on a derived logger.
|
||||
func (m *Mutator) note(ctx context.Context, lvl slog.Level, ns, pod, reason string, kv ...any) {
|
||||
attrs := append([]any{"namespace", ns, "pod", pod, "reason", reason}, kv...)
|
||||
m.log.Log(ctx, lvl, "pod identity skipped", attrs...)
|
||||
}
|
||||
|
||||
// containerOps patches one container, or returns nil when it already carries
|
||||
// the role, region and regional STS settings.
|
||||
func (m *Mutator) containerOps(path string, c containerObject, in injection) []PatchOp {
|
||||
has := map[string]bool{}
|
||||
for _, e := range c.Env {
|
||||
has[e.Name] = true
|
||||
}
|
||||
hasRole := has[envRoleARN] || has[envTokenFile]
|
||||
hasRegion := m.cfg.Region == "" || has[envRegion] || has[envDefRegion]
|
||||
hasSTS := has[envRegionalS]
|
||||
if hasRole && hasRegion && hasSTS {
|
||||
return nil
|
||||
}
|
||||
|
||||
var vars []envVar
|
||||
add := func(name, value string) { vars = append(vars, envVar{name, value}) }
|
||||
if !hasRole {
|
||||
add(envRoleARN, in.roleARN)
|
||||
add(envTokenFile, TokenMountPath+"/"+TokenPath)
|
||||
}
|
||||
if !hasSTS && in.regional {
|
||||
add(envRegionalS, "regional")
|
||||
}
|
||||
if !hasRegion {
|
||||
add(envRegion, m.cfg.Region)
|
||||
add(envDefRegion, m.cfg.Region)
|
||||
}
|
||||
caPath := CAMountPath + "/" + CABundleKey
|
||||
for _, x := range []struct{ name, value string }{
|
||||
{envSTSURL, m.cfg.STSEndpoint}, {envCABundle, caPath}, {envSSLCert, caPath},
|
||||
} {
|
||||
if x.value != "" && !has[x.name] {
|
||||
add(x.name, x.value)
|
||||
}
|
||||
}
|
||||
|
||||
mounted := map[string]bool{}
|
||||
for _, v := range c.VolumeMounts {
|
||||
mounted[v.Name] = true
|
||||
}
|
||||
var mounts []volumeMount
|
||||
if !mounted[TokenVolumeName] {
|
||||
mounts = append(mounts, volumeMount{TokenVolumeName, TokenMountPath, true})
|
||||
}
|
||||
if !mounted[CAVolumeName] {
|
||||
mounts = append(mounts, volumeMount{CAVolumeName, CAMountPath, true})
|
||||
}
|
||||
|
||||
ops := appendOps(nil, path+"/env", len(c.Env) > 0, vars)
|
||||
ops = appendOps(ops, path+"/volumeMounts", len(c.VolumeMounts) > 0, mounts)
|
||||
if ops == nil {
|
||||
ops = []PatchOp{}
|
||||
}
|
||||
return ops
|
||||
}
|
||||
|
||||
// appendOps adds items to an array, creating the array first when it is absent
|
||||
// (an add to "/-" on a missing array is not a valid JSON patch).
|
||||
func appendOps[T any](ops []PatchOp, path string, exists bool, items []T) []PatchOp {
|
||||
switch {
|
||||
case len(items) == 0:
|
||||
return ops
|
||||
case !exists:
|
||||
return append(ops, PatchOp{Op: "add", Path: path, Value: items})
|
||||
}
|
||||
for _, it := range items {
|
||||
ops = append(ops, PatchOp{Op: "add", Path: path + "/-", Value: it})
|
||||
}
|
||||
return ops
|
||||
}
|
||||
|
||||
func volumeOps(pod podObject, expiry int64, audience string) []PatchOp {
|
||||
have := map[string]bool{}
|
||||
for _, v := range pod.Spec.Volumes {
|
||||
have[v.Name] = true
|
||||
}
|
||||
var vols []map[string]any
|
||||
if !have[TokenVolumeName] {
|
||||
vols = append(vols, map[string]any{
|
||||
"name": TokenVolumeName,
|
||||
"projected": map[string]any{"sources": []map[string]any{{
|
||||
"serviceAccountToken": map[string]any{"audience": audience, "expirationSeconds": expiry, "path": TokenPath},
|
||||
}}},
|
||||
})
|
||||
}
|
||||
if !have[CAVolumeName] {
|
||||
vols = append(vols, map[string]any{"name": CAVolumeName, "configMap": map[string]any{"name": CAConfigMapName}})
|
||||
}
|
||||
return appendOps(nil, "/spec/volumes", len(pod.Spec.Volumes) > 0, vols)
|
||||
}
|
||||
|
||||
// tokenExpiry takes the pod annotation over the ServiceAccount's, clamps it to
|
||||
// the allowed range and jitters the default so tokens do not all expire together.
|
||||
func tokenExpiry(sa *ServiceAccount, podAnn map[string]string, dom string) int64 {
|
||||
key := dom + "/token-expiration"
|
||||
for _, raw := range []string{podAnn[key], sa.Annotations[key]} {
|
||||
if raw == "" {
|
||||
continue
|
||||
}
|
||||
if n, err := strconv.ParseInt(raw, 10, 64); err == nil {
|
||||
return min(max(n, MinTokenExpiration), MaxTokenExpiration)
|
||||
}
|
||||
}
|
||||
return DefaultMinTokenExpiration + rand.Int64N(MaxTokenExpiration-DefaultMinTokenExpiration+1) //nolint:gosec // jitter, not security
|
||||
}
|
||||
|
||||
type admissionReview struct {
|
||||
APIVersion string `json:"apiVersion"`
|
||||
Kind string `json:"kind"`
|
||||
Request *AdmissionRequest `json:"request,omitempty"`
|
||||
Response *admissionResponse `json:"response,omitempty"`
|
||||
}
|
||||
|
||||
type admissionResponse struct {
|
||||
UID string `json:"uid"`
|
||||
Allowed bool `json:"allowed"`
|
||||
PatchType string `json:"patchType,omitempty"`
|
||||
Patch string `json:"patch,omitempty"`
|
||||
}
|
||||
|
||||
// ServeHTTP serves an AdmissionReview v1 request.
|
||||
func (m *Mutator) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
http.Error(w, "POST required", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, maxReviewSize))
|
||||
if err != nil {
|
||||
http.Error(w, "read body", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
var in admissionReview
|
||||
if err := json.Unmarshal(body, &in); err != nil || in.Request == nil {
|
||||
http.Error(w, "not an AdmissionReview", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
res := m.Mutate(r.Context(), *in.Request)
|
||||
out := admissionReview{
|
||||
APIVersion: "admission.k8s.io/v1",
|
||||
Kind: "AdmissionReview",
|
||||
Response: &admissionResponse{UID: res.UID, Allowed: res.Allowed},
|
||||
}
|
||||
if len(res.Patch) > 0 {
|
||||
patch, err := json.Marshal(res.Patch)
|
||||
if err != nil {
|
||||
http.Error(w, "encode patch", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
out.Response.PatchType = "JSONPatch"
|
||||
out.Response.Patch = base64.StdEncoding.EncodeToString(patch)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if err := json.NewEncoder(w).Encode(out); err != nil {
|
||||
m.log.Warn("pod identity response write failed", "error", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,630 @@
|
||||
package podidentity
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"maps"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const (
|
||||
testRoleARN = "arn:aws:iam::000000000001:role/app"
|
||||
testRegion = "ap-southeast-2"
|
||||
testSTS = "https://sts.cluster.example:9999"
|
||||
annRole = "eks.amazonaws.com/role-arn"
|
||||
annAudience = "eks.amazonaws.com/audience"
|
||||
annExpiry = "eks.amazonaws.com/token-expiration"
|
||||
annRegionalS = "eks.amazonaws.com/sts-regional-endpoints"
|
||||
annSkip = "eks.amazonaws.com/skip-containers"
|
||||
)
|
||||
|
||||
type fakeSAs struct {
|
||||
sas map[string]*ServiceAccount
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeSAs) GetServiceAccount(_ context.Context, ns, name string) (*ServiceAccount, error) {
|
||||
if f.err != nil {
|
||||
return nil, f.err
|
||||
}
|
||||
if sa, ok := f.sas[ns+"/"+name]; ok {
|
||||
return sa, nil
|
||||
}
|
||||
return nil, ErrServiceAccountNotFound
|
||||
}
|
||||
|
||||
// logRecords captures slog records for assertions.
|
||||
type logRecords struct {
|
||||
mu sync.Mutex
|
||||
recs []map[string]string
|
||||
}
|
||||
|
||||
func (l *logRecords) Enabled(context.Context, slog.Level) bool { return true }
|
||||
func (l *logRecords) WithAttrs([]slog.Attr) slog.Handler { return l }
|
||||
func (l *logRecords) WithGroup(string) slog.Handler { return l }
|
||||
func (l *logRecords) Handle(_ context.Context, r slog.Record) error {
|
||||
m := map[string]string{"msg": r.Message, "level": r.Level.String()}
|
||||
r.Attrs(func(a slog.Attr) bool {
|
||||
m[a.Key] = fmt.Sprint(a.Value.Any())
|
||||
return true
|
||||
})
|
||||
l.mu.Lock()
|
||||
l.recs = append(l.recs, m)
|
||||
l.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// withReason returns the records at warn level or above that carry reason.
|
||||
func (l *logRecords) withReason(reason string) []map[string]string {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
var out []map[string]string
|
||||
for _, r := range l.recs {
|
||||
if r["reason"] == reason {
|
||||
out = append(out, r)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func newMutator(sas ...*ServiceAccount) (*Mutator, *logRecords, *fakeSAs) {
|
||||
f := &fakeSAs{sas: map[string]*ServiceAccount{}}
|
||||
for i, sa := range sas {
|
||||
f.sas[fmt.Sprintf("app/sa%d", i)] = sa
|
||||
}
|
||||
rec := &logRecords{}
|
||||
return NewMutator(Config{Region: testRegion, STSEndpoint: testSTS}, f, slog.New(rec)), rec, f
|
||||
}
|
||||
|
||||
func roleSA(extra map[string]string) *ServiceAccount {
|
||||
ann := map[string]string{annRole: testRoleARN}
|
||||
maps.Copy(ann, extra)
|
||||
return &ServiceAccount{Annotations: ann}
|
||||
}
|
||||
|
||||
type env struct {
|
||||
Name string `json:"name"`
|
||||
Value string `json:"value"`
|
||||
}
|
||||
|
||||
func ctr(name string, envs ...env) map[string]any {
|
||||
c := map[string]any{"name": name, "image": "registry.example/app:1"}
|
||||
if len(envs) > 0 {
|
||||
var e []map[string]any
|
||||
for _, v := range envs {
|
||||
e = append(e, map[string]any{"name": v.Name, "value": v.Value})
|
||||
}
|
||||
c["env"] = e
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func podOf(sa string, ann map[string]string, containers ...map[string]any) map[string]any {
|
||||
spec := map[string]any{"containers": containers}
|
||||
if sa != "" {
|
||||
spec["serviceAccountName"] = sa
|
||||
}
|
||||
meta := map[string]any{"name": "web-1", "namespace": "app"}
|
||||
if ann != nil {
|
||||
meta["annotations"] = ann
|
||||
}
|
||||
return map[string]any{"apiVersion": "v1", "kind": "Pod", "metadata": meta, "spec": spec}
|
||||
}
|
||||
|
||||
// The pod after the patch, decoded for assertions.
|
||||
type outPod struct {
|
||||
Spec struct {
|
||||
Containers []outCtr `json:"containers"`
|
||||
InitContainers []outCtr `json:"initContainers"`
|
||||
Volumes []struct {
|
||||
Name string `json:"name"`
|
||||
Projected *struct {
|
||||
Sources []struct {
|
||||
SAT *struct {
|
||||
Audience string `json:"audience"`
|
||||
ExpirationSeconds int64 `json:"expirationSeconds"`
|
||||
Path string `json:"path"`
|
||||
} `json:"serviceAccountToken"`
|
||||
} `json:"sources"`
|
||||
} `json:"projected"`
|
||||
ConfigMap *struct {
|
||||
Name string `json:"name"`
|
||||
} `json:"configMap"`
|
||||
} `json:"volumes"`
|
||||
} `json:"spec"`
|
||||
}
|
||||
|
||||
type outCtr struct {
|
||||
Name string `json:"name"`
|
||||
Env []env `json:"env"`
|
||||
VM []struct {
|
||||
Name string `json:"name"`
|
||||
MountPath string `json:"mountPath"`
|
||||
ReadOnly bool `json:"readOnly"`
|
||||
} `json:"volumeMounts"`
|
||||
}
|
||||
|
||||
func (c outCtr) envMap() map[string]string {
|
||||
m := map[string]string{}
|
||||
for _, e := range c.Env {
|
||||
m[e.Name] = e.Value
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func (c outCtr) mounts(name string) int {
|
||||
n := 0
|
||||
for _, v := range c.VM {
|
||||
if v.Name == name {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func (p outPod) volumes(name string) int {
|
||||
n := 0
|
||||
for _, v := range p.Spec.Volumes {
|
||||
if v.Name == name {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func (p outPod) tokenExpiry(t *testing.T) (aud string, exp int64) {
|
||||
t.Helper()
|
||||
for _, v := range p.Spec.Volumes {
|
||||
if v.Name == TokenVolumeName && v.Projected != nil && len(v.Projected.Sources) == 1 && v.Projected.Sources[0].SAT != nil {
|
||||
s := v.Projected.Sources[0].SAT
|
||||
assert.Equal(t, TokenPath, s.Path)
|
||||
return s.Audience, s.ExpirationSeconds
|
||||
}
|
||||
}
|
||||
require.Failf(t, "no projected token volume", "volumes: %+v", p.Spec.Volumes)
|
||||
return "", 0
|
||||
}
|
||||
|
||||
// run mutates pod in namespace app and returns the patched pod and the response.
|
||||
func run(t *testing.T, m *Mutator, pod map[string]any) (outPod, AdmissionResponse) {
|
||||
t.Helper()
|
||||
raw, err := json.Marshal(pod)
|
||||
require.NoError(t, err)
|
||||
resp := m.Mutate(context.Background(), AdmissionRequest{UID: "u1", Operation: "CREATE", Namespace: "app", Object: raw})
|
||||
require.True(t, resp.Allowed, "the mutator must never deny a pod")
|
||||
assert.Equal(t, "u1", resp.UID)
|
||||
patched := applyPatch(t, raw, resp.Patch)
|
||||
var out outPod
|
||||
require.NoError(t, json.Unmarshal(patched, &out))
|
||||
return out, resp
|
||||
}
|
||||
|
||||
func TestMutate_InjectsTheAWSSetAndTheSpinifexExtras(t *testing.T) {
|
||||
m, _, _ := newMutator(roleSA(map[string]string{annRegionalS: "true"}))
|
||||
out, _ := run(t, m, podOf("sa0", nil, ctr("app")))
|
||||
|
||||
aud, exp := out.tokenExpiry(t)
|
||||
assert.Equal(t, "sts.amazonaws.com", aud)
|
||||
assert.GreaterOrEqual(t, exp, DefaultMinTokenExpiration)
|
||||
assert.LessOrEqual(t, exp, MaxTokenExpiration)
|
||||
|
||||
require.Len(t, out.Spec.Containers, 1)
|
||||
c := out.Spec.Containers[0]
|
||||
assert.Equal(t, map[string]string{
|
||||
"AWS_ROLE_ARN": testRoleARN,
|
||||
"AWS_WEB_IDENTITY_TOKEN_FILE": TokenMountPath + "/" + TokenPath,
|
||||
"AWS_STS_REGIONAL_ENDPOINTS": "regional",
|
||||
"AWS_REGION": testRegion,
|
||||
"AWS_DEFAULT_REGION": testRegion,
|
||||
"AWS_ENDPOINT_URL_STS": testSTS,
|
||||
"AWS_CA_BUNDLE": CAMountPath + "/" + CABundleKey,
|
||||
"SSL_CERT_FILE": CAMountPath + "/" + CABundleKey,
|
||||
}, c.envMap())
|
||||
|
||||
assert.Equal(t, 1, c.mounts(TokenVolumeName))
|
||||
assert.Equal(t, 1, c.mounts(CAVolumeName))
|
||||
for _, v := range c.VM {
|
||||
assert.True(t, v.ReadOnly, "mount %s must be read-only", v.Name)
|
||||
switch v.Name {
|
||||
case TokenVolumeName:
|
||||
assert.Equal(t, TokenMountPath, v.MountPath)
|
||||
case CAVolumeName:
|
||||
assert.Equal(t, CAMountPath, v.MountPath)
|
||||
}
|
||||
}
|
||||
assert.Equal(t, 1, out.volumes(CAVolumeName))
|
||||
for _, v := range out.Spec.Volumes {
|
||||
if v.Name == CAVolumeName {
|
||||
require.NotNil(t, v.ConfigMap)
|
||||
assert.Equal(t, CAConfigMapName, v.ConfigMap.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMutate_RegionalSTSOnlyWhenTheServiceAccountAsksForIt(t *testing.T) {
|
||||
m, _, _ := newMutator(roleSA(nil), roleSA(map[string]string{annRegionalS: "false"}))
|
||||
for _, sa := range []string{"sa0", "sa1"} {
|
||||
out, _ := run(t, m, podOf(sa, nil, ctr("app")))
|
||||
require.Len(t, out.Spec.Containers, 1, sa)
|
||||
assert.NotContains(t, out.Spec.Containers[0].envMap(), "AWS_STS_REGIONAL_ENDPOINTS", sa)
|
||||
assert.Equal(t, testRoleARN, out.Spec.Containers[0].envMap()["AWS_ROLE_ARN"], sa)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMutate_KeepsACallersOwnTrustSettings(t *testing.T) {
|
||||
m, _, _ := newMutator(roleSA(nil))
|
||||
out, _ := run(t, m, podOf("sa0", nil,
|
||||
ctr("a", env{"SSL_CERT_FILE", "/etc/ssl/own.pem"}),
|
||||
ctr("b", env{"AWS_CA_BUNDLE", "/etc/ssl/own-aws.pem"}, env{"AWS_ENDPOINT_URL_STS", "https://own"})))
|
||||
|
||||
a := out.Spec.Containers[0].envMap()
|
||||
assert.Equal(t, "/etc/ssl/own.pem", a["SSL_CERT_FILE"], "an existing SSL_CERT_FILE replaces the image trust store and must be left alone")
|
||||
assert.Equal(t, CAMountPath+"/"+CABundleKey, a["AWS_CA_BUNDLE"])
|
||||
b := out.Spec.Containers[1].envMap()
|
||||
assert.Equal(t, "/etc/ssl/own-aws.pem", b["AWS_CA_BUNDLE"])
|
||||
assert.Equal(t, "https://own", b["AWS_ENDPOINT_URL_STS"])
|
||||
assert.Equal(t, testRoleARN, b["AWS_ROLE_ARN"])
|
||||
}
|
||||
|
||||
func TestMutate_InjectsInitContainersToo(t *testing.T) {
|
||||
m, _, _ := newMutator(roleSA(nil))
|
||||
pod := podOf("sa0", nil, ctr("app"))
|
||||
pod["spec"].(map[string]any)["initContainers"] = []map[string]any{ctr("setup")}
|
||||
out, _ := run(t, m, pod)
|
||||
|
||||
require.Len(t, out.Spec.InitContainers, 1)
|
||||
assert.Equal(t, testRoleARN, out.Spec.InitContainers[0].envMap()["AWS_ROLE_ARN"])
|
||||
assert.Equal(t, 1, out.Spec.InitContainers[0].mounts(TokenVolumeName))
|
||||
}
|
||||
|
||||
func TestMutate_LeavesAContainerWithAllThreeKeySetsUntouched(t *testing.T) {
|
||||
m, rec, _ := newMutator(roleSA(nil))
|
||||
complete := ctr("done",
|
||||
env{"AWS_ROLE_ARN", "arn:aws:iam::000000000001:role/own"},
|
||||
env{"AWS_WEB_IDENTITY_TOKEN_FILE", "/own/token"},
|
||||
env{"AWS_REGION", "us-east-1"},
|
||||
env{"AWS_STS_REGIONAL_ENDPOINTS", "legacy"})
|
||||
out, _ := run(t, m, podOf("sa0", nil, complete, ctr("plain")))
|
||||
|
||||
got := out.Spec.Containers[0]
|
||||
assert.Len(t, got.Env, 4, "a complete container must not gain variables")
|
||||
assert.Zero(t, got.mounts(TokenVolumeName)+got.mounts(CAVolumeName))
|
||||
assert.Equal(t, testRoleARN, out.Spec.Containers[1].envMap()["AWS_ROLE_ARN"], "its neighbour is still injected")
|
||||
assert.NotEmpty(t, rec.withReason(ReasonAlreadyConfigured))
|
||||
}
|
||||
|
||||
func TestMutate_AddsOnlyTheMissingKeysToAPartiallyConfiguredContainer(t *testing.T) {
|
||||
m, _, _ := newMutator(roleSA(map[string]string{annRegionalS: "true"}))
|
||||
partial := ctr("partial", env{"AWS_ROLE_ARN", "arn:aws:iam::000000000001:role/own"})
|
||||
out, _ := run(t, m, podOf("sa0", nil, partial))
|
||||
|
||||
e := out.Spec.Containers[0].envMap()
|
||||
assert.Equal(t, "arn:aws:iam::000000000001:role/own", e["AWS_ROLE_ARN"], "the caller's role is kept")
|
||||
assert.NotContains(t, e, "AWS_WEB_IDENTITY_TOKEN_FILE", "role keys already defined are not completed")
|
||||
assert.Equal(t, testRegion, e["AWS_REGION"])
|
||||
assert.Equal(t, testRegion, e["AWS_DEFAULT_REGION"])
|
||||
assert.Equal(t, "regional", e["AWS_STS_REGIONAL_ENDPOINTS"])
|
||||
assert.Equal(t, 1, out.Spec.Containers[0].mounts(TokenVolumeName))
|
||||
count := 0
|
||||
for _, kv := range out.Spec.Containers[0].Env {
|
||||
if kv.Name == "AWS_ROLE_ARN" {
|
||||
count++
|
||||
}
|
||||
}
|
||||
assert.Equal(t, 1, count)
|
||||
}
|
||||
|
||||
func TestMutate_DoesNotDuplicateAnExistingTokenVolume(t *testing.T) {
|
||||
m, _, _ := newMutator(roleSA(nil))
|
||||
pod := podOf("sa0", nil, ctr("app"))
|
||||
pod["spec"].(map[string]any)["volumes"] = []map[string]any{{
|
||||
"name": TokenVolumeName,
|
||||
"projected": map[string]any{"sources": []map[string]any{{
|
||||
"serviceAccountToken": map[string]any{"audience": "own", "expirationSeconds": 7200, "path": "token"},
|
||||
}}},
|
||||
}}
|
||||
out, _ := run(t, m, pod)
|
||||
|
||||
assert.Equal(t, 1, out.volumes(TokenVolumeName))
|
||||
aud, exp := out.tokenExpiry(t)
|
||||
assert.Equal(t, "own", aud, "the caller's volume is kept")
|
||||
assert.Equal(t, int64(7200), exp)
|
||||
assert.Equal(t, 1, out.Spec.Containers[0].mounts(TokenVolumeName))
|
||||
}
|
||||
|
||||
func TestMutate_AudienceAndExpiry(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
saAnn map[string]string
|
||||
podAnn map[string]string
|
||||
wantAud string
|
||||
wantExp int64
|
||||
}{
|
||||
{"service account audience", map[string]string{annAudience: "custom.example"}, nil, "custom.example", 0},
|
||||
{"service account expiry", map[string]string{annExpiry: "7200"}, nil, "sts.amazonaws.com", 7200},
|
||||
{"pod expiry wins over service account", map[string]string{annExpiry: "7200"}, map[string]string{annExpiry: "1200"}, "sts.amazonaws.com", 1200},
|
||||
{"below the minimum is raised to 600", nil, map[string]string{annExpiry: "100"}, "sts.amazonaws.com", 600},
|
||||
{"above the maximum is lowered to 86400", map[string]string{annExpiry: "999999"}, nil, "sts.amazonaws.com", 86400},
|
||||
{"invalid expiry falls back to the default", map[string]string{annExpiry: "soon"}, nil, "sts.amazonaws.com", 0},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
m, _, _ := newMutator(roleSA(tc.saAnn))
|
||||
out, _ := run(t, m, podOf("sa0", tc.podAnn, ctr("app")))
|
||||
aud, exp := out.tokenExpiry(t)
|
||||
assert.Equal(t, tc.wantAud, aud)
|
||||
if tc.wantExp != 0 {
|
||||
assert.Equal(t, tc.wantExp, exp)
|
||||
return
|
||||
}
|
||||
assert.GreaterOrEqual(t, exp, MaxTokenExpiration-7200)
|
||||
assert.LessOrEqual(t, exp, MaxTokenExpiration)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMutate_DefaultExpiryIsJitteredWithinTwoHours(t *testing.T) {
|
||||
m, _, _ := newMutator(roleSA(nil))
|
||||
seen := map[int64]bool{}
|
||||
for range 60 {
|
||||
out, _ := run(t, m, podOf("sa0", nil, ctr("app")))
|
||||
_, exp := out.tokenExpiry(t)
|
||||
require.GreaterOrEqual(t, exp, DefaultMinTokenExpiration)
|
||||
require.LessOrEqual(t, exp, MaxTokenExpiration)
|
||||
seen[exp] = true
|
||||
}
|
||||
assert.Greater(t, len(seen), 1, "60 pods all got the same default expiry; it is not jittered")
|
||||
}
|
||||
|
||||
func TestMutate_SkipContainersAnnotation(t *testing.T) {
|
||||
m, rec, _ := newMutator(roleSA(nil))
|
||||
out, _ := run(t, m, podOf("sa0", map[string]string{annSkip: "sidecar, mesh"},
|
||||
ctr("app"), ctr("sidecar"), ctr("mesh")))
|
||||
|
||||
assert.Equal(t, testRoleARN, out.Spec.Containers[0].envMap()["AWS_ROLE_ARN"])
|
||||
for _, i := range []int{1, 2} {
|
||||
c := out.Spec.Containers[i]
|
||||
assert.Empty(t, c.Env, c.Name)
|
||||
assert.Zero(t, c.mounts(TokenVolumeName), c.Name)
|
||||
}
|
||||
assert.Len(t, rec.withReason(ReasonSkipContainer), 2)
|
||||
}
|
||||
|
||||
func TestMutate_LeavesPodsOfUnannotatedServiceAccountsAlone(t *testing.T) {
|
||||
m, _, f := newMutator(&ServiceAccount{Annotations: map[string]string{"other": "x"}}, &ServiceAccount{})
|
||||
f.sas["app/default"] = &ServiceAccount{}
|
||||
for _, sa := range []string{"sa0", "sa1", ""} {
|
||||
_, resp := run(t, m, podOf(sa, nil, ctr("app")))
|
||||
assert.Empty(t, resp.Patch, "service account %q", sa)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMutate_UsesTheDefaultServiceAccountWhenThePodNamesNone(t *testing.T) {
|
||||
m, _, f := newMutator()
|
||||
f.sas["app/default"] = roleSA(nil)
|
||||
out, _ := run(t, m, podOf("", nil, ctr("app")))
|
||||
assert.Equal(t, testRoleARN, out.Spec.Containers[0].envMap()["AWS_ROLE_ARN"])
|
||||
}
|
||||
|
||||
// A pod of a managed add-on as render_addon writes it: the web identity,
|
||||
// region and regional sts settings, the token volume and the gateway CA.
|
||||
func TestMutate_LeavesARenderedManagedAddonPodUnchanged(t *testing.T) {
|
||||
m, _, _ := newMutator(roleSA(map[string]string{annRegionalS: "true"}))
|
||||
c := ctr("controller",
|
||||
env{"AWS_ROLE_ARN", testRoleARN},
|
||||
env{"AWS_WEB_IDENTITY_TOKEN_FILE", TokenMountPath + "/token"},
|
||||
env{"AWS_STS_REGIONAL_ENDPOINTS", "regional"},
|
||||
env{"AWS_REGION", testRegion},
|
||||
env{"AWS_DEFAULT_REGION", testRegion},
|
||||
env{"AWS_ENDPOINT_URL_STS", testSTS},
|
||||
env{"AWS_CA_BUNDLE", "/etc/spinifex/gateway-ca/ca.pem"},
|
||||
env{"SSL_CERT_FILE", "/etc/spinifex/gateway-ca/ca.pem"})
|
||||
c["volumeMounts"] = []map[string]any{
|
||||
{"name": TokenVolumeName, "mountPath": TokenMountPath, "readOnly": true},
|
||||
{"name": "spinifex-gateway-ca", "mountPath": "/etc/spinifex/gateway-ca", "readOnly": true},
|
||||
}
|
||||
pod := podOf("sa0", nil, c)
|
||||
pod["spec"].(map[string]any)["volumes"] = []map[string]any{
|
||||
{"name": TokenVolumeName, "projected": map[string]any{"sources": []map[string]any{{
|
||||
"serviceAccountToken": map[string]any{"audience": "sts.amazonaws.com", "expirationSeconds": 86400, "path": "token"}}}}},
|
||||
{"name": "spinifex-gateway-ca", "secret": map[string]any{"secretName": "spinifex-gateway-ca"}},
|
||||
}
|
||||
_, resp := run(t, m, pod)
|
||||
assert.Empty(t, resp.Patch, "a managed add-on pod must come out byte for byte as it went in")
|
||||
}
|
||||
|
||||
func TestMutate_OnlyActsOnPodCreation(t *testing.T) {
|
||||
m, _, _ := newMutator(roleSA(nil))
|
||||
raw, err := json.Marshal(podOf("sa0", nil, ctr("app")))
|
||||
require.NoError(t, err)
|
||||
resp := m.Mutate(context.Background(), AdmissionRequest{UID: "u", Operation: "UPDATE", Namespace: "app", Object: raw})
|
||||
assert.True(t, resp.Allowed)
|
||||
assert.Empty(t, resp.Patch)
|
||||
}
|
||||
|
||||
func TestMutate_LogsEverySkipAndFailureWithNamespacePodAndReason(t *testing.T) {
|
||||
t.Run("lookup failure fails open", func(t *testing.T) {
|
||||
m, rec, f := newMutator(roleSA(nil))
|
||||
f.err = errors.New("apiserver unavailable")
|
||||
_, resp := run(t, m, podOf("sa0", nil, ctr("app")))
|
||||
assert.Empty(t, resp.Patch)
|
||||
got := rec.withReason(ReasonServiceAccountLookup)
|
||||
require.Len(t, got, 1)
|
||||
assert.Equal(t, "app", got[0]["namespace"])
|
||||
assert.Equal(t, "web-1", got[0]["pod"])
|
||||
assert.Contains(t, got[0]["error"], "apiserver unavailable")
|
||||
})
|
||||
t.Run("missing service account", func(t *testing.T) {
|
||||
m, rec, _ := newMutator()
|
||||
_, resp := run(t, m, podOf("ghost", nil, ctr("app")))
|
||||
assert.Empty(t, resp.Patch)
|
||||
require.Len(t, rec.withReason(ReasonServiceAccountMissing), 1)
|
||||
})
|
||||
t.Run("undecodable object", func(t *testing.T) {
|
||||
m, rec, _ := newMutator()
|
||||
resp := m.Mutate(context.Background(), AdmissionRequest{UID: "u", Operation: "CREATE", Namespace: "app", Object: json.RawMessage(`{"spec":`)})
|
||||
assert.True(t, resp.Allowed)
|
||||
assert.Empty(t, resp.Patch)
|
||||
got := rec.withReason(ReasonDecode)
|
||||
require.Len(t, got, 1)
|
||||
assert.Equal(t, "app", got[0]["namespace"])
|
||||
})
|
||||
t.Run("a pod with only generateName is named by it", func(t *testing.T) {
|
||||
m, rec, _ := newMutator(roleSA(nil))
|
||||
pod := podOf("sa0", map[string]string{annSkip: "app"}, ctr("app"))
|
||||
meta := pod["metadata"].(map[string]any)
|
||||
delete(meta, "name")
|
||||
meta["generateName"] = "web-"
|
||||
run(t, m, pod)
|
||||
got := rec.withReason(ReasonSkipContainer)
|
||||
require.Len(t, got, 1)
|
||||
assert.True(t, strings.HasPrefix(got[0]["pod"], "web-"), "pod attr %q", got[0]["pod"])
|
||||
})
|
||||
t.Run("an unannotated pod is not a skip worth logging", func(t *testing.T) {
|
||||
m, rec, _ := newMutator(&ServiceAccount{})
|
||||
run(t, m, podOf("sa0", nil, ctr("app")))
|
||||
for _, r := range rec.recs {
|
||||
assert.NotEqual(t, "WARN", r["level"], "%v", r)
|
||||
assert.NotEqual(t, "ERROR", r["level"], "%v", r)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestServeHTTP_AnswersAnAdmissionReview(t *testing.T) {
|
||||
m, _, _ := newMutator(roleSA(nil))
|
||||
obj, err := json.Marshal(podOf("sa0", nil, ctr("app")))
|
||||
require.NoError(t, err)
|
||||
body, err := json.Marshal(map[string]any{
|
||||
"apiVersion": "admission.k8s.io/v1", "kind": "AdmissionReview",
|
||||
"request": map[string]any{"uid": "abc-123", "operation": "CREATE", "namespace": "app", "object": json.RawMessage(obj)},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
rr := httptest.NewRecorder()
|
||||
m.ServeHTTP(rr, httptest.NewRequest(http.MethodPost, "/mutate", bytes.NewReader(body)))
|
||||
require.Equal(t, http.StatusOK, rr.Code)
|
||||
|
||||
var review struct {
|
||||
APIVersion string `json:"apiVersion"`
|
||||
Kind string `json:"kind"`
|
||||
Response struct {
|
||||
UID string `json:"uid"`
|
||||
Allowed bool `json:"allowed"`
|
||||
PatchType string `json:"patchType"`
|
||||
Patch string `json:"patch"`
|
||||
} `json:"response"`
|
||||
}
|
||||
require.NoError(t, json.Unmarshal(rr.Body.Bytes(), &review))
|
||||
assert.Equal(t, "admission.k8s.io/v1", review.APIVersion)
|
||||
assert.Equal(t, "AdmissionReview", review.Kind)
|
||||
assert.Equal(t, "abc-123", review.Response.UID)
|
||||
assert.True(t, review.Response.Allowed)
|
||||
assert.Equal(t, "JSONPatch", review.Response.PatchType)
|
||||
patch, err := base64.StdEncoding.DecodeString(review.Response.Patch)
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, string(patch), "AWS_ROLE_ARN")
|
||||
}
|
||||
|
||||
// applyPatch applies add, replace and remove operations to raw.
|
||||
func applyPatch(t *testing.T, raw []byte, ops []PatchOp) []byte {
|
||||
t.Helper()
|
||||
var doc any
|
||||
require.NoError(t, json.Unmarshal(raw, &doc))
|
||||
for _, op := range ops {
|
||||
parts := strings.Split(strings.TrimPrefix(op.Path, "/"), "/")
|
||||
for i := range parts {
|
||||
parts[i] = strings.ReplaceAll(strings.ReplaceAll(parts[i], "~1", "/"), "~0", "~")
|
||||
}
|
||||
var val any
|
||||
if op.Value != nil {
|
||||
b, err := json.Marshal(op.Value)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, json.Unmarshal(b, &val))
|
||||
}
|
||||
doc = patchAt(t, doc, parts, op.Op, val)
|
||||
}
|
||||
out, err := json.Marshal(doc)
|
||||
require.NoError(t, err)
|
||||
return out
|
||||
}
|
||||
|
||||
func patchAt(t *testing.T, node any, parts []string, op string, val any) any {
|
||||
t.Helper()
|
||||
if len(parts) == 1 {
|
||||
key := parts[0]
|
||||
switch n := node.(type) {
|
||||
case map[string]any:
|
||||
switch op {
|
||||
case "add", "replace":
|
||||
n[key] = val
|
||||
case "remove":
|
||||
delete(n, key)
|
||||
default:
|
||||
require.Failf(t, "unsupported op", "%s", op)
|
||||
}
|
||||
return n
|
||||
case []any:
|
||||
if key == "-" {
|
||||
return append(n, val)
|
||||
}
|
||||
i, err := strconv.Atoi(key)
|
||||
require.NoError(t, err)
|
||||
switch op {
|
||||
case "add":
|
||||
n = append(n, nil)
|
||||
copy(n[i+1:], n[i:])
|
||||
n[i] = val
|
||||
case "replace":
|
||||
n[i] = val
|
||||
case "remove":
|
||||
n = append(n[:i], n[i+1:]...)
|
||||
}
|
||||
return n
|
||||
}
|
||||
require.Fail(t, "patch parent is neither object nor array")
|
||||
}
|
||||
switch n := node.(type) {
|
||||
case map[string]any:
|
||||
child, ok := n[parts[0]]
|
||||
require.True(t, ok, "patch path parent %q missing", parts[0])
|
||||
n[parts[0]] = patchAt(t, child, parts[1:], op, val)
|
||||
return n
|
||||
case []any:
|
||||
i, err := strconv.Atoi(parts[0])
|
||||
require.NoError(t, err)
|
||||
n[i] = patchAt(t, n[i], parts[1:], op, val)
|
||||
return n
|
||||
}
|
||||
require.Fail(t, "patch path crosses a scalar")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Upstream skips a container only when the regional STS key is present too: a
|
||||
// container with role and region but no STS key, on a ServiceAccount that does
|
||||
// not ask for regional endpoints, still gets the mounts but no new STS variable.
|
||||
func TestMutate_RoleAndRegionWithoutSTSKeyStillGetsMountsNotSTSEnv(t *testing.T) {
|
||||
m, _, _ := newMutator(roleSA(nil))
|
||||
c := ctr("app",
|
||||
env{"AWS_ROLE_ARN", "arn:aws:iam::000000000001:role/own"},
|
||||
env{"AWS_WEB_IDENTITY_TOKEN_FILE", "/own/token"},
|
||||
env{"AWS_REGION", "us-east-1"})
|
||||
out, _ := run(t, m, podOf("sa0", nil, c))
|
||||
|
||||
got := out.Spec.Containers[0]
|
||||
assert.Equal(t, 1, got.mounts(TokenVolumeName))
|
||||
assert.NotContains(t, got.envMap(), "AWS_STS_REGIONAL_ENDPOINTS")
|
||||
assert.Equal(t, "arn:aws:iam::000000000001:role/own", got.envMap()["AWS_ROLE_ARN"])
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
// Package podidentity injects IRSA credentials into pods of annotated
|
||||
// ServiceAccounts like the AWS pod identity webhook, plus the Spinifex STS
|
||||
// endpoint and gateway CA. It uses no Kubernetes client; the cluster is reached through interfaces.
|
||||
package podidentity
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log/slog"
|
||||
)
|
||||
|
||||
const (
|
||||
// DefaultAnnotationDomain is the annotation domain AWS uses.
|
||||
DefaultAnnotationDomain = "eks.amazonaws.com"
|
||||
// TokenVolumeName and TokenMountPath are where AWS mounts the projected token.
|
||||
TokenVolumeName = "aws-iam-token" //nolint:gosec // volume name, not a credential
|
||||
TokenMountPath = "/var/run/secrets/eks.amazonaws.com/serviceaccount" //nolint:gosec // mount path, not a credential
|
||||
TokenPath = "token"
|
||||
// DefaultAudience is the token audience when the ServiceAccount sets none.
|
||||
DefaultAudience = "sts.amazonaws.com"
|
||||
// MinTokenExpiration and MaxTokenExpiration bound a requested expiry in seconds.
|
||||
MinTokenExpiration int64 = 600
|
||||
MaxTokenExpiration int64 = 86400
|
||||
// DefaultMinTokenExpiration is the low end of the jittered default expiry.
|
||||
DefaultMinTokenExpiration int64 = 79200
|
||||
|
||||
// CAVolumeName, CAMountPath and CABundleKey describe the CA bundle ConfigMap
|
||||
// mount; CAConfigMapName is the ConfigMap published into every namespace.
|
||||
CAConfigMapName = "spinifex-ca-bundle"
|
||||
CAVolumeName = "spinifex-ca-bundle"
|
||||
CAMountPath = "/etc/spinifex/ca-bundle"
|
||||
CABundleKey = "ca.pem"
|
||||
)
|
||||
|
||||
// Reasons logged, with the namespace and pod, when injection is skipped or fails.
|
||||
const (
|
||||
ReasonDecode = "admission object could not be decoded"
|
||||
ReasonServiceAccountLookup = "serviceaccount lookup failed"
|
||||
ReasonServiceAccountMissing = "serviceaccount not found"
|
||||
ReasonAlreadyConfigured = "container already has web identity, region and regional sts settings"
|
||||
ReasonSkipContainer = "container named in skip-containers annotation"
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrServiceAccountNotFound is returned by a ServiceAccountGetter when the
|
||||
// ServiceAccount does not exist.
|
||||
ErrServiceAccountNotFound = errors.New("serviceaccount not found")
|
||||
// ErrNotFound, ErrAlreadyExists and ErrConflict are the ConfigMapStore errors.
|
||||
ErrNotFound = errors.New("not found")
|
||||
ErrAlreadyExists = errors.New("already exists")
|
||||
ErrConflict = errors.New("conflict")
|
||||
)
|
||||
|
||||
// Config is the mutator's configuration.
|
||||
type Config struct {
|
||||
// Region is injected as AWS_REGION and AWS_DEFAULT_REGION when non-empty.
|
||||
Region string
|
||||
// STSEndpoint is injected as AWS_ENDPOINT_URL_STS when non-empty.
|
||||
STSEndpoint string
|
||||
// AnnotationDomain defaults to DefaultAnnotationDomain.
|
||||
AnnotationDomain string
|
||||
}
|
||||
|
||||
// ServiceAccount is the part of a ServiceAccount the mutator reads.
|
||||
type ServiceAccount struct {
|
||||
Annotations map[string]string
|
||||
}
|
||||
|
||||
// ServiceAccountGetter looks up a ServiceAccount.
|
||||
type ServiceAccountGetter interface {
|
||||
GetServiceAccount(ctx context.Context, namespace, name string) (*ServiceAccount, error)
|
||||
}
|
||||
|
||||
// AdmissionRequest is the part of an AdmissionReview request the mutator reads.
|
||||
type AdmissionRequest struct {
|
||||
UID string `json:"uid"`
|
||||
Operation string `json:"operation"`
|
||||
Namespace string `json:"namespace"`
|
||||
Object json.RawMessage `json:"object"`
|
||||
}
|
||||
|
||||
// PatchOp is one RFC 6902 operation.
|
||||
type PatchOp struct {
|
||||
Op string `json:"op"`
|
||||
Path string `json:"path"`
|
||||
Value any `json:"value,omitempty"`
|
||||
}
|
||||
|
||||
// AdmissionResponse is the mutator's answer. Patch is empty when the pod is
|
||||
// left unchanged. The mutator never denies a pod.
|
||||
type AdmissionResponse struct {
|
||||
UID string
|
||||
Allowed bool
|
||||
Patch []PatchOp
|
||||
}
|
||||
|
||||
// Mutator injects IRSA credentials into pods.
|
||||
type Mutator struct {
|
||||
cfg Config
|
||||
sas ServiceAccountGetter
|
||||
log *slog.Logger
|
||||
}
|
||||
|
||||
// NewMutator returns a Mutator. A nil log uses slog.Default.
|
||||
func NewMutator(cfg Config, sas ServiceAccountGetter, log *slog.Logger) *Mutator {
|
||||
if log == nil {
|
||||
log = slog.Default()
|
||||
}
|
||||
return &Mutator{cfg: cfg, sas: sas, log: log}
|
||||
}
|
||||
|
||||
// ConfigMap is the part of a ConfigMap the publisher manages.
|
||||
type ConfigMap struct {
|
||||
Namespace string
|
||||
Name string
|
||||
Data map[string]string
|
||||
// ResourceVersion is opaque and set by the store.
|
||||
ResourceVersion string
|
||||
}
|
||||
|
||||
// ConfigMapStore reads and writes ConfigMaps. Get returns ErrNotFound, Create
|
||||
// returns ErrAlreadyExists, and Update returns ErrConflict or ErrNotFound.
|
||||
type ConfigMapStore interface {
|
||||
Get(ctx context.Context, namespace, name string) (*ConfigMap, error)
|
||||
Create(ctx context.Context, cm ConfigMap) error
|
||||
Update(ctx context.Context, cm ConfigMap) error
|
||||
}
|
||||
|
||||
// NamespaceSource lists namespaces and reports ones created later.
|
||||
type NamespaceSource interface {
|
||||
List(ctx context.Context) ([]string, error)
|
||||
// Watch sends the name of every namespace created after the call until ctx ends.
|
||||
Watch(ctx context.Context) (<-chan string, error)
|
||||
}
|
||||
|
||||
// PublisherConfig configures a Publisher.
|
||||
type PublisherConfig struct {
|
||||
// Bundle is the CA bundle published to every namespace as CABundleKey.
|
||||
Bundle []byte
|
||||
}
|
||||
|
||||
// Publisher keeps the CA bundle ConfigMap present in every namespace.
|
||||
type Publisher struct {
|
||||
cfg PublisherConfig
|
||||
ns NamespaceSource
|
||||
cms ConfigMapStore
|
||||
log *slog.Logger
|
||||
}
|
||||
|
||||
// NewPublisher returns a Publisher. A nil log uses slog.Default.
|
||||
func NewPublisher(cfg PublisherConfig, ns NamespaceSource, cms ConfigMapStore, log *slog.Logger) *Publisher {
|
||||
if log == nil {
|
||||
log = slog.Default()
|
||||
}
|
||||
return &Publisher{cfg: cfg, ns: ns, cms: cms, log: log}
|
||||
}
|
||||
|
||||
// WebhookConfig describes the MutatingWebhookConfiguration to render.
|
||||
type WebhookConfig struct {
|
||||
Name string
|
||||
// URL is the https URL the API server calls, on 127.0.0.1.
|
||||
URL string
|
||||
// CA is the PEM the API server trusts for URL.
|
||||
CA []byte
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
package podidentity
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"maps"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultResync = 5 * time.Minute
|
||||
reconcileAttempts = 3
|
||||
)
|
||||
|
||||
// Reconcile makes the CA bundle ConfigMap in one namespace match the bundle.
|
||||
// Losing a create or update race to another publisher is not an error.
|
||||
func (p *Publisher) Reconcile(ctx context.Context, namespace string) error {
|
||||
want := string(p.cfg.Bundle)
|
||||
var err error
|
||||
for range reconcileAttempts {
|
||||
var cm *ConfigMap
|
||||
cm, err = p.cms.Get(ctx, namespace, CAConfigMapName)
|
||||
switch {
|
||||
case errors.Is(err, ErrNotFound):
|
||||
err = p.cms.Create(ctx, ConfigMap{Namespace: namespace, Name: CAConfigMapName, Data: map[string]string{CABundleKey: want}})
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if !errors.Is(err, ErrAlreadyExists) {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
case err != nil:
|
||||
return err
|
||||
case cm.Data[CABundleKey] == want:
|
||||
return nil
|
||||
}
|
||||
|
||||
data := maps.Clone(cm.Data)
|
||||
if data == nil {
|
||||
data = map[string]string{}
|
||||
}
|
||||
data[CABundleKey] = want
|
||||
err = p.cms.Update(ctx, ConfigMap{Namespace: namespace, Name: CAConfigMapName, Data: data, ResourceVersion: cm.ResourceVersion})
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if !errors.Is(err, ErrConflict) && !errors.Is(err, ErrNotFound) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Run publishes to every existing namespace, then to each new one, and
|
||||
// re-checks every namespace each resync interval until ctx ends.
|
||||
func (p *Publisher) Run(ctx context.Context, resync time.Duration) error {
|
||||
if resync <= 0 {
|
||||
resync = defaultResync
|
||||
}
|
||||
created, err := p.ns.Watch(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
p.reconcileAll(ctx)
|
||||
|
||||
tick := time.NewTicker(resync)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case ns, ok := <-created:
|
||||
if !ok {
|
||||
created = nil
|
||||
continue
|
||||
}
|
||||
p.reconcileOne(ctx, ns)
|
||||
case <-tick.C:
|
||||
if created == nil {
|
||||
if ch, err := p.ns.Watch(ctx); err != nil {
|
||||
p.log.Warn("namespace watch restart failed", "error", err)
|
||||
} else {
|
||||
created = ch
|
||||
}
|
||||
}
|
||||
p.reconcileAll(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Publisher) reconcileAll(ctx context.Context) {
|
||||
names, err := p.ns.List(ctx)
|
||||
if err != nil {
|
||||
p.log.Warn("namespace list failed", "error", err)
|
||||
return
|
||||
}
|
||||
for _, ns := range names {
|
||||
p.reconcileOne(ctx, ns)
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Publisher) reconcileOne(ctx context.Context, ns string) {
|
||||
if err := p.Reconcile(ctx, ns); err != nil && ctx.Err() == nil {
|
||||
p.log.Warn("ca bundle publish failed", "namespace", ns, "error", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,321 @@
|
||||
package podidentity
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"math/big"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// memStore is an in-memory ConfigMapStore with the API server's create and
|
||||
// update semantics: Create fails if present, Update needs the current version.
|
||||
type memStore struct {
|
||||
mu sync.Mutex
|
||||
cms map[string]ConfigMap
|
||||
version int
|
||||
creates int
|
||||
updates int
|
||||
}
|
||||
|
||||
func newMemStore() *memStore { return &memStore{cms: map[string]ConfigMap{}} }
|
||||
|
||||
func (s *memStore) Get(_ context.Context, ns, name string) (*ConfigMap, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
cm, ok := s.cms[ns+"/"+name]
|
||||
if !ok {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
return &cm, nil
|
||||
}
|
||||
|
||||
func (s *memStore) Create(_ context.Context, cm ConfigMap) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if _, ok := s.cms[cm.Namespace+"/"+cm.Name]; ok {
|
||||
return ErrAlreadyExists
|
||||
}
|
||||
s.version++
|
||||
cm.ResourceVersion = strconv.Itoa(s.version)
|
||||
s.cms[cm.Namespace+"/"+cm.Name] = cm
|
||||
s.creates++
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *memStore) Update(_ context.Context, cm ConfigMap) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
cur, ok := s.cms[cm.Namespace+"/"+cm.Name]
|
||||
if !ok {
|
||||
return ErrNotFound
|
||||
}
|
||||
if cur.ResourceVersion != cm.ResourceVersion {
|
||||
return ErrConflict
|
||||
}
|
||||
s.version++
|
||||
cm.ResourceVersion = strconv.Itoa(s.version)
|
||||
s.cms[cm.Namespace+"/"+cm.Name] = cm
|
||||
s.updates++
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *memStore) remove(ns, name string) {
|
||||
s.mu.Lock()
|
||||
delete(s.cms, ns+"/"+name)
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
func (s *memStore) get(ns string) (ConfigMap, bool) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
cm, ok := s.cms[ns+"/"+CAConfigMapName]
|
||||
return cm, ok
|
||||
}
|
||||
|
||||
// fakeNamespaces lists a fixed set and replays created ones on Watch.
|
||||
type fakeNamespaces struct {
|
||||
mu sync.Mutex
|
||||
names []string
|
||||
watcher chan string
|
||||
}
|
||||
|
||||
func (f *fakeNamespaces) List(context.Context) ([]string, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return append([]string(nil), f.names...), nil
|
||||
}
|
||||
|
||||
func (f *fakeNamespaces) Watch(context.Context) (<-chan string, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.watcher = make(chan string, 8)
|
||||
return f.watcher, nil
|
||||
}
|
||||
|
||||
func (f *fakeNamespaces) create(name string) {
|
||||
f.mu.Lock()
|
||||
f.names = append(f.names, name)
|
||||
w := f.watcher
|
||||
f.mu.Unlock()
|
||||
if w != nil {
|
||||
w <- name
|
||||
}
|
||||
}
|
||||
|
||||
var testBundle = []byte("-----BEGIN CERTIFICATE-----\nQUJD\n-----END CERTIFICATE-----\n")
|
||||
|
||||
func TestPublisher_ReconcileCreatesTheBundleConfigMap(t *testing.T) {
|
||||
st := newMemStore()
|
||||
p := NewPublisher(PublisherConfig{Bundle: testBundle}, &fakeNamespaces{}, st, nil)
|
||||
|
||||
require.NoError(t, p.Reconcile(context.Background(), "team-a"))
|
||||
|
||||
cm, ok := st.get("team-a")
|
||||
require.True(t, ok, "no %s ConfigMap in team-a", CAConfigMapName)
|
||||
assert.Equal(t, string(testBundle), cm.Data[CABundleKey])
|
||||
}
|
||||
|
||||
func TestPublisher_ReconcileIsIdempotentAndRepairsDrift(t *testing.T) {
|
||||
st := newMemStore()
|
||||
p := NewPublisher(PublisherConfig{Bundle: testBundle}, &fakeNamespaces{}, st, nil)
|
||||
ctx := context.Background()
|
||||
|
||||
require.NoError(t, p.Reconcile(ctx, "team-a"))
|
||||
require.NoError(t, p.Reconcile(ctx, "team-a"))
|
||||
assert.Equal(t, 1, st.creates)
|
||||
assert.Zero(t, st.updates, "an up-to-date ConfigMap must not be rewritten")
|
||||
|
||||
require.NoError(t, st.Update(ctx, ConfigMap{Namespace: "team-a", Name: CAConfigMapName,
|
||||
Data: map[string]string{CABundleKey: "tampered"}, ResourceVersion: "1"}))
|
||||
require.NoError(t, p.Reconcile(ctx, "team-a"))
|
||||
cm, _ := st.get("team-a")
|
||||
assert.Equal(t, string(testBundle), cm.Data[CABundleKey])
|
||||
}
|
||||
|
||||
func TestPublisher_RunPublishesToEveryNamespaceIncludingLaterOnes(t *testing.T) {
|
||||
st := newMemStore()
|
||||
nss := &fakeNamespaces{names: []string{"default", "kube-system", "team-a"}}
|
||||
p := NewPublisher(PublisherConfig{Bundle: testBundle}, nss, st, nil)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- p.Run(ctx, time.Hour) }()
|
||||
|
||||
for _, ns := range []string{"default", "kube-system", "team-a"} {
|
||||
require.Eventually(t, func() bool { _, ok := st.get(ns); return ok }, 3*time.Second, 10*time.Millisecond, "namespace %s", ns)
|
||||
}
|
||||
nss.create("created-later")
|
||||
require.Eventually(t, func() bool { _, ok := st.get("created-later"); return ok }, 3*time.Second, 10*time.Millisecond)
|
||||
|
||||
cancel()
|
||||
select {
|
||||
case err := <-done:
|
||||
assert.NoError(t, err)
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("Run did not return after its context ended")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublisher_RunRecreatesADeletedConfigMap(t *testing.T) {
|
||||
st := newMemStore()
|
||||
nss := &fakeNamespaces{names: []string{"team-a"}}
|
||||
p := NewPublisher(PublisherConfig{Bundle: testBundle}, nss, st, nil)
|
||||
|
||||
ctx := t.Context()
|
||||
go func() { _ = p.Run(ctx, 20*time.Millisecond) }()
|
||||
|
||||
require.Eventually(t, func() bool { _, ok := st.get("team-a"); return ok }, 3*time.Second, 10*time.Millisecond)
|
||||
st.remove("team-a", CAConfigMapName)
|
||||
require.Eventually(t, func() bool { _, ok := st.get("team-a"); return ok }, 3*time.Second, 10*time.Millisecond,
|
||||
"the ConfigMap stayed deleted; pods created now would hang on the missing volume")
|
||||
}
|
||||
|
||||
// All three control-plane servers run a publisher, so racing is the normal case.
|
||||
func TestPublisher_ConcurrentPublishersDoNotFight(t *testing.T) {
|
||||
st := newMemStore()
|
||||
var wg sync.WaitGroup
|
||||
errs := make(chan error, 12)
|
||||
for range 3 {
|
||||
p := NewPublisher(PublisherConfig{Bundle: testBundle}, &fakeNamespaces{}, st, nil)
|
||||
for range 4 {
|
||||
wg.Go(func() {
|
||||
errs <- p.Reconcile(context.Background(), "team-a")
|
||||
})
|
||||
}
|
||||
}
|
||||
wg.Wait()
|
||||
close(errs)
|
||||
for err := range errs {
|
||||
assert.NoError(t, err, "losing a create race is not an error")
|
||||
}
|
||||
cm, ok := st.get("team-a")
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, string(testBundle), cm.Data[CABundleKey])
|
||||
assert.Equal(t, 1, st.creates)
|
||||
assert.Zero(t, st.updates, "publishers with the same bundle must not rewrite each other's ConfigMap")
|
||||
}
|
||||
|
||||
func certPEM(t *testing.T, cn string) []byte {
|
||||
t.Helper()
|
||||
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
require.NoError(t, err)
|
||||
tmpl := &x509.Certificate{
|
||||
SerialNumber: big.NewInt(time.Now().UnixNano()),
|
||||
Subject: pkix.Name{CommonName: cn},
|
||||
NotBefore: time.Now().Add(-time.Hour),
|
||||
NotAfter: time.Now().Add(time.Hour),
|
||||
IsCA: true,
|
||||
BasicConstraintsValid: true,
|
||||
KeyUsage: x509.KeyUsageCertSign,
|
||||
}
|
||||
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
|
||||
require.NoError(t, err)
|
||||
return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
|
||||
}
|
||||
|
||||
func subjects(t *testing.T, bundle []byte) []string {
|
||||
t.Helper()
|
||||
var out []string
|
||||
for rest := bundle; ; {
|
||||
var b *pem.Block
|
||||
b, rest = pem.Decode(rest)
|
||||
if b == nil {
|
||||
return out
|
||||
}
|
||||
c, err := x509.ParseCertificate(b.Bytes)
|
||||
require.NoError(t, err)
|
||||
out = append(out, c.Subject.CommonName)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildCABundle_HoldsTheSystemRootsAndTheGatewayCAOnce(t *testing.T) {
|
||||
root1, root2, gw := certPEM(t, "root-1"), certPEM(t, "root-2"), certPEM(t, "gateway")
|
||||
system := append(append([]byte{}, root1...), root2...)
|
||||
|
||||
got := subjects(t, BuildCABundle(system, gw))
|
||||
assert.ElementsMatch(t, []string{"root-1", "root-2", "gateway"}, got)
|
||||
|
||||
again := subjects(t, BuildCABundle(append(append([]byte{}, system...), gw...), gw))
|
||||
assert.ElementsMatch(t, []string{"root-1", "root-2", "gateway"}, again, "a certificate in both inputs is listed once")
|
||||
}
|
||||
|
||||
func TestRenderWebhookConfiguration(t *testing.T) {
|
||||
ca := certPEM(t, "k3s-server-ca")
|
||||
raw, err := RenderWebhookConfiguration(WebhookConfig{
|
||||
Name: "spinifex-pod-identity",
|
||||
URL: "https://127.0.0.1:8445/mutate",
|
||||
CA: ca,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
var cfg struct {
|
||||
APIVersion string `json:"apiVersion"`
|
||||
Kind string `json:"kind"`
|
||||
Metadata struct {
|
||||
Name string `json:"name"`
|
||||
} `json:"metadata"`
|
||||
Webhooks []struct {
|
||||
Name string `json:"name"`
|
||||
FailurePolicy string `json:"failurePolicy"`
|
||||
SideEffects string `json:"sideEffects"`
|
||||
AdmissionReviewVersions []string `json:"admissionReviewVersions"`
|
||||
ClientConfig struct {
|
||||
URL string `json:"url"`
|
||||
Service any `json:"service"`
|
||||
CABundle string `json:"caBundle"`
|
||||
} `json:"clientConfig"`
|
||||
Rules []struct {
|
||||
Operations []string `json:"operations"`
|
||||
Resources []string `json:"resources"`
|
||||
} `json:"rules"`
|
||||
} `json:"webhooks"`
|
||||
}
|
||||
require.NoError(t, json.Unmarshal(raw, &cfg))
|
||||
assert.Equal(t, "admissionregistration.k8s.io/v1", cfg.APIVersion)
|
||||
assert.Equal(t, "MutatingWebhookConfiguration", cfg.Kind)
|
||||
assert.Equal(t, "spinifex-pod-identity", cfg.Metadata.Name)
|
||||
|
||||
require.Len(t, cfg.Webhooks, 1)
|
||||
w := cfg.Webhooks[0]
|
||||
assert.Equal(t, "Ignore", w.FailurePolicy, "an unreachable injector must never block pod creation")
|
||||
assert.Equal(t, "None", w.SideEffects)
|
||||
assert.Contains(t, w.AdmissionReviewVersions, "v1")
|
||||
assert.Nil(t, w.ClientConfig.Service, "a Service reference cannot be routed from the control plane")
|
||||
assert.True(t, strings.HasPrefix(w.ClientConfig.URL, "https://127.0.0.1:"), "url %q", w.ClientConfig.URL)
|
||||
got, err := base64.StdEncoding.DecodeString(w.ClientConfig.CABundle)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, string(ca), string(got), "caBundle must be the k3s server CA")
|
||||
require.Len(t, w.Rules, 1)
|
||||
assert.Equal(t, []string{"CREATE"}, w.Rules[0].Operations)
|
||||
assert.Equal(t, []string{"pods"}, w.Rules[0].Resources)
|
||||
}
|
||||
|
||||
func TestRenderWebhookConfiguration_RejectsAURLThatIsNotLoopbackHTTPS(t *testing.T) {
|
||||
ca := certPEM(t, "k3s-server-ca")
|
||||
for _, url := range []string{
|
||||
"http://127.0.0.1:8445/mutate",
|
||||
"https://10.0.0.5:8445/mutate",
|
||||
"https://webhook.kube-system.svc/mutate",
|
||||
"",
|
||||
} {
|
||||
_, err := RenderWebhookConfiguration(WebhookConfig{Name: "n", URL: url, CA: ca})
|
||||
assert.Error(t, err, "url %q", url)
|
||||
}
|
||||
_, err := RenderWebhookConfiguration(WebhookConfig{Name: "n", URL: "https://127.0.0.1:8445/mutate"})
|
||||
assert.Error(t, err, "a missing CA must not render a webhook the API server cannot verify")
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
package podidentity
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/pem"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func freeAddr(t *testing.T) string {
|
||||
t.Helper()
|
||||
l, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = l.Close() }()
|
||||
return l.Addr().String()
|
||||
}
|
||||
|
||||
// startAPI serves a fake kube-apiserver whose certificate is signed by the k3s
|
||||
// server CA, as the real one is.
|
||||
func startAPI(t *testing.T, cfg CertConfig) *httptest.Server {
|
||||
t.Helper()
|
||||
other := cfg
|
||||
other.Dir = t.TempDir()
|
||||
certs, err := EnsureCerts(other)
|
||||
require.NoError(t, err)
|
||||
pair, err := tls.LoadX509KeyPair(certs.ServingCert, certs.ServingKey)
|
||||
require.NoError(t, err)
|
||||
api := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Query().Get("watch") != "" {
|
||||
return
|
||||
}
|
||||
_, _ = io.WriteString(w, `{"items":[]}`)
|
||||
}))
|
||||
api.TLS = &tls.Config{Certificates: []tls.Certificate{pair}, ClientAuth: tls.RequireAnyClientCert, MinVersion: tls.VersionTLS12}
|
||||
api.StartTLS()
|
||||
t.Cleanup(api.Close)
|
||||
return api
|
||||
}
|
||||
|
||||
func TestRun_ServesTheWebhookStagesManifestsAndStopsOnCancel(t *testing.T) {
|
||||
cfg, tlsDir := certTestConfig(t)
|
||||
api := startAPI(t, cfg)
|
||||
dir := t.TempDir()
|
||||
roots := filepath.Join(dir, "roots.pem")
|
||||
require.NoError(t, os.WriteFile(roots, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: api.Certificate().Raw}), 0o644))
|
||||
c := ServiceConfig{
|
||||
Addr: freeAddr(t), Dir: cfg.Dir, TLSDir: tlsDir, APIServer: api.URL,
|
||||
Region: "ap-southeast-2", STSEndpoint: "https://gw:9999",
|
||||
SystemRoots: roots, ManifestDir: filepath.Join(dir, "manifests"), Resync: time.Hour,
|
||||
}
|
||||
require.NoError(t, os.MkdirAll(c.ManifestDir, 0o755))
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- Run(ctx, c, slog.New(slog.DiscardHandler)) }()
|
||||
|
||||
// The API server calls the webhook over TLS, trusting only the server CA.
|
||||
caPEM, err := os.ReadFile(filepath.Join(tlsDir, "server-ca.crt"))
|
||||
require.NoError(t, err)
|
||||
pool := x509.NewCertPool()
|
||||
require.True(t, pool.AppendCertsFromPEM(caPEM))
|
||||
client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}}}
|
||||
require.Eventually(t, func() bool {
|
||||
resp, err := client.Get("https://" + c.Addr + "/healthz")
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
return resp.StatusCode == http.StatusOK
|
||||
}, 10*time.Second, 50*time.Millisecond)
|
||||
|
||||
_, err = os.Stat(filepath.Join(c.ManifestDir, manifestFile))
|
||||
assert.NoError(t, err, "the RBAC and webhook configuration are staged for k3s")
|
||||
|
||||
cancel()
|
||||
select {
|
||||
case err := <-done:
|
||||
assert.NoError(t, err)
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("Run did not stop after cancel")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRun_FailsWhenTheServerCAIsMissing(t *testing.T) {
|
||||
cfg, tlsDir := certTestConfig(t)
|
||||
require.NoError(t, os.Remove(filepath.Join(tlsDir, "server-ca.key")))
|
||||
c := ServiceConfig{
|
||||
Addr: freeAddr(t), Dir: cfg.Dir, TLSDir: tlsDir, APIServer: "https://127.0.0.1:1",
|
||||
Region: "r", STSEndpoint: "https://gw", SystemRoots: filepath.Join(t.TempDir(), "none"), ManifestDir: t.TempDir(),
|
||||
}
|
||||
err := Run(context.Background(), c, slog.New(slog.DiscardHandler))
|
||||
assert.Error(t, err, "the process exits so the supervisor retries until k3s has written its CAs")
|
||||
}
|
||||
|
||||
func TestRun_RejectsANonLoopbackAddress(t *testing.T) {
|
||||
err := Run(context.Background(), ServiceConfig{Addr: "0.0.0.0:8445", Region: "r", STSEndpoint: "s"}, slog.New(slog.DiscardHandler))
|
||||
assert.Error(t, err)
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
package podidentity
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
// DefaultAddr is the loopback listener; 8443 belongs to eks-token-webhook.
|
||||
DefaultAddr = "127.0.0.1:8445"
|
||||
// MutatePath is the URL path the API server calls.
|
||||
MutatePath = "/mutate"
|
||||
|
||||
defaultStateDir = "/var/lib/spinifex-eks/pod-identity"
|
||||
defaultTLSDir = "/var/lib/rancher/k3s/server/tls"
|
||||
defaultAPIServer = "https://127.0.0.1:6443"
|
||||
defaultSystemRoot = "/etc/ssl/certs/ca-certificates.crt"
|
||||
defaultManifests = "/var/lib/rancher/k3s/server/manifests"
|
||||
|
||||
manifestFile = "eks-pod-identity-webhook.yaml"
|
||||
|
||||
certCheckEvery = 24 * time.Hour
|
||||
)
|
||||
|
||||
// ErrCertRenewal is returned by Run when a certificate entered its renew window.
|
||||
// The process exits non-zero so the supervisor restarts it and EnsureCerts reissues.
|
||||
var ErrCertRenewal = errors.New("certificate inside renew window")
|
||||
|
||||
// ServiceConfig is everything the webhook process needs.
|
||||
type ServiceConfig struct {
|
||||
Addr string
|
||||
Dir string
|
||||
TLSDir string
|
||||
APIServer string
|
||||
Region string
|
||||
STSEndpoint string
|
||||
GatewayCAFile string
|
||||
SystemRoots string
|
||||
ManifestDir string
|
||||
Resync time.Duration
|
||||
}
|
||||
|
||||
// ParseServiceConfig reads flags, defaulting region, STS endpoint and gateway
|
||||
// CA from the environment the node already has (the cloud-init first-boot env
|
||||
// that mulga-eks-addon-sync renders the managed add-ons from).
|
||||
func ParseServiceConfig(args []string, getenv func(string) string, errOut io.Writer) (ServiceConfig, error) {
|
||||
var c ServiceConfig
|
||||
fs := flag.NewFlagSet("eks-pod-identity-webhook", flag.ContinueOnError)
|
||||
fs.SetOutput(errOut)
|
||||
fs.StringVar(&c.Addr, "addr", DefaultAddr, "loopback listen address")
|
||||
fs.StringVar(&c.Dir, "dir", defaultStateDir, "directory holding the webhook's certificates")
|
||||
fs.StringVar(&c.TLSDir, "k3s-tls-dir", defaultTLSDir, "k3s server tls directory")
|
||||
fs.StringVar(&c.APIServer, "apiserver", defaultAPIServer, "kube-apiserver URL")
|
||||
fs.StringVar(&c.Region, "region", getenv("EKS_REGION"), "region injected as AWS_REGION")
|
||||
fs.StringVar(&c.STSEndpoint, "sts-endpoint", getenv("EKS_ADDON_GATEWAY_URL"), "endpoint injected as AWS_ENDPOINT_URL_STS")
|
||||
fs.StringVar(&c.GatewayCAFile, "gateway-ca", getenv("EKS_GATEWAY_CA"), "gateway CA PEM added to the published bundle")
|
||||
fs.StringVar(&c.SystemRoots, "system-roots", defaultSystemRoot, "system root certificates PEM")
|
||||
fs.StringVar(&c.ManifestDir, "manifest-dir", defaultManifests, "k3s auto-deploy manifests directory")
|
||||
fs.DurationVar(&c.Resync, "resync", 5*time.Minute, "how often every namespace's CA ConfigMap is re-checked")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return ServiceConfig{}, err
|
||||
}
|
||||
return c, c.Validate()
|
||||
}
|
||||
|
||||
// Validate rejects a configuration that would expose the webhook beyond the
|
||||
// node or run it with nothing to inject.
|
||||
func (c ServiceConfig) Validate() error {
|
||||
host, _, err := net.SplitHostPort(c.Addr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("addr %q: %w", c.Addr, err)
|
||||
}
|
||||
if ip := net.ParseIP(host); ip == nil || !ip.IsLoopback() {
|
||||
return fmt.Errorf("addr %q must be a loopback IP: only the local kube-apiserver may call the webhook", c.Addr)
|
||||
}
|
||||
if c.Region == "" || c.STSEndpoint == "" {
|
||||
return errors.New("region and sts endpoint are required (EKS_REGION, EKS_ADDON_GATEWAY_URL)")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CertConfig returns the certificate settings that follow from the k3s tls directory.
|
||||
func (c ServiceConfig) CertConfig() CertConfig {
|
||||
return CertConfig{
|
||||
Dir: c.Dir,
|
||||
ServerCACert: c.TLSDir + "/server-ca.crt", ServerCAKey: c.TLSDir + "/server-ca.key",
|
||||
ClientCACert: c.TLSDir + "/client-ca.crt", ClientCAKey: c.TLSDir + "/client-ca.key",
|
||||
}
|
||||
}
|
||||
|
||||
// Run serves the webhook on the loopback address and keeps the CA ConfigMaps
|
||||
// published until ctx ends.
|
||||
func Run(ctx context.Context, c ServiceConfig, log *slog.Logger) error {
|
||||
if err := c.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
certs, err := EnsureCerts(c.CertConfig())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
bundle, err := c.caBundle()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.stageManifests(); err != nil {
|
||||
return err
|
||||
}
|
||||
kube, err := NewKubeClient(c.APIServer, c.TLSDir+"/server-ca.crt", certs.ClientCert, certs.ClientKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pair, err := tls.LoadX509KeyPair(certs.ServingCert, certs.ServingKey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("load serving certificate: %w", err)
|
||||
}
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle(MutatePath, NewMutator(Config{Region: c.Region, STSEndpoint: c.STSEndpoint}, kube, log))
|
||||
mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { _, _ = io.WriteString(w, "ok") })
|
||||
srv := &http.Server{
|
||||
Handler: mux,
|
||||
TLSConfig: &tls.Config{Certificates: []tls.Certificate{pair}, MinVersion: tls.VersionTLS12},
|
||||
ReadHeaderTimeout: 5 * time.Second,
|
||||
ReadTimeout: 10 * time.Second,
|
||||
WriteTimeout: 10 * time.Second,
|
||||
}
|
||||
ln, err := net.Listen("tcp", c.Addr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
errc := make(chan error, 1)
|
||||
go func() { errc <- srv.ServeTLS(ln, "", "") }()
|
||||
go func() {
|
||||
if err := NewPublisher(PublisherConfig{Bundle: bundle}, kube, kube, log).Run(ctx, c.Resync); err != nil {
|
||||
log.Error("ca bundle publisher stopped", "error", err)
|
||||
}
|
||||
}()
|
||||
log.Info("pod identity webhook listening", "addr", c.Addr)
|
||||
|
||||
renew := time.NewTicker(certCheckEvery)
|
||||
defer renew.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-renew.C:
|
||||
if !NeedsRenewal(certs, time.Now()) {
|
||||
continue
|
||||
}
|
||||
log.Info("certificate inside renew window; exiting so the supervisor restarts and reissues")
|
||||
shutdown, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_ = srv.Shutdown(shutdown)
|
||||
return ErrCertRenewal
|
||||
case <-ctx.Done():
|
||||
shutdown, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
return srv.Shutdown(shutdown)
|
||||
case err := <-errc:
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// caBundle is the system roots plus the gateway CA, so a client that trusts only
|
||||
// this bundle still reaches public endpoints as well as the gateway.
|
||||
func (c ServiceConfig) caBundle() ([]byte, error) {
|
||||
roots, err := os.ReadFile(c.SystemRoots)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read system roots: %w", err)
|
||||
}
|
||||
var gw []byte
|
||||
if c.GatewayCAFile != "" {
|
||||
if gw, err = os.ReadFile(c.GatewayCAFile); err != nil {
|
||||
return nil, fmt.Errorf("read gateway ca: %w", err)
|
||||
}
|
||||
}
|
||||
b := BuildCABundle(roots, gw)
|
||||
if len(b) == 0 {
|
||||
return nil, errors.New("ca bundle is empty")
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// stageManifests writes the RBAC and webhook configuration into the k3s
|
||||
// auto-deploy directory. Every server renders the same bytes (the CA is shared),
|
||||
// and the file is only replaced when its content changes, so k3s re-applies it
|
||||
// only on a real change.
|
||||
func (c ServiceConfig) stageManifests() error {
|
||||
ca, err := os.ReadFile(c.TLSDir + "/server-ca.crt")
|
||||
if err != nil {
|
||||
return fmt.Errorf("read server ca: %w", err)
|
||||
}
|
||||
body, err := RenderManifests(WebhookConfig{Name: WebhookName, URL: "https://" + c.Addr + MutatePath, CA: ca})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
path := c.ManifestDir + "/" + manifestFile
|
||||
if cur, err := os.ReadFile(path); err == nil && bytes.Equal(cur, body) {
|
||||
return nil
|
||||
}
|
||||
return writeAtomic(path, body, 0o600)
|
||||
}
|
||||
@@ -0,0 +1,305 @@
|
||||
package podidentity
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"math/big"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/agents/eks/konnectivitycert"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// writeCA writes a self-signed CA cert and key as <name>.crt and <name>.key.
|
||||
func writeCA(t *testing.T, dir, name string) *x509.Certificate {
|
||||
t.Helper()
|
||||
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
require.NoError(t, err)
|
||||
tmpl := x509.Certificate{
|
||||
SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: name},
|
||||
NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().AddDate(10, 0, 0),
|
||||
IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign,
|
||||
}
|
||||
der, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &key.PublicKey, key)
|
||||
require.NoError(t, err)
|
||||
kd, err := x509.MarshalECPrivateKey(key)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, name+".crt"), pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o644))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, name+".key"), pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: kd}), 0o600))
|
||||
ca, err := x509.ParseCertificate(der)
|
||||
require.NoError(t, err)
|
||||
return ca
|
||||
}
|
||||
|
||||
func certTestConfig(t *testing.T) (CertConfig, string) {
|
||||
t.Helper()
|
||||
tls := t.TempDir()
|
||||
writeCA(t, tls, "server-ca")
|
||||
writeCA(t, tls, "client-ca")
|
||||
return CertConfig{
|
||||
Dir: filepath.Join(t.TempDir(), "state"),
|
||||
ServerCACert: filepath.Join(tls, "server-ca.crt"), ServerCAKey: filepath.Join(tls, "server-ca.key"),
|
||||
ClientCACert: filepath.Join(tls, "client-ca.crt"), ClientCAKey: filepath.Join(tls, "client-ca.key"),
|
||||
}, tls
|
||||
}
|
||||
|
||||
func parseCert(t *testing.T, path string) *x509.Certificate {
|
||||
t.Helper()
|
||||
b, err := os.ReadFile(path)
|
||||
require.NoError(t, err)
|
||||
blk, _ := pem.Decode(b)
|
||||
require.NotNil(t, blk)
|
||||
c, err := x509.ParseCertificate(blk.Bytes)
|
||||
require.NoError(t, err)
|
||||
return c
|
||||
}
|
||||
|
||||
func TestEnsureCerts_ServingCertIsForLoopbackAndSignedByTheServerCA(t *testing.T) {
|
||||
cfg, _ := certTestConfig(t)
|
||||
out, err := EnsureCerts(cfg)
|
||||
require.NoError(t, err)
|
||||
|
||||
serverCA := parseCert(t, cfg.ServerCACert)
|
||||
serving := parseCert(t, out.ServingCert)
|
||||
require.NoError(t, serving.CheckSignatureFrom(serverCA))
|
||||
require.Len(t, serving.IPAddresses, 1)
|
||||
assert.True(t, serving.IPAddresses[0].IsLoopback())
|
||||
assert.Equal(t, []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, serving.ExtKeyUsage)
|
||||
|
||||
roots := x509.NewCertPool()
|
||||
roots.AddCert(serverCA)
|
||||
_, err = serving.Verify(x509.VerifyOptions{Roots: roots, DNSName: "127.0.0.1"})
|
||||
assert.NoError(t, err, "the caBundle (server CA) must validate the serving cert for 127.0.0.1")
|
||||
}
|
||||
|
||||
func TestEnsureCerts_ClientCertIsSignedByTheClientCAWithTheWebhookIdentity(t *testing.T) {
|
||||
cfg, _ := certTestConfig(t)
|
||||
out, err := EnsureCerts(cfg)
|
||||
require.NoError(t, err)
|
||||
|
||||
client := parseCert(t, out.ClientCert)
|
||||
require.NoError(t, client.CheckSignatureFrom(parseCert(t, cfg.ClientCACert)))
|
||||
assert.Equal(t, ClientCN, client.Subject.CommonName)
|
||||
assert.Equal(t, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, client.ExtKeyUsage)
|
||||
}
|
||||
|
||||
func TestEnsureCerts_KeysAreOwnerOnly(t *testing.T) {
|
||||
cfg, _ := certTestConfig(t)
|
||||
out, err := EnsureCerts(cfg)
|
||||
require.NoError(t, err)
|
||||
for _, k := range []string{out.ServingKey, out.ClientKey} {
|
||||
st, err := os.Stat(k)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, os.FileMode(0o600), st.Mode().Perm(), k)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureCerts_ReusesValidCertsAndReissuesWhenTheCAChanges(t *testing.T) {
|
||||
cfg, tls := certTestConfig(t)
|
||||
first, err := EnsureCerts(cfg)
|
||||
require.NoError(t, err)
|
||||
before, err := os.ReadFile(first.ServingCert)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = EnsureCerts(cfg)
|
||||
require.NoError(t, err)
|
||||
same, err := os.ReadFile(first.ServingCert)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, bytes.Equal(before, same), "a valid cert must be reused so restarts do not churn it")
|
||||
|
||||
writeCA(t, tls, "server-ca") // the cluster CA is replaced
|
||||
_, err = EnsureCerts(cfg)
|
||||
require.NoError(t, err)
|
||||
after, err := os.ReadFile(first.ServingCert)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, bytes.Equal(before, after), "a cert signed by a retired CA must be reissued")
|
||||
require.NoError(t, parseCert(t, first.ServingCert).CheckSignatureFrom(parseCert(t, cfg.ServerCACert)))
|
||||
}
|
||||
|
||||
func TestEnsureCerts_MissingCAFails(t *testing.T) {
|
||||
cfg, _ := certTestConfig(t)
|
||||
cfg.ServerCACert = filepath.Join(t.TempDir(), "absent.crt")
|
||||
_, err := EnsureCerts(cfg)
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func TestRenderManifests_GrantsLeastPrivilegeAndEmbedsTheWebhook(t *testing.T) {
|
||||
out, err := RenderManifests(WebhookConfig{Name: "n", URL: "https://127.0.0.1:8445/mutate", CA: []byte("ca")})
|
||||
require.NoError(t, err)
|
||||
docs := strings.Split(string(out), "---\n")
|
||||
require.Len(t, docs, 3)
|
||||
|
||||
var role struct {
|
||||
Rules []struct {
|
||||
Resources []string `json:"resources"`
|
||||
ResourceNames []string `json:"resourceNames"`
|
||||
Verbs []string `json:"verbs"`
|
||||
} `json:"rules"`
|
||||
}
|
||||
require.NoError(t, json.Unmarshal([]byte(docs[0]), &role))
|
||||
got := map[string][]string{}
|
||||
for _, r := range role.Rules {
|
||||
for _, res := range r.Resources {
|
||||
key := res
|
||||
if len(r.ResourceNames) > 0 {
|
||||
key += "/named"
|
||||
}
|
||||
got[key] = append(got[key], r.Verbs...)
|
||||
}
|
||||
assert.NotContains(t, r.Verbs, "*")
|
||||
assert.NotContains(t, r.Verbs, "delete")
|
||||
}
|
||||
assert.Equal(t, map[string][]string{
|
||||
"serviceaccounts": {"get"},
|
||||
"namespaces": {"list", "watch"},
|
||||
"configmaps": {"create"},
|
||||
"configmaps/named": {"get", "update"},
|
||||
}, got)
|
||||
|
||||
var binding struct {
|
||||
Subjects []struct {
|
||||
Name string `json:"name"`
|
||||
} `json:"subjects"`
|
||||
}
|
||||
require.NoError(t, json.Unmarshal([]byte(docs[1]), &binding))
|
||||
assert.Equal(t, ClientCN, binding.Subjects[0].Name)
|
||||
|
||||
assert.Contains(t, docs[2], `"failurePolicy": "Ignore"`)
|
||||
}
|
||||
|
||||
func TestRenderManifests_IsDeterministic(t *testing.T) {
|
||||
w := WebhookConfig{Name: "n", URL: "https://127.0.0.1:8445/mutate", CA: []byte("ca")}
|
||||
a, err := RenderManifests(w)
|
||||
require.NoError(t, err)
|
||||
b, err := RenderManifests(w)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, a, b)
|
||||
}
|
||||
|
||||
func TestParseServiceConfig_DefaultsFromTheNodeEnvironment(t *testing.T) {
|
||||
env := map[string]string{"EKS_REGION": "ap-southeast-2", "EKS_ADDON_GATEWAY_URL": "https://gw:9999", "EKS_GATEWAY_CA": "/etc/gw.pem"}
|
||||
c, err := ParseServiceConfig(nil, func(k string) string { return env[k] }, &bytes.Buffer{})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "ap-southeast-2", c.Region)
|
||||
assert.Equal(t, "https://gw:9999", c.STSEndpoint)
|
||||
assert.Equal(t, "/etc/gw.pem", c.GatewayCAFile)
|
||||
assert.Equal(t, DefaultAddr, c.Addr)
|
||||
}
|
||||
|
||||
func TestParseServiceConfig_RejectsANonLoopbackListener(t *testing.T) {
|
||||
vars := map[string]string{"EKS_REGION": "r", "EKS_ADDON_GATEWAY_URL": "https://gw"}
|
||||
env := func(k string) string { return vars[k] }
|
||||
for _, addr := range []string{"0.0.0.0:8445", ":8445", "10.0.0.5:8445", "somehost:8445"} {
|
||||
_, err := ParseServiceConfig([]string{"--addr=" + addr}, env, &bytes.Buffer{})
|
||||
assert.Error(t, err, addr)
|
||||
}
|
||||
_, err := ParseServiceConfig([]string{"--addr=127.0.0.1:9000"}, env, &bytes.Buffer{})
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestParseServiceConfig_RequiresRegionAndSTSEndpoint(t *testing.T) {
|
||||
_, err := ParseServiceConfig(nil, func(string) string { return "" }, &bytes.Buffer{})
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func TestStageManifests_WritesTheServerCAAndIsIdempotent(t *testing.T) {
|
||||
cfg, tls := certTestConfig(t)
|
||||
c := ServiceConfig{Addr: DefaultAddr, TLSDir: tls, ManifestDir: t.TempDir()}
|
||||
require.NoError(t, c.stageManifests())
|
||||
|
||||
path := filepath.Join(c.ManifestDir, manifestFile)
|
||||
first, err := os.ReadFile(path)
|
||||
require.NoError(t, err)
|
||||
ca, err := os.ReadFile(cfg.ServerCACert)
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, string(first), base64Of(ca), "caBundle must be the k3s server CA")
|
||||
assert.Contains(t, string(first), "https://127.0.0.1:8445/mutate")
|
||||
|
||||
st1, err := os.Stat(path)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, c.stageManifests())
|
||||
st2, err := os.Stat(path)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, st1.ModTime().Equal(st2.ModTime()), "an unchanged manifest must not be rewritten")
|
||||
}
|
||||
|
||||
func TestCABundle_IsSystemRootsPlusGatewayCA(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeCA(t, dir, "sys")
|
||||
writeCA(t, dir, "gw")
|
||||
c := ServiceConfig{SystemRoots: filepath.Join(dir, "sys.crt"), GatewayCAFile: filepath.Join(dir, "gw.crt")}
|
||||
b, err := c.caBundle()
|
||||
require.NoError(t, err)
|
||||
pool := x509.NewCertPool()
|
||||
require.True(t, pool.AppendCertsFromPEM(b))
|
||||
assert.Len(t, pool.Subjects(), 2) //nolint:staticcheck // counting entries only
|
||||
}
|
||||
|
||||
func base64Of(b []byte) string { return base64.StdEncoding.EncodeToString(b) }
|
||||
|
||||
func TestEnsureCerts_AreValidForOneYear(t *testing.T) {
|
||||
cfg, _ := certTestConfig(t)
|
||||
out, err := EnsureCerts(cfg)
|
||||
require.NoError(t, err)
|
||||
for _, p := range []string{out.ServingCert, out.ClientCert} {
|
||||
c := parseCert(t, p)
|
||||
assert.InDelta(t, 365*24, c.NotAfter.Sub(c.NotBefore).Hours()-1, 24, p)
|
||||
assert.Less(t, time.Until(c.NotAfter), 366*24*time.Hour, p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNeedsRenewal(t *testing.T) {
|
||||
cfg, _ := certTestConfig(t)
|
||||
out, err := EnsureCerts(cfg)
|
||||
require.NoError(t, err)
|
||||
now := time.Now()
|
||||
|
||||
assert.False(t, NeedsRenewal(out, now), "fresh certs are outside the window")
|
||||
assert.False(t, NeedsRenewal(out, now.Add(300*24*time.Hour)))
|
||||
assert.True(t, NeedsRenewal(out, now.Add(340*24*time.Hour)), "inside 30 days of expiry")
|
||||
assert.True(t, NeedsRenewal(out, now.Add(400*24*time.Hour)), "already expired")
|
||||
|
||||
require.NoError(t, os.Remove(out.ClientCert))
|
||||
assert.True(t, NeedsRenewal(out, now), "an unreadable certificate needs renewal")
|
||||
}
|
||||
|
||||
func TestEnsureCerts_ReissuesACertInsideTheRenewWindow(t *testing.T) {
|
||||
cfg, _ := certTestConfig(t)
|
||||
out, err := EnsureCerts(cfg)
|
||||
require.NoError(t, err)
|
||||
// Replace the serving cert with one that expires in 10 days.
|
||||
ca, caKey, err := konnectivitycert.LoadCA(cfg.ServerCACert, cfg.ServerCAKey)
|
||||
require.NoError(t, err)
|
||||
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
require.NoError(t, err)
|
||||
tmpl := x509.Certificate{
|
||||
SerialNumber: big.NewInt(7), Subject: pkix.Name{CommonName: "pod-identity-webhook"},
|
||||
NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(10 * 24 * time.Hour),
|
||||
KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
|
||||
IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1)},
|
||||
}
|
||||
der, err := x509.CreateCertificate(rand.Reader, &tmpl, ca, &key.PublicKey, caKey)
|
||||
require.NoError(t, err)
|
||||
kd, err := x509.MarshalECPrivateKey(key)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, os.WriteFile(out.ServingKey, pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: kd}), 0o600))
|
||||
require.NoError(t, os.WriteFile(out.ServingCert, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o644))
|
||||
require.True(t, NeedsRenewal(out, time.Now()))
|
||||
|
||||
_, err = EnsureCerts(cfg)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, NeedsRenewal(out, time.Now()), "the restart path must leave certs outside the window")
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package podidentity
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
)
|
||||
|
||||
// WebhookName is the fully qualified name of the webhook entry.
|
||||
const WebhookName = "pod-identity.spinifex.mulgadc.com"
|
||||
|
||||
// RenderWebhookConfiguration returns the MutatingWebhookConfiguration as JSON.
|
||||
// The URL must be https on 127.0.0.1: each API server calls its own webhook, and
|
||||
// failurePolicy Ignore keeps an unreachable injector from blocking pod creation.
|
||||
func RenderWebhookConfiguration(c WebhookConfig) ([]byte, error) {
|
||||
if c.Name == "" {
|
||||
return nil, errors.New("webhook configuration needs a name")
|
||||
}
|
||||
u, err := url.Parse(c.URL)
|
||||
if err != nil || u.Scheme != "https" || u.Hostname() != "127.0.0.1" {
|
||||
return nil, fmt.Errorf("webhook url %q must be https on 127.0.0.1", c.URL)
|
||||
}
|
||||
if len(c.CA) == 0 {
|
||||
return nil, errors.New("webhook configuration needs a CA bundle")
|
||||
}
|
||||
cfg := map[string]any{
|
||||
"apiVersion": "admissionregistration.k8s.io/v1",
|
||||
"kind": "MutatingWebhookConfiguration",
|
||||
"metadata": map[string]any{"name": c.Name},
|
||||
"webhooks": []map[string]any{{
|
||||
"name": WebhookName,
|
||||
"clientConfig": map[string]any{
|
||||
"url": c.URL,
|
||||
"caBundle": base64.StdEncoding.EncodeToString(c.CA),
|
||||
},
|
||||
"rules": []map[string]any{{
|
||||
"operations": []string{"CREATE"},
|
||||
"apiGroups": []string{""},
|
||||
"apiVersions": []string{"v1"},
|
||||
"resources": []string{"pods"},
|
||||
"scope": "Namespaced",
|
||||
}},
|
||||
"failurePolicy": "Ignore",
|
||||
"sideEffects": "None",
|
||||
"timeoutSeconds": 5,
|
||||
"admissionReviewVersions": []string{"v1"},
|
||||
}},
|
||||
}
|
||||
return json.Marshal(cfg)
|
||||
}
|
||||
|
||||
// BuildCABundle returns the system roots plus the gateway CA as one PEM bundle,
|
||||
// without duplicate certificates.
|
||||
func BuildCABundle(system, gateway []byte) []byte {
|
||||
seen := map[[sha256.Size]byte]bool{}
|
||||
var out []byte
|
||||
for _, src := range [][]byte{system, gateway} {
|
||||
for rest := src; ; {
|
||||
var b *pem.Block
|
||||
b, rest = pem.Decode(rest)
|
||||
if b == nil {
|
||||
break
|
||||
}
|
||||
if b.Type != "CERTIFICATE" {
|
||||
continue
|
||||
}
|
||||
sum := sha256.Sum256(b.Bytes)
|
||||
if seen[sum] {
|
||||
continue
|
||||
}
|
||||
seen[sum] = true
|
||||
out = append(out, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: b.Bytes})...)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package workerboot
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/mulgadc/spinifex/internal/sqsworkergw"
|
||||
"github.com/mulgadc/spinifex/spinifex/agents/sqs/workerkey"
|
||||
)
|
||||
|
||||
// GatewayClaimer adapts the signed gateway client to the receipt key decision.
|
||||
type GatewayClaimer struct{ Client *sqsworkergw.Client }
|
||||
|
||||
var _ workerkey.Claimer = GatewayClaimer{}
|
||||
|
||||
func (c GatewayClaimer) Claim(ctx context.Context) (workerkey.Decision, error) {
|
||||
k, err := c.Client.ClaimReceiptKey(ctx)
|
||||
return workerkey.Decision{Pending: k.Pending, Fingerprint: k.Fingerprint, HostNodes: k.HostNodes}, err
|
||||
}
|
||||
|
||||
func (c GatewayClaimer) RecordFingerprint(ctx context.Context, fingerprint string) error {
|
||||
return c.Client.RecordReceiptKey(ctx, fingerprint)
|
||||
}
|
||||
@@ -3,22 +3,14 @@ package workerboot
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/hex"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"math/big"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -33,20 +25,16 @@ import (
|
||||
testutil "github.com/mulgadc/spinifex/internal/testkit"
|
||||
"github.com/mulgadc/spinifex/spinifex/agents/sqs/workerkey"
|
||||
sqsworkerapi "github.com/mulgadc/spinifex/spinifex/domains/sqs/awsapi/worker"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/firststore"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/storelaunch"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/workerident"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/workerrole"
|
||||
awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/clustersize"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
|
||||
"github.com/nats-io/nkeys"
|
||||
)
|
||||
|
||||
const (
|
||||
gwInstance = "i-0abc12345def67890"
|
||||
gwSuccessor = "i-0fedcba9876543210"
|
||||
gwVolume = "vol-0a1b2c3d4e5f60718"
|
||||
gwInstance = "i-0abc12345def67890"
|
||||
gwVolume = "vol-0a1b2c3d4e5f60718"
|
||||
)
|
||||
|
||||
// session is one instance-role credential: its secret and the instance ID
|
||||
@@ -230,58 +218,55 @@ func (g *localGateway) reply(code string) {
|
||||
}
|
||||
}
|
||||
|
||||
func (g *localGateway) sentBodies() []string {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
return slices.Clone(g.bodies)
|
||||
}
|
||||
|
||||
// gwCluster is the domain's real worker-identity and first-store records.
|
||||
// gwCluster is the domain's real worker-identity record behind the gateway.
|
||||
type gwCluster struct {
|
||||
ids workerident.Store
|
||||
fence firststore.Store
|
||||
stores storelaunch.Records
|
||||
gw *localGateway
|
||||
ids workerident.Store
|
||||
gw *localGateway
|
||||
}
|
||||
|
||||
func newGWCluster(t *testing.T) gwCluster {
|
||||
// newGWCluster hands the gateway's work actions to work and has it report
|
||||
// hostNodes as its cluster size; nil hostNodes is the declared size.
|
||||
func newGWCluster(t *testing.T, work sqsworkerapi.Work, hostNodes func() int) gwCluster {
|
||||
t.Helper()
|
||||
clustersize.DeclareForTest(t, 1)
|
||||
_, _, js := testutil.StartTestJetStream(t)
|
||||
ctx := context.Background()
|
||||
ids, err := workerident.Open(ctx, js)
|
||||
ids, err := workerident.Open(context.Background(), js)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fence, err := firststore.Open(ctx, js)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stores, err := storelaunch.Open(ctx, js)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return gwCluster{ids: ids, fence: fence, stores: stores, gw: newLocalGateway(t, sqsworkerapi.JetStreamDeps(js))}
|
||||
deps := sqsworkerapi.JetStreamDeps(js)
|
||||
deps.Work = work
|
||||
deps.HostNodes = hostNodes
|
||||
return gwCluster{ids: ids, gw: newLocalGateway(t, deps)}
|
||||
}
|
||||
|
||||
func (c gwCluster) launch(t *testing.T, claim string, supersedes uint64, inst string) workerident.Intent {
|
||||
// launch records a worker on a newly created volume, so its receipt key is
|
||||
// pending, and binds it to inst.
|
||||
func (c gwCluster) launch(t *testing.T, inst string) workerident.Intent {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
in, err := c.ids.BeginLaunch(ctx, testRegion, workerident.LaunchRequest{InstanceType: "t3.small",
|
||||
ClaimID: claim, Supersedes: supersedes, NodeID: "node-a", DataVolumeID: gwVolume, DataVolumeEncrypted: true,
|
||||
in, err := c.ids.BeginLaunch(ctx, testRegion, workerident.LaunchRequest{
|
||||
InstanceType: "t3.small", ClaimID: "claim-0001", NodeID: "node-a", DataVolumeEncrypted: true, SizeGiB: 8,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.ids.BindInstance(ctx, in, inst); err != nil {
|
||||
t.Fatal(err)
|
||||
for _, step := range []func() error{
|
||||
func() error { return c.ids.MarkVolumeAttempted(ctx, in) },
|
||||
func() error { return c.ids.RecordVolume(ctx, in, gwVolume) },
|
||||
func() error { return c.ids.MarkDispatched(ctx, in) },
|
||||
func() error { return c.ids.BindInstance(ctx, in, inst) },
|
||||
} {
|
||||
if err := step(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return in
|
||||
}
|
||||
|
||||
// guestFor points the fake guest's boot at the local gateway, signing with
|
||||
// the instance's own credential through the real client.
|
||||
func (c gwCluster) guestFor(t *testing.T, g *guest, accessKey, inst string) {
|
||||
func (c gwCluster) guestFor(t *testing.T, g *guest, accessKey, inst string) *gwsign.Signer {
|
||||
t.Helper()
|
||||
signer := c.gw.issue(accessKey, "secret-"+accessKey, inst)
|
||||
g.fw("data-serial", testSerial)
|
||||
@@ -289,388 +274,14 @@ func (c gwCluster) guestFor(t *testing.T, g *guest, accessKey, inst string) {
|
||||
g.fw("gateway-url", c.gw.srv.URL)
|
||||
g.cfg.Wait = 10 * time.Millisecond
|
||||
g.cfg.NewClaimer = func(region, url string) (workerkey.Claimer, error) {
|
||||
return sqsworkergw.New(url, region, c.gw.caPEM(), signer)
|
||||
cl, err := sqsworkergw.New(url, region, c.gw.caPEM(), signer)
|
||||
return GatewayClaimer{Client: cl}, err
|
||||
}
|
||||
return signer
|
||||
}
|
||||
|
||||
func (c gwCluster) assertNoSeedSent(t *testing.T, seed string) {
|
||||
t.Helper()
|
||||
seed = strings.TrimSpace(seed)
|
||||
for _, b := range c.gw.sentBodies() {
|
||||
if strings.Contains(b, seed) {
|
||||
t.Fatal("a request carried seed material")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A bound worker boots: the gateway authorizes creation, the guest creates
|
||||
// its key on the mounted disk and records only the public key; reboots get
|
||||
// that same key back from the domain.
|
||||
func TestGateway_FirstBootCreatesKeyAndRebootsReuseIt(t *testing.T) {
|
||||
c := newGWCluster(t)
|
||||
c.launch(t, "claim-0001", 0, gwInstance)
|
||||
g := newGuest(t)
|
||||
c.guestFor(t, g, "ASIAWORKER1", gwInstance)
|
||||
|
||||
pub, err := g.boot()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seed := g.seed()
|
||||
for boot := 2; boot <= 3; boot++ {
|
||||
g.resetMounts()
|
||||
got, err := g.boot()
|
||||
if err != nil || got != pub || g.seed() != seed {
|
||||
t.Fatalf("reboot %d = %q, %v", boot, got, err)
|
||||
}
|
||||
}
|
||||
records := 0
|
||||
for _, b := range c.gw.sentBodies() {
|
||||
if strings.Contains(b, "Action=RecordWorkerPublicKey") {
|
||||
records++
|
||||
if !strings.Contains(b, "PublicKey="+pub) {
|
||||
t.Fatal("a record carried a different key")
|
||||
}
|
||||
}
|
||||
}
|
||||
if records != 1 {
|
||||
t.Fatalf("records = %d, want 1", records)
|
||||
}
|
||||
c.assertNoSeedSent(t, seed)
|
||||
g.assertNoSeedLeak(nil, seed)
|
||||
}
|
||||
|
||||
// The record's reply is lost after it landed: the boot retries, the domain
|
||||
// answers with the recorded key, and the same key is used.
|
||||
func TestGateway_LostRecordReplyKeepsTheKey(t *testing.T) {
|
||||
c := newGWCluster(t)
|
||||
c.launch(t, "claim-0001", 0, gwInstance)
|
||||
g := newGuest(t)
|
||||
c.guestFor(t, g, "ASIAWORKER1", gwInstance)
|
||||
var dropped bool
|
||||
c.gw.dropReply = func(action string) bool {
|
||||
if action == "RecordWorkerPublicKey" && !dropped {
|
||||
dropped = true
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
pub, err := g.boot()
|
||||
if err != nil || !dropped {
|
||||
t.Fatalf("boot = %q, %v (dropped=%v)", pub, err, dropped)
|
||||
}
|
||||
g.resetMounts()
|
||||
if again, err := g.boot(); err != nil || again != pub {
|
||||
t.Fatalf("reboot = %q, %v", again, err)
|
||||
}
|
||||
c.assertNoSeedSent(t, g.seed())
|
||||
}
|
||||
|
||||
// After the key is recorded, a lost guest copy fails closed: the domain
|
||||
// answers with the recorded key and no replacement is created or sent.
|
||||
func TestGateway_KeyLostAfterRecordingFailsClosed(t *testing.T) {
|
||||
c := newGWCluster(t)
|
||||
c.launch(t, "claim-0001", 0, gwInstance)
|
||||
g := newGuest(t)
|
||||
c.guestFor(t, g, "ASIAWORKER1", gwInstance)
|
||||
if _, err := g.boot(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Remove(g.keyFile()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := len(c.gw.sentBodies())
|
||||
g.resetMounts()
|
||||
if _, err := g.boot(); !errors.Is(err, workerkey.ErrKeyMissing) {
|
||||
t.Fatalf("err = %v, want ErrKeyMissing", err)
|
||||
}
|
||||
if _, err := os.Lstat(g.keyFile()); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatal("a replacement key was created")
|
||||
}
|
||||
for _, b := range c.gw.sentBodies()[before:] {
|
||||
if strings.Contains(b, "RecordWorkerPublicKey") {
|
||||
t.Fatal("a replacement key was offered")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// An instance the domain never bound, or one already replaced, is refused
|
||||
// at the first call and creates no key; AccessDenied is not retried.
|
||||
func TestGateway_UnboundOrReplacedInstanceIsRefused(t *testing.T) {
|
||||
t.Run("unbound", func(t *testing.T) {
|
||||
c := newGWCluster(t)
|
||||
c.launch(t, "claim-0001", 0, gwInstance)
|
||||
g := newGuest(t)
|
||||
c.guestFor(t, g, "ASIAOTHER", gwSuccessor)
|
||||
_, err := g.boot()
|
||||
api, ok := errors.AsType[*sqsworkergw.APIError](err)
|
||||
if !ok || api.Code != "AccessDenied" {
|
||||
t.Fatalf("err = %v, want AccessDenied", err)
|
||||
}
|
||||
if n := len(c.gw.sentBodies()); n != g.cfg.Attempts {
|
||||
t.Fatalf("calls = %d, want the %d attempts and no more", n, g.cfg.Attempts)
|
||||
}
|
||||
g.assertNoKey()
|
||||
})
|
||||
t.Run("replaced", func(t *testing.T) {
|
||||
c := newGWCluster(t)
|
||||
first := c.launch(t, "claim-0001", 0, gwInstance)
|
||||
ctx := context.Background()
|
||||
if err := c.ids.RecordStopped(ctx, first, gwInstance); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.fence.Fence(ctx, testRegion, first.Generation+1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.launch(t, "claim-0002", first.Generation, gwSuccessor)
|
||||
g := newGuest(t)
|
||||
c.guestFor(t, g, "ASIAOLD", gwInstance)
|
||||
if _, err := g.boot(); err == nil {
|
||||
t.Fatal("the replaced instance booted")
|
||||
}
|
||||
g.assertNoKey()
|
||||
})
|
||||
}
|
||||
|
||||
// A bad signature is refused before any decision, and creates no key.
|
||||
func TestGateway_BadSignatureCreatesNoKey(t *testing.T) {
|
||||
c := newGWCluster(t)
|
||||
c.launch(t, "claim-0001", 0, gwInstance)
|
||||
g := newGuest(t)
|
||||
c.guestFor(t, g, "ASIAWORKER1", gwInstance)
|
||||
forged := gwsign.NewStatic("ASIAWORKER1", "not-the-secret")
|
||||
g.cfg.NewClaimer = func(region, url string) (workerkey.Claimer, error) {
|
||||
return sqsworkergw.New(url, region, c.gw.caPEM(), forged)
|
||||
}
|
||||
if _, err := g.boot(); err == nil {
|
||||
t.Fatal("a forged signature booted")
|
||||
}
|
||||
g.assertNoKey()
|
||||
}
|
||||
|
||||
// The client trusts only the cluster CA it was given: a gateway whose
|
||||
// certificate another CA signed is never sent a request.
|
||||
func TestGateway_UntrustedServerIsNotContacted(t *testing.T) {
|
||||
c := newGWCluster(t)
|
||||
c.launch(t, "claim-0001", 0, gwInstance)
|
||||
g := newGuest(t)
|
||||
c.guestFor(t, g, "ASIAWORKER1", gwInstance)
|
||||
g.cfg.Attempts = 1
|
||||
signer := c.gw.issue("ASIAWORKER1", "secret-ASIAWORKER1", gwInstance)
|
||||
g.cfg.NewClaimer = func(region, url string) (workerkey.Claimer, error) {
|
||||
return sqsworkergw.New(url, region, otherCA(t), signer)
|
||||
}
|
||||
if _, err := g.boot(); err == nil || !strings.Contains(err.Error(), "certificate") {
|
||||
t.Fatalf("err = %v, want a TLS refusal", err)
|
||||
}
|
||||
if len(c.gw.sentBodies()) != 0 {
|
||||
t.Fatal("a request reached a server outside the trusted CA")
|
||||
}
|
||||
g.assertNoKey()
|
||||
}
|
||||
|
||||
// otherCA is a fresh self-signed CA unrelated to the test server's.
|
||||
func otherCA(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tmpl := &x509.Certificate{
|
||||
SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "other"},
|
||||
NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour),
|
||||
IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign,
|
||||
}
|
||||
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
|
||||
}
|
||||
|
||||
// The worker VM starts before its launch binds the instance, so its first
|
||||
// calls are denied. It creates nothing while denied, and boots once bound.
|
||||
func TestGateway_BootBeforeBindSucceedsAfterBinding(t *testing.T) {
|
||||
c := newGWCluster(t)
|
||||
ctx := context.Background()
|
||||
intent, err := c.ids.BeginLaunch(ctx, testRegion, workerident.LaunchRequest{InstanceType: "t3.small",
|
||||
ClaimID: "claim-0001", NodeID: "node-a", DataVolumeID: gwVolume, DataVolumeEncrypted: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
g := newGuest(t)
|
||||
c.guestFor(t, g, "ASIAWORKER1", gwInstance)
|
||||
g.cfg.Attempts = 1000
|
||||
c.gw.replied = make(chan string, 2000)
|
||||
|
||||
type result struct {
|
||||
pub string
|
||||
err error
|
||||
}
|
||||
done := make(chan result, 1)
|
||||
go func() {
|
||||
pub, err := g.boot()
|
||||
done <- result{pub, err}
|
||||
}()
|
||||
if code := <-c.gw.replied; code != "AccessDenied" {
|
||||
t.Fatalf("first reply = %q, want AccessDenied before binding", code)
|
||||
}
|
||||
g.assertNoKey()
|
||||
if err := c.ids.BindInstance(ctx, intent, gwInstance); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := <-done
|
||||
if r.err != nil || r.pub == "" {
|
||||
t.Fatalf("boot after binding = %q, %v", r.pub, r.err)
|
||||
}
|
||||
if g.seed() == "" {
|
||||
t.Fatal("booted without a key on the worker disk")
|
||||
}
|
||||
}
|
||||
|
||||
// A replaced worker is never bound again: it keeps being denied, creates
|
||||
// nothing and gives up within its attempts.
|
||||
func TestGateway_StaleWorkerGivesUpWithinTheBound(t *testing.T) {
|
||||
c := newGWCluster(t)
|
||||
ctx := context.Background()
|
||||
first := c.launch(t, "claim-0001", 0, gwInstance)
|
||||
if err := c.ids.RecordStopped(ctx, first, gwInstance); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.fence.Fence(ctx, testRegion, first.Generation+1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.launch(t, "claim-0002", first.Generation, gwSuccessor)
|
||||
g := newGuest(t)
|
||||
c.guestFor(t, g, "ASIAOLD", gwInstance)
|
||||
g.cfg.Attempts = 5
|
||||
_, err := g.boot()
|
||||
if api, ok := errors.AsType[*sqsworkergw.APIError](err); !ok || !api.Pending() {
|
||||
t.Fatalf("err = %v, want the denial", err)
|
||||
}
|
||||
if n := len(c.gw.sentBodies()); n != 5 {
|
||||
t.Fatalf("calls = %d, want exactly the 5 attempts", n)
|
||||
}
|
||||
g.assertNoKey()
|
||||
}
|
||||
|
||||
// bindStore runs the store launch record through to bound for pub at address,
|
||||
// as the operator's EnsureSQSStore would.
|
||||
func (c gwCluster) bindStore(t *testing.T, pub, address string) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
const claim = "store-claim-0001"
|
||||
steps := []func() error{
|
||||
func() error { return c.stores.Begin(ctx, testRegion, claim, "node-a", pub, "t3.medium", 8) },
|
||||
func() error { return c.stores.MarkVolumeAttempted(ctx, testRegion, claim) },
|
||||
func() error { return c.stores.RecordVolume(ctx, testRegion, claim, "vol-0b1c2d3e4f5a60718") },
|
||||
func() error { return c.stores.MarkNetworkAttempted(ctx, testRegion, claim) },
|
||||
func() error { return c.stores.MarkDispatched(ctx, testRegion, claim) },
|
||||
func() error { return c.stores.Bind(ctx, testRegion, claim, "i-0123456789abcdef0", address) },
|
||||
}
|
||||
for i, step := range steps {
|
||||
if err := step(); err != nil {
|
||||
t.Fatalf("store launch step %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// readyFor points the guest's readiness at the local gateway and a store on port.
|
||||
func (c gwCluster) readyFor(g *guest, accessKey, inst string, port int) ReadyConfig {
|
||||
signer := c.gw.issue(accessKey, "secret-"+accessKey, inst)
|
||||
return ReadyConfig{
|
||||
Config: g.cfg,
|
||||
NewDescriber: func(region, url string) (StoreDescriber, error) {
|
||||
cl, err := sqsworkergw.New(url, region, c.gw.caPEM(), signer)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return gwDescriber{cl}, nil
|
||||
},
|
||||
Checker: NATSChecker{Port: port, Dial: time.Second},
|
||||
}
|
||||
}
|
||||
|
||||
type gwDescriber struct{ c *sqsworkergw.Client }
|
||||
|
||||
func (d gwDescriber) DescribeStore(ctx context.Context) (Store, error) {
|
||||
st, err := d.c.DescribeStore(ctx)
|
||||
return Store(st), err
|
||||
}
|
||||
|
||||
// The whole path with real components: the worker boots and records its key,
|
||||
// waits while the operator has not launched the store, then reaches the real
|
||||
// store's JetStream with its own key once the store is bound; after a reboot
|
||||
// it is ready again with the same key. No request carries the seed.
|
||||
func TestGateway_ReadyAfterADelayedStoreLaunch(t *testing.T) {
|
||||
c := newGWCluster(t)
|
||||
c.launch(t, "claim-0001", 0, gwInstance)
|
||||
g := newGuest(t)
|
||||
c.guestFor(t, g, "ASIAWORKER1", gwInstance)
|
||||
pub, err := g.boot()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
port := storeServer(t, pub)
|
||||
err = g.watch(c.readyFor(g, "ASIAWORKER1", gwInstance, port), func(call int, ready bool) bool {
|
||||
if ready {
|
||||
if call < 4 {
|
||||
t.Fatalf("ready at call %d, before the store launch", call)
|
||||
}
|
||||
return true
|
||||
}
|
||||
if call == 3 {
|
||||
c.bindStore(t, pub, "127.0.0.1")
|
||||
}
|
||||
return false
|
||||
})
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("watch = %v", err)
|
||||
}
|
||||
for boot := 1; boot <= 2; boot++ {
|
||||
g.resetMounts()
|
||||
if again, err := g.boot(); err != nil || again != pub {
|
||||
t.Fatalf("reboot %d = %q %v", boot, again, err)
|
||||
}
|
||||
if err := g.watch(c.readyFor(g, "ASIAWORKER1", gwInstance, port), func(_ int, ready bool) bool { return ready }); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("ready after reboot %d: %v", boot, err)
|
||||
}
|
||||
}
|
||||
c.assertNoSeedSent(t, g.seed())
|
||||
g.assertNoSeedLeak(nil, g.seed())
|
||||
}
|
||||
|
||||
// A store bound to a key other than this worker's is never checked, and a
|
||||
// replaced worker cannot read the store at all; neither is ever ready.
|
||||
func TestGateway_ReadyRefusesWrongKeyAndReplacedWorker(t *testing.T) {
|
||||
c := newGWCluster(t)
|
||||
first := c.launch(t, "claim-0001", 0, gwInstance)
|
||||
g := newGuest(t)
|
||||
c.guestFor(t, g, "ASIAWORKER1", gwInstance)
|
||||
if _, err := g.boot(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
other, _ := nkeys.CreateUser()
|
||||
otherPub, _ := other.PublicKey()
|
||||
c.bindStore(t, otherPub, "127.0.0.1")
|
||||
port := storeServer(t, otherPub)
|
||||
never := func(call int, ready bool) bool {
|
||||
if ready {
|
||||
t.Fatal("ready against a store for another key")
|
||||
}
|
||||
return call == 3
|
||||
}
|
||||
_ = g.watch(c.readyFor(g, "ASIAWORKER1", gwInstance, port), never)
|
||||
if !slices.ContainsFunc(g.logs, func(l string) bool { return strings.Contains(l, ErrStoreKeyMismatch.Error()) }) {
|
||||
t.Fatalf("no mismatch reported: %v", g.logs)
|
||||
}
|
||||
|
||||
if err := c.ids.RecordStopped(context.Background(), first, gwInstance); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = g.watch(c.readyFor(g, "ASIAWORKER1", gwInstance, port), never)
|
||||
if !strings.Contains(g.logs[len(g.logs)-1], "AccessDenied") {
|
||||
t.Fatalf("stopped worker: %v", g.logs[len(g.logs)-1])
|
||||
}
|
||||
func (g *localGateway) sentBodies() []string {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
return slices.Clone(g.bodies)
|
||||
}
|
||||
|
||||
@@ -10,15 +10,14 @@ import (
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/nats-io/nkeys"
|
||||
)
|
||||
|
||||
// StorePort is the port the store guest's NATS listens on.
|
||||
// StorePort is the port the guest's own SQS NATS server listens on, bound
|
||||
// to loopback only; nothing outside the guest can reach it.
|
||||
const StorePort = 4222
|
||||
|
||||
// NATSChecker connects with the worker key and asks JetStream for its account
|
||||
// info. It authenticates the client only: there is no TLS, so a pass says
|
||||
// something answering at the address accepted the key.
|
||||
// NATSChecker asks the guest's loopback JetStream for its account info. A
|
||||
// pass says the local store is up and serving JetStream.
|
||||
type NATSChecker struct {
|
||||
// Port overrides StorePort, for tests against a local server.
|
||||
Port int
|
||||
@@ -35,36 +34,22 @@ func (e *StoreUnreachableError) Error() string { return "store not answering:
|
||||
func (e *StoreUnreachableError) Unwrap() error { return e.Err }
|
||||
func (e *StoreUnreachableError) Retryable() bool { return true }
|
||||
|
||||
// ErrStoreRefusedKey is a store that answered but refused the worker key.
|
||||
var ErrStoreRefusedKey = errors.New("the store refused the worker key")
|
||||
|
||||
// Check passes once JetStream answers for the worker key.
|
||||
func (c NATSChecker) Check(ctx context.Context, address string, kp nkeys.KeyPair) error {
|
||||
if ip := net.ParseIP(address); ip == nil || ip.To4() == nil {
|
||||
return errors.New("store address is not an IPv4 address")
|
||||
}
|
||||
pub, err := kp.PublicKey()
|
||||
if err != nil {
|
||||
return fmt.Errorf("derive worker public key: %w", err)
|
||||
}
|
||||
port := c.Port
|
||||
func localURL(port int) string {
|
||||
if port == 0 {
|
||||
port = StorePort
|
||||
}
|
||||
return "nats://" + net.JoinHostPort("127.0.0.1", strconv.Itoa(port))
|
||||
}
|
||||
|
||||
// Check passes once the local JetStream answers.
|
||||
func (c NATSChecker) Check(ctx context.Context) error {
|
||||
dial := c.Dial
|
||||
if dial == 0 {
|
||||
dial = 5 * time.Second
|
||||
}
|
||||
nc, err := nats.Connect("nats://"+net.JoinHostPort(address, strconv.Itoa(port)),
|
||||
nats.Nkey(pub, kp.Sign),
|
||||
nats.Timeout(dial),
|
||||
nats.NoReconnect(),
|
||||
nats.Name("spinifex-sqs-worker-readiness"),
|
||||
)
|
||||
nc, err := nats.Connect(localURL(c.Port), nats.Timeout(dial), nats.NoReconnect(),
|
||||
nats.Name("spinifex-sqs-worker-readiness"))
|
||||
if err != nil {
|
||||
if errors.Is(err, nats.ErrAuthorization) || errors.Is(err, nats.ErrAuthExpired) {
|
||||
return fmt.Errorf("%w: %w", ErrStoreRefusedKey, err)
|
||||
}
|
||||
return &StoreUnreachableError{Err: err}
|
||||
}
|
||||
defer nc.Close()
|
||||
@@ -79,3 +64,5 @@ func (c NATSChecker) Check(ctx context.Context, address string, kp nkeys.KeyPair
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var errNoChecker = errors.New("no store checker is configured")
|
||||
|
||||
@@ -9,37 +9,19 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/agents/sqs/workerkey"
|
||||
"github.com/nats-io/nkeys"
|
||||
)
|
||||
|
||||
// Store is the Region's store launch as the gateway reports it. Bound means a
|
||||
// VM was launched for the store, not that it serves.
|
||||
type Store struct {
|
||||
Bound bool
|
||||
Address string
|
||||
PublicKey string
|
||||
}
|
||||
|
||||
// StoreDescriber asks the gateway for this worker's store.
|
||||
type StoreDescriber interface {
|
||||
DescribeStore(ctx context.Context) (Store, error)
|
||||
}
|
||||
|
||||
// NewDescriber builds the gateway client the store's address comes from.
|
||||
type NewDescriber func(region, gatewayURL string) (StoreDescriber, error)
|
||||
|
||||
// Checker proves the store's JetStream answers this worker's key at address.
|
||||
// Only an error with Retryable() true is asked again.
|
||||
// Checker proves the guest's local JetStream answers. Only an error with
|
||||
// Retryable() true is asked again.
|
||||
type Checker interface {
|
||||
Check(ctx context.Context, address string, kp nkeys.KeyPair) error
|
||||
Check(ctx context.Context) error
|
||||
}
|
||||
|
||||
// ReadyConfig is Config plus the readiness capabilities and cadence.
|
||||
type ReadyConfig struct {
|
||||
Config
|
||||
|
||||
NewDescriber NewDescriber
|
||||
Checker Checker
|
||||
Checker Checker
|
||||
// CallTimeout bounds one describe-and-check pass; zero is 30 seconds.
|
||||
CallTimeout time.Duration
|
||||
// Refresh is the wait after a pass before checking again; zero is 30s.
|
||||
@@ -47,18 +29,15 @@ type ReadyConfig struct {
|
||||
// Refused is the wait after a failure that cannot clear by itself, such
|
||||
// as a refused or mismatched key; zero is five minutes.
|
||||
Refused time.Duration
|
||||
// Serve, when set, also serves SQS owner calls while Watch runs.
|
||||
Serve *ServeConfig
|
||||
}
|
||||
|
||||
// ErrStoreKeyMismatch is a bound store launched for a key other than the one
|
||||
// on this worker's disk; it is asked again only at the Refused cadence.
|
||||
var ErrStoreKeyMismatch = errors.New("the store was launched for a different worker key")
|
||||
|
||||
const readyFile = "store-ready"
|
||||
|
||||
// Watch keeps the readiness marker true until ctx ends: it re-proves the disk,
|
||||
// then checks the store with the worker's existing key every pass, writing the
|
||||
// marker on a pass and removing it on any failure and on return. It never
|
||||
// creates a key. NKey authenticates this client only, not the store server.
|
||||
// then checks the guest's local store every pass, writing the marker on a
|
||||
// pass and removing it on any failure and on return. It never creates a key.
|
||||
func Watch(ctx context.Context, cfg ReadyConfig) error {
|
||||
marker := filepath.Join(cfg.RunDir, readyFile)
|
||||
defer func() { _ = os.Remove(marker) }()
|
||||
@@ -72,14 +51,19 @@ func Watch(ctx context.Context, cfg ReadyConfig) error {
|
||||
if err := mountWorkerDisk(cfg.Config, c.serial); err != nil {
|
||||
return err
|
||||
}
|
||||
if cfg.NewDescriber == nil || cfg.Checker == nil {
|
||||
return errors.New("no gateway client or store checker is configured")
|
||||
}
|
||||
d, err := cfg.NewDescriber(c.region, c.gatewayURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("build gateway client: %w", err)
|
||||
if cfg.Checker == nil {
|
||||
return errNoChecker
|
||||
}
|
||||
keys := workerkey.Store{Dir: filepath.Join(cfg.DataDir, keyDirName)}
|
||||
if cfg.Serve != nil {
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
serve(ctx, cfg, c, keys)
|
||||
}()
|
||||
// Calls in flight finish before the marker is removed and Watch returns.
|
||||
defer func() { <-done }()
|
||||
}
|
||||
callTimeout := orDefault(cfg.CallTimeout, 30*time.Second)
|
||||
refresh := orDefault(cfg.Refresh, 30*time.Second)
|
||||
refused := orDefault(cfg.Refused, 5*time.Minute)
|
||||
@@ -87,21 +71,21 @@ func Watch(ctx context.Context, cfg ReadyConfig) error {
|
||||
last := ""
|
||||
for {
|
||||
callCtx, cancel := context.WithTimeout(ctx, callTimeout)
|
||||
st, err := readyOnce(callCtx, cfg, d, keys)
|
||||
err := cfg.Checker.Check(callCtx)
|
||||
cancel()
|
||||
delay, state := refresh, ""
|
||||
if err == nil {
|
||||
err = writeReady(cfg.RunDir, st)
|
||||
err = writeReady(cfg.RunDir)
|
||||
}
|
||||
switch {
|
||||
case ctx.Err() != nil:
|
||||
return ctx.Err()
|
||||
case err == nil:
|
||||
state = fmt.Sprintf("store at %s answered the worker key (client-authenticated only; the store server is not authenticated)", st.Address)
|
||||
state = "local SQS store is serving JetStream"
|
||||
default:
|
||||
_ = os.Remove(marker)
|
||||
delay = cfg.Wait
|
||||
if !pending(err) && !retryable(err) && !errors.Is(err, errStorePending) {
|
||||
if !pending(err) && !retryable(err) {
|
||||
delay = refused
|
||||
}
|
||||
state = fmt.Sprintf("store not ready: %v", err)
|
||||
@@ -126,37 +110,12 @@ func orDefault(d, def time.Duration) time.Duration {
|
||||
return d
|
||||
}
|
||||
|
||||
var errStorePending = errors.New("the store is not yet bound")
|
||||
|
||||
func readyOnce(ctx context.Context, cfg ReadyConfig, d StoreDescriber, keys workerkey.Store) (Store, error) {
|
||||
st, err := d.DescribeStore(ctx)
|
||||
if err != nil {
|
||||
return Store{}, err
|
||||
}
|
||||
if !st.Bound {
|
||||
return Store{}, errStorePending
|
||||
}
|
||||
kp, err := keys.ForExistingStore(st.PublicKey)
|
||||
if err != nil {
|
||||
if errors.Is(err, workerkey.ErrKeyMismatch) {
|
||||
return Store{}, fmt.Errorf("%w: %w", ErrStoreKeyMismatch, err)
|
||||
}
|
||||
return Store{}, err
|
||||
}
|
||||
defer kp.Wipe()
|
||||
if err := cfg.Checker.Check(ctx, st.Address, kp); err != nil {
|
||||
return Store{}, err
|
||||
}
|
||||
return st, nil
|
||||
}
|
||||
|
||||
// writeReady publishes only the address and public key, never the seed.
|
||||
func writeReady(runDir string, st Store) error {
|
||||
// writeReady publishes that the local store answered; it holds no secret.
|
||||
func writeReady(runDir string) error {
|
||||
if err := os.MkdirAll(runDir, 0o750); err != nil {
|
||||
return fmt.Errorf("create %s: %w", runDir, err)
|
||||
}
|
||||
body := "address=" + st.Address + "\npublic=" + st.PublicKey + "\n"
|
||||
if err := os.WriteFile(filepath.Join(runDir, readyFile), []byte(body), 0o600); err != nil {
|
||||
if err := os.WriteFile(filepath.Join(runDir, readyFile), []byte("ready\n"), 0o600); err != nil {
|
||||
return fmt.Errorf("publish store readiness: %w", err)
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -1,438 +0,0 @@
|
||||
package workerboot
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nkeys"
|
||||
)
|
||||
|
||||
// describer is the gateway's DescribeStore, answering from a script.
|
||||
type describer struct {
|
||||
answers []describeAnswer
|
||||
calls int
|
||||
}
|
||||
|
||||
type describeAnswer struct {
|
||||
st Store
|
||||
err error
|
||||
}
|
||||
|
||||
func (d *describer) DescribeStore(context.Context) (Store, error) {
|
||||
d.calls++
|
||||
a := d.answers[min(d.calls, len(d.answers))-1]
|
||||
return a.st, a.err
|
||||
}
|
||||
|
||||
type checker struct {
|
||||
errs []error
|
||||
calls int
|
||||
seen []string
|
||||
}
|
||||
|
||||
func (c *checker) Check(_ context.Context, address string, kp nkeys.KeyPair) error {
|
||||
c.calls++
|
||||
pub, _ := kp.PublicKey()
|
||||
c.seen = append(c.seen, address+"|"+pub)
|
||||
if len(c.errs) > 0 {
|
||||
err := c.errs[0]
|
||||
c.errs = c.errs[1:]
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// bootedGuest has run the boot step, so its disk holds a recorded key.
|
||||
func bootedGuest(t *testing.T) (*guest, string) {
|
||||
t.Helper()
|
||||
g := newGuest(t)
|
||||
g.contract()
|
||||
pub, err := g.boot()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return g, pub
|
||||
}
|
||||
|
||||
func (g *guest) readyCfg(d StoreDescriber, c Checker) ReadyConfig {
|
||||
return ReadyConfig{
|
||||
Config: g.cfg,
|
||||
NewDescriber: func(string, string) (StoreDescriber, error) { return d, nil },
|
||||
Checker: c,
|
||||
}
|
||||
}
|
||||
|
||||
// hooked asks stop before each describe, after the previous pass has written
|
||||
// or removed the marker, and ends the watch when stop says so.
|
||||
type hooked struct {
|
||||
d StoreDescriber
|
||||
marker string
|
||||
stop func(call int, ready bool) bool
|
||||
cancel context.CancelFunc
|
||||
calls int
|
||||
}
|
||||
|
||||
func (h *hooked) DescribeStore(ctx context.Context) (Store, error) {
|
||||
h.calls++
|
||||
_, err := os.Stat(h.marker)
|
||||
if h.stop(h.calls, err == nil) {
|
||||
h.cancel()
|
||||
return Store{}, context.Canceled
|
||||
}
|
||||
return h.d.DescribeStore(ctx)
|
||||
}
|
||||
|
||||
// watch runs Watch with millisecond cadences until stop ends it, then checks
|
||||
// the marker was removed on the way out.
|
||||
func (g *guest) watch(cfg ReadyConfig, stop func(call int, ready bool) bool) error {
|
||||
g.t.Helper()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
inner := cfg.NewDescriber
|
||||
cfg.NewDescriber = func(region, url string) (StoreDescriber, error) {
|
||||
d, err := inner(region, url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &hooked{d: d, marker: filepath.Join(cfg.RunDir, readyFile), stop: stop, cancel: cancel}, nil
|
||||
}
|
||||
for _, p := range []*time.Duration{&cfg.Wait, &cfg.Refresh, &cfg.Refused} {
|
||||
if *p == 0 {
|
||||
*p = time.Millisecond
|
||||
}
|
||||
}
|
||||
err := Watch(ctx, cfg)
|
||||
if errors.Is(ctx.Err(), context.DeadlineExceeded) {
|
||||
g.t.Fatal("the watch never reached its stop condition")
|
||||
}
|
||||
g.assertNotReady()
|
||||
return err
|
||||
}
|
||||
|
||||
func bound(pub string) describeAnswer {
|
||||
return describeAnswer{st: Store{Bound: true, Address: "10.240.0.9", PublicKey: pub}}
|
||||
}
|
||||
|
||||
func (g *guest) marker() string {
|
||||
g.t.Helper()
|
||||
b, _ := os.ReadFile(filepath.Join(g.cfg.RunDir, readyFile))
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// A store launched long after the worker booted, far past the old 15-minute
|
||||
// bound of 180 attempts, still becomes ready; nothing is published before.
|
||||
func TestWatch_StoreLaunchedLateBecomesReady(t *testing.T) {
|
||||
g, pub := bootedGuest(t)
|
||||
answers := make([]describeAnswer, 201)
|
||||
answers[200] = bound(pub)
|
||||
d := &describer{answers: answers}
|
||||
c := &checker{errs: []error{&StoreUnreachableError{Err: errors.New("connection refused")}}}
|
||||
published := ""
|
||||
err := g.watch(g.readyCfg(d, c), func(call int, ready bool) bool {
|
||||
if ready {
|
||||
published = g.marker()
|
||||
return true
|
||||
}
|
||||
if call > 202 {
|
||||
t.Fatalf("not ready by call %d", call)
|
||||
}
|
||||
return false
|
||||
})
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("watch = %v", err)
|
||||
}
|
||||
if d.calls != 202 || c.calls != 2 || c.seen[0] != "10.240.0.9|"+pub {
|
||||
t.Fatalf("describe %d, check %d, seen %v", d.calls, c.calls, c.seen)
|
||||
}
|
||||
if published != "address=10.240.0.9\npublic="+pub+"\n" {
|
||||
t.Fatalf("published %q", published)
|
||||
}
|
||||
notReady := 0
|
||||
for _, l := range g.logs {
|
||||
if strings.HasPrefix(l, "store not ready") {
|
||||
notReady++
|
||||
}
|
||||
}
|
||||
if notReady > 2 {
|
||||
t.Fatalf("%d not-ready lines for two distinct states", notReady)
|
||||
}
|
||||
}
|
||||
|
||||
// After a pass, the marker goes as soon as a later check fails, and returns
|
||||
// once the store answers again.
|
||||
func TestWatch_LossAfterReadyRemovesTheMarker(t *testing.T) {
|
||||
cases := map[string]func(pub string) (*describer, *checker){
|
||||
"store stops answering": func(pub string) (*describer, *checker) {
|
||||
return &describer{answers: []describeAnswer{bound(pub)}},
|
||||
&checker{errs: []error{nil, &StoreUnreachableError{Err: errors.New("connection refused")}, nil}}
|
||||
},
|
||||
"gateway reply lost": func(pub string) (*describer, *checker) {
|
||||
return &describer{answers: []describeAnswer{bound(pub), {err: transient{}}, bound(pub)}}, &checker{}
|
||||
},
|
||||
"store no longer bound": func(pub string) (*describer, *checker) {
|
||||
return &describer{answers: []describeAnswer{bound(pub), {}, bound(pub)}}, &checker{}
|
||||
},
|
||||
"store now refuses the key": func(pub string) (*describer, *checker) {
|
||||
return &describer{answers: []describeAnswer{bound(pub)}}, &checker{errs: []error{nil, ErrStoreRefusedKey, nil}}
|
||||
},
|
||||
}
|
||||
for name, mk := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
g, pub := bootedGuest(t)
|
||||
d, c := mk(pub)
|
||||
var seen []bool
|
||||
if err := g.watch(g.readyCfg(d, c), func(call int, ready bool) bool {
|
||||
seen = append(seen, ready)
|
||||
return call == 4
|
||||
}); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("watch = %v", err)
|
||||
}
|
||||
if want := []bool{false, true, false, true}; !slices.Equal(seen, want) {
|
||||
t.Fatalf("marker before each pass = %v, want %v", seen, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A marker left by an earlier run is cleared before the first pass.
|
||||
func TestWatch_ClearsAStaleMarker(t *testing.T) {
|
||||
g, pub := bootedGuest(t)
|
||||
mustWrite(t, filepath.Join(g.cfg.RunDir, readyFile), "address=10.0.0.1\n")
|
||||
d := &describer{answers: []describeAnswer{{}}}
|
||||
_ = g.watch(g.readyCfg(d, &checker{}), func(call int, ready bool) bool {
|
||||
if call == 1 && ready {
|
||||
t.Fatal("a stale marker survived into the first pass")
|
||||
}
|
||||
return call == 2
|
||||
})
|
||||
_ = pub
|
||||
}
|
||||
|
||||
// Failures that cannot clear by themselves wait the Refused cadence: in the
|
||||
// window, each is tried once and never published.
|
||||
func TestWatch_FinalFailuresDoNotSpin(t *testing.T) {
|
||||
other, _ := nkeys.CreateUser()
|
||||
otherPub, _ := other.PublicKey()
|
||||
cases := map[string]struct {
|
||||
d func(pub string) *describer
|
||||
c *checker
|
||||
checks int
|
||||
}{
|
||||
"store refuses the key": {func(pub string) *describer { return &describer{answers: []describeAnswer{bound(pub)}} },
|
||||
&checker{errs: []error{ErrStoreRefusedKey}}, 1},
|
||||
"store launched for another key": {func(string) *describer { return &describer{answers: []describeAnswer{bound(otherPub)}} },
|
||||
&checker{}, 0},
|
||||
"unclassified gateway refusal": {func(string) *describer {
|
||||
return &describer{answers: []describeAnswer{{err: errors.New("InvalidParameterValue")}}}
|
||||
}, &checker{}, 0},
|
||||
}
|
||||
for name, tc := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
g, pub := bootedGuest(t)
|
||||
g.cfg.Wait = time.Millisecond
|
||||
d := tc.d(pub)
|
||||
cfg := g.readyCfg(d, tc.c)
|
||||
cfg.Refresh, cfg.Refused = time.Millisecond, time.Hour
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond)
|
||||
defer cancel()
|
||||
if err := Watch(ctx, cfg); !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("watch = %v", err)
|
||||
}
|
||||
if d.calls != 1 || tc.c.calls != tc.checks {
|
||||
t.Fatalf("describe %d, check %d in the window", d.calls, tc.c.calls)
|
||||
}
|
||||
g.assertNotReady()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A mismatched store is asked again at the Refused cadence, so a store the
|
||||
// operator relaunches for this worker's key is picked up.
|
||||
func TestWatch_RecoversWhenTheStoreIsRelaunchedForThisKey(t *testing.T) {
|
||||
g, pub := bootedGuest(t)
|
||||
other, _ := nkeys.CreateUser()
|
||||
otherPub, _ := other.PublicKey()
|
||||
d := &describer{answers: []describeAnswer{bound(otherPub), bound(pub)}}
|
||||
c := &checker{}
|
||||
if err := g.watch(g.readyCfg(d, c), func(_ int, ready bool) bool { return ready }); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("watch = %v", err)
|
||||
}
|
||||
if d.calls != 2 || c.calls != 1 {
|
||||
t.Fatalf("describe %d, check %d", d.calls, c.calls)
|
||||
}
|
||||
}
|
||||
|
||||
// A key missing from the disk is never replaced, however often it is asked.
|
||||
func TestWatch_LostKeyIsNeverReplaced(t *testing.T) {
|
||||
g, pub := bootedGuest(t)
|
||||
dir := filepath.Join(g.cfg.DataDir, keyDirName)
|
||||
entries, _ := os.ReadDir(dir)
|
||||
for _, e := range entries {
|
||||
if err := os.Remove(filepath.Join(dir, e.Name())); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
d := &describer{answers: []describeAnswer{bound(pub)}}
|
||||
c := &checker{}
|
||||
_ = g.watch(g.readyCfg(d, c), func(call int, ready bool) bool {
|
||||
if ready {
|
||||
t.Fatal("ready without a key")
|
||||
}
|
||||
return call == 5
|
||||
})
|
||||
if c.calls != 0 {
|
||||
t.Fatalf("checked %d times without a key", c.calls)
|
||||
}
|
||||
if after, _ := os.ReadDir(dir); len(after) != 0 {
|
||||
t.Fatal("a replacement key was written")
|
||||
}
|
||||
}
|
||||
|
||||
// Readiness never runs on a disk that is not proven mounted.
|
||||
func TestWatch_RefusesWithoutTheMountedWorkerDisk(t *testing.T) {
|
||||
g, pub := bootedGuest(t)
|
||||
g.resetMounts()
|
||||
g.disk.noopMount = true
|
||||
d := &describer{answers: []describeAnswer{bound(pub)}}
|
||||
if err := Watch(context.Background(), g.readyCfg(d, &checker{})); err == nil || d.calls != 0 {
|
||||
t.Fatalf("err = %v, describe %d", err, d.calls)
|
||||
}
|
||||
}
|
||||
|
||||
// After a restart the same key reaches readiness again with no new key.
|
||||
func TestWatch_SurvivesARestart(t *testing.T) {
|
||||
g, pub := bootedGuest(t)
|
||||
for range 2 {
|
||||
d := &describer{answers: []describeAnswer{bound(pub)}}
|
||||
if err := g.watch(g.readyCfg(d, &checker{}), func(_ int, ready bool) bool { return ready }); !errors.Is(err, context.Canceled) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
g.resetMounts()
|
||||
again, err := g.boot()
|
||||
if err != nil || again != pub {
|
||||
t.Fatalf("reboot = %q %v", again, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (g *guest) assertNotReady() {
|
||||
g.t.Helper()
|
||||
if _, err := os.Stat(filepath.Join(g.cfg.RunDir, readyFile)); !os.IsNotExist(err) {
|
||||
g.t.Fatalf("readiness was published: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func startStore(t *testing.T, pub string, port int) *server.Server {
|
||||
t.Helper()
|
||||
opts := &server.Options{Host: "127.0.0.1", Port: port, JetStream: true, StoreDir: t.TempDir(), NoLog: true,
|
||||
NoSigs: true, Nkeys: []*server.NkeyUser{{Nkey: pub}}}
|
||||
ns, err := server.NewServer(opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go ns.Start()
|
||||
if !ns.ReadyForConnections(10 * time.Second) {
|
||||
t.Fatal("store server did not start")
|
||||
}
|
||||
t.Cleanup(ns.Shutdown)
|
||||
return ns
|
||||
}
|
||||
|
||||
// End to end on the guest with a real store: not ready before it serves,
|
||||
// ready once it does, not ready again once it stops, and the marker never
|
||||
// holds the seed.
|
||||
func TestWatch_RealStoreStartsLateThenStops(t *testing.T) {
|
||||
g, pub := bootedGuest(t)
|
||||
seedFile, _ := os.ReadDir(filepath.Join(g.cfg.DataDir, keyDirName))
|
||||
key, _ := os.ReadFile(filepath.Join(g.cfg.DataDir, keyDirName, seedFile[0].Name()))
|
||||
l, _ := net.Listen("tcp", "127.0.0.1:0")
|
||||
port := l.Addr().(*net.TCPAddr).Port
|
||||
_ = l.Close()
|
||||
d := &describer{answers: []describeAnswer{{st: Store{Bound: true, Address: "127.0.0.1", PublicKey: pub}}}}
|
||||
cfg := g.readyCfg(d, NATSChecker{Port: port, Dial: time.Second})
|
||||
var ns *server.Server
|
||||
phase := 0
|
||||
err := g.watch(cfg, func(call int, ready bool) bool {
|
||||
switch {
|
||||
case phase == 0 && call == 3:
|
||||
if ready {
|
||||
t.Fatal("ready before the store served")
|
||||
}
|
||||
ns, phase = startStore(t, pub, port), 1
|
||||
case phase == 1 && ready:
|
||||
if strings.Contains(g.marker(), strings.TrimSpace(string(key))) {
|
||||
t.Fatal("the seed was published")
|
||||
}
|
||||
ns.Shutdown()
|
||||
ns.WaitForShutdown()
|
||||
phase = 2
|
||||
case phase == 2:
|
||||
if ready {
|
||||
t.Fatal("still ready after the store stopped")
|
||||
}
|
||||
return true
|
||||
}
|
||||
return false
|
||||
})
|
||||
if !errors.Is(err, context.Canceled) || phase != 2 {
|
||||
t.Fatalf("watch = %v, phase %d", err, phase)
|
||||
}
|
||||
}
|
||||
|
||||
// storeServer is a real NATS server with JetStream that admits only pub.
|
||||
func storeServer(t *testing.T, pub string) int {
|
||||
t.Helper()
|
||||
opts := &server.Options{
|
||||
Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true,
|
||||
Nkeys: []*server.NkeyUser{{Nkey: pub}},
|
||||
}
|
||||
ns, err := server.NewServer(opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go ns.Start()
|
||||
if !ns.ReadyForConnections(10 * time.Second) {
|
||||
t.Fatal("store server did not start")
|
||||
}
|
||||
t.Cleanup(ns.Shutdown)
|
||||
return ns.Addr().(*net.TCPAddr).Port
|
||||
}
|
||||
|
||||
// The real checker passes only with the admitted key; a wrong key is refused
|
||||
// as final, and nothing listening is retryable.
|
||||
func TestNATSChecker_AgainstARealStore(t *testing.T) {
|
||||
kp, _ := nkeys.CreateUser()
|
||||
pub, _ := kp.PublicKey()
|
||||
port := storeServer(t, pub)
|
||||
c := NATSChecker{Port: port, Dial: 2 * time.Second}
|
||||
if err := c.Check(context.Background(), "127.0.0.1", kp); err != nil {
|
||||
t.Fatalf("admitted key: %v", err)
|
||||
}
|
||||
other, _ := nkeys.CreateUser()
|
||||
err := c.Check(context.Background(), "127.0.0.1", other)
|
||||
if !errors.Is(err, ErrStoreRefusedKey) || retryable(err) {
|
||||
t.Fatalf("wrong key: %v", err)
|
||||
}
|
||||
seed, _ := kp.Seed()
|
||||
if strings.Contains(err.Error(), string(seed)) {
|
||||
t.Fatal("the error echoes a seed")
|
||||
}
|
||||
l, _ := net.Listen("tcp", "127.0.0.1:0")
|
||||
closed := l.Addr().(*net.TCPAddr).Port
|
||||
_ = l.Close()
|
||||
err = NATSChecker{Port: closed, Dial: time.Second}.Check(context.Background(), "127.0.0.1", kp)
|
||||
if !retryable(err) {
|
||||
t.Fatalf("nothing listening: %v is not retryable", err)
|
||||
}
|
||||
if err := c.Check(context.Background(), "store.example", kp); err == nil || retryable(err) {
|
||||
t.Fatalf("hostname accepted: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
package workerboot
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/agents/sqs/workerkey"
|
||||
)
|
||||
|
||||
// ServeConfig makes the ready step also serve SQS owner calls.
|
||||
type ServeConfig struct {
|
||||
Connector Connector
|
||||
// Run serves until ctx ends against the store's JetStream, with the
|
||||
// receipt key from the worker disk; it must not retain the key.
|
||||
Run func(ctx context.Context, region, gatewayURL string, js jetstream.JetStream, receiptKey []byte) error
|
||||
}
|
||||
|
||||
// Connector opens the guest's local JetStream.
|
||||
type Connector interface {
|
||||
Connect(ctx context.Context) (jetstream.JetStream, func(), error)
|
||||
}
|
||||
|
||||
// NATSConnector connects to the guest's loopback SQS NATS server.
|
||||
type NATSConnector struct {
|
||||
// Port overrides StorePort, for tests against a local server.
|
||||
Port int
|
||||
}
|
||||
|
||||
var _ Connector = NATSConnector{}
|
||||
|
||||
// ErrUnsupportedTopology refuses serving, and so creating any store state,
|
||||
// unless the gateway reports a single-node host cluster.
|
||||
var ErrUnsupportedTopology = errors.New("SQS serving supports only a single-node host cluster")
|
||||
|
||||
// ErrReceiptPending refuses serving while the platform has not recorded the
|
||||
// receipt key; the boot step records it before the store first starts.
|
||||
var ErrReceiptPending = errors.New("the receipt key is not yet recorded")
|
||||
|
||||
func (c NATSConnector) Connect(_ context.Context) (jetstream.JetStream, func(), error) {
|
||||
nc, err := nats.Connect(localURL(c.Port), nats.Timeout(5*time.Second), nats.MaxReconnects(-1),
|
||||
nats.Name("spinifex-sqs-worker"))
|
||||
if err != nil {
|
||||
return nil, nil, &StoreUnreachableError{Err: err}
|
||||
}
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
nc.Close()
|
||||
return nil, nil, &StoreUnreachableError{Err: err}
|
||||
}
|
||||
return js, nc.Close, nil
|
||||
}
|
||||
|
||||
// serve runs the owners until ctx ends, starting again after any failure to
|
||||
// open them. It never creates a key: the receipt key must be on the disk.
|
||||
func serve(ctx context.Context, cfg ReadyConfig, c contract, keys workerkey.Store) {
|
||||
last := ""
|
||||
for ctx.Err() == nil {
|
||||
err := serveOnce(ctx, cfg, c, keys)
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if state := fmt.Sprintf("SQS serving stopped: %v", err); state != last {
|
||||
cfg.Log(state)
|
||||
last = state
|
||||
}
|
||||
delay := cfg.Wait
|
||||
if !pending(err) && !retryable(err) && !errors.Is(err, ErrReceiptPending) {
|
||||
delay = orDefault(cfg.Refused, 5*time.Minute)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(max(delay, time.Millisecond)):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func serveOnce(ctx context.Context, cfg ReadyConfig, c contract, keys workerkey.Store) error {
|
||||
callCtx, cancel := context.WithTimeout(ctx, orDefault(cfg.CallTimeout, 30*time.Second))
|
||||
defer cancel()
|
||||
if cfg.NewClaimer == nil {
|
||||
return errors.New("no gateway client is configured")
|
||||
}
|
||||
claimer, err := cfg.NewClaimer(c.region, c.gatewayURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("build gateway client: %w", err)
|
||||
}
|
||||
d, err := claimer.Claim(callCtx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.HostNodes != 1 {
|
||||
return fmt.Errorf("%w: the gateway reports %d host nodes", ErrUnsupportedTopology, d.HostNodes)
|
||||
}
|
||||
if d.Pending {
|
||||
return ErrReceiptPending
|
||||
}
|
||||
receipt, err := keys.ForRecorded(d.Fingerprint)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer clear(receipt)
|
||||
js, closeConn, err := cfg.Serve.Connector.Connect(callCtx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closeConn()
|
||||
cfg.Log("serving SQS owner calls from the local store")
|
||||
return cfg.Serve.Run(ctx, c.region, c.gatewayURL, js, receipt)
|
||||
}
|
||||
@@ -0,0 +1,434 @@
|
||||
package workerboot
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/sqs"
|
||||
sqstypes "github.com/aws/aws-sdk-go-v2/service/sqs/types"
|
||||
"github.com/mulgadc/bluebottle/pkg/iampolicy"
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/mulgadc/spinifex/internal/gwsign"
|
||||
"github.com/mulgadc/spinifex/internal/sqsworkergw"
|
||||
"github.com/mulgadc/spinifex/spinifex/agents/sqs/workerkey"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/awsapi"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/relay"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/workerident"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/workerserve"
|
||||
awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/envelope"
|
||||
)
|
||||
|
||||
const tenantAccount = "123456789012"
|
||||
|
||||
// publicSQS serves the public SQS registration over the relay, as the
|
||||
// ingress would after authenticating the tenant; tenant SigV4 and IAM are
|
||||
// not under test here.
|
||||
func publicSQS(t *testing.T, r *relay.Relay) *sqs.Client {
|
||||
t.Helper()
|
||||
var reg dispatch.Registration
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||||
err := reg.Dispatch(w, dispatch.Invocation{
|
||||
Request: req, AccountID: tenantAccount, Region: testRegion, Service: awsapi.ServiceName,
|
||||
Authorize: func(string, string, []string, iampolicy.ConditionKeys) error { return nil },
|
||||
Actor: func() (dispatch.Actor, error) {
|
||||
return dispatch.Actor{UserID: "AIDAPOLLER", AccountID: tenantAccount}, nil
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
code, msg, _ := awserrors.ResolveErrorDetail(err)
|
||||
if code == "" {
|
||||
code = err.Error()
|
||||
}
|
||||
meta := awserrors.LookupErrorMessage(awsapi.ServiceName, code)
|
||||
if msg == "" {
|
||||
msg = meta.Message
|
||||
}
|
||||
qc := code
|
||||
if q, ok := reg.QueryErrorCodes[code]; ok {
|
||||
qc = q
|
||||
}
|
||||
envelope.WriteJSON(w, envelope.ProtocolJSON10Query, envelope.JSONError{
|
||||
Code: code, Verbatim: true, Message: msg, Status: meta.HTTPCode, Namespace: reg.ErrorNamespace, QueryCode: qc,
|
||||
})
|
||||
}
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
reg = awsapi.NewRegistration(awsapi.Deps{
|
||||
Queues: r, Messages: r, Receives: r, Deletes: r, Visibility: r, Endpoint: srv.URL,
|
||||
})
|
||||
return sqs.New(sqs.Options{
|
||||
BaseEndpoint: aws.String(srv.URL), Region: testRegion, RetryMaxAttempts: 1,
|
||||
Credentials: aws.AnonymousCredentials{},
|
||||
})
|
||||
}
|
||||
|
||||
// localStore is the guest's own SQS NATS server: loopback only, no auth,
|
||||
// JetStream kept under the worker disk so a restart finds its data.
|
||||
func localStore(t *testing.T, g *guest, port int) *server.Server {
|
||||
t.Helper()
|
||||
if port == 0 {
|
||||
port = -1
|
||||
}
|
||||
ns, err := server.NewServer(&server.Options{
|
||||
Host: "127.0.0.1", Port: port, JetStream: true, StoreDir: g.cfg.DataDir, NoLog: true, NoSigs: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go ns.Start()
|
||||
if !ns.ReadyForConnections(10 * time.Second) {
|
||||
t.Fatal("store server did not start")
|
||||
}
|
||||
t.Cleanup(func() { ns.Shutdown(); ns.WaitForShutdown() })
|
||||
return ns
|
||||
}
|
||||
|
||||
func storePort(ns *server.Server) int { return ns.Addr().(*net.TCPAddr).Port }
|
||||
|
||||
func freePort(t *testing.T) int {
|
||||
t.Helper()
|
||||
l, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer l.Close()
|
||||
return l.Addr().(*net.TCPAddr).Port
|
||||
}
|
||||
|
||||
// servingWorker is the guest's ready step serving through the local
|
||||
// gateway's signed work actions against the local store on port.
|
||||
type servingWorker struct {
|
||||
cancel context.CancelFunc
|
||||
done chan struct{}
|
||||
once sync.Once
|
||||
mu sync.Mutex
|
||||
logs []string
|
||||
g *guest
|
||||
}
|
||||
|
||||
func (w *servingWorker) log(msg string) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.logs = append(w.logs, msg)
|
||||
}
|
||||
|
||||
func (w *servingWorker) logged(substr string) bool {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
return slices.ContainsFunc(w.logs, func(l string) bool { return strings.Contains(l, substr) })
|
||||
}
|
||||
|
||||
func (w *servingWorker) ready() bool {
|
||||
_, err := os.Stat(filepath.Join(w.g.cfg.RunDir, readyFile))
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func (c gwCluster) startServing(t *testing.T, g *guest, signer *gwsign.Signer, port int) *servingWorker {
|
||||
t.Helper()
|
||||
cfg := ReadyConfig{Config: g.cfg, Checker: NATSChecker{Port: port, Dial: time.Second}}
|
||||
cfg.Serve = &ServeConfig{
|
||||
Connector: NATSConnector{Port: port},
|
||||
Run: func(ctx context.Context, region, url string, js jetstream.JetStream, receiptKey []byte) error {
|
||||
gw, err := sqsworkergw.New(url, region, c.gw.caPEM(), signer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return workerserve.Run(ctx, gw, js, receiptKey, cfg.Log)
|
||||
},
|
||||
}
|
||||
cfg.Wait, cfg.Refresh, cfg.Refused, cfg.CallTimeout = 20*time.Millisecond, 50*time.Millisecond, 50*time.Millisecond, 5*time.Second
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
w := &servingWorker{cancel: cancel, done: make(chan struct{}), g: g}
|
||||
cfg.Log = w.log
|
||||
go func() {
|
||||
defer close(w.done)
|
||||
_ = Watch(ctx, cfg)
|
||||
}()
|
||||
t.Cleanup(w.stop)
|
||||
return w
|
||||
}
|
||||
|
||||
func (w *servingWorker) stop() {
|
||||
w.once.Do(w.cancel)
|
||||
<-w.done
|
||||
}
|
||||
|
||||
// eventually repeats call until it succeeds.
|
||||
func eventually(t *testing.T, call func() error) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(15 * time.Second)
|
||||
for {
|
||||
err := call()
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("never succeeded: %v", err)
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
// bootedWorker launches a worker on a new volume and boots its guest, which
|
||||
// creates and records the receipt key; it returns the guest and its signer.
|
||||
func (c gwCluster) bootedWorker(t *testing.T) (*guest, *gwsign.Signer, workerident.Intent) {
|
||||
t.Helper()
|
||||
in := c.launch(t, gwInstance)
|
||||
g := newGuest(t)
|
||||
signer := c.guestFor(t, g, "ASIAWORKER1", gwInstance)
|
||||
if _, err := g.boot(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return g, signer, in
|
||||
}
|
||||
|
||||
// The SDK path through one SQS VM: the worker serves create, send, receive
|
||||
// and delete from its local store; after the store and worker restart on the
|
||||
// same disk the queue is still there and an earlier handle still works.
|
||||
func TestServe_SDKPathThroughTheWorkerSurvivesRestart(t *testing.T) {
|
||||
r := &relay.Relay{}
|
||||
c := newGWCluster(t, r, nil)
|
||||
g, signer, _ := c.bootedWorker(t)
|
||||
ns := localStore(t, g, 0)
|
||||
port := storePort(ns)
|
||||
w := c.startServing(t, g, signer, port)
|
||||
cl := publicSQS(t, r)
|
||||
ctx := context.Background()
|
||||
|
||||
var url string
|
||||
eventually(t, func() error {
|
||||
out, err := cl.CreateQueue(ctx, &sqs.CreateQueueInput{QueueName: aws.String("orders")})
|
||||
if err == nil {
|
||||
url = aws.ToString(out.QueueUrl)
|
||||
}
|
||||
return err
|
||||
})
|
||||
body := strings.Repeat(`"quoted" \ body `, 4096)
|
||||
sent, err := cl.SendMessage(ctx, &sqs.SendMessageInput{
|
||||
QueueUrl: aws.String(url), MessageBody: aws.String(body),
|
||||
MessageAttributes: map[string]sqstypes.MessageAttributeValue{
|
||||
"kind": {DataType: aws.String("String"), StringValue: aws.String("new\norder")},
|
||||
"blob": {DataType: aws.String("Binary"), BinaryValue: []byte{0, 1, 2, 255}},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := cl.ReceiveMessage(ctx, &sqs.ReceiveMessageInput{
|
||||
QueueUrl: aws.String(url), MaxNumberOfMessages: 10, WaitTimeSeconds: 2,
|
||||
MessageAttributeNames: []string{"All"}, VisibilityTimeout: 60,
|
||||
})
|
||||
if err != nil || len(got.Messages) != 1 {
|
||||
t.Fatalf("receive = %v, %v", got, err)
|
||||
}
|
||||
m := got.Messages[0]
|
||||
if aws.ToString(m.MessageId) != aws.ToString(sent.MessageId) || aws.ToString(m.Body) != body ||
|
||||
aws.ToString(m.MD5OfMessageAttributes) != aws.ToString(sent.MD5OfMessageAttributes) ||
|
||||
aws.ToString(m.MessageAttributes["kind"].StringValue) != "new\norder" {
|
||||
t.Fatalf("received a different message: %+v", m)
|
||||
}
|
||||
if _, err := cl.DeleteMessage(ctx, &sqs.DeleteMessageInput{QueueUrl: aws.String(url), ReceiptHandle: m.ReceiptHandle}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := cl.SendMessage(ctx, &sqs.SendMessageInput{QueueUrl: aws.String(url), MessageBody: aws.String("second")}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
held, err := cl.ReceiveMessage(ctx, &sqs.ReceiveMessageInput{QueueUrl: aws.String(url), WaitTimeSeconds: 2, VisibilityTimeout: 60})
|
||||
if err != nil || len(held.Messages) != 1 {
|
||||
t.Fatalf("receive before restart = %v, %v", held, err)
|
||||
}
|
||||
|
||||
w.stop()
|
||||
ns.Shutdown()
|
||||
ns.WaitForShutdown()
|
||||
g.resetMounts()
|
||||
if _, err := g.boot(); err != nil {
|
||||
t.Fatalf("reboot: %v", err)
|
||||
}
|
||||
localStore(t, g, port)
|
||||
c.startServing(t, g, signer, port)
|
||||
eventually(t, func() error {
|
||||
_, err := cl.GetQueueUrl(ctx, &sqs.GetQueueUrlInput{QueueName: aws.String("orders")})
|
||||
return err
|
||||
})
|
||||
if _, err := cl.DeleteMessage(ctx, &sqs.DeleteMessageInput{QueueUrl: aws.String(url), ReceiptHandle: held.Messages[0].ReceiptHandle}); err != nil {
|
||||
t.Fatalf("handle from before the restart: %v", err)
|
||||
}
|
||||
empty, err := cl.ReceiveMessage(ctx, &sqs.ReceiveMessageInput{QueueUrl: aws.String(url)})
|
||||
if err != nil || len(empty.Messages) != 0 {
|
||||
t.Fatalf("queue after delete = %v, %v", empty, err)
|
||||
}
|
||||
|
||||
key, err := workerkey.Store{Dir: g.keyDir()}.ReceiptKey()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, b := range c.gw.sentBodies() {
|
||||
if strings.Contains(b, string(key)) {
|
||||
t.Fatal("a request carried key material")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The store comes up before the worker serves: while the local store is not
|
||||
// answering the guest is not ready and no call is served; once it starts,
|
||||
// readiness and serving follow.
|
||||
func TestServe_UnavailableStoreIsNeverReady(t *testing.T) {
|
||||
r := &relay.Relay{CallLimit: time.Second, PollerStale: 200 * time.Millisecond}
|
||||
c := newGWCluster(t, r, nil)
|
||||
g, signer, _ := c.bootedWorker(t)
|
||||
port := freePort(t)
|
||||
w := c.startServing(t, g, signer, port)
|
||||
cl := publicSQS(t, r)
|
||||
ctx := context.Background()
|
||||
eventually(t, func() error {
|
||||
if !w.logged("store not ready") {
|
||||
return errors.New("no failed check yet")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if w.ready() {
|
||||
t.Fatal("ready without a store")
|
||||
}
|
||||
if _, err := cl.ListQueues(ctx, &sqs.ListQueuesInput{}); err == nil {
|
||||
t.Fatal("a call was served without a store")
|
||||
}
|
||||
localStore(t, g, port)
|
||||
eventually(t, func() error {
|
||||
_, err := cl.ListQueues(ctx, &sqs.ListQueuesInput{})
|
||||
return err
|
||||
})
|
||||
eventually(t, func() error {
|
||||
if !w.ready() {
|
||||
return errors.New("not ready")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// A worker whose recorded receipt key is gone, or replaced, never serves and
|
||||
// never makes a new key, so handles issued earlier are not invalidated.
|
||||
func TestServe_ReceiptKeyLossOrChangeNeverServes(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
mutate func(t *testing.T, path string)
|
||||
want error
|
||||
}{
|
||||
{"missing", func(t *testing.T, path string) {
|
||||
if err := os.Remove(path); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}, workerkey.ErrReceiptKeyMissing},
|
||||
{"changed", func(t *testing.T, path string) {
|
||||
if err := os.WriteFile(path, []byte(strings.Repeat("k", workerkey.ReceiptKeyLen)), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}, workerkey.ErrKeyMismatch},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
r := &relay.Relay{CallLimit: time.Second, PollerStale: 200 * time.Millisecond}
|
||||
c := newGWCluster(t, r, nil)
|
||||
g, signer, _ := c.bootedWorker(t)
|
||||
port := storePort(localStore(t, g, 0))
|
||||
w := c.startServing(t, g, signer, port)
|
||||
cl := publicSQS(t, r)
|
||||
eventually(t, func() error {
|
||||
_, err := cl.ListQueues(context.Background(), &sqs.ListQueuesInput{})
|
||||
return err
|
||||
})
|
||||
w.stop()
|
||||
|
||||
tc.mutate(t, g.keyFile())
|
||||
again := c.startServing(t, g, signer, port)
|
||||
eventually(t, func() error {
|
||||
if again.logged(tc.want.Error()) {
|
||||
return nil
|
||||
}
|
||||
return errors.New("refusal not logged yet")
|
||||
})
|
||||
if _, err := cl.ListQueues(context.Background(), &sqs.ListQueuesInput{}); err == nil {
|
||||
t.Fatal("a call was served without the recorded receipt key")
|
||||
}
|
||||
g.resetMounts()
|
||||
if _, err := g.boot(); !errors.Is(err, tc.want) {
|
||||
t.Fatalf("reboot = %v, want %v", err, tc.want)
|
||||
}
|
||||
if tc.name == "missing" {
|
||||
if _, err := os.Lstat(g.keyFile()); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatal("a replacement receipt key was created")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A worker the domain has recorded as stopped cannot take work.
|
||||
func TestServe_StoppedWorkerTakesNoWork(t *testing.T) {
|
||||
r := &relay.Relay{CallLimit: time.Second, PollerStale: 200 * time.Millisecond}
|
||||
c := newGWCluster(t, r, nil)
|
||||
g, signer, in := c.bootedWorker(t)
|
||||
w := c.startServing(t, g, signer, storePort(localStore(t, g, 0)))
|
||||
cl := publicSQS(t, r)
|
||||
ctx := context.Background()
|
||||
eventually(t, func() error {
|
||||
_, err := cl.CreateQueue(ctx, &sqs.CreateQueueInput{QueueName: aws.String("orders")})
|
||||
return err
|
||||
})
|
||||
if err := c.ids.RecordStopped(ctx, in, gwInstance); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
eventually(t, func() error {
|
||||
if _, err := cl.ListQueues(ctx, &sqs.ListQueuesInput{}); err == nil {
|
||||
return errors.New("still served")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if !w.logged("AccessDenied") {
|
||||
t.Fatal("the worker's polls were not refused")
|
||||
}
|
||||
}
|
||||
|
||||
// A gateway reporting a multi-node host cluster is refused before the worker
|
||||
// opens its store, so no queue or marker state is created.
|
||||
func TestServe_MultiNodeHostCreatesNoStoreState(t *testing.T) {
|
||||
c := newGWCluster(t, &relay.Relay{}, func() int { return 3 })
|
||||
g, signer, _ := c.bootedWorker(t)
|
||||
port := storePort(localStore(t, g, 0))
|
||||
w := c.startServing(t, g, signer, port)
|
||||
eventually(t, func() error {
|
||||
if !w.logged(ErrUnsupportedTopology.Error()) {
|
||||
return errors.New("not yet refused")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
w.stop()
|
||||
|
||||
js, closeConn, err := NATSConnector{Port: port}.Connect(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer closeConn()
|
||||
names := js.StreamNames(context.Background())
|
||||
var got []string
|
||||
for n := range names.Name() {
|
||||
got = append(got, n)
|
||||
}
|
||||
if names.Err() != nil || len(got) != 0 {
|
||||
t.Fatalf("store streams = %v, %v; want none", got, names.Err())
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
// Package workerboot prepares the shared SQS worker guest: it selects exactly
|
||||
// the expected data disk by serial, checks and mounts it, proves that disk is
|
||||
// what is mounted, and only then loads or creates the worker's store key on it
|
||||
// under the current first-store decision the gateway returns.
|
||||
// Package workerboot prepares the SQS guest: it selects exactly the expected
|
||||
// data disk by serial, checks and mounts it, proves that disk is what is
|
||||
// mounted, and only then settles the receipt key under the gateway's decision,
|
||||
// before the guest's own SQS store first starts on that disk.
|
||||
package workerboot
|
||||
|
||||
import (
|
||||
@@ -18,11 +18,10 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/agents/sqs/workerkey"
|
||||
"github.com/nats-io/nkeys"
|
||||
)
|
||||
|
||||
// Guest contract, read from fw_cfg under opt/spinifex/sqs-worker/. None of it
|
||||
// is a decision: whether a store key may be created is asked of the gateway.
|
||||
// is a decision: whether a receipt key may be created is asked of the gateway.
|
||||
//
|
||||
// data-serial virtio-blk serial of the worker's own ext4 data disk
|
||||
// region the Region the worker signs its gateway calls for
|
||||
@@ -30,11 +29,15 @@ import (
|
||||
const (
|
||||
keyDirName = "sqs-worker-key"
|
||||
maxFwCfg = 128
|
||||
// StoreDirName is the store's JetStream directory on the data disk. Its
|
||||
// presence is SQS data, so a pending decision never creates a key there.
|
||||
StoreDirName = "jetstream"
|
||||
receiptReady = "receipt-ready"
|
||||
)
|
||||
|
||||
var regionPattern = regexp.MustCompile(`^[a-z]{2}(-[a-z]+)+-[0-9]+$`)
|
||||
|
||||
// NewClaimer builds the gateway client the first-store decision comes from.
|
||||
// NewClaimer builds the gateway client the receipt key decision comes from.
|
||||
type NewClaimer func(region, gatewayURL string) (workerkey.Claimer, error)
|
||||
|
||||
// Disk checks and mounts the worker data disk. The guest never formats.
|
||||
@@ -67,9 +70,9 @@ type contract struct {
|
||||
serial, region, gatewayURL string
|
||||
}
|
||||
|
||||
// Boot runs the boot step and returns the worker's public key. The key is
|
||||
// loaded or created only under the gateway's current decision; any missing,
|
||||
// malformed or ambiguous input or answer refuses.
|
||||
// Boot runs the boot step and returns the receipt key's fingerprint. The key
|
||||
// is loaded, or created on a new volume, only under the gateway's current
|
||||
// decision; any missing, malformed or ambiguous input or answer refuses.
|
||||
func Boot(ctx context.Context, cfg Config) (string, error) {
|
||||
c, err := readContract(cfg.FwCfgDir)
|
||||
if err != nil {
|
||||
@@ -86,20 +89,26 @@ func Boot(ctx context.Context, cfg Config) (string, error) {
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("build gateway client: %w", err)
|
||||
}
|
||||
kp, err := acquire(ctx, cfg, workerkey.Store{Dir: filepath.Join(cfg.DataDir, keyDirName)}, claimer)
|
||||
key, err := acquire(ctx, cfg, workerkey.Store{Dir: filepath.Join(cfg.DataDir, keyDirName)}, claimer)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer kp.Wipe()
|
||||
pub, err := kp.PublicKey()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("derive worker public key: %w", err)
|
||||
}
|
||||
if err := writePublicKey(cfg.RunDir, pub); err != nil {
|
||||
fp := workerkey.Fingerprint(key)
|
||||
clear(key)
|
||||
if err := writeMarker(cfg.RunDir, receiptReady, fp); err != nil {
|
||||
return "", err
|
||||
}
|
||||
cfg.Log("store key ready: public=" + pub)
|
||||
return pub, nil
|
||||
cfg.Log("receipt key ready: fingerprint=" + fp)
|
||||
return fp, nil
|
||||
}
|
||||
|
||||
// hasStoreData reports whether the disk already holds the store's directory.
|
||||
func hasStoreData(dataDir string) (bool, error) {
|
||||
_, err := os.Lstat(filepath.Join(dataDir, StoreDirName))
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return false, nil
|
||||
}
|
||||
return err == nil, err
|
||||
}
|
||||
|
||||
// mountWorkerDisk selects the disk with serial, checks and mounts it at
|
||||
@@ -176,20 +185,21 @@ func readContract(dir string) (contract, error) {
|
||||
// acquire runs the decision and key step, repeating it only on an error that
|
||||
// says it is safe to repeat, including a denial a not-yet-bound worker gets.
|
||||
// A denial creates nothing, and each repeat re-asks the gateway.
|
||||
func acquire(ctx context.Context, cfg Config, store workerkey.Store, c workerkey.Claimer) (kp nkeys.KeyPair, err error) {
|
||||
func acquire(ctx context.Context, cfg Config, store workerkey.Store, c workerkey.Claimer) (key []byte, err error) {
|
||||
attempts := max(cfg.Attempts, 1)
|
||||
hasData := func() (bool, error) { return hasStoreData(cfg.DataDir) }
|
||||
for i := range attempts {
|
||||
kp, err = store.Acquire(ctx, c)
|
||||
key, err = store.Acquire(ctx, c, hasData)
|
||||
if err == nil || i == attempts-1 {
|
||||
return kp, err
|
||||
return key, err
|
||||
}
|
||||
switch {
|
||||
case pending(err):
|
||||
cfg.Log(fmt.Sprintf("worker not yet authorized, asking again: %v", err))
|
||||
case retryable(err):
|
||||
cfg.Log(fmt.Sprintf("first-store decision not settled, retrying: %v", err))
|
||||
cfg.Log(fmt.Sprintf("receipt key decision not settled, retrying: %v", err))
|
||||
default:
|
||||
return kp, err
|
||||
return key, err
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
@@ -200,6 +210,13 @@ func acquire(ctx context.Context, cfg Config, store workerkey.Store, c workerkey
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Transient reports whether a failed Boot may be run again: the decision was
|
||||
// still pending or retryable when its bound ran out. Every other refusal,
|
||||
// including a missing or different key, is final.
|
||||
func Transient(err error) bool {
|
||||
return pending(err) || retryable(err) || errors.Is(err, context.DeadlineExceeded)
|
||||
}
|
||||
|
||||
func pending(err error) bool {
|
||||
p, ok := errors.AsType[interface {
|
||||
error
|
||||
@@ -333,29 +350,25 @@ func readTrimmed(path string) (string, error) {
|
||||
return strings.TrimSpace(string(b)), nil
|
||||
}
|
||||
|
||||
// writePublicKey publishes the public key for later reporting; it is not secret.
|
||||
func writePublicKey(runDir, pub string) error {
|
||||
// writeMarker publishes a non-secret value under runDir atomically.
|
||||
func writeMarker(runDir, name, value string) error {
|
||||
if err := os.MkdirAll(runDir, 0o750); err != nil {
|
||||
return fmt.Errorf("create %s: %w", runDir, err)
|
||||
}
|
||||
tmp, err := os.CreateTemp(runDir, ".public-key-*")
|
||||
tmp, err := os.CreateTemp(runDir, "."+name+"-*")
|
||||
if err != nil {
|
||||
return fmt.Errorf("write public key: %w", err)
|
||||
return fmt.Errorf("write %s: %w", name, err)
|
||||
}
|
||||
defer os.Remove(tmp.Name())
|
||||
if _, err := tmp.WriteString(pub + "\n"); err != nil {
|
||||
tmp.Close()
|
||||
return fmt.Errorf("write public key: %w", err)
|
||||
_, werr := tmp.WriteString(value + "\n")
|
||||
if cerr := tmp.Close(); werr == nil {
|
||||
werr = cerr
|
||||
}
|
||||
if err := tmp.Chmod(0o644); err != nil {
|
||||
tmp.Close()
|
||||
return fmt.Errorf("write public key: %w", err)
|
||||
if werr != nil {
|
||||
return fmt.Errorf("write %s: %w", name, werr)
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
return fmt.Errorf("write public key: %w", err)
|
||||
}
|
||||
if err := os.Rename(tmp.Name(), filepath.Join(runDir, "public-key")); err != nil {
|
||||
return fmt.Errorf("write public key: %w", err)
|
||||
if err := os.Rename(tmp.Name(), filepath.Join(runDir, name)); err != nil {
|
||||
return fmt.Errorf("write %s: %w", name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -2,6 +2,9 @@ package workerboot
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
@@ -11,7 +14,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/agents/sqs/workerkey"
|
||||
"github.com/nats-io/nkeys"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -32,12 +34,12 @@ type guest struct {
|
||||
logs []string
|
||||
}
|
||||
|
||||
// authority stands for the domain's first-store record behind the gateway:
|
||||
// it authorizes creation until a key is recorded, then only that key.
|
||||
// authority stands for the worker record behind the gateway: pending until a
|
||||
// receipt key fingerprint is recorded, then exactly that fingerprint.
|
||||
type authority struct {
|
||||
recorded string
|
||||
claimErrs []error
|
||||
// recordErrs fail RecordPublicKey; landed ones record before failing,
|
||||
// recordErrs fail RecordFingerprint; landed ones record before failing,
|
||||
// as a reply lost after the write.
|
||||
recordErrs []recordErr
|
||||
claims int
|
||||
@@ -63,30 +65,30 @@ type notBoundError struct{}
|
||||
func (notBoundError) Error() string { return "AccessDenied" }
|
||||
func (notBoundError) Pending() bool { return true }
|
||||
|
||||
func (a *authority) Claim(context.Context) (bool, string, error) {
|
||||
func (a *authority) Claim(context.Context) (workerkey.Decision, error) {
|
||||
a.claims++
|
||||
if len(a.claimErrs) > 0 {
|
||||
err := a.claimErrs[0]
|
||||
a.claimErrs = a.claimErrs[1:]
|
||||
return false, "", err
|
||||
return workerkey.Decision{}, err
|
||||
}
|
||||
return a.recorded == "", a.recorded, nil
|
||||
return workerkey.Decision{Pending: a.recorded == "", Fingerprint: a.recorded, HostNodes: 1}, nil
|
||||
}
|
||||
|
||||
func (a *authority) RecordPublicKey(_ context.Context, pub string) error {
|
||||
a.sent = append(a.sent, pub)
|
||||
if a.recorded != "" && a.recorded != pub {
|
||||
func (a *authority) RecordFingerprint(_ context.Context, fp string) error {
|
||||
a.sent = append(a.sent, fp)
|
||||
if a.recorded != "" && a.recorded != fp {
|
||||
return errors.New("InvalidParameterValue")
|
||||
}
|
||||
if len(a.recordErrs) > 0 {
|
||||
e := a.recordErrs[0]
|
||||
a.recordErrs = a.recordErrs[1:]
|
||||
if e.landed {
|
||||
a.recorded = pub
|
||||
a.recorded = fp
|
||||
}
|
||||
return e.err
|
||||
}
|
||||
a.recorded = pub
|
||||
a.recorded = fp
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -188,34 +190,26 @@ func (g *guest) boot() (string, error) {
|
||||
return Boot(context.Background(), g.cfg)
|
||||
}
|
||||
|
||||
// assertSentOnlyPublicKeys checks nothing but valid public keys left the guest.
|
||||
func (g *guest) assertSentOnlyPublicKeys(seed string) {
|
||||
// recordKey stands for a receipt key created on this volume earlier and
|
||||
// recorded by the platform; it returns that key.
|
||||
func (g *guest) recordKey() []byte {
|
||||
g.t.Helper()
|
||||
for _, v := range g.auth.sent {
|
||||
if !nkeys.IsValidPublicUserKey(v) || (seed != "" && strings.Contains(v, seed)) {
|
||||
g.t.Fatal("the guest sent something other than a public key")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// seedKey stands for a worker key created when the store was first made and
|
||||
// recorded by the domain; it returns that key's public half.
|
||||
func (g *guest) seedKey() string {
|
||||
g.t.Helper()
|
||||
kp, err := workerkey.Store{Dir: g.keyDir()}.ForNewStore()
|
||||
if err != nil {
|
||||
g.t.Fatalf("seed worker key: %v", err)
|
||||
}
|
||||
defer kp.Wipe()
|
||||
pub, err := kp.PublicKey()
|
||||
if err != nil {
|
||||
key := make([]byte, workerkey.ReceiptKeyLen)
|
||||
if _, err := rand.Read(key); err != nil {
|
||||
g.t.Fatal(err)
|
||||
}
|
||||
g.auth.recorded = pub
|
||||
return pub
|
||||
mustMkdir(g.t, g.keyDir())
|
||||
if err := os.Chmod(g.keyDir(), 0o700); err != nil {
|
||||
g.t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(g.keyFile(), key, 0o600); err != nil {
|
||||
g.t.Fatal(err)
|
||||
}
|
||||
g.auth.recorded = workerkey.Fingerprint(key)
|
||||
return key
|
||||
}
|
||||
|
||||
func (g *guest) keyFile() string { return filepath.Join(g.keyDir(), "store-client.nk") }
|
||||
func (g *guest) keyFile() string { return filepath.Join(g.keyDir(), "receipt.key") }
|
||||
|
||||
// resetMounts is a reboot: only the initramfs root is mounted.
|
||||
func (g *guest) resetMounts() {
|
||||
@@ -235,12 +229,12 @@ func (g *guest) appendMount(majMin, root, dir, fstype, source string) {
|
||||
|
||||
func (g *guest) keyDir() string { return filepath.Join(g.cfg.DataDir, keyDirName) }
|
||||
|
||||
func (g *guest) seed() string {
|
||||
b, err := os.ReadFile(filepath.Join(g.keyDir(), "store-client.nk"))
|
||||
func (g *guest) key() []byte {
|
||||
b, err := os.ReadFile(g.keyFile())
|
||||
if err != nil {
|
||||
g.t.Fatalf("read key: %v", err)
|
||||
}
|
||||
return string(b)
|
||||
return b
|
||||
}
|
||||
|
||||
func (g *guest) assertNoKey() {
|
||||
@@ -248,145 +242,130 @@ func (g *guest) assertNoKey() {
|
||||
if _, err := os.Lstat(g.keyDir()); !errors.Is(err, os.ErrNotExist) {
|
||||
g.t.Fatalf("a key directory exists although boot refused: %v", err)
|
||||
}
|
||||
if _, err := os.Lstat(filepath.Join(g.cfg.RunDir, "public-key")); !errors.Is(err, os.ErrNotExist) {
|
||||
g.t.Fatal("a public key was published although boot refused")
|
||||
if _, err := os.Lstat(filepath.Join(g.cfg.RunDir, receiptReady)); !errors.Is(err, os.ErrNotExist) {
|
||||
g.t.Fatal("receipt readiness was published although boot refused")
|
||||
}
|
||||
}
|
||||
|
||||
func (g *guest) assertNoSeedLeak(err error, seed string) {
|
||||
// assertNoKeyLeak checks no error or log line carries the key's bytes.
|
||||
func (g *guest) assertNoKeyLeak(err error) {
|
||||
g.t.Helper()
|
||||
if seed == "" {
|
||||
b, rerr := os.ReadFile(g.keyFile())
|
||||
if rerr != nil || len(b) == 0 {
|
||||
return
|
||||
}
|
||||
if err != nil && strings.Contains(err.Error(), seed) {
|
||||
g.t.Fatal("an error exposes the seed")
|
||||
}
|
||||
for _, l := range g.logs {
|
||||
if strings.Contains(l, seed) {
|
||||
g.t.Fatal("a log line exposes the seed")
|
||||
for _, enc := range []string{string(b), hex.EncodeToString(b), base64.StdEncoding.EncodeToString(b)} {
|
||||
if err != nil && strings.Contains(err.Error(), enc) {
|
||||
g.t.Fatal("an error exposes the receipt key")
|
||||
}
|
||||
for _, l := range g.logs {
|
||||
if strings.Contains(l, enc) {
|
||||
g.t.Fatal("a log line exposes the receipt key")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// otherPub is a valid store key the worker does not hold.
|
||||
func otherPub(t *testing.T) string {
|
||||
t.Helper()
|
||||
pub, _ := userKey(t)
|
||||
return pub
|
||||
}
|
||||
|
||||
func userKey(t *testing.T) (pub, seed string) {
|
||||
t.Helper()
|
||||
kp, err := nkeys.CreateUser()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p, err := kp.PublicKey()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s, err := kp.Seed()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p, string(s)
|
||||
}
|
||||
|
||||
func TestBoot_FirstStoreCreatesKeyThenReusesItAcrossReboots(t *testing.T) {
|
||||
func TestBoot_NewVolumeCreatesKeyThenReusesItAcrossReboots(t *testing.T) {
|
||||
g := newGuest(t)
|
||||
g.contract()
|
||||
pub, err := g.boot()
|
||||
fp, err := g.boot()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seed := g.seed()
|
||||
if g.auth.recorded != pub || len(g.auth.sent) != 1 {
|
||||
t.Fatalf("recorded %q after %d sends, want %s once", g.auth.recorded, len(g.auth.sent), pub)
|
||||
key := g.key()
|
||||
if fp != workerkey.Fingerprint(key) || g.auth.recorded != fp || len(g.auth.sent) != 1 {
|
||||
t.Fatalf("recorded %q after %d sends, want %s once", g.auth.recorded, len(g.auth.sent), fp)
|
||||
}
|
||||
// The store has since written data; a recorded key still reloads.
|
||||
mustMkdir(t, filepath.Join(g.cfg.DataDir, StoreDirName))
|
||||
for boot := 2; boot <= 3; boot++ {
|
||||
g.resetMounts()
|
||||
got, err := g.boot()
|
||||
if err != nil || got != pub || g.seed() != seed {
|
||||
t.Fatalf("boot %d = %q, %v; the worker key must not change", boot, got, err)
|
||||
if err != nil || got != fp || string(g.key()) != string(key) {
|
||||
t.Fatalf("boot %d = %q, %v; the receipt key must not change", boot, got, err)
|
||||
}
|
||||
}
|
||||
if len(g.auth.sent) != 1 {
|
||||
t.Fatalf("a recorded key was sent again: %d sends", len(g.auth.sent))
|
||||
t.Fatalf("a recorded fingerprint was sent again: %d sends", len(g.auth.sent))
|
||||
}
|
||||
if g.disk.mounts != 3 {
|
||||
t.Fatalf("mounts = %d, want one per boot", g.disk.mounts)
|
||||
}
|
||||
published, err := os.ReadFile(filepath.Join(g.cfg.RunDir, "public-key"))
|
||||
if err != nil || strings.TrimSpace(string(published)) != pub {
|
||||
t.Fatalf("published key = %q, %v", published, err)
|
||||
published, err := os.ReadFile(filepath.Join(g.cfg.RunDir, receiptReady))
|
||||
if err != nil || strings.TrimSpace(string(published)) != fp {
|
||||
t.Fatalf("published fingerprint = %q, %v", published, err)
|
||||
}
|
||||
g.assertSentOnlyPublicKeys(seed)
|
||||
g.assertNoSeedLeak(nil, seed)
|
||||
g.assertNoKeyLeak(nil)
|
||||
}
|
||||
|
||||
// Once the domain holds a key, losing the guest's copy never yields a new
|
||||
// one, whatever a replayed first-start fw_cfg says.
|
||||
// Once the platform holds a fingerprint, losing the disk's key never yields a
|
||||
// new one; the boot refuses without asking again.
|
||||
func TestBoot_KeyLostAfterRecordingIsNeverReplaced(t *testing.T) {
|
||||
g := newGuest(t)
|
||||
g.contract()
|
||||
g.seedKey()
|
||||
if err := os.Remove(g.keyFile()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
g.fw("store-status", "new\n")
|
||||
g.resetMounts()
|
||||
g.recordKey()
|
||||
mustRemove(t, g.keyFile())
|
||||
_, err := g.boot()
|
||||
if !errors.Is(err, workerkey.ErrKeyMissing) {
|
||||
t.Fatalf("err = %v, want ErrKeyMissing", err)
|
||||
if !errors.Is(err, workerkey.ErrReceiptKeyMissing) {
|
||||
t.Fatalf("err = %v, want ErrReceiptKeyMissing", err)
|
||||
}
|
||||
if _, err := os.Lstat(g.keyFile()); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatal("a replacement key was created for an existing store")
|
||||
t.Fatal("a replacement key was created for a recorded volume")
|
||||
}
|
||||
if len(g.auth.sent) != 0 || g.auth.claims != 1 {
|
||||
t.Fatalf("sends=%d claims=%d; a final refusal must not be retried or answered", len(g.auth.sent), g.auth.claims)
|
||||
}
|
||||
if _, err := os.Lstat(filepath.Join(g.cfg.RunDir, "public-key")); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatal("a public key was published")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBoot_RecordedKeyMismatchRefuses(t *testing.T) {
|
||||
g := newGuest(t)
|
||||
g.contract()
|
||||
g.seedKey()
|
||||
seed := g.seed()
|
||||
g.auth.recorded = otherPub(t)
|
||||
key := g.recordKey()
|
||||
g.auth.recorded = workerkey.Fingerprint(append([]byte("x"), key[1:]...))
|
||||
_, err := g.boot()
|
||||
if !errors.Is(err, workerkey.ErrKeyMismatch) {
|
||||
t.Fatalf("err = %v, want ErrKeyMismatch", err)
|
||||
}
|
||||
if g.seed() != seed || len(g.auth.sent) != 0 {
|
||||
t.Fatal("a mismatch changed or sent the worker key")
|
||||
if string(g.key()) != string(key) || len(g.auth.sent) != 0 {
|
||||
t.Fatal("a mismatch changed or sent the receipt key")
|
||||
}
|
||||
g.assertNoSeedLeak(err, seed)
|
||||
g.assertNoKeyLeak(err)
|
||||
}
|
||||
|
||||
// A reply lost after the key was recorded is retried, and the retry is
|
||||
// answered with that same key: one key, recorded once.
|
||||
// A pending decision on a disk that already holds the store's directory
|
||||
// creates no key: the platform and the disk disagree about a new volume.
|
||||
func TestBoot_PendingWithExistingStoreDataCreatesNoKey(t *testing.T) {
|
||||
g := newGuest(t)
|
||||
g.contract()
|
||||
g.appendMount(testMajMin, "/", g.cfg.DataDir, "ext4", "/dev/vda")
|
||||
mustMkdir(t, filepath.Join(g.cfg.DataDir, StoreDirName))
|
||||
if _, err := g.boot(); !errors.Is(err, workerkey.ErrDiskHasData) {
|
||||
t.Fatalf("err = %v, want ErrDiskHasData", err)
|
||||
}
|
||||
g.assertNoKey()
|
||||
}
|
||||
|
||||
// A reply lost after recording is retried with the same key: one key,
|
||||
// recorded once.
|
||||
func TestBoot_LostRecordReplyKeepsTheSameKey(t *testing.T) {
|
||||
for _, landed := range []bool{true, false} {
|
||||
t.Run(fmt.Sprintf("landed=%v", landed), func(t *testing.T) {
|
||||
g := newGuest(t)
|
||||
g.contract()
|
||||
g.auth.recordErrs = []recordErr{{err: transient{}, landed: landed}}
|
||||
pub, err := g.boot()
|
||||
fp, err := g.boot()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if g.auth.recorded != pub || g.auth.claims != 2 {
|
||||
if g.auth.recorded != fp || g.auth.claims != 2 {
|
||||
t.Fatalf("recorded=%q claims=%d", g.auth.recorded, g.auth.claims)
|
||||
}
|
||||
for _, sent := range g.auth.sent {
|
||||
if sent != pub {
|
||||
t.Fatal("a second key was sent")
|
||||
if sent != fp {
|
||||
t.Fatal("a second key was recorded")
|
||||
}
|
||||
}
|
||||
g.assertSentOnlyPublicKeys(g.seed())
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -512,7 +491,7 @@ func TestBoot_DiskRefusalsNeverAskTheGateway(t *testing.T) {
|
||||
func TestBoot_AlreadyMountedWorkerDiskIsReused(t *testing.T) {
|
||||
g := newGuest(t)
|
||||
g.contract()
|
||||
g.seedKey()
|
||||
g.recordKey()
|
||||
g.appendMount(testMajMin, "/", g.cfg.DataDir, "ext4", "/dev/vda")
|
||||
if _, err := g.boot(); err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -3,43 +3,73 @@ package workerkey
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/nats-io/nkeys"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// Claimer is the SQS domain's first-store decision for this worker identity.
|
||||
type Claimer interface {
|
||||
// Claim returns createKey, or the one public key the store authorizes.
|
||||
Claim(ctx context.Context) (createKey bool, publicKey string, err error)
|
||||
RecordPublicKey(ctx context.Context, publicKey string) error
|
||||
// Decision is the platform's receipt-key initialization state for this worker
|
||||
// volume: Pending authorizes creating or resuming the key, otherwise exactly
|
||||
// the recorded fingerprint is accepted.
|
||||
type Decision struct {
|
||||
Pending bool
|
||||
Fingerprint string
|
||||
// HostNodes is the gateway's declared cluster node count, zero if unknown.
|
||||
HostNodes int
|
||||
}
|
||||
|
||||
// Acquire returns the worker's store key under a current domain decision. A
|
||||
// new key is usable only once its public key is recorded; a recorded key is
|
||||
// never replaced, so a missing or different key on disk fails closed.
|
||||
func (s Store) Acquire(ctx context.Context, c Claimer) (nkeys.KeyPair, error) {
|
||||
create, authorized, err := c.Claim(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !create {
|
||||
return s.ForExistingStore(authorized)
|
||||
}
|
||||
if authorized != "" {
|
||||
return nil, errors.New("first-store decision is contradictory")
|
||||
}
|
||||
// After a crash before recording, this loads the key already on disk.
|
||||
kp, err := s.ForNewStore()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pub, err := kp.PublicKey()
|
||||
if err == nil {
|
||||
err = c.RecordPublicKey(ctx, pub)
|
||||
}
|
||||
if err != nil {
|
||||
kp.Wipe()
|
||||
return nil, err
|
||||
}
|
||||
return kp, nil
|
||||
// Claimer reads and completes the platform's receipt-key initialization.
|
||||
type Claimer interface {
|
||||
Claim(ctx context.Context) (Decision, error)
|
||||
RecordFingerprint(ctx context.Context, fingerprint string) error
|
||||
}
|
||||
|
||||
// ErrDiskHasData refuses a pending initialization on a disk that already holds
|
||||
// SQS data: the platform and the disk disagree about whether it is new.
|
||||
var ErrDiskHasData = errors.New("receipt key initialization is pending but the disk already holds SQS data")
|
||||
|
||||
// Acquire returns the receipt key under the current platform decision. A
|
||||
// pending decision creates or resumes the key only while hasData reports no
|
||||
// SQS data, then records its fingerprint; the caller must clear the key.
|
||||
func (s Store) Acquire(ctx context.Context, c Claimer, hasData func() (bool, error)) ([]byte, error) {
|
||||
d, err := c.Claim(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !d.Pending {
|
||||
return s.ForRecorded(d.Fingerprint)
|
||||
}
|
||||
if d.Fingerprint != "" {
|
||||
return nil, errors.New("receipt key decision is contradictory")
|
||||
}
|
||||
data, err := hasData()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("check the disk for SQS data: %w", err)
|
||||
}
|
||||
if data {
|
||||
return nil, ErrDiskHasData
|
||||
}
|
||||
key, err := s.ensureReceipt()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := c.RecordFingerprint(ctx, Fingerprint(key)); err != nil {
|
||||
clear(key)
|
||||
return nil, err
|
||||
}
|
||||
return key, nil
|
||||
}
|
||||
|
||||
// ForRecorded returns the disk's receipt key only if it matches fingerprint.
|
||||
func (s Store) ForRecorded(fingerprint string) ([]byte, error) {
|
||||
if len(fingerprint) != 64 {
|
||||
return nil, errors.New("recorded receipt key fingerprint is malformed")
|
||||
}
|
||||
key, err := s.ReceiptKey()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if Fingerprint(key) != fingerprint {
|
||||
clear(key)
|
||||
return nil, ErrKeyMismatch
|
||||
}
|
||||
return key, nil
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package workerkey
|
||||
package workerkey_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -7,260 +7,126 @@ import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
testutil "github.com/mulgadc/spinifex/internal/testkit"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/firststore"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/nats-io/nkeys"
|
||||
"github.com/mulgadc/spinifex/spinifex/agents/sqs/workerkey"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const region = "ap-southeast-2"
|
||||
|
||||
// domainClaimer drives the real first-store state machine for one worker
|
||||
// identity. A failing record either drops the write or applies it and loses
|
||||
// the reply.
|
||||
type domainClaimer struct {
|
||||
s firststore.Store
|
||||
id firststore.Identity
|
||||
failRecord bool
|
||||
applied bool
|
||||
// platform stands for the worker record behind the gateway: pending until a
|
||||
// fingerprint is recorded, then exactly that fingerprint.
|
||||
type platform struct {
|
||||
recorded string
|
||||
recordErr error
|
||||
landed bool
|
||||
records int
|
||||
}
|
||||
|
||||
func (c *domainClaimer) Claim(ctx context.Context) (bool, string, error) {
|
||||
d, err := c.s.Claim(ctx, region, c.id)
|
||||
return d.CreateKey, d.PublicKey, err
|
||||
func (p *platform) Claim(context.Context) (workerkey.Decision, error) {
|
||||
return workerkey.Decision{Pending: p.recorded == "", Fingerprint: p.recorded, HostNodes: 1}, nil
|
||||
}
|
||||
|
||||
func (c *domainClaimer) RecordPublicKey(ctx context.Context, pub string) error {
|
||||
if c.failRecord {
|
||||
c.failRecord = false
|
||||
if c.applied {
|
||||
c.s.RecordPublicKey(ctx, region, c.id, pub)
|
||||
func (p *platform) RecordFingerprint(_ context.Context, fp string) error {
|
||||
p.records++
|
||||
if p.recordErr != nil {
|
||||
if p.landed {
|
||||
p.recorded = fp
|
||||
}
|
||||
return errors.New("nats: timeout")
|
||||
return p.recordErr
|
||||
}
|
||||
return c.s.RecordPublicKey(ctx, region, c.id, pub)
|
||||
p.recorded = fp
|
||||
return nil
|
||||
}
|
||||
|
||||
func domain(t *testing.T) (firststore.Store, jetstream.KeyValue) {
|
||||
func newStore(t *testing.T) workerkey.Store {
|
||||
t.Helper()
|
||||
_, _, js := testutil.StartTestJetStream(t)
|
||||
kv, err := js.CreateKeyValue(context.Background(), firststore.BucketConfig(1))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return firststore.Store{JS: js, KV: kv}, kv
|
||||
return workerkey.Store{Dir: filepath.Join(t.TempDir(), "key")}
|
||||
}
|
||||
|
||||
func worker(gen uint64, n byte) firststore.Identity {
|
||||
return firststore.Identity{InstanceID: "i-0000000000000000" + string('0'+n), ClaimID: "claim-" + string('a'+n) + "-0001", Generation: gen}
|
||||
func noData() (bool, error) { return false, nil }
|
||||
|
||||
func TestAcquire_NewVolumeCreatesAndRecordsThenReloads(t *testing.T) {
|
||||
s, p := newStore(t), &platform{}
|
||||
key, err := s.Acquire(t.Context(), p, noData)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, key, workerkey.ReceiptKeyLen)
|
||||
assert.Equal(t, workerkey.Fingerprint(key), p.recorded)
|
||||
|
||||
again, err := s.Acquire(t.Context(), p, func() (bool, error) { return true, nil })
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, key, again, "a recorded key is reloaded, store data or not")
|
||||
assert.Equal(t, 1, p.records)
|
||||
}
|
||||
|
||||
func workerDisk(t *testing.T) Store {
|
||||
return Store{Dir: filepath.Join(t.TempDir(), "sqs-worker-key")}
|
||||
}
|
||||
// A reply lost after the write, or a crash before recording, resumes with the
|
||||
// key already on disk rather than a new one.
|
||||
func TestAcquire_LostRecordReplyResumesTheSameKey(t *testing.T) {
|
||||
for _, landed := range []bool{false, true} {
|
||||
s := newStore(t)
|
||||
p := &platform{recordErr: errors.New("connection reset"), landed: landed}
|
||||
_, err := s.Acquire(t.Context(), p, noData)
|
||||
require.Error(t, err)
|
||||
first, err := s.ReceiptKey()
|
||||
require.NoError(t, err)
|
||||
|
||||
func keyPath(s Store) string { return filepath.Join(s.Dir, keyFile) }
|
||||
|
||||
func pubOf(t *testing.T, kp nkeys.KeyPair) string {
|
||||
t.Helper()
|
||||
p, err := kp.PublicKey()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func recorded(t *testing.T, s firststore.Store, id firststore.Identity) string {
|
||||
t.Helper()
|
||||
d, err := s.Claim(context.Background(), region, id)
|
||||
if err != nil || d.CreateKey {
|
||||
t.Fatalf("domain decision = %+v, %v; want a recorded key", d, err)
|
||||
}
|
||||
return d.PublicKey
|
||||
}
|
||||
|
||||
func TestAcquire_CrashBeforeRecordingReusesTheKeyOnDisk(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
fs, _ := domain(t)
|
||||
disk := workerDisk(t)
|
||||
c := &domainClaimer{s: fs, id: worker(1, 1), failRecord: true}
|
||||
if _, err := disk.Acquire(ctx, c); err == nil {
|
||||
t.Fatal("a key was returned before its public key was recorded")
|
||||
}
|
||||
before, err := os.ReadFile(keyPath(disk))
|
||||
if err != nil {
|
||||
t.Fatalf("key not synced to disk before recording: %v", err)
|
||||
}
|
||||
kp, err := disk.Acquire(ctx, c)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer kp.Wipe()
|
||||
after, _ := os.ReadFile(keyPath(disk))
|
||||
if string(before) != string(after) {
|
||||
t.Fatal("the retry replaced the key")
|
||||
}
|
||||
if recorded(t, fs, c.id) != pubOf(t, kp) {
|
||||
t.Fatal("the recorded key is not the key on disk")
|
||||
p.recordErr = nil
|
||||
key, err := s.Acquire(t.Context(), p, noData)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, first, key)
|
||||
assert.Equal(t, workerkey.Fingerprint(first), p.recorded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcquire_LostRecordReplyIsRecoveredOnReboot(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
fs, _ := domain(t)
|
||||
disk := workerDisk(t)
|
||||
c := &domainClaimer{s: fs, id: worker(1, 1), failRecord: true, applied: true}
|
||||
if _, err := disk.Acquire(ctx, c); err == nil {
|
||||
t.Fatal("expected the lost reply to surface")
|
||||
}
|
||||
kp, err := disk.Acquire(ctx, c)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer kp.Wipe()
|
||||
if recorded(t, fs, c.id) != pubOf(t, kp) {
|
||||
t.Fatal("reboot did not use the recorded key")
|
||||
}
|
||||
// A pending decision on a disk that already holds store data creates nothing:
|
||||
// the platform and the disk disagree about whether the volume is new.
|
||||
func TestAcquire_PendingWithStoreDataCreatesNothing(t *testing.T) {
|
||||
s, p := newStore(t), &platform{}
|
||||
_, err := s.Acquire(t.Context(), p, func() (bool, error) { return true, nil })
|
||||
require.ErrorIs(t, err, workerkey.ErrDiskHasData)
|
||||
_, err = os.Stat(s.Dir)
|
||||
assert.ErrorIs(t, err, os.ErrNotExist)
|
||||
assert.Zero(t, p.records)
|
||||
}
|
||||
|
||||
func TestAcquire_AfterRecordingKeyLossOrMismatchFailsClosed(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
setup := func(t *testing.T, launch bool) (firststore.Store, Store, *domainClaimer, string) {
|
||||
fs, _ := domain(t)
|
||||
disk := workerDisk(t)
|
||||
c := &domainClaimer{s: fs, id: worker(1, 1)}
|
||||
kp, err := disk.Acquire(ctx, c)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pub := pubOf(t, kp)
|
||||
kp.Wipe()
|
||||
if launch {
|
||||
if err := fs.RecordLaunchIntent(ctx, region, c.id, pub); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return fs, disk, c, pub
|
||||
}
|
||||
for _, launch := range []bool{false, true} {
|
||||
name := map[bool]string{false: "key recorded", true: "launch intent"}[launch]
|
||||
t.Run(name+"/key lost then reboot", func(t *testing.T) {
|
||||
fs, disk, c, pub := setup(t, launch)
|
||||
os.Remove(keyPath(disk))
|
||||
// A relaunched worker generation gets the same answer.
|
||||
for _, id := range []firststore.Identity{c.id, worker(2, 2)} {
|
||||
c.id = id
|
||||
if _, err := disk.Acquire(ctx, c); !errors.Is(err, ErrKeyMissing) {
|
||||
t.Fatalf("acquire = %v, want ErrKeyMissing", err)
|
||||
}
|
||||
if _, err := os.Lstat(keyPath(disk)); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatal("a replacement key was created")
|
||||
}
|
||||
}
|
||||
if recorded(t, fs, c.id) != pub {
|
||||
t.Fatal("the recorded key changed")
|
||||
}
|
||||
})
|
||||
t.Run(name+"/different key on disk", func(t *testing.T) {
|
||||
_, _, c, _ := setup(t, launch)
|
||||
other := workerDisk(t)
|
||||
kp, err := other.ForNewStore()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kp.Wipe()
|
||||
before, _ := os.ReadFile(keyPath(other))
|
||||
if _, err := other.Acquire(ctx, c); !errors.Is(err, ErrKeyMismatch) {
|
||||
t.Fatalf("acquire = %v, want ErrKeyMismatch", err)
|
||||
}
|
||||
after, _ := os.ReadFile(keyPath(other))
|
||||
if string(before) != string(after) {
|
||||
t.Fatal("the mismatched key was replaced")
|
||||
}
|
||||
})
|
||||
}
|
||||
// Once recorded, a missing, changed or damaged key refuses and is never
|
||||
// regenerated: a new key would invalidate every handle already issued.
|
||||
func TestAcquire_RecordedKeyIsNeverRegenerated(t *testing.T) {
|
||||
s, p := newStore(t), &platform{}
|
||||
key, err := s.Acquire(t.Context(), p, noData)
|
||||
require.NoError(t, err)
|
||||
path := filepath.Join(s.Dir, "receipt.key")
|
||||
|
||||
require.NoError(t, os.Remove(path))
|
||||
_, err = s.Acquire(t.Context(), p, noData)
|
||||
require.ErrorIs(t, err, workerkey.ErrReceiptKeyMissing)
|
||||
_, err = os.Stat(path)
|
||||
assert.ErrorIs(t, err, os.ErrNotExist, "no replacement key is written")
|
||||
|
||||
other := append([]byte(nil), key...)
|
||||
other[0] ^= 0xff
|
||||
require.NoError(t, os.WriteFile(path, other, 0o600))
|
||||
_, err = s.Acquire(t.Context(), p, noData)
|
||||
require.ErrorIs(t, err, workerkey.ErrKeyMismatch)
|
||||
|
||||
require.NoError(t, os.WriteFile(path, key[:8], 0o600))
|
||||
_, err = s.Acquire(t.Context(), p, noData)
|
||||
require.Error(t, err)
|
||||
got, rerr := os.ReadFile(path)
|
||||
require.NoError(t, rerr)
|
||||
assert.Equal(t, key[:8], got, "a damaged key is refused, not replaced")
|
||||
assert.Equal(t, 1, p.records)
|
||||
}
|
||||
|
||||
func TestAcquire_RefusedClaimCreatesNoKey(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cases := map[string]func(t *testing.T, fs firststore.Store, kv jetstream.KeyValue) firststore.Identity{
|
||||
"rival holds the claim": func(t *testing.T, fs firststore.Store, _ jetstream.KeyValue) firststore.Identity {
|
||||
fs.Claim(ctx, region, worker(1, 1))
|
||||
return worker(1, 2)
|
||||
},
|
||||
"stale replayed generation": func(t *testing.T, fs firststore.Store, _ jetstream.KeyValue) firststore.Identity {
|
||||
fs.Claim(ctx, region, worker(1, 1))
|
||||
fs.Claim(ctx, region, worker(2, 2))
|
||||
return worker(1, 1)
|
||||
},
|
||||
"unreadable record": func(t *testing.T, _ firststore.Store, kv jetstream.KeyValue) firststore.Identity {
|
||||
kv.Put(ctx, "region."+region, []byte("{"))
|
||||
return worker(1, 1)
|
||||
},
|
||||
}
|
||||
for name, setup := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
fs, kv := domain(t)
|
||||
id := setup(t, fs, kv)
|
||||
disk := workerDisk(t)
|
||||
if _, err := disk.Acquire(ctx, &domainClaimer{s: fs, id: id}); err == nil {
|
||||
t.Fatal("a refused claim returned a key")
|
||||
}
|
||||
if _, err := os.Lstat(keyPath(disk)); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatal("a refused claim created a key")
|
||||
}
|
||||
})
|
||||
}
|
||||
func TestAcquire_ContradictoryDecisionRefuses(t *testing.T) {
|
||||
s := newStore(t)
|
||||
_, err := s.Acquire(t.Context(), contradictory{}, noData)
|
||||
require.Error(t, err)
|
||||
_, err = os.Stat(s.Dir)
|
||||
assert.ErrorIs(t, err, os.ErrNotExist)
|
||||
}
|
||||
|
||||
type contradictoryClaimer struct{ pub string }
|
||||
type contradictory struct{}
|
||||
|
||||
func (c contradictoryClaimer) Claim(context.Context) (bool, string, error) { return true, c.pub, nil }
|
||||
func (contradictoryClaimer) RecordPublicKey(context.Context, string) error {
|
||||
return errors.New("unexpected")
|
||||
}
|
||||
|
||||
func TestAcquire_ContradictoryDecisionCreatesNoKey(t *testing.T) {
|
||||
kp, _ := nkeys.CreateUser()
|
||||
disk := workerDisk(t)
|
||||
if _, err := disk.Acquire(context.Background(), contradictoryClaimer{pub: pubOf(t, kp)}); err == nil {
|
||||
t.Fatal("a contradictory decision returned a key")
|
||||
}
|
||||
if _, err := os.Lstat(keyPath(disk)); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatal("a contradictory decision created a key")
|
||||
}
|
||||
}
|
||||
|
||||
// A replica that has applied the claim but not the key record must not lead a
|
||||
// worker whose key is lost to create a replacement.
|
||||
func TestAcquire_StaleClaimedReadAfterRecordingCreatesNoKey(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
fs, kv := domain(t)
|
||||
disk := workerDisk(t)
|
||||
id := worker(1, 1)
|
||||
fs.Claim(ctx, region, id)
|
||||
claimed, err := kv.Get(ctx, "region."+region)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kp, err := disk.Acquire(ctx, &domainClaimer{s: fs, id: id})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kp.Wipe()
|
||||
os.Remove(keyPath(disk))
|
||||
|
||||
_, _, replicaJS := testutil.StartTestJetStream(t)
|
||||
replica, err := replicaJS.CreateKeyValue(ctx, firststore.BucketConfig(1))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
replica.Put(ctx, "region."+region, claimed.Value())
|
||||
stale := firststore.Store{JS: replicaJS, KV: fs.KV}
|
||||
if _, err := disk.Acquire(ctx, &domainClaimer{s: stale, id: id}); err == nil {
|
||||
t.Fatal("a stale claimed read returned a key")
|
||||
}
|
||||
if _, err := os.Lstat(keyPath(disk)); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatal("a stale claimed read created a replacement key")
|
||||
}
|
||||
func (contradictory) Claim(context.Context) (workerkey.Decision, error) {
|
||||
return workerkey.Decision{Pending: true, Fingerprint: "ab"}, nil
|
||||
}
|
||||
func (contradictory) RecordFingerprint(context.Context, string) error { return nil }
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
package workerkey
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
const (
|
||||
receiptFile = "receipt.key"
|
||||
// ReceiptKeyLen is the receipt-signing key's length in bytes.
|
||||
ReceiptKeyLen = 32
|
||||
)
|
||||
|
||||
// ErrReceiptKeyMissing is a worker disk without the receipt-signing key once
|
||||
// its fingerprint is recorded. It is never regenerated: a new key would
|
||||
// invalidate every handle already issued.
|
||||
var ErrReceiptKeyMissing = errors.New("worker receipt-signing key is missing")
|
||||
|
||||
func (s Store) receiptPath() string { return filepath.Join(s.Dir, receiptFile) }
|
||||
|
||||
// ReceiptKey returns the receipt-signing key already on disk; it never
|
||||
// creates one. The caller must not log or retain it beyond use.
|
||||
func (s Store) ReceiptKey() ([]byte, error) {
|
||||
unlock, err := s.prepareDir()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer unlock()
|
||||
key, err := s.loadReceipt()
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, ErrReceiptKeyMissing
|
||||
}
|
||||
return key, err
|
||||
}
|
||||
|
||||
// ensureReceipt loads the receipt-signing key, creating it only if absent.
|
||||
// Only a pending initialization calls it, so a crash before the fingerprint
|
||||
// is recorded resumes with the key already written.
|
||||
func (s Store) ensureReceipt() ([]byte, error) {
|
||||
unlock, err := s.prepareDir()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer unlock()
|
||||
key, err := s.loadReceipt()
|
||||
if err == nil {
|
||||
if err := s.syncNames(); err != nil {
|
||||
clear(key)
|
||||
return nil, err
|
||||
}
|
||||
return key, nil
|
||||
}
|
||||
if !errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, err
|
||||
}
|
||||
key = make([]byte, ReceiptKeyLen)
|
||||
if _, err := rand.Read(key); err != nil {
|
||||
clear(key)
|
||||
return nil, fmt.Errorf("generate worker receipt key: %w", err)
|
||||
}
|
||||
if err := s.install(key); err != nil {
|
||||
clear(key)
|
||||
return nil, err
|
||||
}
|
||||
return key, nil
|
||||
}
|
||||
|
||||
// install links a synced temporary file into place, so the key appears whole
|
||||
// or not at all, then makes both directory entries durable.
|
||||
func (s Store) install(key []byte) error {
|
||||
tmp, err := os.CreateTemp(s.Dir, tempPrefix+"*"+tempSuffix)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create worker receipt key file: %w", err)
|
||||
}
|
||||
tmpName := tmp.Name()
|
||||
defer os.Remove(tmpName)
|
||||
if err := writeSynced(tmp, key); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Link(tmpName, s.receiptPath()); err != nil {
|
||||
return fmt.Errorf("install worker receipt key file: %w", err)
|
||||
}
|
||||
return s.syncNames()
|
||||
}
|
||||
|
||||
// Fingerprint is the SHA-256 of a receipt key, hex encoded. It identifies the
|
||||
// key without revealing it.
|
||||
func Fingerprint(key []byte) string {
|
||||
sum := sha256.Sum256(key)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// loadReceipt refuses a file that is present but unusable rather than
|
||||
// replacing it.
|
||||
func (s Store) loadReceipt() ([]byte, error) {
|
||||
fi, err := os.Lstat(s.receiptPath())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !fi.Mode().IsRegular() || fi.Mode().Perm()&0o077 != 0 {
|
||||
return nil, ErrUnsafe
|
||||
}
|
||||
if fi.Size() != ReceiptKeyLen {
|
||||
return nil, errors.New("worker receipt key file has the wrong length")
|
||||
}
|
||||
key, err := os.ReadFile(s.receiptPath())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read worker receipt key: %w", err)
|
||||
}
|
||||
if len(key) != ReceiptKeyLen {
|
||||
clear(key)
|
||||
return nil, errors.New("worker receipt key file has the wrong length")
|
||||
}
|
||||
return key, nil
|
||||
}
|
||||
@@ -1,34 +1,26 @@
|
||||
// Package workerkey keeps the shared SQS worker's store-client NKey on the
|
||||
// worker's own persistent disk. The seed is generated and held only inside the
|
||||
// worker guest; the platform learns the public key alone.
|
||||
// Package workerkey keeps the SQS guest's receipt-signing key on its own
|
||||
// persistent disk. The key is generated and held only inside the guest; the
|
||||
// platform learns its fingerprint alone.
|
||||
package workerkey
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
|
||||
"github.com/nats-io/nkeys"
|
||||
)
|
||||
|
||||
const (
|
||||
keyFile = "store-client.nk"
|
||||
tempPrefix = ".store-client-"
|
||||
tempPrefix = ".receipt-"
|
||||
tempSuffix = ".tmp"
|
||||
// A user seed is 58 bytes; anything much larger is not one.
|
||||
maxKeyFileSize = 256
|
||||
)
|
||||
|
||||
// Each refusal is fail-closed: the worker never replaces a key it cannot use.
|
||||
// Each refusal is fail-closed: the guest never replaces a key it cannot use.
|
||||
var (
|
||||
ErrKeyMissing = errors.New("worker store key is missing but the store already exists")
|
||||
ErrKeyMismatch = errors.New("worker store key does not match the store's authorized key")
|
||||
ErrUnsafe = errors.New("worker store key path is not owner-only")
|
||||
ErrCorrupt = errors.New("worker store key file does not hold a user NKey seed")
|
||||
ErrKeyMismatch = errors.New("worker receipt key does not match the recorded fingerprint")
|
||||
ErrUnsafe = errors.New("worker receipt key path is not owner-only")
|
||||
)
|
||||
|
||||
// Store is the key directory on the worker disk. Workerkey cannot tell a
|
||||
@@ -38,66 +30,12 @@ type Store struct {
|
||||
Dir string
|
||||
}
|
||||
|
||||
// ForNewStore returns the worker's store-client key for a store that the
|
||||
// caller has positively established does not exist yet, generating it on first
|
||||
// use. An unreadable or absent store record is not that proof.
|
||||
func (s Store) ForNewStore() (nkeys.KeyPair, error) {
|
||||
return s.ensure("")
|
||||
}
|
||||
|
||||
// ForExistingStore returns the worker's key only if it is already on disk and
|
||||
// is the one the store authorizes; it never generates a key.
|
||||
func (s Store) ForExistingStore(authorized string) (nkeys.KeyPair, error) {
|
||||
if !nkeys.IsValidPublicUserKey(authorized) {
|
||||
return nil, errors.New("authorized store key is not a user public NKey")
|
||||
}
|
||||
return s.ensure(authorized)
|
||||
}
|
||||
|
||||
func (s Store) ensure(expected string) (nkeys.KeyPair, error) {
|
||||
unlock, err := s.prepareDir()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer unlock()
|
||||
kp, err := s.load()
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
if expected != "" {
|
||||
return nil, ErrKeyMissing
|
||||
}
|
||||
kp, err = s.create()
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if expected != "" {
|
||||
pub, err := kp.PublicKey()
|
||||
if err != nil {
|
||||
kp.Wipe()
|
||||
return nil, ErrCorrupt
|
||||
}
|
||||
if pub != expected {
|
||||
kp.Wipe()
|
||||
return nil, fmt.Errorf("%w: have %s, store authorizes %s", ErrKeyMismatch, pub, expected)
|
||||
}
|
||||
}
|
||||
// Re-synced on every return, so a key whose earlier sync failed is never
|
||||
// handed out before its name and directory are durable.
|
||||
if err := s.syncNames(); err != nil {
|
||||
kp.Wipe()
|
||||
return nil, err
|
||||
}
|
||||
return kp, nil
|
||||
}
|
||||
|
||||
func (s Store) path() string { return filepath.Join(s.Dir, keyFile) }
|
||||
|
||||
// prepareDir creates only the key directory, inside a parent that must already
|
||||
// exist, refuses one that is not owner-only, locks it and removes temporary
|
||||
// files a crash left behind.
|
||||
func (s Store) prepareDir() (unlock func(), err error) {
|
||||
if s.Dir == "" {
|
||||
return nil, errors.New("worker store key directory is not set")
|
||||
return nil, errors.New("worker receipt key directory is not set")
|
||||
}
|
||||
parent, err := os.Stat(filepath.Dir(s.Dir))
|
||||
if err != nil || !parent.IsDir() {
|
||||
@@ -136,69 +74,6 @@ func (s Store) prepareDir() (unlock func(), err error) {
|
||||
return unlock, nil
|
||||
}
|
||||
|
||||
// load reads an existing key. Any file that is present but unusable refuses
|
||||
// rather than being replaced.
|
||||
func (s Store) load() (nkeys.KeyPair, error) {
|
||||
fi, err := os.Lstat(s.path())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !fi.Mode().IsRegular() || fi.Mode().Perm()&0o077 != 0 {
|
||||
return nil, ErrUnsafe
|
||||
}
|
||||
if fi.Size() > maxKeyFileSize {
|
||||
return nil, ErrCorrupt
|
||||
}
|
||||
seed, err := os.ReadFile(s.path())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read worker store key: %w", err)
|
||||
}
|
||||
kp, err := nkeys.FromSeed(bytes.TrimSpace(seed))
|
||||
if err != nil {
|
||||
return nil, ErrCorrupt
|
||||
}
|
||||
pub, err := kp.PublicKey()
|
||||
if err != nil || !nkeys.IsValidPublicUserKey(pub) {
|
||||
kp.Wipe()
|
||||
return nil, ErrCorrupt
|
||||
}
|
||||
return kp, nil
|
||||
}
|
||||
|
||||
// create writes a new seed to a synced temporary file and links it into place
|
||||
// without replacing anything; the caller syncs the directory names.
|
||||
func (s Store) create() (nkeys.KeyPair, error) {
|
||||
kp, err := nkeys.CreateUser()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("generate worker store key: %w", err)
|
||||
}
|
||||
seed, err := kp.Seed()
|
||||
if err != nil {
|
||||
kp.Wipe()
|
||||
return nil, fmt.Errorf("encode worker store key: %w", err)
|
||||
}
|
||||
|
||||
tmp, err := os.CreateTemp(s.Dir, tempPrefix+"*"+tempSuffix)
|
||||
if err != nil {
|
||||
kp.Wipe()
|
||||
return nil, fmt.Errorf("create worker store key file: %w", err)
|
||||
}
|
||||
tmpName := tmp.Name()
|
||||
defer os.Remove(tmpName)
|
||||
if err := writeSynced(tmp, seed); err != nil {
|
||||
kp.Wipe()
|
||||
return nil, err
|
||||
}
|
||||
if err := os.Link(tmpName, s.path()); err != nil {
|
||||
kp.Wipe()
|
||||
if errors.Is(err, fs.ErrExist) {
|
||||
return s.load()
|
||||
}
|
||||
return nil, fmt.Errorf("install worker store key file: %w", err)
|
||||
}
|
||||
return kp, nil
|
||||
}
|
||||
|
||||
// syncNames makes the key's directory entry and the key directory's own entry
|
||||
// in its parent durable.
|
||||
func (s Store) syncNames() error {
|
||||
@@ -211,18 +86,18 @@ func (s Store) syncNames() error {
|
||||
func writeSynced(f *os.File, data []byte) error {
|
||||
if err := f.Chmod(0o600); err != nil {
|
||||
f.Close()
|
||||
return fmt.Errorf("restrict worker store key file: %w", err)
|
||||
return fmt.Errorf("restrict worker receipt key file: %w", err)
|
||||
}
|
||||
if _, err := f.Write(data); err != nil {
|
||||
f.Close()
|
||||
return fmt.Errorf("write worker store key file: %w", err)
|
||||
return fmt.Errorf("write worker receipt key file: %w", err)
|
||||
}
|
||||
if err := f.Sync(); err != nil {
|
||||
f.Close()
|
||||
return fmt.Errorf("sync worker store key file: %w", err)
|
||||
return fmt.Errorf("sync worker receipt key file: %w", err)
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
return fmt.Errorf("close worker store key file: %w", err)
|
||||
return fmt.Errorf("close worker receipt key file: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,423 +0,0 @@
|
||||
package workerkey
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nkeys"
|
||||
)
|
||||
|
||||
func newStore(t *testing.T) Store {
|
||||
t.Helper()
|
||||
return Store{Dir: filepath.Join(t.TempDir(), "keys")}
|
||||
}
|
||||
|
||||
func publicKey(t *testing.T, kp nkeys.KeyPair) string {
|
||||
t.Helper()
|
||||
pub, err := kp.PublicKey()
|
||||
if err != nil {
|
||||
t.Fatalf("public key: %v", err)
|
||||
}
|
||||
return pub
|
||||
}
|
||||
|
||||
func otherUserKey(t *testing.T) string {
|
||||
t.Helper()
|
||||
kp, err := nkeys.CreateUser()
|
||||
if err != nil {
|
||||
t.Fatalf("create user: %v", err)
|
||||
}
|
||||
return publicKey(t, kp)
|
||||
}
|
||||
|
||||
func readFile(t *testing.T, path string) []byte {
|
||||
t.Helper()
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read %s: %v", path, err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func assertNoSeed(t *testing.T, err error, seed []byte) {
|
||||
t.Helper()
|
||||
if err != nil && strings.Contains(err.Error(), string(seed)) {
|
||||
t.Fatalf("error exposes the seed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsure_FirstBootPersistsAndRestartReusesKey(t *testing.T) {
|
||||
s := newStore(t)
|
||||
kp, err := s.ForNewStore()
|
||||
if err != nil {
|
||||
t.Fatalf("first boot: %v", err)
|
||||
}
|
||||
pub := publicKey(t, kp)
|
||||
|
||||
dirInfo, err := os.Stat(s.Dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := dirInfo.Mode().Perm(); got != 0o700 {
|
||||
t.Fatalf("directory mode = %o, want 700", got)
|
||||
}
|
||||
fileInfo, err := os.Stat(s.path())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := fileInfo.Mode().Perm(); got != 0o600 {
|
||||
t.Fatalf("key file mode = %o, want 600", got)
|
||||
}
|
||||
onDisk, err := nkeys.FromSeed(readFile(t, s.path()))
|
||||
if err != nil {
|
||||
t.Fatalf("persisted file is not a seed: %v", err)
|
||||
}
|
||||
if publicKey(t, onDisk) != pub {
|
||||
t.Fatal("persisted seed does not match the returned key")
|
||||
}
|
||||
|
||||
// A restart is a fresh Store over the same disk; neither path may
|
||||
// generate a replacement.
|
||||
again, err := Store{Dir: s.Dir}.ForExistingStore(pub)
|
||||
if err != nil {
|
||||
t.Fatalf("restart for the existing store: %v", err)
|
||||
}
|
||||
if publicKey(t, again) != pub {
|
||||
t.Fatal("restart for the existing store changed the key")
|
||||
}
|
||||
again, err = Store{Dir: s.Dir}.ForNewStore()
|
||||
if err != nil {
|
||||
t.Fatalf("repeated first start: %v", err)
|
||||
}
|
||||
if publicKey(t, again) != pub {
|
||||
t.Fatal("repeated first start replaced the key")
|
||||
}
|
||||
entries, err := os.ReadDir(s.Dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(entries) != 1 {
|
||||
t.Fatalf("key directory holds %d entries, want only the key", len(entries))
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsure_ExistingStoreWithMissingKeyFailsClosed(t *testing.T) {
|
||||
s := newStore(t)
|
||||
_, err := s.ForExistingStore(otherUserKey(t))
|
||||
if !errors.Is(err, ErrKeyMissing) {
|
||||
t.Fatalf("err = %v, want ErrKeyMissing", err)
|
||||
}
|
||||
if _, err := os.Lstat(s.path()); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("a key was generated for an existing store: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsure_MismatchedKeyFailsClosedAndKeepsFile(t *testing.T) {
|
||||
s := newStore(t)
|
||||
if _, err := s.ForNewStore(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := readFile(t, s.path())
|
||||
|
||||
_, err := Store{Dir: s.Dir}.ForExistingStore(otherUserKey(t))
|
||||
if !errors.Is(err, ErrKeyMismatch) {
|
||||
t.Fatalf("err = %v, want ErrKeyMismatch", err)
|
||||
}
|
||||
assertNoSeed(t, err, before)
|
||||
if string(readFile(t, s.path())) != string(before) {
|
||||
t.Fatal("mismatch replaced the key file")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsure_UnusableKeyFileIsNeverReplaced(t *testing.T) {
|
||||
seedKP, err := nkeys.CreateUser()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
userSeed, err := seedKP.Seed()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
accountKP, err := nkeys.CreateAccount()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
accountSeed, err := accountKP.Seed()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
content []byte
|
||||
mode os.FileMode
|
||||
want error
|
||||
}{
|
||||
{"empty", nil, 0o600, ErrCorrupt},
|
||||
{"garbage", []byte("not a seed"), 0o600, ErrCorrupt},
|
||||
{"account seed", accountSeed, 0o600, ErrCorrupt},
|
||||
{"oversized", make([]byte, maxKeyFileSize+1), 0o600, ErrCorrupt},
|
||||
{"group readable", userSeed, 0o640, ErrUnsafe},
|
||||
{"world readable", userSeed, 0o604, ErrUnsafe},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
s := newStore(t)
|
||||
if err := os.MkdirAll(s.Dir, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(s.path(), tc.content, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Chmod(s.path(), tc.mode); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err := s.ForNewStore()
|
||||
if !errors.Is(err, tc.want) {
|
||||
t.Fatalf("err = %v, want %v", err, tc.want)
|
||||
}
|
||||
assertNoSeed(t, err, userSeed)
|
||||
assertNoSeed(t, err, accountSeed)
|
||||
if string(readFile(t, s.path())) != string(tc.content) {
|
||||
t.Fatal("an unusable key file was replaced")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsure_RefusesUnsafeDirectoryAndSymlink(t *testing.T) {
|
||||
t.Run("open directory", func(t *testing.T) {
|
||||
s := newStore(t)
|
||||
if err := os.MkdirAll(s.Dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Chmod(s.Dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.ForNewStore(); !errors.Is(err, ErrUnsafe) {
|
||||
t.Fatalf("err = %v, want ErrUnsafe", err)
|
||||
}
|
||||
if _, err := os.Lstat(s.path()); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatal("a key was written into an open directory")
|
||||
}
|
||||
})
|
||||
t.Run("symlinked key", func(t *testing.T) {
|
||||
s := newStore(t)
|
||||
target := newStore(t)
|
||||
if _, err := target.ForNewStore(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.MkdirAll(s.Dir, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Symlink(target.path(), s.path()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.ForNewStore(); !errors.Is(err, ErrUnsafe) {
|
||||
t.Fatalf("err = %v, want ErrUnsafe", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestEnsure_CrashLeftoverTempIsNotAKey(t *testing.T) {
|
||||
s := newStore(t)
|
||||
if err := os.MkdirAll(s.Dir, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stale := filepath.Join(s.Dir, tempPrefix+"123"+tempSuffix)
|
||||
if err := os.WriteFile(stale, []byte("partial"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// A crash before the link means no store was authorized for it.
|
||||
if _, err := s.ForExistingStore(otherUserKey(t)); !errors.Is(err, ErrKeyMissing) {
|
||||
t.Fatalf("err = %v, want ErrKeyMissing", err)
|
||||
}
|
||||
if _, err := os.Lstat(stale); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatal("stale temporary key file was kept")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsure_ConcurrentFirstBootAgreesOnOneKey(t *testing.T) {
|
||||
const rounds, n = 20, 16
|
||||
for range rounds {
|
||||
s := newStore(t)
|
||||
pubs := make([]string, n)
|
||||
errs := make([]error, n)
|
||||
start := make(chan struct{})
|
||||
var wg sync.WaitGroup
|
||||
for i := range n {
|
||||
wg.Go(func() {
|
||||
<-start
|
||||
kp, err := Store{Dir: s.Dir}.ForNewStore()
|
||||
if err != nil {
|
||||
errs[i] = err
|
||||
return
|
||||
}
|
||||
pubs[i], errs[i] = kp.PublicKey()
|
||||
})
|
||||
}
|
||||
close(start)
|
||||
wg.Wait()
|
||||
onDisk, err := nkeys.FromSeed(readFile(t, s.path()))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := publicKey(t, onDisk)
|
||||
for i := range n {
|
||||
if errs[i] != nil {
|
||||
t.Fatalf("call %d: %v", i, errs[i])
|
||||
}
|
||||
if pubs[i] != want {
|
||||
t.Fatalf("call %d returned a key that is not the persisted one", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestForExistingStore_RejectsInvalidAuthorizedKey(t *testing.T) {
|
||||
kp, err := nkeys.CreateUser()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seed, err := kp.Seed()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// An empty key is what an unreadable store record would yield; it must not
|
||||
// fall through to first-start generation.
|
||||
for _, authorized := range []string{"", string(seed)} {
|
||||
s := newStore(t)
|
||||
_, err := s.ForExistingStore(authorized)
|
||||
if err == nil {
|
||||
t.Fatalf("authorized key %q was accepted", authorized[:min(len(authorized), 2)])
|
||||
}
|
||||
assertNoSeed(t, err, seed)
|
||||
if _, err := os.Lstat(s.Dir); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatal("an invalid authorized key still touched the disk")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestForNewStore_RefusesMissingParent(t *testing.T) {
|
||||
s := Store{Dir: filepath.Join(t.TempDir(), "missing-parent", "keys")}
|
||||
if _, err := s.ForNewStore(); err == nil {
|
||||
t.Fatal("a key directory was created without its parent")
|
||||
}
|
||||
if _, err := os.Lstat(filepath.Dir(s.Dir)); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatal("the missing parent was created")
|
||||
}
|
||||
}
|
||||
|
||||
// hookSyncs records every directory sync and fails call number failAt
|
||||
// (counted from one); zero fails none.
|
||||
func hookSyncs(t *testing.T, failAt int) *[]string {
|
||||
t.Helper()
|
||||
var synced []string
|
||||
orig := syncDirFn
|
||||
t.Cleanup(func() { syncDirFn = orig })
|
||||
syncDirFn = func(dir string) error {
|
||||
synced = append(synced, dir)
|
||||
if len(synced) == failAt {
|
||||
return errors.New("injected directory sync failure")
|
||||
}
|
||||
return orig(dir)
|
||||
}
|
||||
return &synced
|
||||
}
|
||||
|
||||
func TestEnsure_RetryAfterFailedDirectorySyncKeepsKeyAndSyncs(t *testing.T) {
|
||||
retries := map[string]func(s Store, pub string) (nkeys.KeyPair, error){
|
||||
"first start": func(s Store, _ string) (nkeys.KeyPair, error) { return s.ForNewStore() },
|
||||
"existing store": func(s Store, pub string) (nkeys.KeyPair, error) {
|
||||
return s.ForExistingStore(pub)
|
||||
},
|
||||
}
|
||||
for name, retry := range retries {
|
||||
for failAt := 1; failAt <= 2; failAt++ {
|
||||
t.Run(fmt.Sprintf("%s after sync %d failed", name, failAt), func(t *testing.T) {
|
||||
s := newStore(t)
|
||||
hookSyncs(t, failAt)
|
||||
if _, err := s.ForNewStore(); err == nil {
|
||||
t.Fatal("a key was returned although a directory sync failed")
|
||||
}
|
||||
first := readFile(t, s.path())
|
||||
onDisk, err := nkeys.FromSeed(first)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pub := publicKey(t, onDisk)
|
||||
|
||||
synced := hookSyncs(t, 0)
|
||||
kp, err := retry(Store{Dir: s.Dir}, pub)
|
||||
if err != nil {
|
||||
t.Fatalf("retry: %v", err)
|
||||
}
|
||||
if string(readFile(t, s.path())) != string(first) {
|
||||
t.Fatal("retry replaced the key")
|
||||
}
|
||||
if publicKey(t, kp) != pub {
|
||||
t.Fatal("retry returned a key other than the persisted one")
|
||||
}
|
||||
want := []string{s.Dir, filepath.Dir(s.Dir)}
|
||||
if len(*synced) != 2 || (*synced)[0] != want[0] || (*synced)[1] != want[1] {
|
||||
t.Fatalf("retry synced %v, want %v", *synced, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Another holder of the directory lock, such as a second worker process on
|
||||
// the same disk, must finish before this one reads, cleans or creates a key.
|
||||
func TestEnsure_WaitsForDirectoryLock(t *testing.T) {
|
||||
s := newStore(t)
|
||||
if err := os.MkdirAll(s.Dir, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
holder, err := os.Open(s.Dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer holder.Close()
|
||||
if err := syscall.Flock(int(holder.Fd()), syscall.LOCK_EX); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := Store{Dir: s.Dir}.ForNewStore()
|
||||
done <- err
|
||||
}()
|
||||
deadline := time.Now().Add(300 * time.Millisecond)
|
||||
for time.Now().Before(deadline) {
|
||||
select {
|
||||
case err := <-done:
|
||||
t.Fatalf("Ensure returned while another holder had the lock: %v", err)
|
||||
default:
|
||||
}
|
||||
if _, err := os.Lstat(s.path()); err == nil {
|
||||
t.Fatal("a key was created while another holder had the lock")
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
|
||||
if err := syscall.Flock(int(holder.Fd()), syscall.LOCK_UN); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Fatalf("Ensure after unlock: %v", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("Ensure did not proceed after the lock was released")
|
||||
}
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/mulgadc/spinifex/internal/testkit"
|
||||
"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
|
||||
awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
@@ -479,13 +480,13 @@ func TestSealSurvivesServiceReconstruction(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
ctx := context.Background()
|
||||
|
||||
svc, err := NewAccountSettingsServiceImplWithNATS(ctx, nil, nc)
|
||||
svc, err := NewAccountSettingsServiceImplWithNATS(ctx, &config.Config{}, nc)
|
||||
require.NoError(t, err)
|
||||
svc.SetAccountTerminationChecker(terminatingChecker(testAccountID))
|
||||
first, err := svc.SealAccount(ctx, testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
rebuilt, err := NewAccountSettingsServiceImplWithNATS(ctx, nil, nc)
|
||||
rebuilt, err := NewAccountSettingsServiceImplWithNATS(ctx, &config.Config{}, nc)
|
||||
require.NoError(t, err)
|
||||
|
||||
entry, err := rebuilt.settingsKV.Get(ctx, testAccountID)
|
||||
@@ -546,14 +547,14 @@ func TestSealLeavesOtherAccountsUntouched(t *testing.T) {
|
||||
svc := setupTestAccountService(t)
|
||||
ctx := context.Background()
|
||||
accountB := "222222222222"
|
||||
_, err := svc.EnableEbsEncryptionByDefault(ctx, &ec2.EnableEbsEncryptionByDefaultInput{}, accountB)
|
||||
_, err := svc.EnableSerialConsoleAccess(ctx, &ec2.EnableSerialConsoleAccessInput{}, accountB)
|
||||
require.NoError(t, err)
|
||||
|
||||
svc.SetAccountTerminationChecker(terminatingChecker(testAccountID))
|
||||
_, err = svc.SealAccount(ctx, testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
out, err := svc.GetEbsEncryptionByDefault(ctx, &ec2.GetEbsEncryptionByDefaultInput{}, accountB)
|
||||
out, err := svc.GetSerialConsoleAccessStatus(ctx, &ec2.GetSerialConsoleAccessStatusInput{}, accountB)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, *out.EbsEncryptionByDefault)
|
||||
assert.True(t, *out.SerialConsoleAccessEnabled)
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
|
||||
ebsencryption "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/encryption"
|
||||
awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
|
||||
@@ -170,23 +171,48 @@ func (s *AccountSettingsServiceImpl) getSettings(ctx context.Context, accountID
|
||||
return &settings, nil
|
||||
}
|
||||
|
||||
// platformEncrypts reports whether every new volume is encrypted, which is
|
||||
// what EBS encryption by default means on AWS. The platform decides it, not
|
||||
// the account.
|
||||
func (s *AccountSettingsServiceImpl) platformEncrypts(ctx context.Context) (bool, error) {
|
||||
encrypted, err := ebsencryption.Enabled(s.config.Viperblock.EncryptionKeyFile)
|
||||
if err != nil {
|
||||
slog.ErrorContext(ctx, "Failed to load EBS encryption key", "err", err)
|
||||
return false, errors.New(awserrors.ErrorInternalError)
|
||||
}
|
||||
return encrypted, nil
|
||||
}
|
||||
|
||||
// EnableEbsEncryptionByDefault enables EBS encryption by default for the account.
|
||||
// It reports the platform's posture, so it answers false where no key is configured.
|
||||
func (s *AccountSettingsServiceImpl) EnableEbsEncryptionByDefault(ctx context.Context, input *ec2.EnableEbsEncryptionByDefaultInput, accountID string) (*ec2.EnableEbsEncryptionByDefaultOutput, error) {
|
||||
slog.InfoContext(ctx, "EnableEbsEncryptionByDefault called", "accountID", accountID)
|
||||
|
||||
encrypted, err := s.platformEncrypts(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.mutateSettings(ctx, accountID, func(rec *AccountSettingsRecord) { rec.EbsEncryptionByDefault = true }); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &ec2.EnableEbsEncryptionByDefaultOutput{
|
||||
EbsEncryptionByDefault: aws.Bool(true),
|
||||
EbsEncryptionByDefault: aws.Bool(encrypted),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// DisableEbsEncryptionByDefault disables EBS encryption by default for the account.
|
||||
// It is refused while the platform encrypts every volume, since it could not take effect.
|
||||
func (s *AccountSettingsServiceImpl) DisableEbsEncryptionByDefault(ctx context.Context, input *ec2.DisableEbsEncryptionByDefaultInput, accountID string) (*ec2.DisableEbsEncryptionByDefaultOutput, error) {
|
||||
slog.InfoContext(ctx, "DisableEbsEncryptionByDefault called", "accountID", accountID)
|
||||
|
||||
encrypted, err := s.platformEncrypts(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if encrypted {
|
||||
return nil, awserrors.Errorf(awserrors.ErrorOperationNotPermitted, "EBS encryption by default cannot be disabled: this platform encrypts every volume")
|
||||
}
|
||||
if err := s.mutateSettings(ctx, accountID, func(rec *AccountSettingsRecord) { rec.EbsEncryptionByDefault = false }); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -200,13 +226,16 @@ func (s *AccountSettingsServiceImpl) DisableEbsEncryptionByDefault(ctx context.C
|
||||
func (s *AccountSettingsServiceImpl) GetEbsEncryptionByDefault(ctx context.Context, input *ec2.GetEbsEncryptionByDefaultInput, accountID string) (*ec2.GetEbsEncryptionByDefaultOutput, error) {
|
||||
slog.InfoContext(ctx, "GetEbsEncryptionByDefault called", "accountID", accountID)
|
||||
|
||||
settings, err := s.getSettings(ctx, accountID)
|
||||
if _, err := s.getSettings(ctx, accountID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
encrypted, err := s.platformEncrypts(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &ec2.GetEbsEncryptionByDefaultOutput{
|
||||
EbsEncryptionByDefault: aws.Bool(settings.EbsEncryptionByDefault),
|
||||
EbsEncryptionByDefault: aws.Bool(encrypted),
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -2,13 +2,18 @@ package account
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/json"
|
||||
awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/mulgadc/spinifex/internal/testkit"
|
||||
"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
|
||||
awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -27,47 +32,88 @@ func TestNewAccountSettingsServiceImplWithNATS_NoJetStream(t *testing.T) {
|
||||
var _ AccountSettingsService = (*AccountSettingsServiceImpl)(nil)
|
||||
|
||||
func setupTestAccountService(t *testing.T) *AccountSettingsServiceImpl {
|
||||
t.Helper()
|
||||
return setupTestAccountServiceWithConfig(t, &config.Config{})
|
||||
}
|
||||
|
||||
func setupTestAccountServiceWithConfig(t *testing.T, cfg *config.Config) *AccountSettingsServiceImpl {
|
||||
t.Helper()
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
|
||||
svc, err := NewAccountSettingsServiceImplWithNATS(t.Context(), nil, nc)
|
||||
svc, err := NewAccountSettingsServiceImplWithNATS(t.Context(), cfg, nc)
|
||||
require.NoError(t, err)
|
||||
return svc
|
||||
}
|
||||
|
||||
// setupEncryptingAccountService configures a Viperblock master key, so the
|
||||
// platform encrypts every volume it creates.
|
||||
func setupEncryptingAccountService(t *testing.T) *AccountSettingsServiceImpl {
|
||||
t.Helper()
|
||||
raw := make([]byte, 32)
|
||||
_, err := rand.Read(raw)
|
||||
require.NoError(t, err)
|
||||
keyFile := filepath.Join(t.TempDir(), "master.key")
|
||||
require.NoError(t, os.WriteFile(keyFile, raw, 0o600))
|
||||
return setupTestAccountServiceWithConfig(t, &config.Config{Viperblock: config.ViperblockConfig{EncryptionKeyFile: keyFile}})
|
||||
}
|
||||
|
||||
const testAccountID = "111111111111"
|
||||
|
||||
// EBS Encryption tests
|
||||
|
||||
func TestEbsEncryption_DefaultOff(t *testing.T) {
|
||||
func TestEbsEncryption_NoKeyReportsFalse(t *testing.T) {
|
||||
svc := setupTestAccountService(t)
|
||||
out, err := svc.GetEbsEncryptionByDefault(context.Background(), &ec2.GetEbsEncryptionByDefaultInput{}, testAccountID)
|
||||
ctx := context.Background()
|
||||
|
||||
enableOut, err := svc.EnableEbsEncryptionByDefault(ctx, &ec2.EnableEbsEncryptionByDefaultInput{}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, *out.EbsEncryptionByDefault)
|
||||
}
|
||||
assert.False(t, *enableOut.EbsEncryptionByDefault, "enabling cannot encrypt volumes without a key")
|
||||
|
||||
func TestEbsEncryption_EnableDisable(t *testing.T) {
|
||||
svc := setupTestAccountService(t)
|
||||
|
||||
// Enable
|
||||
enableOut, err := svc.EnableEbsEncryptionByDefault(context.Background(), &ec2.EnableEbsEncryptionByDefaultInput{}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, *enableOut.EbsEncryptionByDefault)
|
||||
|
||||
// Verify
|
||||
getOut, err := svc.GetEbsEncryptionByDefault(context.Background(), &ec2.GetEbsEncryptionByDefaultInput{}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, *getOut.EbsEncryptionByDefault)
|
||||
|
||||
// Disable
|
||||
disableOut, err := svc.DisableEbsEncryptionByDefault(context.Background(), &ec2.DisableEbsEncryptionByDefaultInput{}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, *disableOut.EbsEncryptionByDefault)
|
||||
|
||||
// Verify
|
||||
getOut, err = svc.GetEbsEncryptionByDefault(context.Background(), &ec2.GetEbsEncryptionByDefaultInput{}, testAccountID)
|
||||
getOut, err := svc.GetEbsEncryptionByDefault(ctx, &ec2.GetEbsEncryptionByDefaultInput{}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, *getOut.EbsEncryptionByDefault)
|
||||
|
||||
disableOut, err := svc.DisableEbsEncryptionByDefault(ctx, &ec2.DisableEbsEncryptionByDefaultInput{}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, *disableOut.EbsEncryptionByDefault)
|
||||
}
|
||||
|
||||
func TestEbsEncryption_PlatformEncryptsReportsTrue(t *testing.T) {
|
||||
svc := setupEncryptingAccountService(t)
|
||||
ctx := context.Background()
|
||||
|
||||
getOut, err := svc.GetEbsEncryptionByDefault(ctx, &ec2.GetEbsEncryptionByDefaultInput{}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, *getOut.EbsEncryptionByDefault, "never-configured account still reads true")
|
||||
|
||||
enableOut, err := svc.EnableEbsEncryptionByDefault(ctx, &ec2.EnableEbsEncryptionByDefaultInput{}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, *enableOut.EbsEncryptionByDefault)
|
||||
}
|
||||
|
||||
func TestEbsEncryption_DisableRefusedWhilePlatformEncrypts(t *testing.T) {
|
||||
svc := setupEncryptingAccountService(t)
|
||||
ctx := context.Background()
|
||||
|
||||
_, err := svc.DisableEbsEncryptionByDefault(ctx, &ec2.DisableEbsEncryptionByDefaultInput{}, testAccountID)
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorOperationNotPermitted), "got %v", err)
|
||||
|
||||
_, err = svc.settingsKV.Get(ctx, testAccountID)
|
||||
assert.ErrorIs(t, err, jetstream.ErrKeyNotFound, "a refused disable writes nothing")
|
||||
|
||||
getOut, err := svc.GetEbsEncryptionByDefault(ctx, &ec2.GetEbsEncryptionByDefaultInput{}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, *getOut.EbsEncryptionByDefault)
|
||||
}
|
||||
|
||||
// A key file that is configured but cannot be loaded is a fault, not a
|
||||
// report that volumes are unencrypted.
|
||||
func TestEbsEncryption_UnloadableKeyIsInternalError(t *testing.T) {
|
||||
missing := filepath.Join(t.TempDir(), "absent.key")
|
||||
svc := setupTestAccountServiceWithConfig(t, &config.Config{Viperblock: config.ViperblockConfig{EncryptionKeyFile: missing}})
|
||||
|
||||
_, err := svc.GetEbsEncryptionByDefault(context.Background(), &ec2.GetEbsEncryptionByDefaultInput{}, testAccountID)
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInternalError), "got %v", err)
|
||||
}
|
||||
|
||||
// Serial Console tests
|
||||
@@ -101,26 +147,6 @@ func TestSerialConsole_EnableDisable(t *testing.T) {
|
||||
|
||||
// Multi-account isolation tests
|
||||
|
||||
func TestEbsEncryption_MultiAccountIsolation(t *testing.T) {
|
||||
svc := setupTestAccountService(t)
|
||||
accountA := "111111111111"
|
||||
accountB := "222222222222"
|
||||
|
||||
// Enable EBS encryption for account A only
|
||||
_, err := svc.EnableEbsEncryptionByDefault(context.Background(), &ec2.EnableEbsEncryptionByDefaultInput{}, accountA)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Account A should have encryption enabled
|
||||
outA, err := svc.GetEbsEncryptionByDefault(context.Background(), &ec2.GetEbsEncryptionByDefaultInput{}, accountA)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, *outA.EbsEncryptionByDefault)
|
||||
|
||||
// Account B should still have encryption disabled (default)
|
||||
outB, err := svc.GetEbsEncryptionByDefault(context.Background(), &ec2.GetEbsEncryptionByDefaultInput{}, accountB)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, *outB.EbsEncryptionByDefault)
|
||||
}
|
||||
|
||||
func TestSerialConsole_MultiAccountIsolation(t *testing.T) {
|
||||
svc := setupTestAccountService(t)
|
||||
accountA := "111111111111"
|
||||
@@ -151,12 +177,12 @@ func TestSettingsKey_EmptyAccountIDFallback(t *testing.T) {
|
||||
func TestRecordWithoutStateIsActive(t *testing.T) {
|
||||
svc := setupTestAccountService(t)
|
||||
ctx := context.Background()
|
||||
_, err := svc.settingsKV.Put(ctx, testAccountID, []byte(`{"ebs_encryption_by_default":true,"serial_console_access":false}`))
|
||||
_, err := svc.settingsKV.Put(ctx, testAccountID, []byte(`{"ebs_encryption_by_default":false,"serial_console_access":true}`))
|
||||
require.NoError(t, err)
|
||||
|
||||
out, err := svc.GetEbsEncryptionByDefault(ctx, &ec2.GetEbsEncryptionByDefaultInput{}, testAccountID)
|
||||
out, err := svc.GetSerialConsoleAccessStatus(ctx, &ec2.GetSerialConsoleAccessStatusInput{}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, *out.EbsEncryptionByDefault)
|
||||
assert.True(t, *out.SerialConsoleAccessEnabled)
|
||||
|
||||
_, err = svc.EnableSerialConsoleAccess(ctx, &ec2.EnableSerialConsoleAccessInput{}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -1215,6 +1215,19 @@ func TestHTTP_RoleCredentials(t *testing.T) {
|
||||
assert.Contains(t, rec.Body.String(), `"Code":"Success"`)
|
||||
}
|
||||
|
||||
func TestHTTP_RoleCredentialsMintRefusedYieldsFailedBody(t *testing.T) {
|
||||
res := &fakeResolver{eni: testENI(), inst: &instanceFacts{iamInstanceProfileArn: "arn:aws:iam::111122223333:instance-profile/app-profile"}}
|
||||
assumer := &fakeAssumer{err: errors.New(awserrors.ErrorAccessDenied)}
|
||||
svc, _ := newTestService(res, &fakeIAM{profile: profileFixture(), roleARN: "arn:aws:iam::111122223333:role/app-role"}, assumer)
|
||||
h := withTapENI(svc.httpHandler(), testENI())
|
||||
token := issueToken(t, h)
|
||||
|
||||
rec := get(t, h, prefixSecurityCreds+"app-role", token)
|
||||
assert.Equal(t, http.StatusOK, rec.Code)
|
||||
assert.Contains(t, rec.Body.String(), `"Code":"Failed"`)
|
||||
assert.NotContains(t, rec.Body.String(), "AccessKeyId")
|
||||
}
|
||||
|
||||
func TestHTTP_RoleCredentialsWrongRole404(t *testing.T) {
|
||||
res := &fakeResolver{eni: testENI(), inst: &instanceFacts{iamInstanceProfileArn: "arn:aws:iam::111122223333:instance-profile/app-profile"}}
|
||||
svc, _ := newTestService(res, &fakeIAM{profile: profileFixture(), roleARN: "arn:aws:iam::111122223333:role/app-role"}, &fakeAssumer{})
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
package systeminstance
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
)
|
||||
|
||||
// Kind selects the guest a direct-boot system instance runs. The empty kind
|
||||
// is the load balancer; every other kind is launched only on its own subject.
|
||||
type Kind string
|
||||
|
||||
// ErrNonLoadBalancerKind refuses a non-load-balancer guest on the legacy path.
|
||||
var ErrNonLoadBalancerKind = errors.New("sysinstance: guest kind must use its own launch subject")
|
||||
|
||||
// volumeIDPattern is the platform's volume ID shape. The guest finds its disk
|
||||
// by the ID without its dash, which must fit the 20-byte virtio-blk serial.
|
||||
var volumeIDPattern = regexp.MustCompile(`^vol-[0-9a-f]{17}$`)
|
||||
|
||||
// RefuseNonLoadBalancerKind keeps every non-load-balancer guest off the
|
||||
// legacy launch path, which would boot the load balancer image instead.
|
||||
func (in *SystemInstanceInput) RefuseNonLoadBalancerKind() error {
|
||||
if in.Kind != "" || in.SQSWorker != nil {
|
||||
return fmt.Errorf("%w: kind %q", ErrNonLoadBalancerKind, in.Kind)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user