mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-29 23:46:00 +00:00
Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Josh Siviour
|
488f1c7bf9 | ||
|
Ben Duncan
|
3617e4a045 | ||
|
Josh Siviour
|
24281cb860 | ||
|
Josh Siviour
|
e58838bc67 |
@@ -102,9 +102,9 @@ jobs:
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
|
||||
- name: Build microVM artifacts (lb-agent + kernel + initramfs + qboot)
|
||||
- name: Build microVM artifacts (lb-agent + kernel + initramfs + qboot + SQS store image)
|
||||
if: steps.version.outputs.exists != 'true'
|
||||
run: cd spinifex && make build-microvm-image
|
||||
run: cd spinifex && make build-microvm-image build-sqs-store-microvm-image
|
||||
|
||||
- name: Build amd64 Tarball
|
||||
if: steps.version.outputs.exists != 'true'
|
||||
|
||||
@@ -57,8 +57,8 @@ jobs:
|
||||
go-version-file: spinifex/go.mod
|
||||
cache: false
|
||||
|
||||
- name: Build microVM artifacts (lb-agent + kernel + initramfs + qboot)
|
||||
run: cd spinifex && make build-microvm-image
|
||||
- name: Build microVM artifacts (lb-agent + kernel + initramfs + qboot + SQS store image)
|
||||
run: cd spinifex && make build-microvm-image build-sqs-store-microvm-image
|
||||
|
||||
- name: Build amd64 Tarball
|
||||
env:
|
||||
|
||||
@@ -21,6 +21,9 @@ build/microvm/vmlinuz
|
||||
build/microvm/initramfs.cpio.gz
|
||||
build/microvm/qboot.rom
|
||||
build/microvm/qboot.LICENSE
|
||||
build/microvm/sqs-store/vmlinuz
|
||||
build/microvm/sqs-store/initramfs.cpio.gz
|
||||
build/microvm/sqs-store/.stage.*/
|
||||
build/qboot/
|
||||
!spinifex/runtime/roles/spinifexui/frontend/dist/
|
||||
tmp/
|
||||
|
||||
@@ -140,6 +140,23 @@ $(MICROVM_ARTIFACTS) &: $(MICROVM_INPUTS)
|
||||
bin/lb-agent:
|
||||
$(MAKE) build-lb-agent
|
||||
|
||||
# The SQS store microVM image: a separate artifact from the LB image above, not
|
||||
# installed or released by any target yet.
|
||||
SQS_STORE_MICROVM_OUT_DIR := $(MICROVM_OUT_DIR)/sqs-store
|
||||
SQS_STORE_MICROVM_ARTIFACTS := $(SQS_STORE_MICROVM_OUT_DIR)/vmlinuz $(SQS_STORE_MICROVM_OUT_DIR)/initramfs.cpio.gz
|
||||
SQS_STORE_MICROVM_INPUTS := scripts/build-sqs-store-microvm-image.sh scripts/lib/sqs-store-microvm-publish.sh $(SQS_STORE_MICROVM_OUT_DIR)/init.sh $(SQS_STORE_MICROVM_OUT_DIR)/inittab $(addprefix $(SQS_STORE_MICROVM_OUT_DIR)/,sqs-store-lib.sh sqs-store-prepare sqs-store-run sqs-store-stop) bin/nats-server
|
||||
|
||||
$(SQS_STORE_MICROVM_ARTIFACTS) &: $(SQS_STORE_MICROVM_INPUTS)
|
||||
./scripts/build-sqs-store-microvm-image.sh
|
||||
|
||||
# The dedicated SQS store server, built from the go.mod-pinned module so the
|
||||
# image ships exactly the reviewed version. Static for the musl guest.
|
||||
bin/nats-server: go.mod go.sum
|
||||
@mkdir -p bin
|
||||
GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags '-s -w' -o $@ github.com/nats-io/nats-server/v2
|
||||
|
||||
build-sqs-store-microvm-image: $(SQS_STORE_MICROVM_ARTIFACTS) ## Build the SQS store microVM kernel and initramfs
|
||||
|
||||
# qboot is the microvm firmware. The source tarball ships with each release
|
||||
# because qboot is GPL-2.0.
|
||||
QBOOT_COMMIT := $(shell sed -n 's/^QBOOT_COMMIT="\(.*\)"$$/\1/p' scripts/build-qboot.sh)
|
||||
@@ -151,6 +168,12 @@ $(QBOOT_ARTIFACTS) &: scripts/build-qboot.sh
|
||||
|
||||
build-microvm-image: $(MICROVM_ARTIFACTS) $(QBOOT_ARTIFACTS) ## Build microVM kernel, initramfs and qboot firmware (incremental — skips when up to date)
|
||||
|
||||
install-sqs-store-microvm: $(SQS_STORE_MICROVM_ARTIFACTS) ## Install the SQS store microVM image to /usr/share/spinifex/microvm/sqs-store/
|
||||
sudo install -d /usr/share/spinifex/microvm/sqs-store
|
||||
sudo rm -f /usr/share/spinifex/microvm/sqs-store/initramfs.cpio.gz
|
||||
sudo install -m 0644 $(SQS_STORE_MICROVM_OUT_DIR)/vmlinuz /usr/share/spinifex/microvm/sqs-store/vmlinuz
|
||||
sudo install -m 0644 $(SQS_STORE_MICROVM_OUT_DIR)/initramfs.cpio.gz /usr/share/spinifex/microvm/sqs-store/initramfs.cpio.gz
|
||||
|
||||
install-microvm: $(MICROVM_ARTIFACTS) $(QBOOT_ARTIFACTS) ## Install microVM artifacts to /usr/share/spinifex/microvm/
|
||||
sudo install -d /usr/share/spinifex/microvm
|
||||
sudo install -m 0644 $(MICROVM_OUT_DIR)/vmlinuz /usr/share/spinifex/microvm/vmlinuz
|
||||
@@ -177,7 +200,7 @@ preflight:
|
||||
# reason: between them they install the software, wire OVN, erase the state and
|
||||
# remove the install, so a shellcheck warning in one is an outage or a wrong
|
||||
# deletion, not a style note.
|
||||
NODE_SCRIPTS := scripts/setup.sh scripts/setup-ovn.sh scripts/install-node.sh scripts/smoke-test.sh scripts/node-reset.sh scripts/uninstall-spx.sh scripts/lib/distro.sh
|
||||
NODE_SCRIPTS := scripts/setup.sh scripts/setup-ovn.sh scripts/install-node.sh scripts/smoke-test.sh scripts/node-reset.sh scripts/uninstall-spx.sh scripts/lib/distro.sh scripts/lib/stage-sqs-store-microvm.sh
|
||||
|
||||
test-build-scripts:
|
||||
@echo -e "\n....Running build/scripts/**/*_test.sh...."
|
||||
@@ -451,6 +474,7 @@ distro-amd64:
|
||||
echo "[distro-amd64] WARNING: microvm artifacts missing — tarball will not include them"; \
|
||||
echo "[distro-amd64] run 'make build-microvm-image' before 'make distro-amd64'"; \
|
||||
fi
|
||||
./scripts/lib/stage-sqs-store-microvm.sh $(SQS_STORE_MICROVM_OUT_DIR) dist/amd64
|
||||
tar -czf dist/spinifex-$(VERSION)-linux-amd64.tar.gz -C dist/amd64 .
|
||||
sha256sum dist/spinifex-$(VERSION)-linux-amd64.tar.gz > dist/spinifex-$(VERSION)-linux-amd64.tar.gz.sha256
|
||||
|
||||
@@ -469,7 +493,7 @@ distro-arm64:
|
||||
distro-clean:
|
||||
rm -rf dist/
|
||||
|
||||
.PHONY: test-package-check build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench fuzz test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle \
|
||||
.PHONY: test-package-check build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image build-sqs-store-microvm-image install-microvm install-sqs-store-microvm go_build preflight test test-cover test-race diff-coverage bench fuzz test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle \
|
||||
deploy reinstall clean \
|
||||
install-system install-go install-aws quickinstall \
|
||||
lint fix govulncheck nilaway \
|
||||
|
||||
Executable
+96
@@ -0,0 +1,96 @@
|
||||
#!/bin/sh
|
||||
mount -t proc proc /proc
|
||||
mount -t sysfs sys /sys
|
||||
mount -t devtmpfs dev /dev
|
||||
|
||||
# virtio_mmio is built-in (CONFIG_VIRTIO_MMIO=y) so virtio-net devices are
|
||||
# already enumerated by the time PID 1 runs — no modprobe needed.
|
||||
# qemu_fw_cfg and virtio_net are =m in stock Alpine linux-virt, so load them.
|
||||
#
|
||||
# modprobe needs a usable modules.dep; when the image is built on a host whose
|
||||
# depmod can't produce one, busybox modprobe silently no-ops and the module
|
||||
# never loads — fw_cfg then never appears and init panics. insmod by explicit
|
||||
# path needs no modules.dep, so fall back to it. Handles .ko and .ko.gz.
|
||||
load_mod() {
|
||||
modprobe "$1" 2>/dev/null && return 0
|
||||
for ko in $(find /lib/modules -name "$1.ko" -o -name "$1.ko.gz" 2>/dev/null); do
|
||||
insmod "$ko" 2>/dev/null && return 0
|
||||
case "$ko" in
|
||||
*.gz) gunzip -c "$ko" >/tmp/mod.ko 2>/dev/null \
|
||||
&& insmod /tmp/mod.ko 2>/dev/null && return 0 ;;
|
||||
esac
|
||||
done
|
||||
return 1
|
||||
}
|
||||
load_mod qemu_fw_cfg || load_mod fw_cfg_sysfs || true
|
||||
# virtio_net depends on net_failover, which depends on failover. The load_mod
|
||||
# insmod fallback loads a single .ko with no dependency resolution, and
|
||||
# modules.dep is absent when the image is built on a kmod-less host (depmod
|
||||
# skipped), so modprobe can't resolve the chain either. Load it explicitly
|
||||
# bottom-up so the NIC comes up regardless of how the image was built —
|
||||
# without this the agent has no network and the LB hangs in provisioning.
|
||||
load_mod failover || true
|
||||
load_mod net_failover || true
|
||||
load_mod virtio_net || true
|
||||
|
||||
NETCFG=/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw
|
||||
i=0
|
||||
while [ "$i" -lt 25 ]; do
|
||||
[ -f "$NETCFG" ] && break
|
||||
sleep 0.04
|
||||
i=$((i + 1))
|
||||
done
|
||||
|
||||
if [ ! -f "$NETCFG" ]; then
|
||||
# Dump host/guest fw_cfg state to the serial console before init exits so a
|
||||
# "netcfg not available" panic is diagnosable from the captured console log
|
||||
# (module-not-loaded vs qemu-not-exposing-by_name vs wrong machine type).
|
||||
echo "[init] FATAL: fw_cfg netcfg not available" >&2
|
||||
echo "[init] dmi: vendor=$(cat /sys/class/dmi/id/sys_vendor 2>/dev/null) product=$(cat /sys/class/dmi/id/product_name 2>/dev/null) bios=$(cat /sys/class/dmi/id/bios_version 2>/dev/null)" >&2
|
||||
echo "[init] qemu_fw_cfg dir present: $([ -d /sys/firmware/qemu_fw_cfg ] && echo yes || echo NO)" >&2
|
||||
echo "[init] modules: $(lsmod 2>/dev/null | awk 'NR>1{print $1}' | tr '\n' ',')" >&2
|
||||
echo "[init] by_name listing:" >&2
|
||||
ls -la /sys/firmware/qemu_fw_cfg/by_name/ 2>&1 | sed 's/^/[init] /' >&2
|
||||
ls -la /sys/firmware/qemu_fw_cfg/by_name/opt/ 2>&1 | sed 's/^/[init] /' >&2
|
||||
echo "[init] dmesg fw_cfg:" >&2
|
||||
dmesg 2>/dev/null | grep -i fw_cfg | sed 's/^/[init] /' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
. "$NETCFG"
|
||||
|
||||
for n in 0 1 2 3 4 5; do
|
||||
eval mac=\$NIC${n}_MAC
|
||||
[ -z "$mac" ] && continue
|
||||
|
||||
iface=""
|
||||
for attempt in 1 2 3 4 5; do
|
||||
iface=$(ip -o link | awk -v m="$mac" 'tolower($0) ~ tolower(m) {print $2}' | tr -d :)
|
||||
[ -n "$iface" ] && break
|
||||
sleep 0.02
|
||||
done
|
||||
|
||||
if [ -z "$iface" ]; then
|
||||
echo "[init] WARNING: no interface found for MAC $mac (NIC$n), skipping" >&2
|
||||
continue
|
||||
fi
|
||||
|
||||
eval cidr=\$NIC${n}_CIDR
|
||||
eval gw=\$NIC${n}_GW
|
||||
eval default=\$NIC${n}_DEFAULT
|
||||
eval rdst=\$NIC${n}_ROUTE_DST
|
||||
eval rvia=\$NIC${n}_ROUTE_VIA
|
||||
|
||||
ip link set "$iface" up
|
||||
[ -n "$cidr" ] && ip addr add "$cidr" dev "$iface"
|
||||
[ "$default" = "1" ] && [ -n "$gw" ] && ip route add default via "$gw" dev "$iface"
|
||||
[ -n "$rdst" ] && [ -n "$rvia" ] && ip route add "$rdst" via "$rvia" dev "$iface"
|
||||
done
|
||||
|
||||
# The store image takes no lb-agent environment. A CA bundle is optional here:
|
||||
# nothing in this image dials out yet.
|
||||
CA_CERT=/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/ca-cert/raw
|
||||
mkdir -p /etc/ssl/certs
|
||||
[ -f "$CA_CERT" ] && cp "$CA_CERT" /etc/ssl/certs/ca-certificates.crt
|
||||
|
||||
exec /sbin/init
|
||||
@@ -0,0 +1,10 @@
|
||||
# The power button loads here, not in /init: busybox init has its SIGUSR2
|
||||
# (poweroff) handler installed before sysinit runs, so a press can never reach
|
||||
# the pre-exec shell PID 1. A press before this line is lost; the host re-sends.
|
||||
::sysinit:/sbin/modprobe tiny_power_button power_signal=12
|
||||
::sysinit:/usr/local/lib/sqs-store/sqs-store-prepare
|
||||
::respawn:/usr/local/lib/sqs-store/sqs-store-run
|
||||
::ctrlaltdel:/sbin/reboot
|
||||
# One action: the store stops and its disk unmounts first, and the generic
|
||||
# umount -a -r runs only if the store is gone.
|
||||
::shutdown:/usr/local/lib/sqs-store/sqs-store-stop
|
||||
@@ -0,0 +1,215 @@
|
||||
# sqs-store-lib.sh — shared by the SQS store guest's prepare, run and stop
|
||||
# steps. Sourced, never executed. Every path and command is overridable so the
|
||||
# host test can drive the same code against a fake fw_cfg, sysfs and tools.
|
||||
#
|
||||
# Guest contract, read from fw_cfg under opt/spinifex/sqs-store/:
|
||||
# data-serial virtio-blk serial of the already-formatted ext4 data disk
|
||||
# listen-addr IPv4 address on a configured guest NIC; the only listener
|
||||
# client-nkey NATS user NKey public key allowed to connect; never a seed
|
||||
# Anything absent, malformed or ambiguous refuses the store. Nothing formats.
|
||||
|
||||
SQS_FWCFG=${SQS_FWCFG:-/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/sqs-store}
|
||||
SQS_SYSBLOCK=${SQS_SYSBLOCK:-/sys/block}
|
||||
SQS_DEVDIR=${SQS_DEVDIR:-/dev}
|
||||
# The host test cannot create block nodes, so it relaxes this to -e.
|
||||
SQS_NODE_TEST=${SQS_NODE_TEST:--b}
|
||||
SQS_RUN=${SQS_RUN:-/run/sqs-store}
|
||||
SQS_DATA=${SQS_DATA:-/data}
|
||||
SQS_CONSOLE=${SQS_CONSOLE:-/dev/console}
|
||||
SQS_MOUNTS=${SQS_MOUNTS:-/proc/mounts}
|
||||
SQS_PORT=${SQS_PORT:-4222}
|
||||
SQS_STOP_BUDGET_TENTHS=${SQS_STOP_BUDGET_TENTHS:-120}
|
||||
NATS_BIN=${NATS_BIN:-/usr/local/bin/nats-server}
|
||||
E2FSCK=${E2FSCK:-e2fsck}
|
||||
MOUNT=${MOUNT:-mount}
|
||||
UMOUNT=${UMOUNT:-umount}
|
||||
IP=${IP:-ip}
|
||||
|
||||
sqs_log() { echo "[sqs-store] $*" >> "$SQS_CONSOLE"; }
|
||||
sqs_fatal() { sqs_log "FATAL: $*"; return 1; }
|
||||
|
||||
# sqs_fwcfg <name>: print one fw_cfg value, refusing a missing, empty, oversized
|
||||
# or multi-token file. The value is not echoed on failure.
|
||||
sqs_fwcfg() {
|
||||
f="$SQS_FWCFG/$1/raw"
|
||||
[ -f "$f" ] || { sqs_fatal "fw_cfg $1 not supplied"; return 1; }
|
||||
[ "$(wc -c < "$f")" -le 128 ] || { sqs_fatal "fw_cfg $1 is oversized"; return 1; }
|
||||
v=$(cat "$f")
|
||||
case "$v" in
|
||||
"" | *[[:space:]]*) sqs_fatal "fw_cfg $1 must be exactly one value"; return 1 ;;
|
||||
esac
|
||||
printf '%s' "$v"
|
||||
}
|
||||
|
||||
# A volume serial is the volume ID with dashes stripped: vol + 17 hex.
|
||||
sqs_valid_serial() {
|
||||
[ "${#1}" -eq 20 ] || return 1
|
||||
case "$1" in vol*) ;; *) return 1 ;; esac
|
||||
case "${1#vol}" in *[!0-9a-f]*) return 1 ;; esac
|
||||
}
|
||||
|
||||
# A NATS user public NKey: "U" then 55 base32 characters. The checksum is
|
||||
# left to the pinned server's own config test.
|
||||
sqs_valid_user_nkey() {
|
||||
[ "${#1}" -eq 56 ] || return 1
|
||||
case "$1" in U*) ;; *) return 1 ;; esac
|
||||
case "$1" in *[!A-Z2-7]*) return 1 ;; esac
|
||||
}
|
||||
|
||||
sqs_valid_ipv4() {
|
||||
case "$1" in *[!0-9.]* | .* | *. | *..*) return 1 ;; esac
|
||||
old_ifs=$IFS; IFS=.
|
||||
# shellcheck disable=SC2086 # split on dots on purpose
|
||||
set -- $1
|
||||
IFS=$old_ifs
|
||||
[ "$#" -eq 4 ] || return 1
|
||||
for o in "$@"; do
|
||||
[ "${#o}" -le 3 ] && [ "$o" -le 255 ] || return 1
|
||||
case "$o" in 0?*) return 1 ;; esac
|
||||
done
|
||||
}
|
||||
|
||||
# sqs_find_disk <serial>: print the /dev node of the one block device whose
|
||||
# sysfs serial matches exactly. Zero or several matches refuse.
|
||||
sqs_find_disk() {
|
||||
found="" n=0
|
||||
for b in "$SQS_SYSBLOCK"/*; do
|
||||
[ -r "$b/serial" ] || continue
|
||||
[ "$(cat "$b/serial")" = "$1" ] || continue
|
||||
found=${b##*/}
|
||||
n=$((n + 1))
|
||||
done
|
||||
[ "$n" -eq 1 ] || { sqs_fatal "expected one disk with serial $1, found $n"; return 1; }
|
||||
test "$SQS_NODE_TEST" "$SQS_DEVDIR/$found" || { sqs_fatal "disk $found has no block device node"; return 1; }
|
||||
printf '%s' "$SQS_DEVDIR/$found"
|
||||
}
|
||||
|
||||
# sqs_has_addr <ipv4>: the address is assigned to a non-loopback interface.
|
||||
sqs_has_addr() {
|
||||
"$IP" -o -4 addr show 2>/dev/null \
|
||||
| awk -v a="$1" '$2 != "lo" { split($4, p, "/"); if (p[1] == a) f = 1 } END { exit !f }'
|
||||
}
|
||||
|
||||
# sqs_alive <pid>: running or stopped, not exited (a zombie counts as exited).
|
||||
sqs_alive() { [ -r "/proc/$1/status" ] && ! grep -q '^State:[[:space:]]*Z' "/proc/$1/status" 2>/dev/null; }
|
||||
|
||||
# sqs_starttime <pid>: kernel start time (stat field 22). It survives exec, and
|
||||
# pairing it with the pid reduces PID-reuse risk; without a pidfd the check and
|
||||
# the signal stay separate steps, so it is not an absolute guarantee.
|
||||
sqs_starttime() { sed 's/.*) //' "/proc/$1/stat" 2>/dev/null | cut -d' ' -f20; }
|
||||
|
||||
# sqs_same <pid> <starttime>: the recorded process is still alive.
|
||||
sqs_same() { [ -n "$2" ] && sqs_alive "$1" && [ "$(sqs_starttime "$1")" = "$2" ]; }
|
||||
|
||||
# sqs_prepare: validate the contract, check and mount the data disk, write the
|
||||
# server config and mark the store ready. Returns non-zero, unready, on any refusal.
|
||||
sqs_prepare() {
|
||||
rm -f "$SQS_RUN/ready"
|
||||
mkdir -p "$SQS_RUN"
|
||||
serial=$(sqs_fwcfg data-serial) || return 1
|
||||
sqs_valid_serial "$serial" || { sqs_fatal "data-serial is not a volume serial"; return 1; }
|
||||
addr=$(sqs_fwcfg listen-addr) || return 1
|
||||
sqs_valid_ipv4 "$addr" || { sqs_fatal "listen-addr is not an IPv4 address"; return 1; }
|
||||
nkey=$(sqs_fwcfg client-nkey) || return 1
|
||||
sqs_valid_user_nkey "$nkey" || { sqs_fatal "client-nkey is not a user public NKey"; return 1; }
|
||||
sqs_has_addr "$addr" || { sqs_fatal "listen-addr $addr is not on any guest NIC"; return 1; }
|
||||
|
||||
# The server checks the config, including the NKey checksum, before the
|
||||
# disk is touched.
|
||||
umask 077
|
||||
cat > "$SQS_RUN/nats.conf" <<EOF
|
||||
listen: "$addr:$SQS_PORT"
|
||||
server_name: "sqs-store"
|
||||
jetstream {
|
||||
store_dir: "$SQS_DATA"
|
||||
sync_interval: always
|
||||
}
|
||||
authorization {
|
||||
users = [ { nkey: "$nkey" } ]
|
||||
}
|
||||
EOF
|
||||
if ! "$NATS_BIN" -t -c "$SQS_RUN/nats.conf" > "$SQS_RUN/config-test.out" 2>&1; then
|
||||
sqs_fatal "server config test failed: $(sed 's/U[A-Z2-7]\{55\}/<nkey>/g' "$SQS_RUN/config-test.out" | tr '\n' ' ')"
|
||||
return 1
|
||||
fi
|
||||
|
||||
dev=$(sqs_find_disk "$serial") || return 1
|
||||
rc=0
|
||||
"$E2FSCK" -p "$dev" > "$SQS_RUN/fsck.out" 2>&1 || rc=$?
|
||||
sqs_log "e2fsck -p $dev rc=$rc"
|
||||
# 0 clean, 1 corrected: proceed. Anything higher, including a disk that is
|
||||
# not ext4 at all, refuses; the guest never formats.
|
||||
[ "$rc" -le 1 ] || { sqs_fatal "e2fsck refused $dev: $(tr '\n' ' ' < "$SQS_RUN/fsck.out")"; return 1; }
|
||||
mkdir -p "$SQS_DATA"
|
||||
"$MOUNT" -t ext4 -o nodev,nosuid,noexec "$dev" "$SQS_DATA" || { sqs_fatal "mount $dev failed"; return 1; }
|
||||
: > "$SQS_RUN/ready"
|
||||
sqs_log "prepared: $dev mounted at $SQS_DATA, config written for $addr:$SQS_PORT; server not started yet"
|
||||
}
|
||||
|
||||
# sqs_run: the respawned supervisor body. Its identity is recorded before the
|
||||
# stopping check: a stop that misses it set stopping first, so this parks; a
|
||||
# stop that reads it signals this process whether or not it has exec'd yet.
|
||||
sqs_run() {
|
||||
mkdir -p "$SQS_RUN"
|
||||
echo "$$ $(sqs_starttime $$)" > "$SQS_RUN/pid"
|
||||
if [ -e "$SQS_RUN/stopping" ]; then
|
||||
sqs_log "run: shutdown in progress, not starting"
|
||||
return 2
|
||||
fi
|
||||
if [ ! -e "$SQS_RUN/ready" ]; then
|
||||
sqs_log "run: store not prepared, not starting"
|
||||
return 2
|
||||
fi
|
||||
sqs_log "run: starting pid=$$"
|
||||
exec "$NATS_BIN" -c "$SQS_RUN/nats.conf"
|
||||
}
|
||||
|
||||
# sqs_stop: first shutdown action. Block any respawn, SIGTERM the recorded run
|
||||
# process (a server, or a start not yet exec'd), SIGKILL only after the budget,
|
||||
# then sync and unmount. Each signal rechecks the identity; a survivor keeps the disk mounted.
|
||||
sqs_stop() {
|
||||
mkdir -p "$SQS_RUN"
|
||||
: > "$SQS_RUN/stopping"
|
||||
sqs_log "stop: begin"
|
||||
rec=$(cat "$SQS_RUN/pid" 2>/dev/null || true)
|
||||
pid=${rec%% *} st=${rec#* }
|
||||
[ "$st" != "$rec" ] || st=""
|
||||
if [ -n "$pid" ] && sqs_same "$pid" "$st"; then
|
||||
kill -TERM "$pid" 2>/dev/null || true
|
||||
sqs_log "stop: SIGTERM sent to run pid=$pid"
|
||||
i=0
|
||||
while sqs_same "$pid" "$st" && [ "$i" -lt "$SQS_STOP_BUDGET_TENTHS" ]; do sleep 0.1; i=$((i + 1)); done
|
||||
if sqs_same "$pid" "$st"; then
|
||||
sqs_log "stop: run pid=$pid still running after $((SQS_STOP_BUDGET_TENTHS / 10))s, SIGKILL"
|
||||
kill -KILL "$pid" 2>/dev/null || true
|
||||
j=0
|
||||
while sqs_same "$pid" "$st" && [ "$j" -lt 30 ]; do sleep 0.1; j=$((j + 1)); done
|
||||
fi
|
||||
if sqs_same "$pid" "$st"; then
|
||||
sqs_log "stop: run pid=$pid STILL ALIVE, leaving $SQS_DATA mounted"
|
||||
return 1
|
||||
fi
|
||||
sqs_log "stop: store exited after $((i / 10)).$((i % 10))s"
|
||||
else
|
||||
sqs_log "stop: no running store"
|
||||
fi
|
||||
sync
|
||||
if grep -q " $SQS_DATA " "$SQS_MOUNTS" 2>/dev/null; then
|
||||
rc=0
|
||||
"$UMOUNT" "$SQS_DATA" || rc=$?
|
||||
sqs_log "stop: umount $SQS_DATA rc=$rc"
|
||||
fi
|
||||
}
|
||||
|
||||
# sqs_shutdown: the only shutdown action. The generic read-only remount runs
|
||||
# only after the store stopped; a surviving writer keeps every mount as it is.
|
||||
sqs_shutdown() {
|
||||
if ! sqs_stop; then
|
||||
sync
|
||||
sqs_log "shutdown: store survived, skipping umount -a -r"
|
||||
return 1
|
||||
fi
|
||||
rc=0
|
||||
"$UMOUNT" -a -r || rc=$?
|
||||
echo "[init] shutdown: umount -a -r rc=$rc" >> "$SQS_CONSOLE"
|
||||
}
|
||||
Executable
+9
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
# sysinit: load the disk drivers, then validate the guest contract, check and
|
||||
# mount the data disk and write the server config. A refusal leaves the store
|
||||
# unprepared, so the run step never starts it.
|
||||
. /usr/local/lib/sqs-store/sqs-store-lib.sh
|
||||
modprobe virtio_blk 2>/dev/null || true
|
||||
modprobe ext4 2>/dev/null || true
|
||||
sqs_prepare || sqs_log "store refused; it will not start"
|
||||
exit 0
|
||||
Executable
+6
@@ -0,0 +1,6 @@
|
||||
#!/bin/sh
|
||||
# Respawned by init. A store that must not run parks here instead of exiting,
|
||||
# so init does not respawn it in a tight loop.
|
||||
. /usr/local/lib/sqs-store/sqs-store-lib.sh
|
||||
sqs_run
|
||||
exec sleep 2147483647
|
||||
Executable
+5
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
# The only shutdown action: stop the store and unmount its disk, then run the
|
||||
# generic read-only remount only if the store is gone.
|
||||
. /usr/local/lib/sqs-store/sqs-store-lib.sh
|
||||
sqs_shutdown
|
||||
+51
-1
@@ -243,6 +243,22 @@ Hot-attaching a volume to a running direct-boot microVM is not supported, becaus
|
||||
|
||||
This covers only requests that are already mounted when the instance starts; nothing yet supplies data volumes to a system instance, and the path is verified with fake-QEMU unit tests, not with a real EBS volume in a guest.
|
||||
|
||||
#### Volume leases
|
||||
|
||||
Before opening an engine on a volume, the Viperblock worker takes a lease: one entry per volume in the `VIPERBLOCK_VOLUME_LEASES` JetStream KV bucket, which expires 45 seconds after its last write (`providers/ebs/viperblock/volume_lease.go`).
|
||||
A renewal becomes eligible once the last confirmed write is 10 seconds old, and the holder checks for that periodically, so the rewrite lands some time after that point.
|
||||
Each rewrite is conditioned on the revision the holder last wrote, and a refused write that finds the entry gone or naming another owner or generation fences the export as a takeover.
|
||||
It may keep writing for 40 seconds after the last write the store acknowledged, measured from when that write was sent, because the server starts the entry's expiry when it applies the write and never earlier.
|
||||
A slow acknowledgement therefore shortens the remaining validity rather than extending it, and an acquisition acknowledged after its validity has already run out is refused as retryable.
|
||||
|
||||
For a holder that cannot renew, a timer is armed for the validity deadline, leaving a nominal 5 seconds before the entry can expire; when it fires, the export's nbdkit is fenced with SIGKILL (`volume_fence.go`).
|
||||
The fence runs on its own goroutine, so Go scheduling, that goroutine and process termination all delay the kill, and enough delay can use up the whole margin; a kill held in uninterruptible sleep, which the fence waits up to 30 seconds for, is the extreme case.
|
||||
Once a lease is lost or has lapsed it stays lost: a renewal that succeeds afterwards is ignored, and another open on the same node cannot share it while the fence runs.
|
||||
|
||||
These rules bound only the holder's own clock.
|
||||
The storage backend does not reject a stale writer, so a paused or partitioned host that resumes after its validity can still write; generation enforcement at the write boundary is required by Q-24 in the umbrella repository's `docs/specs/viperblock/rfc3-1_overview.md` and is not implemented.
|
||||
An export that survives a daemon restart writes without a renewed lease until recovery adopts it, and a survivor that recovery cannot adopt while a guest still uses it is left running without a lease or a fence.
|
||||
|
||||
### Predastore (S3)
|
||||
|
||||
Object storage used for:
|
||||
@@ -274,7 +290,7 @@ storage-provider contract.
|
||||
|
||||
`domains/sqs/message` defines the store-neutral interface the SQS domain reaches its message store through (ADR-0011:S6), with a JetStream adapter in `domains/sqs/message/jetstream` and a shared conformance suite in `domains/sqs/message/conformance`.
|
||||
It is an internal primitive: nothing outside the SQS domain uses it, no daemon or service is wired to it, and it does not itself deliver an SQS API.
|
||||
The dedicated JetStream server ADR-0011 describes is not deployed by this code, and Spinifex ships no host service, unit, user or firewall rule for it.
|
||||
The dedicated JetStream server ADR-0011 describes is not deployed by this code, and Spinifex ships no host service, unit, user or firewall rule for it; the guest that could run it is described under *SQS store system instance (not wired)*.
|
||||
|
||||
The interface covers durable and idempotent append, body and metadata reads, conditional state writes and deletes, per-namespace counts, namespace enumeration, an ordered change feed, a bounded paginated scan, namespace deletion and a sweep of tombstoned bodies.
|
||||
A namespace carries an immutable message-shard and state-shard allocation chosen when the queue is created, so a later change in shard counts never moves an existing queue.
|
||||
@@ -293,6 +309,40 @@ If that consumer delivers nothing for about two seconds while the stream holds a
|
||||
|
||||
Coverage is unit and conformance tests on in-process R1 and R3 servers; there is no end-to-end or multi-host validation of this code, and earlier multi-host evidence predates the final feed.
|
||||
|
||||
### SQS store system instance (not wired)
|
||||
|
||||
Spinifex can build, install, launch, restart and give storage to a dedicated SQS store microVM, but nothing calls any of it.
|
||||
No SQS API, domain service, admin command or daemon start-up path creates a store, so none of this is in production use.
|
||||
|
||||
The guest image is built by `make build-sqs-store-microvm-image`, separate from the load balancer image, and bundles the `nats-server` version pinned in `go.mod` plus the ext4 tools.
|
||||
Release tarballs carry it as `microvm-sqs-store/`, and `setup.sh` installs it to `/usr/share/spinifex/microvm/sqs-store/`.
|
||||
A release without a complete image removes any previously installed one, so a launch is then refused rather than booting a stale image.
|
||||
|
||||
At boot the guest reads three firmware config values: the data disk's serial, an IPv4 address on one of its interfaces, and one NATS user NKey public key.
|
||||
It mounts exactly one matching disk at `/data` after `e2fsck -p`, refuses a missing, duplicate, unformatted or unrepairable disk, and never formats.
|
||||
JetStream keeps its data on `/data` with `sync_interval: always` (ADR-0011:S3), and the server listens only on that address and accepts only that key.
|
||||
On shutdown the store is stopped before `/data` is unmounted; a store that survives SIGKILL leaves every mount in place.
|
||||
|
||||
A launch uses the versioned subject `system.LaunchSQSStore.v1`, which a daemon without this code does not serve, so it fails with no responder instead of booting another guest.
|
||||
The legacy `system.LaunchInstance` path refuses the store kind on both send and receive.
|
||||
Every input check runs before capacity is reserved, and the data volume must be an available system-account volume.
|
||||
It is mounted through the normal EBS path and detached, never deleted, on terminate, which is the protection standard EBS gives today and nothing stronger.
|
||||
CPU mitigations stay enabled for the store, unlike the load balancer.
|
||||
A successful launch means the VM is running, not that NATS is ready.
|
||||
|
||||
The instance record keeps the store's non-secret guest settings so a restart on the node that launched it rewrites the firmware config files.
|
||||
A relaunch on any other node is refused, and cross-node recovery never claims a store.
|
||||
Both checks exist only in daemons with this code: an older daemon can still claim and relaunch a store elsewhere, and an explicit start served by another node claims the instance before its relaunch is refused.
|
||||
|
||||
`domains/sqs/storevolume` prepares a fresh data volume, because the guest never formats.
|
||||
It checks for `mkfs.ext4` and `qemu-img` before allocating, creates the volume in `creating`, writes an empty ext4 image to it over NBD on the local node, unpublishes it, and only then makes it `available`.
|
||||
While a volume is `creating`, attach, delete, modify, force-detach and snapshot are refused.
|
||||
A volume is deleted only after a confirmed unpublish; any publish error, an unconfirmed unpublish or a failed release keeps it and reports its ID, with its state reported as unknown after a failed release.
|
||||
|
||||
Evidence is local only: unit tests, real QEMU boots against disposable scratch disks, and one run of the preparer against the `qemunbdd` provider whose volume kept acknowledged writes across a reboot.
|
||||
Viperblock, volume encryption, the EC2 volume record path, multiple nodes and host reachability of the guest address are not tested.
|
||||
Credential issuance and rotation, store placement and NATS readiness checks do not exist, and there is no failover: a lost node leaves its store unavailable.
|
||||
|
||||
## Configuration
|
||||
|
||||
Cluster configuration (`spinifex/bootstrap/config/config.go`):
|
||||
|
||||
@@ -33,6 +33,7 @@ require (
|
||||
github.com/mulgadc/viperblock v1.23.0
|
||||
github.com/nats-io/nats-server/v2 v2.15.1
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
github.com/nats-io/nkeys v0.4.16
|
||||
github.com/opencontainers/runtime-spec v1.3.0
|
||||
github.com/oracle/oci-go-sdk/v65 v65.128.0
|
||||
github.com/ovn-kubernetes/libovsdb v0.9.0
|
||||
@@ -161,7 +162,6 @@ require (
|
||||
github.com/muesli/termenv v0.16.0 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.2 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||
github.com/opencontainers/image-spec v1.1.1 // indirect
|
||||
|
||||
Executable
+475
@@ -0,0 +1,475 @@
|
||||
#!/bin/bash
|
||||
# build-sqs-store-microvm-image.sh — build the SQS store microVM kernel +
|
||||
# initramfs: the dedicated nats-server, ext4 support, the store's guest steps
|
||||
# and a clean power-down. The guest never formats a disk; build/microvm/
|
||||
# sqs-store/sqs-store-lib.sh states its fw_cfg contract. The LB image
|
||||
# (build-microvm-image.sh) is separate and unchanged.
|
||||
#
|
||||
# Inputs:
|
||||
# $REPO_ROOT/build/microvm/sqs-store/init.sh, inittab
|
||||
# $REPO_ROOT/build/microvm/sqs-store/sqs-store-{lib.sh,prepare,run,stop}
|
||||
# $REPO_ROOT/bin/nats-server (make bin/nats-server, the go.mod-pinned version)
|
||||
#
|
||||
# Outputs:
|
||||
# $REPO_ROOT/build/microvm/sqs-store/vmlinuz
|
||||
# $REPO_ROOT/build/microvm/sqs-store/initramfs.cpio.gz
|
||||
#
|
||||
# Rootfs strategy (first match wins):
|
||||
# 1. apk available on host → install directly into a temp chroot (Alpine CI/CD)
|
||||
# 2. docker available → run alpine container, export filesystem
|
||||
# 3. podman available → same via podman
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
BUILD_DIR="$REPO_ROOT/build/microvm/sqs-store"
|
||||
INIT_SH="$BUILD_DIR/init.sh"
|
||||
INITTAB="$BUILD_DIR/inittab"
|
||||
NATS_SERVER_BIN="$REPO_ROOT/bin/nats-server"
|
||||
|
||||
ALPINE_VERSION="${ALPINE_VERSION:-3.24}"
|
||||
ALPINE_MIRROR="${ALPINE_MIRROR:-https://dl-cdn.alpinelinux.org/alpine}"
|
||||
# e2fsprogs: mkfs.ext4/e2fsck for a guest that mounts an ext4 data volume.
|
||||
# kmod: the image's own depmod/modprobe rebuild module metadata below.
|
||||
ALPINE_PACKAGES="busybox linux-virt iproute2 ca-certificates e2fsprogs kmod"
|
||||
# Modules kept by name on top of the virtio/fw_cfg/failover globs, each with its
|
||||
# full depends= and softdep closure. tiny_power_button turns the ACPI power
|
||||
# button (QMP system_powerdown) into a signal busybox init handles as poweroff.
|
||||
MICROVM_EXTRA_MODULES="${MICROVM_EXTRA_MODULES:-ext4 tiny_power_button}"
|
||||
|
||||
echo "[build-sqs-store-microvm-image] repo root: $REPO_ROOT"
|
||||
echo "[build-sqs-store-microvm-image] build dir: $BUILD_DIR"
|
||||
mkdir -p "$BUILD_DIR"
|
||||
|
||||
# --- Verify non-negotiable tools ---
|
||||
# fakeroot is mandatory: device nodes are written into the cpio archive inside a
|
||||
# faked-root environment so no real device node is ever created on the host
|
||||
# filesystem (which previously required sudo mknod and risked clobbering host
|
||||
# /dev/null on cleanup).
|
||||
for tool in cpio gzip find fakeroot; do
|
||||
if ! command -v "$tool" >/dev/null 2>&1; then
|
||||
echo "ERROR: required tool not found: $tool" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# --- Verify the dedicated nats-server is the go.mod-pinned build ---
|
||||
# A stale or host-installed binary would ship an unreviewed server version.
|
||||
NATS_SERVER_PIN=$(awk '$1 == "github.com/nats-io/nats-server/v2" { print $2; exit }' "$REPO_ROOT/go.mod")
|
||||
if [ -z "$NATS_SERVER_PIN" ]; then
|
||||
echo "ERROR: no github.com/nats-io/nats-server/v2 requirement in $REPO_ROOT/go.mod" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ ! -x "$NATS_SERVER_BIN" ]; then
|
||||
echo "ERROR: $NATS_SERVER_BIN not found — run 'make bin/nats-server'" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! file -b "$NATS_SERVER_BIN" | grep -q "statically linked"; then
|
||||
echo "ERROR: $NATS_SERVER_BIN is not statically linked — it cannot run on the musl guest" >&2
|
||||
exit 1
|
||||
fi
|
||||
nats_server_version=$("$NATS_SERVER_BIN" --version 2>&1 | awk '{ print $NF; exit }')
|
||||
if [ "$nats_server_version" != "$NATS_SERVER_PIN" ]; then
|
||||
echo "ERROR: $NATS_SERVER_BIN reports $nats_server_version, go.mod pins $NATS_SERVER_PIN" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "[build-sqs-store-microvm-image] nats-server: $nats_server_version (matches go.mod)"
|
||||
|
||||
# --- Create chroot ---
|
||||
CHROOT_DIR=$(mktemp -d)
|
||||
# Guard: every destructive op below targets "$CHROOT_DIR/...". An empty or
|
||||
# unexpected value would aim rm -rf at the host (e.g. /dev). Refuse anything that
|
||||
# is not a freshly-created temp dir.
|
||||
case "${CHROOT_DIR:?mktemp -d returned empty}" in
|
||||
/tmp/*|/var/tmp/*|"${TMPDIR:-/nonexistent}"/*) ;;
|
||||
*) echo "ERROR: refusing to use unexpected CHROOT_DIR: $CHROOT_DIR" >&2; exit 1 ;;
|
||||
esac
|
||||
# Outputs are written here first, beside $BUILD_DIR so publishing is a rename.
|
||||
STAGE_DIR=$(mktemp -d "$BUILD_DIR/.stage.XXXXXX")
|
||||
CONTAINER_TOOL=""
|
||||
CONTAINER_CID=""
|
||||
cleanup() {
|
||||
echo "[build-sqs-store-microvm-image] cleaning up chroot: $CHROOT_DIR"
|
||||
rm -rf "$CHROOT_DIR"
|
||||
rm -rf "${STAGE_DIR:?}"
|
||||
if [ -n "$CONTAINER_CID" ] && [ -n "$CONTAINER_TOOL" ]; then
|
||||
echo "[build-sqs-store-microvm-image] cleaning up container: $CONTAINER_CID"
|
||||
$CONTAINER_TOOL rm -f "$CONTAINER_CID" >/dev/null 2>&1 || true
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
# --- Populate rootfs ---
|
||||
# Strategy 1: native apk (Alpine hosts, CI)
|
||||
if command -v apk >/dev/null 2>&1; then
|
||||
echo "[build-sqs-store-microvm-image] rootfs: native apk"
|
||||
mkdir -p \
|
||||
"$CHROOT_DIR/etc/apk" \
|
||||
"$CHROOT_DIR/lib/apk/db" \
|
||||
"$CHROOT_DIR/var/cache/apk"
|
||||
cat > "$CHROOT_DIR/etc/apk/repositories" <<EOF
|
||||
${ALPINE_MIRROR}/v${ALPINE_VERSION}/main
|
||||
${ALPINE_MIRROR}/v${ALPINE_VERSION}/community
|
||||
EOF
|
||||
apk add \
|
||||
--root "$CHROOT_DIR" \
|
||||
--arch x86_64 \
|
||||
--no-cache \
|
||||
--allow-untrusted \
|
||||
--initdb \
|
||||
$ALPINE_PACKAGES
|
||||
|
||||
# Strategy 2: docker -> BuildKit local export.
|
||||
# Do NOT use `docker export` here. On Docker's overlay2 driver it leaks the
|
||||
# container's overlay mount (the mount survives `docker rm`), orphaning layers
|
||||
# under /var/lib/docker that only a daemon restart reclaims; on a busy CI host
|
||||
# this grows without bound. `buildx --output type=local` writes the image
|
||||
# filesystem straight to a directory and releases every mount, so nothing leaks.
|
||||
elif command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then
|
||||
echo "[build-sqs-store-microvm-image] rootfs: docker buildx --output type=local (alpine:${ALPINE_VERSION})"
|
||||
BUILD_CTX=$(mktemp -d)
|
||||
cat > "$BUILD_CTX/Dockerfile" <<DOCKERFILE
|
||||
FROM alpine:${ALPINE_VERSION}
|
||||
RUN apk add --no-cache ${ALPINE_PACKAGES}
|
||||
DOCKERFILE
|
||||
docker buildx build \
|
||||
--platform linux/amd64 \
|
||||
--output "type=local,dest=${CHROOT_DIR}" \
|
||||
"$BUILD_CTX"
|
||||
rm -rf "$BUILD_CTX"
|
||||
|
||||
# Strategy 3: podman -> run + export. Podman's store does not exhibit the docker
|
||||
# overlay2 export-mount leak, and podman has no buildx.
|
||||
elif command -v podman >/dev/null 2>&1; then
|
||||
echo "[build-sqs-store-microvm-image] rootfs: podman export (alpine:${ALPINE_VERSION})"
|
||||
CONTAINER_TOOL=podman
|
||||
CONTAINER_CID=$(podman run -d \
|
||||
"alpine:${ALPINE_VERSION}" \
|
||||
sh -c "apk add --no-cache ${ALPINE_PACKAGES}")
|
||||
|
||||
exit_code=$(podman wait "$CONTAINER_CID")
|
||||
if [ "$exit_code" != "0" ]; then
|
||||
echo "ERROR: package installation failed in container (exit $exit_code)" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
podman export "$CONTAINER_CID" | tar -x -C "$CHROOT_DIR"
|
||||
|
||||
else
|
||||
echo "ERROR: no rootfs build tool available." >&2
|
||||
echo " Install one of: apk (Alpine), docker, podman" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# --- /dev: empty mountpoint only ---
|
||||
# tar extraction without root silently creates 0-byte regular files for device
|
||||
# nodes. Drop them and leave an empty /dev mountpoint. The two static nodes the
|
||||
# kernel needs (/dev/console to wire init's stdio, /dev/null) are written into
|
||||
# the cpio archive under fakeroot at pack time — NOT created on the host. init
|
||||
# mounts devtmpfs over /dev as its first action for everything else at runtime.
|
||||
rm -rf "${CHROOT_DIR:?}/dev"
|
||||
mkdir "$CHROOT_DIR/dev"
|
||||
|
||||
# --- Place init script ---
|
||||
echo "[build-sqs-store-microvm-image] installing init.sh as /init..."
|
||||
if [ ! -f "$INIT_SH" ]; then
|
||||
echo "ERROR: $INIT_SH not found — cannot build initramfs" >&2
|
||||
exit 1
|
||||
fi
|
||||
install -m 0755 "$INIT_SH" "$CHROOT_DIR/init"
|
||||
|
||||
# --- Place inittab ---
|
||||
# Required: without it busybox init runs its default table, not the power-down.
|
||||
if [ ! -f "$INITTAB" ]; then
|
||||
echo "ERROR: $INITTAB not found — cannot build initramfs" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "[build-sqs-store-microvm-image] installing inittab..."
|
||||
install -m 0644 "$INITTAB" "$CHROOT_DIR/etc/inittab"
|
||||
|
||||
# --- Place the store's guest steps ---
|
||||
echo "[build-sqs-store-microvm-image] installing store guest steps..."
|
||||
mkdir -p "$CHROOT_DIR/usr/local/lib/sqs-store"
|
||||
for step in sqs-store-lib.sh sqs-store-prepare sqs-store-run sqs-store-stop; do
|
||||
if [ ! -f "$BUILD_DIR/$step" ]; then
|
||||
echo "ERROR: $BUILD_DIR/$step not found — cannot build initramfs" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
install -m 0644 "$BUILD_DIR/sqs-store-lib.sh" "$CHROOT_DIR/usr/local/lib/sqs-store/"
|
||||
install -m 0755 "$BUILD_DIR/sqs-store-prepare" "$BUILD_DIR/sqs-store-run" "$BUILD_DIR/sqs-store-stop" \
|
||||
"$CHROOT_DIR/usr/local/lib/sqs-store/"
|
||||
|
||||
# --- Copy nats-server binary ---
|
||||
echo "[build-sqs-store-microvm-image] copying nats-server binary..."
|
||||
mkdir -p "$CHROOT_DIR/usr/local/bin"
|
||||
install -m 0755 "$NATS_SERVER_BIN" "$CHROOT_DIR/usr/local/bin/nats-server"
|
||||
|
||||
# --- Create required directories for init ---
|
||||
mkdir -p \
|
||||
"$CHROOT_DIR/proc" \
|
||||
"$CHROOT_DIR/sys" \
|
||||
"$CHROOT_DIR/dev" \
|
||||
"$CHROOT_DIR/etc/ssl/certs"
|
||||
|
||||
# --- Locate kernel ---
|
||||
KERNEL_IMG=$(find "$CHROOT_DIR/boot" -name "vmlinuz*" | sort | tail -1)
|
||||
if [ -z "$KERNEL_IMG" ]; then
|
||||
echo "ERROR: no vmlinuz found in $CHROOT_DIR/boot" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "[build-sqs-store-microvm-image] kernel image: $KERNEL_IMG"
|
||||
|
||||
KERNEL_VER=$(basename "$KERNEL_IMG" | sed 's/vmlinuz-//')
|
||||
KERNEL_CONFIG="$CHROOT_DIR/boot/config-${KERNEL_VER}"
|
||||
|
||||
# --- Assert fw_cfg module ---
|
||||
if ! find "$CHROOT_DIR/lib/modules" \
|
||||
-name "qemu_fw_cfg.ko*" \
|
||||
-o -name "fw_cfg_sysfs.ko*" \
|
||||
| grep -q .; then
|
||||
if ! grep -q "CONFIG_FW_CFG_SYSFS=y" "$KERNEL_CONFIG" 2>/dev/null; then
|
||||
echo "ERROR: qemu_fw_cfg module missing from initramfs — fw_cfg reads will fail at boot" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "[build-sqs-store-microvm-image] fw_cfg: built-in (CONFIG_FW_CFG_SYSFS=y)"
|
||||
else
|
||||
echo "[build-sqs-store-microvm-image] fw_cfg: module found in lib/modules"
|
||||
fi
|
||||
|
||||
# --- Assert ext4 userspace tools (data volume) ---
|
||||
for fs_tool in mkfs.ext4 e2fsck; do
|
||||
if [ ! -x "$CHROOT_DIR/sbin/$fs_tool" ] && [ ! -x "$CHROOT_DIR/usr/sbin/$fs_tool" ]; then
|
||||
echo "ERROR: $fs_tool missing from image — guest cannot check or create an ext4 data volume" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
echo "[build-sqs-store-microvm-image] e2fsprogs: mkfs.ext4 + e2fsck present"
|
||||
|
||||
# --- Strip kernel modules (Phase 2 keep-list approach) ---
|
||||
# Only modules needed by a virtio-mmio microvm guest are kept:
|
||||
# virtio*.ko* — virtio bus, net, blk, rng, console, and helpers
|
||||
# *fw_cfg*.ko* — QEMU fw_cfg sysfs driver (delivers boot blobs)
|
||||
# *failover*.ko* — net_failover + failover (virtio_net deps for live migration)
|
||||
# $MICROVM_EXTRA_MODULES — ext4 by default, plus dependency closure
|
||||
# All other loadable modules are removed. Drivers compiled into the kernel
|
||||
# (CONFIG_VIRTIO_MMIO=y etc.) are unaffected — they live in vmlinuz, not here.
|
||||
|
||||
# module_path prints a module's kernel/ path from the stock modules.dep, by
|
||||
# name or modules.alias; it fails when the module is not loadable.
|
||||
module_path() {
|
||||
local kdir="$1" name="${2//-/_}" path alias_target
|
||||
path=$(awk -F: -v n="$name" '{ f = $1; sub(/.*\//, "", f); sub(/\.ko(\.gz|\.xz|\.zst)?$/, "", f);
|
||||
gsub(/-/, "_", f); if (f == n) { print $1; exit } }' "$kdir/modules.dep")
|
||||
if [ -z "$path" ] && [ -f "$kdir/modules.alias" ]; then
|
||||
alias_target=$(awk -v n="$2" '$1 == "alias" && $2 == n { print $3; exit }' "$kdir/modules.alias")
|
||||
[ -n "$alias_target" ] && [ "$alias_target" != "$2" ] && path=$(module_path "$kdir" "$alias_target")
|
||||
fi
|
||||
[ -n "$path" ] && printf '%s\n' "$path"
|
||||
}
|
||||
|
||||
# module_builtin reports whether a module name is compiled into the kernel.
|
||||
module_builtin() {
|
||||
local kdir="$1" name="${2//-/_}"
|
||||
[ -f "$kdir/modules.builtin" ] || return 1
|
||||
awk -v n="$name" '{ f = $0; sub(/.*\//, "", f); sub(/\.ko$/, "", f); gsub(/-/, "_", f);
|
||||
if (f == n) { found = 1; exit } } END { exit !found }' "$kdir/modules.builtin"
|
||||
}
|
||||
|
||||
# image_kmod runs the image's own kmod (depmod, modprobe) through its musl
|
||||
# loader, so module metadata matches the guest's modprobe without a host kmod.
|
||||
image_kmod() {
|
||||
"$CHROOT_DIR/lib/ld-musl-x86_64.so.1" --library-path "$CHROOT_DIR/lib:$CHROOT_DIR/usr/lib" \
|
||||
--argv0 "$1" "$CHROOT_DIR/bin/kmod" "${@:2}"
|
||||
}
|
||||
|
||||
echo "[build-sqs-store-microvm-image] stripping kernel modules to virtio+fw_cfg+${MICROVM_EXTRA_MODULES} only..."
|
||||
for kver_dir in "$CHROOT_DIR/lib/modules"/*/; do
|
||||
kver="$(basename "$kver_dir")"
|
||||
kernel_dir="$kver_dir/kernel"
|
||||
[ -d "$kernel_dir" ] || continue
|
||||
|
||||
if [ ! -s "$kver_dir/modules.dep" ]; then
|
||||
echo "ERROR: stock ${kver_dir}modules.dep missing — cannot resolve module dependencies" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Extra modules start the closure; the glob families are kept as found.
|
||||
keep_list=$(mktemp)
|
||||
: > "$keep_list"
|
||||
for name in $MICROVM_EXTRA_MODULES; do
|
||||
if ! path=$(module_path "$kver_dir" "$name"); then
|
||||
if module_builtin "$kver_dir" "$name"; then continue; fi
|
||||
echo "ERROR: module $name is neither loadable nor built into $kver" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "$path" >> "$keep_list"
|
||||
done
|
||||
|
||||
# Close over modules.dep hard dependencies and modules.softdep pre:/post:
|
||||
# names until the set stops growing. A softdep must be loadable or built in.
|
||||
while :; do
|
||||
before=$(sort -u "$keep_list" | wc -l)
|
||||
for path in $(sort -u "$keep_list"); do
|
||||
awk -F': ' -v p="$path" '$1 == p { n = split($2, d, " "); for (i = 1; i <= n; i++) print d[i] }' \
|
||||
"$kver_dir/modules.dep" >> "$keep_list"
|
||||
mod=$(basename "$path"); mod="${mod%%.ko*}"; mod="${mod//-/_}"
|
||||
[ -f "$kver_dir/modules.softdep" ] || continue
|
||||
for soft in $(awk -v n="$mod" '{ m = $2; gsub(/-/, "_", m) }
|
||||
$1 == "softdep" && m == n { for (i = 3; i <= NF; i++) if ($i != "pre:" && $i != "post:") print $i }' \
|
||||
"$kver_dir/modules.softdep"); do
|
||||
if soft_path=$(module_path "$kver_dir" "$soft"); then
|
||||
echo "$soft_path" >> "$keep_list"
|
||||
elif ! module_builtin "$kver_dir" "$soft"; then
|
||||
echo "ERROR: softdep $soft of $mod is neither loadable nor built into $kver" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
done
|
||||
after=$(sort -u "$keep_list" | wc -l)
|
||||
[ "$before" -eq "$after" ] && break
|
||||
done
|
||||
|
||||
extra_names=$(sort -u "$keep_list" | while read -r p; do m=$(basename "$p"); m="${m%%.ko*}"; echo "${m//-/_}"; done)
|
||||
(cd "$kver_dir" && find kernel -name "virtio*.ko*" -o -name "*fw_cfg*.ko*" \
|
||||
-o -name "*failover*.ko*") >> "$keep_list"
|
||||
|
||||
# Copy the keeper set out, preserving each kernel/ relative path.
|
||||
tmp_keep=$(mktemp -d)
|
||||
while read -r rel; do
|
||||
rel="${rel#kernel/}"
|
||||
mkdir -p "$tmp_keep/$(dirname "$rel")"
|
||||
cp "$kernel_dir/$rel" "$tmp_keep/$rel"
|
||||
done < <(sort -u "$keep_list")
|
||||
rm -f "$keep_list"
|
||||
|
||||
# Replace kernel module tree with the keeper set.
|
||||
rm -rf "$kernel_dir"
|
||||
mkdir -p "$kernel_dir"
|
||||
# Restore each kept file into the same relative path.
|
||||
(cd "$tmp_keep" && find . -name "*.ko*" | while read -r f; do
|
||||
dest_dir="$kernel_dir/$(dirname "${f#./}")"
|
||||
mkdir -p "$dest_dir"
|
||||
cp "$f" "$dest_dir/"
|
||||
done)
|
||||
rm -rf "$tmp_keep"
|
||||
|
||||
# Decompress kept modules to plain .ko. Alpine ships .ko.gz, but a host
|
||||
# kmod that lacks gzip support (e.g. Debian trixie's kmod 34) silently
|
||||
# produces an EMPTY modules.dep from compressed modules — the guest's
|
||||
# modprobe then no-ops and qemu_fw_cfg never loads. Plain .ko sidesteps the
|
||||
# host-kmod dependency entirely and the guest needs no decompressor either.
|
||||
find "$kernel_dir" -name "*.ko.gz" -exec gunzip -f {} +
|
||||
|
||||
# Independent closure gate: each extra-closure module's own .modinfo depends=
|
||||
# must name a kept module or a built-in, so a resolver bug fails here.
|
||||
kept_names=$(find "$kernel_dir" -name "*.ko" -exec basename {} .ko \; | tr '-' '_' | sort -u)
|
||||
while read -r ko; do
|
||||
ko_name=$(basename "$ko" .ko); ko_name="${ko_name//-/_}"
|
||||
grep -qx "$ko_name" <<< "$extra_names" || continue
|
||||
deps=$(tr '\0' '\n' < "$ko" | grep -a -m1 '^depends=' | cut -d= -f2 | tr ',' ' ' || true)
|
||||
for dep in $deps; do
|
||||
dep="${dep//-/_}"
|
||||
if ! grep -qx "$dep" <<< "$kept_names" && ! module_builtin "$kver_dir" "$dep"; then
|
||||
echo "ERROR: $(basename "$ko") depends on $dep, which is neither kept nor built in" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
done < <(find "$kernel_dir" -name "*.ko")
|
||||
for name in $MICROVM_EXTRA_MODULES; do
|
||||
if ! grep -qx "${name//-/_}" <<< "$kept_names" && ! module_builtin "$kver_dir" "$name"; then
|
||||
echo "ERROR: required module $name missing from the stripped module tree" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
echo "[build-sqs-store-microvm-image] extra module closure: $(echo "$extra_names" | tr '\n' ' ')"
|
||||
echo "[build-sqs-store-microvm-image] modules kept: $(echo "$kept_names" | tr '\n' ' ')"
|
||||
|
||||
# Rebuild module metadata with the image's own kmod, the binary the guest's
|
||||
# modprobe runs, so no host depmod is needed. Stock indexes still name .ko.gz
|
||||
# files, so they are removed first and a failed rebuild cannot ship them.
|
||||
for f in dep dep.bin alias alias.bin softdep weakdep symbols symbols.bin \
|
||||
builtin.bin builtin.alias.bin devname; do
|
||||
rm -f "$kver_dir/modules.$f"
|
||||
done
|
||||
if ! image_kmod depmod -b "$CHROOT_DIR" "$kver"; then
|
||||
echo "ERROR: image depmod failed for $kver" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ ! -s "$kver_dir/modules.dep" ] || [ ! -s "$kver_dir/modules.dep.bin" ]; then
|
||||
echo "ERROR: ${kver_dir}modules.dep or modules.dep.bin missing or empty after depmod" >&2
|
||||
exit 1
|
||||
fi
|
||||
# Resolution gate: the image's modprobe must resolve every required module
|
||||
# to plain .ko files present in the stripped tree.
|
||||
for name in $MICROVM_EXTRA_MODULES qemu_fw_cfg virtio_net; do
|
||||
module_builtin "$kver_dir" "$name" && continue
|
||||
if ! plan=$(image_kmod modprobe -d "$CHROOT_DIR" -S "$kver" --show-depends "$name"); then
|
||||
echo "ERROR: image modprobe cannot resolve $name for $kver" >&2
|
||||
exit 1
|
||||
fi
|
||||
while read -r verb path _; do
|
||||
[ "$verb" = "insmod" ] || continue
|
||||
case "$path" in
|
||||
"$CHROOT_DIR/lib/modules/$kver/kernel/"*.ko) [ -f "$path" ] && continue ;;
|
||||
esac
|
||||
echo "ERROR: modprobe plan for $name names $path, not a kept plain .ko" >&2
|
||||
exit 1
|
||||
done <<< "$plan"
|
||||
echo "[build-sqs-store-microvm-image] modprobe $name: $(awk '{ n = split($2, a, "/"); printf "%s ", a[n] }' <<< "$plan")"
|
||||
done
|
||||
done
|
||||
|
||||
# --- Strip package-manager artifacts (not needed at runtime) ---
|
||||
echo "[build-sqs-store-microvm-image] stripping package manager artifacts..."
|
||||
rm -rf \
|
||||
"$CHROOT_DIR/lib/apk" \
|
||||
"$CHROOT_DIR/var/cache/apk" \
|
||||
"$CHROOT_DIR/etc/apk" \
|
||||
"$CHROOT_DIR/usr/share/man" \
|
||||
"$CHROOT_DIR/usr/share/doc" \
|
||||
"$CHROOT_DIR/usr/share/apk" \
|
||||
"$CHROOT_DIR/usr/include" \
|
||||
"$CHROOT_DIR/usr/lib/pkgconfig"
|
||||
|
||||
# --- Strip debug symbols from binaries ---
|
||||
# nats-server in /usr/local/bin is left as built (-s -w), so the shipped bytes
|
||||
# are the ones whose version was checked above.
|
||||
echo "[build-sqs-store-microvm-image] stripping debug symbols from binaries..."
|
||||
find "$CHROOT_DIR/usr/sbin" "$CHROOT_DIR/usr/bin" \
|
||||
"$CHROOT_DIR/sbin" "$CHROOT_DIR/bin" \
|
||||
-type f 2>/dev/null | while read -r bin; do
|
||||
strip --strip-all "$bin" 2>/dev/null || true
|
||||
done
|
||||
|
||||
# --- Copy vmlinuz out before stripping /boot from chroot ---
|
||||
VMLINUZ_OUT="$STAGE_DIR/vmlinuz"
|
||||
echo "[build-sqs-store-microvm-image] copying kernel: $VMLINUZ_OUT"
|
||||
cp "$KERNEL_IMG" "$VMLINUZ_OUT"
|
||||
|
||||
# Drop /boot from the chroot — vmlinuz, Alpine's stock initramfs-virt, System.map
|
||||
# and kernel config all live here (~25 MiB). None are needed at runtime: the
|
||||
# guest already has the kernel loaded by QEMU's -kernel, and Alpine's initramfs
|
||||
# is unused because we replace it with our own /init. Leaving /boot in the cpio
|
||||
# nearly doubles uncompressed initramfs size and forces a 512 MiB guest just to
|
||||
# survive decompression.
|
||||
rm -rf "${CHROOT_DIR:?}/boot"
|
||||
|
||||
# --- Build initramfs ---
|
||||
INITRAMFS_OUT="$STAGE_DIR/initramfs.cpio.gz"
|
||||
echo "[build-sqs-store-microvm-image] building initramfs: $INITRAMFS_OUT"
|
||||
# Create the static device nodes and pack the archive inside a single fakeroot
|
||||
# session: mknod/cpio see faked device entries and write real char-device
|
||||
# records into the cpio, while the host filesystem never gets an actual node.
|
||||
fakeroot sh -c '
|
||||
cd "$1" || exit 1
|
||||
mknod -m 600 dev/console c 5 1
|
||||
mknod -m 666 dev/null c 1 3
|
||||
find . | cpio --quiet -o -H newc | gzip -9 > "$2"
|
||||
' _ "$CHROOT_DIR" "$INITRAMFS_OUT"
|
||||
|
||||
# --- Gate the staged pair, then publish it ---
|
||||
# Nothing reaches $BUILD_DIR until every gate passes, so a failed build cannot
|
||||
# leave a pair the grouped make target would take as up to date.
|
||||
"$REPO_ROOT/scripts/lib/sqs-store-microvm-publish.sh" "$STAGE_DIR" "$BUILD_DIR"
|
||||
echo ""
|
||||
echo "[build-sqs-store-microvm-image] done."
|
||||
@@ -0,0 +1,288 @@
|
||||
#!/bin/sh
|
||||
# sqs-store-guest_test.sh — the SQS store guest steps start the store only on
|
||||
# a valid contract and one checked, mounted disk, never format, and stop the
|
||||
# store before unmounting it.
|
||||
#
|
||||
# Drives build/microvm/sqs-store/sqs-store-lib.sh against a fake fw_cfg, sysfs
|
||||
# and stub tools. The stop cases signal only test-owned children.
|
||||
# The case settings below are read by the sourced library, not by this file.
|
||||
# shellcheck disable=SC2034
|
||||
set -eu
|
||||
|
||||
ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/../.." && pwd)
|
||||
LIB=${SQS_GUEST_LIB:-"$ROOT/build/microvm/sqs-store/sqs-store-lib.sh"}
|
||||
TMP=$(mktemp -d)
|
||||
CHILDREN=""
|
||||
cleanup() {
|
||||
for p in $CHILDREN; do kill -KILL "$p" 2>/dev/null || true; done
|
||||
rm -rf "$TMP"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
fails=0
|
||||
pass() { echo " ok - $1"; }
|
||||
fail() { echo " FAIL - $1"; fails=$((fails + 1)); }
|
||||
check() { if eval "$2"; then pass "$1"; else fail "$1"; fi; }
|
||||
|
||||
echo "-- build/microvm/sqs-store/sqs-store-lib.sh"
|
||||
|
||||
SERIAL=vol0123456789abcdef0
|
||||
NKEY=UAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAB
|
||||
ADDR=10.11.12.13
|
||||
|
||||
# Stub tools record each call to calls.log; their exit codes come from files.
|
||||
mkdir -p "$TMP/bin"
|
||||
cat > "$TMP/bin/e2fsck" <<'EOF'
|
||||
#!/bin/sh
|
||||
echo "e2fsck $*" >> "$T/calls.log"
|
||||
exit "$(cat "$T/fsck.rc")"
|
||||
EOF
|
||||
cat > "$TMP/bin/mount" <<'EOF'
|
||||
#!/bin/sh
|
||||
echo "mount $*" >> "$T/calls.log"
|
||||
exit "$(cat "$T/mount.rc")"
|
||||
EOF
|
||||
cat > "$TMP/bin/umount" <<'EOF'
|
||||
#!/bin/sh
|
||||
echo "umount $*" >> "$T/calls.log"
|
||||
EOF
|
||||
for f in mkfs mkfs.ext4 mke2fs; do
|
||||
printf '#!/bin/sh\necho "mkfs %s $*" >> "$T/calls.log"\n' "$f" > "$TMP/bin/$f"
|
||||
done
|
||||
cat > "$TMP/bin/ip" <<'EOF'
|
||||
#!/bin/sh
|
||||
cat "$T/ip.out"
|
||||
EOF
|
||||
cat > "$TMP/bin/nats-server" <<'EOF'
|
||||
#!/bin/sh
|
||||
echo "nats-server $*" >> "$T/calls.log"
|
||||
[ "$1" = -t ] && exit "$(cat "$T/configtest.rc")"
|
||||
exit 0
|
||||
EOF
|
||||
chmod +x "$TMP/bin"/*
|
||||
|
||||
# fresh <name>: a case directory with a valid contract and one matching disk
|
||||
# among others. Callers then break one thing.
|
||||
fresh() {
|
||||
T="$TMP/$1"
|
||||
export T
|
||||
mkdir -p "$T/fwcfg" "$T/sys" "$T/dev" "$T/run" "$T/data"
|
||||
for k in data-serial listen-addr client-nkey; do mkdir -p "$T/fwcfg/$k"; done
|
||||
printf '%s\n' "$SERIAL" > "$T/fwcfg/data-serial/raw"
|
||||
printf '%s\n' "$ADDR" > "$T/fwcfg/listen-addr/raw"
|
||||
printf '%s\n' "$NKEY" > "$T/fwcfg/client-nkey/raw"
|
||||
disk vda vol00000000000000000
|
||||
disk vdb "$SERIAL"
|
||||
echo 0 > "$T/fsck.rc"
|
||||
echo 0 > "$T/mount.rc"
|
||||
echo 0 > "$T/configtest.rc"
|
||||
: > "$T/calls.log"
|
||||
: > "$T/console"
|
||||
printf '1: lo inet 127.0.0.1/8 scope host lo\n2: eth0 inet %s/24 brd 10.11.12.255 scope global eth0\n' "$ADDR" > "$T/ip.out"
|
||||
}
|
||||
disk() { mkdir -p "$T/sys/$1"; printf '%s\n' "$2" > "$T/sys/$1/serial"; : > "$T/dev/$1"; }
|
||||
|
||||
# prepare: run sqs_prepare in a subshell with every path pointed at the case.
|
||||
prepare() {
|
||||
(
|
||||
SQS_FWCFG="$T/fwcfg" SQS_SYSBLOCK="$T/sys" SQS_DEVDIR="$T/dev" SQS_NODE_TEST=-e
|
||||
SQS_RUN="$T/run" SQS_DATA="$T/data" SQS_CONSOLE="$T/console"
|
||||
E2FSCK="$TMP/bin/e2fsck" MOUNT="$TMP/bin/mount" IP="$TMP/bin/ip" NATS_BIN="$TMP/bin/nats-server"
|
||||
PATH="$TMP/bin:$PATH"
|
||||
export PATH
|
||||
# shellcheck source=../../build/microvm/sqs-store/sqs-store-lib.sh
|
||||
. "$LIB"
|
||||
sqs_prepare
|
||||
) > /dev/null 2>&1
|
||||
}
|
||||
refused() { [ ! -e "$T/run/ready" ] && grep -q FATAL "$T/console"; }
|
||||
never_mounted() { ! grep -q '^mount ' "$T/calls.log"; }
|
||||
never_formatted() { ! grep -q '^mkfs ' "$T/calls.log"; }
|
||||
never_checked() { ! grep -q '^e2fsck ' "$T/calls.log"; }
|
||||
|
||||
# --- A valid contract prepares the store ---
|
||||
fresh valid
|
||||
check "valid contract is prepared" 'prepare && [ -e "$T/run/ready" ]'
|
||||
check "the matching disk is checked, then mounted at the data dir" \
|
||||
'grep -qx "e2fsck -p $T/dev/vdb" "$T/calls.log" && grep -qx "mount -t ext4 -o nodev,nosuid,noexec $T/dev/vdb $T/data" "$T/calls.log"'
|
||||
check "config listens only on the supplied address" \
|
||||
'grep -c "listen" "$T/run/nats.conf" | grep -qx 1 && grep -qx "listen: \"$ADDR:4222\"" "$T/run/nats.conf"'
|
||||
check "config admits only the supplied user NKey" 'grep -q "users = \[ { nkey: \"$NKEY\" } \]" "$T/run/nats.conf"'
|
||||
check "config syncs JetStream on the data disk" \
|
||||
'grep -q "store_dir: \"$T/data\"" "$T/run/nats.conf" && grep -q "sync_interval: always" "$T/run/nats.conf"'
|
||||
check "config is tested by the server before ready" 'grep -qx "nats-server -t -c $T/run/nats.conf" "$T/calls.log"'
|
||||
check "no auth material on the console" '! grep -q "$NKEY" "$T/console"'
|
||||
check "prepare reports the disk and config, not a listener" 'grep -q "prepared: $T/dev/vdb mounted" "$T/console" && ! grep -qi "listening" "$T/console"'
|
||||
|
||||
fresh fsck-corrected; echo 1 > "$T/fsck.rc"
|
||||
check "e2fsck exit 1 (corrected) proceeds" 'prepare && [ -e "$T/run/ready" ]'
|
||||
|
||||
# --- Disk refusals: nothing mounted, nothing formatted ---
|
||||
fresh no-disk; rm -rf "$T/sys/vdb"
|
||||
check "missing disk is refused before any check or mount" '! prepare && refused && never_checked && never_mounted'
|
||||
|
||||
fresh wrong-disk; printf 'vol0fffffffffffffff0\n' > "$T/fwcfg/data-serial/raw"
|
||||
check "disk with another serial is refused" '! prepare && refused && never_checked && never_mounted'
|
||||
|
||||
fresh two-disks; disk vdc "$SERIAL"
|
||||
check "two disks with the serial are refused" '! prepare && refused && never_checked && never_mounted'
|
||||
|
||||
fresh bad-serial; printf 'vol-0123456789abcdef\n' > "$T/fwcfg/data-serial/raw"
|
||||
check "malformed serial is refused" '! prepare && refused && never_mounted'
|
||||
|
||||
fresh no-node; rm -f "$T/dev/vdb"
|
||||
check "disk without a device node is refused" '! prepare && refused && never_mounted'
|
||||
|
||||
for rc in 2 4 8; do
|
||||
fresh "fsck-$rc"; echo "$rc" > "$T/fsck.rc"
|
||||
check "e2fsck exit $rc is refused without mount or format" '! prepare && refused && never_mounted && never_formatted'
|
||||
done
|
||||
|
||||
fresh mount-fail; echo 32 > "$T/mount.rc"
|
||||
check "failed mount is refused" '! prepare && refused'
|
||||
|
||||
# --- Auth and listener refusals happen before the disk is touched ---
|
||||
fresh no-nkey; rm -rf "$T/fwcfg/client-nkey"
|
||||
check "missing NKey is refused before the disk" '! prepare && refused && never_checked && never_mounted'
|
||||
|
||||
for bad in seed lower short two empty; do
|
||||
fresh "nkey-$bad"
|
||||
case $bad in
|
||||
seed) v=SUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA ;;
|
||||
lower) v=uaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab ;;
|
||||
short) v=UAAAA ;;
|
||||
two) v="$NKEY
|
||||
$NKEY" ;;
|
||||
empty) v="" ;;
|
||||
esac
|
||||
printf '%s\n' "$v" > "$T/fwcfg/client-nkey/raw"
|
||||
check "NKey ($bad) is refused before the disk, without echoing it" \
|
||||
'! prepare && refused && never_checked && { [ -z "$v" ] || ! grep -qF "$v" "$T/console"; }'
|
||||
done
|
||||
|
||||
fresh nkey-checksum; echo 1 > "$T/configtest.rc"
|
||||
check "NKey the server rejects is refused before the disk" '! prepare && refused && never_checked && never_mounted'
|
||||
|
||||
fresh no-addr; rm -rf "$T/fwcfg/listen-addr"
|
||||
check "missing listen address is refused" '! prepare && refused && never_mounted'
|
||||
|
||||
fresh addr-absent; printf '1: lo inet 127.0.0.1/8 scope host lo\n2: eth0 inet 10.11.12.99/24 scope global eth0\n' > "$T/ip.out"
|
||||
check "listen address not on a guest NIC is refused" '! prepare && refused && never_mounted'
|
||||
|
||||
fresh addr-loopback; printf '127.0.0.1\n' > "$T/fwcfg/listen-addr/raw"
|
||||
check "loopback listen address is refused" '! prepare && refused && never_mounted'
|
||||
|
||||
for bad in 0.0.0.0x 1.2.3 1.2.3.256 01.2.3.4; do
|
||||
fresh "addr-$bad"; printf '%s\n' "$bad" > "$T/fwcfg/listen-addr/raw"
|
||||
check "malformed listen address $bad is refused" '! prepare && refused && never_mounted'
|
||||
done
|
||||
|
||||
# --- Run: the store starts only when prepared and not stopping ---
|
||||
run_step() {
|
||||
(
|
||||
SQS_RUN="$T/run" SQS_CONSOLE="$T/console" NATS_BIN="$TMP/bin/nats-server"
|
||||
# shellcheck source=../../build/microvm/sqs-store/sqs-store-lib.sh
|
||||
. "$LIB"
|
||||
sqs_run
|
||||
) > /dev/null 2>&1
|
||||
}
|
||||
fresh run-ready; prepare; : > "$T/calls.log"
|
||||
check "a prepared store starts the server" 'run_step && grep -qx "nats-server -c $T/run/nats.conf" "$T/calls.log"'
|
||||
fresh run-unready
|
||||
check "an unprepared store never starts the server" '! run_step && ! grep -q "^nats-server" "$T/calls.log"'
|
||||
fresh run-stopping; prepare; : > "$T/calls.log"; : > "$T/run/stopping"
|
||||
check "a stopping store never starts the server" '! run_step && ! grep -q "^nats-server" "$T/calls.log"'
|
||||
|
||||
# --- Stop: SIGTERM the verified store, then unmount ---
|
||||
STORE="$TMP/bin/fake-store"
|
||||
cp "$(command -v sleep)" "$STORE"
|
||||
stop_step() {
|
||||
(
|
||||
SQS_RUN="$T/run" SQS_DATA="$T/data" SQS_CONSOLE="$T/console" SQS_MOUNTS="$T/mounts"
|
||||
UMOUNT="$TMP/bin/umount" NATS_BIN="$STORE" SQS_STOP_BUDGET_TENTHS=10
|
||||
# shellcheck source=../../build/microvm/sqs-store/sqs-store-lib.sh
|
||||
. "$LIB"
|
||||
sqs_stop
|
||||
) > /dev/null 2>&1
|
||||
}
|
||||
gone() { [ ! -r "/proc/$1/status" ] || grep -q '^State:[[:space:]]*Z' "/proc/$1/status"; }
|
||||
# record <pid>: write the identity sqs_run would record for that process.
|
||||
# shellcheck source=../../build/microvm/sqs-store/sqs-store-lib.sh
|
||||
record() { echo "$1 $( . "$LIB"; sqs_starttime "$1" )" > "$T/run/pid"; }
|
||||
|
||||
fresh stop-orderly; echo "dev $T/data ext4 rw 0 0" > "$T/mounts"
|
||||
"$STORE" 60 & pid=$!; CHILDREN="$CHILDREN $pid"; record "$pid"
|
||||
stop_step
|
||||
check "stop sends SIGTERM and the store exits" 'gone "$pid" && grep -q "SIGTERM sent" "$T/console" && ! grep -q SIGKILL "$T/console"'
|
||||
check "stop blocks respawn and unmounts after the store exits" \
|
||||
'[ -e "$T/run/stopping" ] && grep -qx "umount $T/data" "$T/calls.log"'
|
||||
|
||||
fresh stop-hung; echo "dev $T/data ext4 rw 0 0" > "$T/mounts"
|
||||
(trap '' TERM; exec "$STORE" 60) & pid=$!; CHILDREN="$CHILDREN $pid"; record "$pid"
|
||||
stop_step
|
||||
check "store ignoring SIGTERM is killed after the budget, then unmounted" \
|
||||
'gone "$pid" && grep -q SIGKILL "$T/console" && grep -qx "umount $T/data" "$T/calls.log"'
|
||||
|
||||
fresh stop-reused; echo "dev $T/data ext4 rw 0 0" > "$T/mounts"
|
||||
"$STORE" 60 & pid=$!; CHILDREN="$CHILDREN $pid"; echo "$pid 1" > "$T/run/pid"
|
||||
stop_step
|
||||
check "a recorded PID with another start time is never signalled" '! gone "$pid" && grep -q "no running store" "$T/console"'
|
||||
kill -KILL "$pid"
|
||||
|
||||
# A start held after its stopping check and before its exec: the run step
|
||||
# blocks writing "run: starting" to a FIFO console until the test reads it.
|
||||
fresh stop-pending-start; echo "dev $T/data ext4 rw 0 0" > "$T/mounts"
|
||||
: > "$T/run/ready"
|
||||
mkfifo "$T/run-console"
|
||||
printf '#!/bin/sh\n: > "%s/started"\nexec "%s" 60\n' "$T" "$STORE" > "$TMP/bin/pending-nats"
|
||||
chmod +x "$TMP/bin/pending-nats"
|
||||
SQS_RUN="$T/run" SQS_CONSOLE="$T/run-console" NATS_BIN="$TMP/bin/pending-nats" \
|
||||
sh -c '. "$1"; sqs_run' sh "$LIB" & pid=$!; CHILDREN="$CHILDREN $pid"
|
||||
i=0
|
||||
while [ "$i" -lt 100 ] && ! grep -q "^$pid " "$T/run/pid" 2>/dev/null; do i=$((i + 1)); sleep 0.05; done
|
||||
check "pending start is held before its exec" 'grep -q "^$pid " "$T/run/pid" && ! gone "$pid" && [ ! -e "$T/started" ]'
|
||||
stop_step
|
||||
# Release the FIFO: a start that survived the stop would exec the server now.
|
||||
cat "$T/run-console" > /dev/null 2>&1 &
|
||||
reader=$!; CHILDREN="$CHILDREN $reader"
|
||||
i=0
|
||||
while [ "$i" -lt 20 ] && ! gone "$pid"; do i=$((i + 1)); sleep 0.05; done
|
||||
check "stop ends a pending start before it can exec the server" 'gone "$pid" && [ ! -e "$T/started" ]'
|
||||
check "stop unmounts only after the pending start is gone" \
|
||||
'grep -q "store exited" "$T/console" && grep -qx "umount $T/data" "$T/calls.log"'
|
||||
kill -KILL "$reader" 2>/dev/null || true
|
||||
|
||||
# --- Shutdown: the generic remount runs only once the store is gone ---
|
||||
# shutdown_step [survivor-pid]: with a pid, that process is reported alive
|
||||
# whatever happens to it, as a writer stuck past SIGKILL would be.
|
||||
shutdown_step() {
|
||||
(
|
||||
SQS_RUN="$T/run" SQS_DATA="$T/data" SQS_CONSOLE="$T/console" SQS_MOUNTS="$T/mounts"
|
||||
UMOUNT="$TMP/bin/umount" SQS_STOP_BUDGET_TENTHS=5
|
||||
# shellcheck source=../../build/microvm/sqs-store/sqs-store-lib.sh
|
||||
. "$LIB"
|
||||
if [ -n "${1:-}" ]; then sqs_same() { [ "$1" = "$survivor" ]; }; survivor=$1; fi
|
||||
sqs_shutdown
|
||||
) > /dev/null 2>&1
|
||||
}
|
||||
|
||||
fresh shutdown-orderly; echo "dev $T/data ext4 rw 0 0" > "$T/mounts"
|
||||
"$STORE" 60 & pid=$!; CHILDREN="$CHILDREN $pid"; record "$pid"
|
||||
check "orderly shutdown succeeds" 'shutdown_step'
|
||||
check "orderly shutdown unmounts the data disk, then runs umount -a -r" \
|
||||
'[ "$(grep "^umount" "$T/calls.log" | tr "\n" "|")" = "umount $T/data|umount -a -r|" ]'
|
||||
|
||||
fresh shutdown-survivor; echo "dev $T/data ext4 rw 0 0" > "$T/mounts"
|
||||
"$STORE" 60 & pid=$!; CHILDREN="$CHILDREN $pid"; record "$pid"
|
||||
check "shutdown with a surviving store reports failure" '! shutdown_step "$pid"'
|
||||
check "a surviving store leaves the data disk and every other mount alone" \
|
||||
'! grep -q "^umount" "$T/calls.log" && grep -q "STILL ALIVE" "$T/console" && grep -q "skipping umount -a -r" "$T/console"'
|
||||
|
||||
fresh stop-unmounted; : > "$T/mounts"
|
||||
stop_step
|
||||
check "stop with no data mount does not unmount" '! grep -q "^umount" "$T/calls.log"'
|
||||
|
||||
if [ "$fails" -ne 0 ]; then
|
||||
echo " $fails failure(s)"
|
||||
exit 1
|
||||
fi
|
||||
Executable
+97
@@ -0,0 +1,97 @@
|
||||
#!/bin/bash
|
||||
# sqs-store-microvm-publish.sh STAGE_DIR OUT_DIR — gate a staged SQS store
|
||||
# microVM kernel + initramfs and move them into OUT_DIR only if every gate passes.
|
||||
#
|
||||
# A failed gate leaves OUT_DIR as it was: a previous valid pair stays, and an
|
||||
# older pair stays older than the build inputs, so make rebuilds next time.
|
||||
set -euo pipefail
|
||||
|
||||
STAGE_DIR="${1:?usage: sqs-store-microvm-publish.sh STAGE_DIR OUT_DIR}"
|
||||
OUT_DIR="${2:?usage: sqs-store-microvm-publish.sh STAGE_DIR OUT_DIR}"
|
||||
VMLINUZ="$STAGE_DIR/vmlinuz"
|
||||
INITRAMFS="$STAGE_DIR/initramfs.cpio.gz"
|
||||
tag="[build-sqs-store-microvm-image]"
|
||||
|
||||
for f in "$VMLINUZ" "$INITRAMFS"; do
|
||||
if [ ! -s "$f" ]; then
|
||||
echo "ERROR: staged artifact $f missing or empty" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# --- Content gate: read the packed archive, not the chroot ---
|
||||
if ! archive_list=$(zcat "$INITRAMFS" | cpio --quiet -t); then
|
||||
echo "ERROR: $INITRAMFS is not a readable gzip cpio archive" >&2
|
||||
exit 1
|
||||
fi
|
||||
for want in init etc/inittab usr/local/bin/nats-server dev/console dev/null \
|
||||
usr/local/lib/sqs-store/sqs-store-lib.sh usr/local/lib/sqs-store/sqs-store-prepare \
|
||||
usr/local/lib/sqs-store/sqs-store-run usr/local/lib/sqs-store/sqs-store-stop; do
|
||||
if ! grep -qx "$want" <<< "$archive_list"; then
|
||||
echo "ERROR: $want missing from $INITRAMFS" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
for fs_tool in mkfs.ext4 e2fsck; do
|
||||
if ! grep -qxE "(usr/)?sbin/$fs_tool" <<< "$archive_list"; then
|
||||
echo "ERROR: $fs_tool missing from $INITRAMFS" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
# Module files may spell a name with dashes (tiny-power-button.ko). Normalise
|
||||
# once: piping into grep -q under pipefail fails when grep exits on a match.
|
||||
module_list=${archive_list//-/_}
|
||||
for want_ko in ext4 tiny_power_button; do
|
||||
if ! grep -q "/$want_ko\.ko\$" <<< "$module_list"; then
|
||||
echo "ERROR: $want_ko missing from $INITRAMFS" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
if grep -qE '(^|/)(lb-agent|haproxy|nginx)$' <<< "$archive_list"; then
|
||||
echo "ERROR: LB data-plane binaries found in the SQS store image" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "$tag content gate: PASS"
|
||||
|
||||
echo ""
|
||||
echo "$tag staged artifacts:"
|
||||
ls -lh "$VMLINUZ" "$INITRAMFS"
|
||||
echo ""
|
||||
|
||||
# --- Artifact size gate (50 MiB total) ---
|
||||
ARTIFACT_MAX_MiB="${MICROVM_ARTIFACT_MAX_MiB:-50}"
|
||||
total_bytes=$(( $(stat -c %s "$VMLINUZ") + $(stat -c %s "$INITRAMFS") ))
|
||||
max_bytes=$(( ARTIFACT_MAX_MiB * 1024 * 1024 ))
|
||||
total_mib=$(( (total_bytes + 1048575) / 1048576 ))
|
||||
if [ "$total_bytes" -gt "$max_bytes" ]; then
|
||||
echo "ERROR: artifact size ${total_mib} MiB exceeds ${ARTIFACT_MAX_MiB} MiB gate" >&2
|
||||
echo " Set MICROVM_ARTIFACT_MAX_MiB=N to override during development." >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "$tag artifact size gate: PASS (${total_mib} MiB / ${ARTIFACT_MAX_MiB} MiB)"
|
||||
|
||||
# --- Guest-RAM initramfs gate (uncompressed) ---
|
||||
# The kernel unpacks the whole initramfs into RAM before init, so the
|
||||
# UNCOMPRESSED size — not the gzip artifact — sets the guest memory floor.
|
||||
UNCOMPRESSED_MAX_MiB="${MICROVM_INITRAMFS_UNCOMPRESSED_MAX_MiB:-96}"
|
||||
uncompressed_bytes=$(zcat "$INITRAMFS" | wc -c)
|
||||
uncompressed_mib=$(( (uncompressed_bytes + 1048575) / 1048576 ))
|
||||
umax_bytes=$(( UNCOMPRESSED_MAX_MiB * 1024 * 1024 ))
|
||||
if [ "$uncompressed_bytes" -gt "$umax_bytes" ]; then
|
||||
echo "ERROR: uncompressed initramfs ${uncompressed_mib} MiB exceeds ${UNCOMPRESSED_MAX_MiB} MiB guest-RAM gate" >&2
|
||||
echo " The kernel unpacks this into guest RAM before init; a larger" >&2
|
||||
echo " footprint panics before userspace. Slim the initramfs, or raise the" >&2
|
||||
echo " instance memory AND MICROVM_INITRAMFS_UNCOMPRESSED_MAX_MiB together." >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "$tag initramfs guest-RAM gate: PASS (${uncompressed_mib} MiB / ${UNCOMPRESSED_MAX_MiB} MiB uncompressed)"
|
||||
|
||||
# --- Publish ---
|
||||
# Two renames are not atomic as a pair, so the old initramfs goes first and the
|
||||
# new one lands last. Stopped in between, the initramfs is absent: no mixed pair
|
||||
# exists and the grouped make target is missing, so make rebuilds.
|
||||
mkdir -p "$OUT_DIR"
|
||||
rm -f "$OUT_DIR/initramfs.cpio.gz"
|
||||
mv -f "$VMLINUZ" "$OUT_DIR/vmlinuz"
|
||||
mv -f "$INITRAMFS" "$OUT_DIR/initramfs.cpio.gz"
|
||||
echo "$tag published: $OUT_DIR/vmlinuz $OUT_DIR/initramfs.cpio.gz"
|
||||
@@ -0,0 +1,196 @@
|
||||
#!/bin/sh
|
||||
# sqs-store-microvm-publish_test.sh — a staged SQS store microVM pair reaches
|
||||
# the output directory only when every gate passes, and a failed build cannot
|
||||
# leave a pair that make then takes as up to date.
|
||||
#
|
||||
# Artifacts are small fake archives listing the gated paths; no image is built.
|
||||
set -eu
|
||||
|
||||
LIB_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
|
||||
PUBLISH="$LIB_DIR/sqs-store-microvm-publish.sh"
|
||||
TMP=$(mktemp -d)
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
|
||||
fails=0
|
||||
pass() { echo " ok - $1"; }
|
||||
fail() { echo " FAIL - $1"; fails=$((fails + 1)); }
|
||||
|
||||
echo "-- lib/sqs-store-microvm-publish.sh"
|
||||
|
||||
for tool in cpio gzip make; do
|
||||
if ! command -v "$tool" >/dev/null 2>&1; then
|
||||
echo " FAIL - required tool not found: $tool"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# stage <dir> [omit-path] — a fake staged pair; omit-path is left out of the archive.
|
||||
stage() {
|
||||
dir=$1 omit=${2:-}
|
||||
root="$TMP/root.$$.$(date +%s%N)"
|
||||
for p in init etc/inittab usr/local/bin/nats-server sbin/mkfs.ext4 sbin/e2fsck \
|
||||
usr/local/lib/sqs-store/sqs-store-lib.sh usr/local/lib/sqs-store/sqs-store-prepare \
|
||||
usr/local/lib/sqs-store/sqs-store-run usr/local/lib/sqs-store/sqs-store-stop \
|
||||
dev/console dev/null lib/modules/6.12-virt/kernel/fs/ext4/ext4.ko \
|
||||
lib/modules/6.12-virt/kernel/drivers/acpi/tiny-power-button.ko; do
|
||||
[ "$p" = "$omit" ] && continue
|
||||
mkdir -p "$root/$(dirname "$p")"
|
||||
echo "$p" > "$root/$p"
|
||||
done
|
||||
# pad=N appends N files after the gated ones, as a real image's listing has.
|
||||
if [ "${pad:-0}" -gt 0 ]; then
|
||||
mkdir -p "$root/zz"
|
||||
(cd "$root/zz" && seq "$pad" | xargs touch)
|
||||
fi
|
||||
mkdir -p "$dir"
|
||||
(cd "$root" && find . -mindepth 1 | sed 's|^\./||' | sort | cpio --quiet -o -H newc) | gzip > "$dir/initramfs.cpio.gz"
|
||||
echo kernel > "$dir/vmlinuz"
|
||||
rm -rf "$root"
|
||||
}
|
||||
|
||||
# fingerprint <dir> — content and mtime of both outputs, or "absent".
|
||||
fingerprint() {
|
||||
for f in "$1/vmlinuz" "$1/initramfs.cpio.gz"; do
|
||||
if [ -e "$f" ]; then
|
||||
printf '%s %s ' "$(cksum < "$f")" "$(stat -c %Y "$f")"
|
||||
else
|
||||
printf 'absent '
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# --- A valid pair is published and leaves the stage ---
|
||||
stage "$TMP/ok/stage"
|
||||
if bash "$PUBLISH" "$TMP/ok/stage" "$TMP/ok/out" >/dev/null 2>&1 \
|
||||
&& [ -s "$TMP/ok/out/vmlinuz" ] && [ -s "$TMP/ok/out/initramfs.cpio.gz" ] \
|
||||
&& [ ! -e "$TMP/ok/stage/vmlinuz" ] && [ ! -e "$TMP/ok/stage/initramfs.cpio.gz" ]; then
|
||||
pass "valid pair is published"
|
||||
else
|
||||
fail "valid pair is published"
|
||||
fi
|
||||
|
||||
# --- A large listing with the modules early still passes the module gate ---
|
||||
pad=20000 stage "$TMP/large/stage"
|
||||
if bash "$PUBLISH" "$TMP/large/stage" "$TMP/large/out" >/dev/null 2>&1; then
|
||||
pass "large archive with early modules is published"
|
||||
else
|
||||
fail "large archive with early modules is published"
|
||||
fi
|
||||
|
||||
# --- Each failing gate keeps the previous valid pair exactly ---
|
||||
for case in nats-server store-stop ext4 size; do
|
||||
d="$TMP/keep-$case"
|
||||
stage "$d/prev"
|
||||
bash "$PUBLISH" "$d/prev" "$d/out" >/dev/null 2>&1
|
||||
before=$(fingerprint "$d/out")
|
||||
max=50
|
||||
case $case in
|
||||
nats-server) stage "$d/stage" usr/local/bin/nats-server ;;
|
||||
store-stop) stage "$d/stage" usr/local/lib/sqs-store/sqs-store-stop ;;
|
||||
ext4) stage "$d/stage" lib/modules/6.12-virt/kernel/fs/ext4/ext4.ko ;;
|
||||
size) stage "$d/stage"; max=0 ;;
|
||||
esac
|
||||
if MICROVM_ARTIFACT_MAX_MiB=$max bash "$PUBLISH" "$d/stage" "$d/out" >/dev/null 2>&1; then
|
||||
fail "$case gate failure is refused"
|
||||
elif [ "$(fingerprint "$d/out")" = "$before" ]; then
|
||||
pass "$case gate failure keeps the previous pair"
|
||||
else
|
||||
fail "$case gate failure keeps the previous pair"
|
||||
fi
|
||||
done
|
||||
|
||||
# --- With no previous pair, a failed gate publishes nothing ---
|
||||
stage "$TMP/none/stage" usr/local/bin/nats-server
|
||||
if bash "$PUBLISH" "$TMP/none/stage" "$TMP/none/out" >/dev/null 2>&1; then
|
||||
fail "failed gate with no previous pair is refused"
|
||||
elif [ "$(fingerprint "$TMP/none/out")" = "absent absent " ]; then
|
||||
pass "failed gate with no previous pair publishes nothing"
|
||||
else
|
||||
fail "failed gate with no previous pair publishes nothing"
|
||||
fi
|
||||
|
||||
# --- make cannot take a failed rebuild's pair as up to date ---
|
||||
# The same grouped-target shape as the Makefile; the builder stages a pair
|
||||
# missing nats-server once BAD exists, the way a failed real build would.
|
||||
m="$TMP/make"
|
||||
mkdir -p "$m/out"
|
||||
cat > "$m/build.sh" <<EOF
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
rm -rf "$m/stage"
|
||||
. "$m/build.sh.lib"
|
||||
if [ -e "$m/BAD" ]; then
|
||||
stage "$m/stage" usr/local/bin/nats-server
|
||||
else
|
||||
stage "$m/stage"
|
||||
fi
|
||||
bash "$PUBLISH" "$m/stage" "$m/out"
|
||||
EOF
|
||||
{
|
||||
echo "TMP=\"$TMP\""
|
||||
sed -n '/^stage() {/,/^}/p' "$0"
|
||||
} > "$m/build.sh.lib"
|
||||
printf 'out/vmlinuz out/initramfs.cpio.gz &: input\n\tsh build.sh\n' > "$m/Makefile"
|
||||
echo v1 > "$m/input"
|
||||
if (cd "$m" && make -s out/vmlinuz >/dev/null 2>&1) && [ -s "$m/out/vmlinuz" ]; then
|
||||
pass "make builds the first valid pair"
|
||||
else
|
||||
fail "make builds the first valid pair"
|
||||
echo " $fails failure(s)"; exit 1
|
||||
fi
|
||||
good=$(fingerprint "$m/out")
|
||||
touch "$m/BAD"
|
||||
# Age the published pair so the input is strictly newer, without a future mtime.
|
||||
past="@$(( $(date +%s) - 10 ))"
|
||||
touch -d "$past" "$m/out/vmlinuz" "$m/out/initramfs.cpio.gz"
|
||||
touch "$m/input"
|
||||
good=$(fingerprint "$m/out")
|
||||
if (cd "$m" && make -s out/vmlinuz >/dev/null 2>&1); then
|
||||
fail "make reports the failed rebuild"
|
||||
else
|
||||
pass "make reports the failed rebuild"
|
||||
fi
|
||||
if (cd "$m" && make -q out/vmlinuz out/initramfs.cpio.gz); then
|
||||
fail "after a failed rebuild make still sees the pair as stale"
|
||||
else
|
||||
pass "after a failed rebuild make still sees the pair as stale"
|
||||
fi
|
||||
if [ "$(fingerprint "$m/out")" = "$good" ]; then
|
||||
pass "failed rebuild leaves the previous valid pair"
|
||||
else
|
||||
fail "failed rebuild leaves the previous valid pair"
|
||||
fi
|
||||
|
||||
# --- A publish stopped after the kernel lands leaves no mixed pair ---
|
||||
# A stub mv lets the first rename through and fails the second: the point at
|
||||
# which an interrupt would leave a new kernel beside an old initramfs.
|
||||
i="$TMP/interrupt"
|
||||
mkdir -p "$i/bin"
|
||||
real_mv=$(command -v mv)
|
||||
{
|
||||
echo '#!/bin/sh'
|
||||
echo "if [ -e '$i/moved' ]; then exit 1; fi"
|
||||
echo ": > '$i/moved'"
|
||||
echo "exec '$real_mv' \"\$@\""
|
||||
} > "$i/bin/mv"
|
||||
chmod +x "$i/bin/mv"
|
||||
stage "$i/prev"
|
||||
bash "$PUBLISH" "$i/prev" "$i/out" >/dev/null 2>&1
|
||||
stage "$i/stage"
|
||||
echo new-kernel > "$i/stage/vmlinuz"
|
||||
if PATH="$i/bin:$PATH" bash "$PUBLISH" "$i/stage" "$i/out" >/dev/null 2>&1; then
|
||||
fail "interrupted publish is reported"
|
||||
elif [ "$(cat "$i/out/vmlinuz")" = new-kernel ] && [ ! -e "$i/out/initramfs.cpio.gz" ]; then
|
||||
pass "interrupted publish leaves no initramfs beside the new kernel"
|
||||
else
|
||||
fail "interrupted publish leaves no initramfs beside the new kernel"
|
||||
fi
|
||||
printf 'out/vmlinuz out/initramfs.cpio.gz &: input\n\ttrue\n' > "$i/Makefile"
|
||||
touch -d "@$(( $(date +%s) - 10 ))" "$i/input"
|
||||
if (cd "$i" && make -q out/vmlinuz out/initramfs.cpio.gz); then
|
||||
fail "make does not accept the interrupted publish"
|
||||
else
|
||||
pass "make does not accept the interrupted publish"
|
||||
fi
|
||||
|
||||
[ "$fails" -eq 0 ] || { echo " $fails failure(s)"; exit 1; }
|
||||
Executable
+18
@@ -0,0 +1,18 @@
|
||||
#!/bin/sh
|
||||
# stage-sqs-store-microvm.sh <image-dir> <dist-root>: copy a built SQS store
|
||||
# microVM image into a release tree as <dist-root>/microvm-sqs-store. Without a
|
||||
# complete image it stages nothing and says so, as the load balancer image does.
|
||||
set -eu
|
||||
src=$1 dest=$2/microvm-sqs-store
|
||||
# An image staged by an earlier build must not ship with this one. Only the
|
||||
# two image files are removed; anything else in the directory is left alone.
|
||||
rm -f "$dest/vmlinuz" "$dest/initramfs.cpio.gz"
|
||||
if [ -s "$src/vmlinuz" ] && [ -s "$src/initramfs.cpio.gz" ]; then
|
||||
echo "[distro] staging the SQS store microVM image into the tarball"
|
||||
mkdir -p "$dest"
|
||||
cp "$src/vmlinuz" "$src/initramfs.cpio.gz" "$dest/"
|
||||
else
|
||||
[ ! -d "$dest" ] || rmdir "$dest" 2> /dev/null || true
|
||||
echo "[distro] WARNING: SQS store microVM image missing; the tarball will not include it"
|
||||
echo "[distro] run 'make build-sqs-store-microvm-image' first"
|
||||
fi
|
||||
@@ -1542,6 +1542,41 @@ install_files() {
|
||||
$SUDO cp "$EXTRACT_DIR/microvm/"* /usr/share/spinifex/microvm/
|
||||
info " /usr/share/spinifex/microvm/*"
|
||||
fi
|
||||
install_sqs_store_image
|
||||
}
|
||||
|
||||
# The SQS store image ships beside the load balancer's, not inside it: an older
|
||||
# installer copies microvm/* without -r and would stop at a subdirectory.
|
||||
install_sqs_store_image() {
|
||||
local src="$EXTRACT_DIR/microvm-sqs-store" f
|
||||
local dest="${SPINIFEX_MICROVM_DIR:-/usr/share/spinifex/microvm}/sqs-store"
|
||||
# A previously installed image belongs to an older release, and this
|
||||
# release's daemon must not boot it. Only the two image files are removed.
|
||||
if [ ! -d "$src" ]; then
|
||||
remove_sqs_store_image "$dest"
|
||||
warn "This release has no SQS store microVM image; SQS store launches will be refused"
|
||||
return 0
|
||||
fi
|
||||
for f in vmlinuz initramfs.cpio.gz; do
|
||||
if [ ! -s "$src/$f" ]; then
|
||||
remove_sqs_store_image "$dest"
|
||||
fatal "The SQS store microVM image in this release is incomplete: $f is missing"
|
||||
fi
|
||||
done
|
||||
# The initramfs goes last, so an interrupted install leaves no pair the
|
||||
# daemon would boot: it refuses a launch while either file is missing.
|
||||
$SUDO install -d "$dest"
|
||||
$SUDO rm -f "$dest/initramfs.cpio.gz"
|
||||
$SUDO install -m 0644 "$src/vmlinuz" "$dest/vmlinuz"
|
||||
$SUDO install -m 0644 "$src/initramfs.cpio.gz" "$dest/initramfs.cpio.gz"
|
||||
info " $dest/vmlinuz, $dest/initramfs.cpio.gz"
|
||||
}
|
||||
|
||||
# Running store VMs keep their already-loaded kernel; the files are read at boot.
|
||||
remove_sqs_store_image() {
|
||||
[ -e "$1/initramfs.cpio.gz" ] || [ -e "$1/vmlinuz" ] || return 0
|
||||
$SUDO rm -f "$1/initramfs.cpio.gz" "$1/vmlinuz"
|
||||
warn "Removed the SQS store microVM image left by an earlier install from $1"
|
||||
}
|
||||
|
||||
# --- Create directories ---
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
#!/bin/sh
|
||||
# setup_sqs_store_image_test.sh — tests for shipping the SQS store microVM
|
||||
# image: staging it into a release tree and installing it with setup.sh.
|
||||
#
|
||||
# setup.sh is sourced with INSTALL_SPINIFEX_LIB_ONLY=1 so main never runs, and
|
||||
# installs into a temporary SPINIFEX_MICROVM_DIR with SUDO cleared.
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
|
||||
SETUP="$SCRIPT_DIR/setup.sh"
|
||||
STAGE="$SCRIPT_DIR/lib/stage-sqs-store-microvm.sh"
|
||||
TMP=$(mktemp -d)
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
|
||||
fails=0
|
||||
pass() { echo " ok - $1"; }
|
||||
fail() { echo " FAIL - $1"; fails=$((fails + 1)); }
|
||||
|
||||
echo "-- SQS store microVM image release and install"
|
||||
|
||||
# image <dir> <tag>: a fake built image whose bytes name its tag.
|
||||
image() {
|
||||
mkdir -p "$1"
|
||||
printf 'kernel-%s\n' "$2" > "$1/vmlinuz"
|
||||
printf 'initramfs-%s\n' "$2" > "$1/initramfs.cpio.gz"
|
||||
}
|
||||
|
||||
# release <dir>: a release tree with the load balancer image, as distro-amd64 builds it.
|
||||
release() {
|
||||
rm -rf "$1"
|
||||
mkdir -p "$1/microvm"
|
||||
printf 'lb-kernel\n' > "$1/microvm/vmlinuz"
|
||||
printf 'lb-initramfs\n' > "$1/microvm/initramfs.cpio.gz"
|
||||
printf 'qboot\n' > "$1/microvm/qboot.rom"
|
||||
}
|
||||
|
||||
# install_image <extract-dir> <microvm-dir> [path-prefix]: runs
|
||||
# install_sqs_store_image and sets RC and OUT.
|
||||
install_image() {
|
||||
RC=0
|
||||
OUT=$(PATH="${3:-}$PATH" EXTRACT_DIR="$1" SPINIFEX_MICROVM_DIR="$2" INSTALL_SPINIFEX_LIB_ONLY=1 \
|
||||
bash -c '. "$1"; SUDO=""; install_sqs_store_image' _ "$SETUP" 2>&1) || RC=$?
|
||||
}
|
||||
|
||||
same() { cmp -s "$1" "$2"; }
|
||||
|
||||
# --- staging ---
|
||||
image "$TMP/built" v1
|
||||
release "$TMP/rel"
|
||||
sh "$STAGE" "$TMP/built" "$TMP/rel" > "$TMP/stage.out"
|
||||
if same "$TMP/built/vmlinuz" "$TMP/rel/microvm-sqs-store/vmlinuz" \
|
||||
&& same "$TMP/built/initramfs.cpio.gz" "$TMP/rel/microvm-sqs-store/initramfs.cpio.gz"; then
|
||||
pass "stages the built image beside the load balancer's"
|
||||
else
|
||||
fail "staged image differs from the build"
|
||||
fi
|
||||
if [ ! -e "$TMP/rel/microvm/sqs-store" ] && [ "$(ls "$TMP/rel/microvm" | wc -l)" -eq 3 ]; then
|
||||
pass "leaves the load balancer's microvm directory as it was"
|
||||
else
|
||||
fail "load balancer directory changed: $(ls "$TMP/rel/microvm")"
|
||||
fi
|
||||
|
||||
release "$TMP/rel-none"
|
||||
mkdir -p "$TMP/half"; printf 'k\n' > "$TMP/half/vmlinuz"
|
||||
OUT=$(sh "$STAGE" "$TMP/half" "$TMP/rel-none")
|
||||
if [ ! -e "$TMP/rel-none/microvm-sqs-store" ] && echo "$OUT" | grep -q 'WARNING: SQS store microVM image missing'; then
|
||||
pass "stages nothing from an incomplete build, and says so"
|
||||
else
|
||||
fail "incomplete build staged or unannounced: $OUT"
|
||||
fi
|
||||
|
||||
# --- an older installer meets a release that carries the store image ---
|
||||
# It copies microvm/* without -r; the sibling directory must not reach it.
|
||||
mkdir -p "$TMP/old-dest"
|
||||
if cp "$TMP/rel/microvm/"* "$TMP/old-dest/" 2> "$TMP/old.err" && [ ! -e "$TMP/old-dest/sqs-store" ]; then
|
||||
pass "an older installer's copy still succeeds and ignores the store image"
|
||||
else
|
||||
fail "older installer copy failed: $(cat "$TMP/old.err")"
|
||||
fi
|
||||
mkdir -p "$TMP/nested/microvm/sqs-store" "$TMP/nested-dest"
|
||||
printf 'x\n' > "$TMP/nested/microvm/vmlinuz"
|
||||
if cp "$TMP/nested/microvm/"* "$TMP/nested-dest/" 2> /dev/null; then
|
||||
fail "control: a nested directory did not break the older copy, so the test proves nothing"
|
||||
else
|
||||
pass "control: a nested directory would have broken the older copy"
|
||||
fi
|
||||
|
||||
# --- install ---
|
||||
install_image "$TMP/rel" "$TMP/mv"
|
||||
if [ "$RC" = 0 ] && same "$TMP/built/vmlinuz" "$TMP/mv/sqs-store/vmlinuz" \
|
||||
&& same "$TMP/built/initramfs.cpio.gz" "$TMP/mv/sqs-store/initramfs.cpio.gz"; then
|
||||
pass "installs the staged image where the daemon looks"
|
||||
else
|
||||
fail "install rc=$RC: $OUT"
|
||||
fi
|
||||
mode=$(stat -c '%a' "$TMP/mv/sqs-store/initramfs.cpio.gz" 2> /dev/null || echo none)
|
||||
if [ "$mode" = 644 ]; then pass "installs the image 0644"; else fail "initramfs mode $mode"; fi
|
||||
|
||||
image "$TMP/built2" v2
|
||||
release "$TMP/rel2"
|
||||
sh "$STAGE" "$TMP/built2" "$TMP/rel2" > /dev/null
|
||||
install_image "$TMP/rel2" "$TMP/mv"
|
||||
if [ "$RC" = 0 ] && same "$TMP/built2/vmlinuz" "$TMP/mv/sqs-store/vmlinuz" \
|
||||
&& same "$TMP/built2/initramfs.cpio.gz" "$TMP/mv/sqs-store/initramfs.cpio.gz"; then
|
||||
pass "an upgrade replaces both files"
|
||||
else
|
||||
fail "upgrade rc=$RC: $OUT"
|
||||
fi
|
||||
|
||||
# A release without the store image: nothing installed, and a clear warning.
|
||||
install_image "$TMP/rel-none" "$TMP/mv-none"
|
||||
if [ "$RC" = 0 ] && [ ! -e "$TMP/mv-none/sqs-store" ] && echo "$OUT" | grep -q 'no SQS store microVM image; SQS store launches will be refused'; then
|
||||
pass "a release without the image installs nothing and warns"
|
||||
else
|
||||
fail "release without image: rc=$RC exists=$([ -e "$TMP/mv-none/sqs-store" ] && echo yes || echo no) $OUT"
|
||||
fi
|
||||
|
||||
# installed <dir>: a store image from an earlier install, plus an unrelated file.
|
||||
installed() {
|
||||
image "$1/sqs-store" old
|
||||
printf 'keep\n' > "$1/sqs-store/operator-note"
|
||||
}
|
||||
gone() { [ ! -e "$1/sqs-store/vmlinuz" ] && [ ! -e "$1/sqs-store/initramfs.cpio.gz" ]; }
|
||||
kept() { [ "$(cat "$1/sqs-store/operator-note" 2> /dev/null)" = keep ]; }
|
||||
|
||||
# Upgrading to a release without the image must not leave the old one usable.
|
||||
installed "$TMP/mv-up"
|
||||
install_image "$TMP/rel-none" "$TMP/mv-up"
|
||||
if [ "$RC" = 0 ] && gone "$TMP/mv-up" && kept "$TMP/mv-up" \
|
||||
&& echo "$OUT" | grep -q 'Removed the SQS store microVM image left by an earlier install' \
|
||||
&& echo "$OUT" | grep -q 'SQS store launches will be refused'; then
|
||||
pass "a release without the image removes the earlier one, keeps other files, and warns"
|
||||
else
|
||||
fail "release without image over an installed one: rc=$RC $OUT"
|
||||
fi
|
||||
|
||||
# An incomplete store image refuses and removes the earlier pair, because the
|
||||
# binaries this release installed before it would otherwise boot the old one.
|
||||
installed "$TMP/mv-bad"
|
||||
cp -r "$TMP/rel2" "$TMP/rel-bad"
|
||||
rm "$TMP/rel-bad/microvm-sqs-store/initramfs.cpio.gz"
|
||||
install_image "$TMP/rel-bad" "$TMP/mv-bad"
|
||||
if [ "$RC" != 0 ] && echo "$OUT" | grep -q 'incomplete: initramfs.cpio.gz is missing' \
|
||||
&& gone "$TMP/mv-bad" && kept "$TMP/mv-bad"; then
|
||||
pass "an incomplete image is refused and the earlier pair is removed"
|
||||
else
|
||||
fail "incomplete image: rc=$RC $OUT"
|
||||
fi
|
||||
|
||||
# Restaging from a missing or incomplete build drops an image staged earlier.
|
||||
for case in missing incomplete; do
|
||||
image "$TMP/built-$case" new
|
||||
release "$TMP/rel-$case"
|
||||
sh "$STAGE" "$TMP/built" "$TMP/rel-$case" > /dev/null
|
||||
if [ "$case" = missing ]; then rm -r "$TMP/built-$case"; else rm "$TMP/built-$case/initramfs.cpio.gz"; fi
|
||||
OUT=$(sh "$STAGE" "$TMP/built-$case" "$TMP/rel-$case")
|
||||
if [ ! -e "$TMP/rel-$case/microvm-sqs-store" ] && echo "$OUT" | grep -q 'tarball will not include it'; then
|
||||
pass "restaging from a $case build ships no earlier image"
|
||||
else
|
||||
fail "restaging from a $case build left: $(ls -R "$TMP/rel-$case/microvm-sqs-store" 2> /dev/null)"
|
||||
fi
|
||||
done
|
||||
mkdir -p "$TMP/rel-other/microvm-sqs-store"
|
||||
image "$TMP/rel-other/microvm-sqs-store" old
|
||||
printf 'keep\n' > "$TMP/rel-other/microvm-sqs-store/note"
|
||||
sh "$STAGE" "$TMP/no-such-build" "$TMP/rel-other" > /dev/null
|
||||
if [ ! -e "$TMP/rel-other/microvm-sqs-store/vmlinuz" ] && [ "$(cat "$TMP/rel-other/microvm-sqs-store/note")" = keep ]; then
|
||||
pass "restaging removes only the image files"
|
||||
else
|
||||
fail "restaging touched more than the image files"
|
||||
fi
|
||||
|
||||
# An install that stops at the initramfs leaves no pair the daemon would boot.
|
||||
mkdir -p "$TMP/bin"
|
||||
cat > "$TMP/bin/install" << 'EOF'
|
||||
#!/bin/sh
|
||||
for a in "$@"; do last=$a; done
|
||||
case "$last" in */initramfs.cpio.gz) echo "install: stopped" >&2; exit 1 ;; esac
|
||||
exec /usr/bin/install "$@"
|
||||
EOF
|
||||
chmod +x "$TMP/bin/install"
|
||||
install_image "$TMP/rel" "$TMP/mv" "$TMP/bin:"
|
||||
if [ "$RC" != 0 ] && same "$TMP/built/vmlinuz" "$TMP/mv/sqs-store/vmlinuz" && [ ! -e "$TMP/mv/sqs-store/initramfs.cpio.gz" ]; then
|
||||
pass "an interrupted install leaves the initramfs missing, not mismatched"
|
||||
else
|
||||
fail "interrupted install: rc=$RC initramfs=$([ -e "$TMP/mv/sqs-store/initramfs.cpio.gz" ] && echo present || echo missing)"
|
||||
fi
|
||||
|
||||
[ "$fails" = 0 ] || { echo "FAIL: $fails"; exit 1; }
|
||||
echo " setup_sqs_store_image_test ok"
|
||||
@@ -0,0 +1,38 @@
|
||||
package snapshot
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// A volume still being prepared has no settled contents, so it cannot be
|
||||
// captured; once available the same volume snapshots normally.
|
||||
func TestCreateSnapshot_RefusesACreatingVolume(t *testing.T) {
|
||||
svc, store := setupTestSnapshotService(t)
|
||||
ctx := context.Background()
|
||||
seedProviderVolume(t, svc, "vol-preparing", 4)
|
||||
seedVolumeDocument(t, store, ebsmetadata.Volume{
|
||||
VolumeID: "vol-preparing", TenantID: testAccountID, CapacityGiB: 4,
|
||||
State: "creating", AvailabilityZone: "us-east-1a",
|
||||
})
|
||||
|
||||
_, err := svc.CreateSnapshot(ctx, &ec2.CreateSnapshotInput{VolumeId: aws.String("vol-preparing")}, testAccountID)
|
||||
assert.EqualError(t, err, awserrors.ErrorIncorrectState)
|
||||
snaps, err := svc.metadata.ListSnapshotsStrict(ctx, testAccountID)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, snaps)
|
||||
|
||||
meta, err := svc.metadata.GetVolume(ctx, testAccountID, "vol-preparing")
|
||||
require.NoError(t, err)
|
||||
meta.State = "available"
|
||||
require.NoError(t, svc.metadata.PutVolume(ctx, meta))
|
||||
_, err = svc.CreateSnapshot(ctx, &ec2.CreateSnapshotInput{VolumeId: aws.String("vol-preparing")}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
@@ -204,6 +204,10 @@ func (s *SnapshotServiceImpl) CreateSnapshot(ctx context.Context, input *ec2.Cre
|
||||
if volume.CapacityGiB == 0 {
|
||||
return nil, errors.New(awserrors.ErrorServerInternal)
|
||||
}
|
||||
// A volume still being prepared has no settled contents to capture.
|
||||
if volume.State == string(ebsprovider.VolumeStateCreating) {
|
||||
return nil, errors.New(awserrors.ErrorIncorrectState)
|
||||
}
|
||||
|
||||
// Flush the writes buffered by whichever node serves the volume, so the
|
||||
// provider reads a current checkpoint. An attached volume that cannot be
|
||||
|
||||
@@ -86,6 +86,11 @@ type SystemInstanceInput struct {
|
||||
// endpoint serves the profile's role credentials. Empty leaves the VM
|
||||
// without a profile (the agent then needs static keys).
|
||||
IamInstanceProfileArn string `json:"iam_instance_profile_arn,omitempty"`
|
||||
|
||||
// Kind selects a non-load-balancer direct-boot guest; empty is the load
|
||||
// balancer. SQSStore carries the KindSQSStore guest's volume and key.
|
||||
Kind Kind `json:"kind,omitempty"`
|
||||
SQSStore *SQSStoreInput `json:"sqs_store,omitempty"`
|
||||
}
|
||||
|
||||
// ExtraENIInput describes an additional pre-created ENI to attach to a system
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
package systeminstance
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
|
||||
"github.com/nats-io/nkeys"
|
||||
)
|
||||
|
||||
// Kind selects the guest a direct-boot system instance runs. The empty kind
|
||||
// is the load balancer; every other kind is launched only on its own subject.
|
||||
type Kind string
|
||||
|
||||
// KindSQSStore is the dedicated SQS store guest on a persistent data volume.
|
||||
const KindSQSStore Kind = "sqs-store"
|
||||
|
||||
// SQSStoreLaunchSubjectRoot is the subject root for SQS store launches. It is
|
||||
// versioned and separate from system.LaunchInstance so a daemon that predates
|
||||
// it has no subscriber and the launch fails instead of booting another guest.
|
||||
const SQSStoreLaunchSubjectRoot = "system.LaunchSQSStore.v1"
|
||||
|
||||
// SQSStoreLaunchSubject is the queue subject for an instance type; with a node
|
||||
// ID it is that node's targeted subject.
|
||||
func SQSStoreLaunchSubject(instanceType, nodeID string) string {
|
||||
if nodeID == "" {
|
||||
return fmt.Sprintf("%s.%s", SQSStoreLaunchSubjectRoot, instanceType)
|
||||
}
|
||||
return fmt.Sprintf("%s.%s.%s", SQSStoreLaunchSubjectRoot, instanceType, nodeID)
|
||||
}
|
||||
|
||||
// SQSStoreInput is what an SQS store guest needs beyond its network: an
|
||||
// existing, already-formatted system-account volume and the one NATS user
|
||||
// public key the store accepts. It never carries a private key or seed.
|
||||
type SQSStoreInput struct {
|
||||
DataVolumeID string `json:"data_volume_id"`
|
||||
ClientNKey string `json:"client_nkey"`
|
||||
}
|
||||
|
||||
// ErrInvalidSQSStoreInput is returned for a malformed SQS store launch.
|
||||
var ErrInvalidSQSStoreInput = errors.New("sysinstance: invalid SQS store launch")
|
||||
|
||||
// volumeIDPattern is the platform's volume ID shape. The guest finds its disk
|
||||
// by the ID without its dash, which must fit the 20-byte virtio-blk serial.
|
||||
var volumeIDPattern = regexp.MustCompile(`^vol-[0-9a-f]{17}$`)
|
||||
|
||||
// ValidateSQSStore checks an SQS store launch's shape. It reads nothing, so a
|
||||
// valid result says only that the volume and key are well formed.
|
||||
func (in *SystemInstanceInput) ValidateSQSStore() error {
|
||||
switch {
|
||||
case in.Kind != KindSQSStore:
|
||||
return fmt.Errorf("%w: kind %q is not %q", ErrInvalidSQSStoreInput, in.Kind, KindSQSStore)
|
||||
case in.BootMode != "" && in.BootMode != BootDirect:
|
||||
return fmt.Errorf("%w: boot mode %q, the store boots direct only", ErrInvalidSQSStoreInput, in.BootMode)
|
||||
case in.SQSStore == nil:
|
||||
return fmt.Errorf("%w: no data volume or client key", ErrInvalidSQSStoreInput)
|
||||
case !volumeIDPattern.MatchString(in.SQSStore.DataVolumeID):
|
||||
return fmt.Errorf("%w: data volume ID %q is malformed", ErrInvalidSQSStoreInput, in.SQSStore.DataVolumeID)
|
||||
case !nkeys.IsValidPublicUserKey(in.SQSStore.ClientNKey):
|
||||
// The value is not echoed: a caller that sent a seed here must not
|
||||
// have it written to a log.
|
||||
return fmt.Errorf("%w: client key is not a NATS user public key", ErrInvalidSQSStoreInput)
|
||||
case in.LBAgentEnv != "" || in.CACert != "" || in.Scheme != "":
|
||||
return fmt.Errorf("%w: load balancer settings on an SQS store launch", ErrInvalidSQSStoreInput)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RefuseNonLoadBalancerKind keeps every non-load-balancer guest off the
|
||||
// legacy launch path, which would boot the load balancer image instead.
|
||||
func (in *SystemInstanceInput) RefuseNonLoadBalancerKind() error {
|
||||
if in.Kind != "" || in.SQSStore != nil {
|
||||
return fmt.Errorf("%w: kind %q must use its own launch subject", ErrInvalidSQSStoreInput, in.Kind)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package systeminstance
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
identifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
|
||||
"github.com/nats-io/nkeys"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func validSQSStoreInput(t *testing.T) (*SystemInstanceInput, []byte) {
|
||||
t.Helper()
|
||||
kp, err := nkeys.CreateUser()
|
||||
require.NoError(t, err)
|
||||
pub, err := kp.PublicKey()
|
||||
require.NoError(t, err)
|
||||
seed, err := kp.Seed()
|
||||
require.NoError(t, err)
|
||||
return &SystemInstanceInput{
|
||||
InstanceType: "sys.micro",
|
||||
Kind: KindSQSStore,
|
||||
SQSStore: &SQSStoreInput{DataVolumeID: "vol-0123456789abcdef0", ClientNKey: pub},
|
||||
}, seed
|
||||
}
|
||||
|
||||
func TestValidateSQSStore(t *testing.T) {
|
||||
in, _ := validSQSStoreInput(t)
|
||||
require.NoError(t, in.ValidateSQSStore())
|
||||
|
||||
_, seed := validSQSStoreInput(t)
|
||||
account, err := nkeys.CreateAccount()
|
||||
require.NoError(t, err)
|
||||
accountPub, err := account.PublicKey()
|
||||
require.NoError(t, err)
|
||||
|
||||
cases := map[string]func(*SystemInstanceInput){
|
||||
"no kind": func(in *SystemInstanceInput) { in.Kind = "" },
|
||||
"other kind": func(in *SystemInstanceInput) { in.Kind = "lambda" },
|
||||
"ami boot": func(in *SystemInstanceInput) { in.BootMode = BootAMI },
|
||||
"no store input": func(in *SystemInstanceInput) { in.SQSStore = nil },
|
||||
"no volume": func(in *SystemInstanceInput) { in.SQSStore.DataVolumeID = "" },
|
||||
"malformed volume": func(in *SystemInstanceInput) { in.SQSStore.DataVolumeID = "vol-XYZ" },
|
||||
"no key": func(in *SystemInstanceInput) { in.SQSStore.ClientNKey = "" },
|
||||
"seed as key": func(in *SystemInstanceInput) { in.SQSStore.ClientNKey = string(seed) },
|
||||
"account key": func(in *SystemInstanceInput) { in.SQSStore.ClientNKey = accountPub },
|
||||
"lb agent env": func(in *SystemInstanceInput) { in.LBAgentEnv = "LB_ID=x" },
|
||||
"lb ca cert": func(in *SystemInstanceInput) { in.CACert = "cert" },
|
||||
"lb scheme": func(in *SystemInstanceInput) { in.Scheme = "internal" },
|
||||
}
|
||||
for name, mutate := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
in, _ := validSQSStoreInput(t)
|
||||
mutate(in)
|
||||
err := in.ValidateSQSStore()
|
||||
require.ErrorIs(t, err, ErrInvalidSQSStoreInput)
|
||||
assert.NotContains(t, err.Error(), string(seed), "a seed must never be echoed")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateSQSStore_VolumeIDMatchesTheGuestSerial(t *testing.T) {
|
||||
accepted := identifiers.GenerateResourceID("vol")
|
||||
in, _ := validSQSStoreInput(t)
|
||||
in.SQSStore.DataVolumeID = accepted
|
||||
require.NoError(t, in.ValidateSQSStore(), "a platform-generated volume ID is accepted")
|
||||
// The guest accepts only vol + 17 lowercase hex, the full 20-byte serial.
|
||||
assert.Len(t, strings.ReplaceAll(accepted, "-", ""), 20)
|
||||
|
||||
for _, id := range []string{
|
||||
// 16 hex
|
||||
"vol-0123456789abcdef",
|
||||
// 18 hex: serial over 20 bytes
|
||||
"vol-0123456789abcdef01",
|
||||
// 8 hex
|
||||
"vol-01234567",
|
||||
// upper case
|
||||
"vol-0123456789ABCDEF0",
|
||||
// no dash
|
||||
"vol0123456789abcdef0",
|
||||
// not a volume
|
||||
"snap-0123456789abcdef0",
|
||||
// trailing newline
|
||||
"vol-0123456789abcdef0\n",
|
||||
} {
|
||||
in, _ := validSQSStoreInput(t)
|
||||
in.SQSStore.DataVolumeID = id
|
||||
require.ErrorIs(t, in.ValidateSQSStore(), ErrInvalidSQSStoreInput, id)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRefuseNonLoadBalancerKind(t *testing.T) {
|
||||
require.NoError(t, (&SystemInstanceInput{InstanceType: "sys.micro"}).RefuseNonLoadBalancerKind())
|
||||
in, _ := validSQSStoreInput(t)
|
||||
require.ErrorIs(t, in.RefuseNonLoadBalancerKind(), ErrInvalidSQSStoreInput)
|
||||
require.Error(t, (&SystemInstanceInput{SQSStore: &SQSStoreInput{}}).RefuseNonLoadBalancerKind())
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
package volume
|
||||
|
||||
//test:in-package — the creating state is asserted on the metadata document,
|
||||
// and the metadata store is unexported.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/mulgadc/spinifex/spinifex/providers/ebs"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func createPreparingVolume(t *testing.T, svc *VolumeServiceImpl) string {
|
||||
t.Helper()
|
||||
vol, err := svc.CreateVolumeForPreparation(context.Background(), &ec2.CreateVolumeInput{
|
||||
Size: aws.Int64(4), AvailabilityZone: aws.String("ap-southeast-2a"),
|
||||
}, testVolAccountID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "creating", aws.StringValue(vol.State))
|
||||
return aws.StringValue(vol.VolumeId)
|
||||
}
|
||||
|
||||
func requireVolumeState(t *testing.T, svc *VolumeServiceImpl, volumeID, state string) {
|
||||
t.Helper()
|
||||
meta, err := svc.metadata.GetVolume(context.Background(), testVolAccountID, volumeID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, state, meta.State)
|
||||
assert.Empty(t, meta.AttachedInstance)
|
||||
assert.Equal(t, uint64(4), meta.CapacityGiB)
|
||||
}
|
||||
|
||||
func TestCreateVolumeForPreparation_IsBornCreatingWhileCreateVolumeStaysAvailable(t *testing.T) {
|
||||
svc := newTestVolumeService("ap-southeast-2a")
|
||||
id := createPreparingVolume(t, svc)
|
||||
requireVolumeState(t, svc, id, "creating")
|
||||
|
||||
described, err := svc.DescribeVolumes(context.Background(), &ec2.DescribeVolumesInput{VolumeIds: []*string{aws.String(id)}}, testVolAccountID)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, described.Volumes, 1)
|
||||
assert.Equal(t, "creating", aws.StringValue(described.Volumes[0].State))
|
||||
|
||||
normal, err := svc.CreateVolume(context.Background(), &ec2.CreateVolumeInput{
|
||||
Size: aws.Int64(4), AvailabilityZone: aws.String("ap-southeast-2a"),
|
||||
}, testVolAccountID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "available", aws.StringValue(normal.State))
|
||||
requireVolumeState(t, svc, aws.StringValue(normal.VolumeId), "available")
|
||||
}
|
||||
|
||||
// Every public path that could take, change or capture the volume refuses it
|
||||
// and leaves it exactly as it was until preparation completes.
|
||||
func TestCreatingVolume_PublicMutationsAreRefused(t *testing.T) {
|
||||
svc := newTestVolumeService("ap-southeast-2a")
|
||||
svc.snapshotKV = setupTestVolumeKV(t)
|
||||
id := createPreparingVolume(t, svc)
|
||||
ctx := context.Background()
|
||||
|
||||
_, err := svc.DeleteVolume(ctx, &ec2.DeleteVolumeInput{VolumeId: aws.String(id)}, testVolAccountID)
|
||||
assert.EqualError(t, err, awserrors.ErrorVolumeInUse)
|
||||
requireVolumeState(t, svc, id, "creating")
|
||||
|
||||
_, err = svc.ModifyVolume(ctx, &ec2.ModifyVolumeInput{VolumeId: aws.String(id), Size: aws.Int64(8)}, testVolAccountID)
|
||||
assert.EqualError(t, err, awserrors.ErrorIncorrectState)
|
||||
requireVolumeState(t, svc, id, "creating")
|
||||
|
||||
_, err = svc.ForceDetachVolume(ctx, &ec2.DetachVolumeInput{VolumeId: aws.String(id), Force: aws.Bool(true)}, testVolAccountID)
|
||||
assert.EqualError(t, err, awserrors.ErrorIncorrectState)
|
||||
requireVolumeState(t, svc, id, "creating")
|
||||
}
|
||||
|
||||
func TestCompleteVolumePreparation_ReleasesTheVolume(t *testing.T) {
|
||||
svc := newTestVolumeService("ap-southeast-2a")
|
||||
svc.snapshotKV = setupTestVolumeKV(t)
|
||||
id := createPreparingVolume(t, svc)
|
||||
ctx := context.Background()
|
||||
|
||||
require.NoError(t, svc.CompleteVolumePreparation(ctx, testVolAccountID, id))
|
||||
requireVolumeState(t, svc, id, "available")
|
||||
|
||||
// Released means ordinary behavior again: modify and delete now proceed.
|
||||
_, err := svc.ModifyVolume(ctx, &ec2.ModifyVolumeInput{VolumeId: aws.String(id), Size: aws.Int64(8)}, testVolAccountID)
|
||||
require.NoError(t, err)
|
||||
_, err = svc.DeleteVolume(ctx, &ec2.DeleteVolumeInput{VolumeId: aws.String(id)}, testVolAccountID)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// Completion only releases an unattached creating volume, so it can never
|
||||
// flip a volume some other path owns to available.
|
||||
func TestCompleteVolumePreparation_RefusesAnythingButAnUnattachedCreatingVolume(t *testing.T) {
|
||||
svc := newTestVolumeService("ap-southeast-2a")
|
||||
ctx := context.Background()
|
||||
|
||||
normal, err := svc.CreateVolume(ctx, &ec2.CreateVolumeInput{
|
||||
Size: aws.Int64(4), AvailabilityZone: aws.String("ap-southeast-2a"),
|
||||
}, testVolAccountID)
|
||||
require.NoError(t, err)
|
||||
err = svc.CompleteVolumePreparation(ctx, testVolAccountID, aws.StringValue(normal.VolumeId))
|
||||
require.ErrorContains(t, err, awserrors.ErrorIncorrectState)
|
||||
|
||||
require.NoError(t, svc.UpdateVolumeState(testVolAccountID, aws.StringValue(normal.VolumeId), "in-use", "i-0123456789abcdef0", "/dev/sdf"))
|
||||
err = svc.CompleteVolumePreparation(ctx, testVolAccountID, aws.StringValue(normal.VolumeId))
|
||||
require.ErrorContains(t, err, awserrors.ErrorIncorrectState)
|
||||
meta, err := svc.metadata.GetVolume(ctx, testVolAccountID, aws.StringValue(normal.VolumeId))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "in-use", meta.State)
|
||||
|
||||
id := createPreparingVolume(t, svc)
|
||||
err = svc.CompleteVolumePreparation(ctx, "000000000099", id)
|
||||
assert.EqualError(t, err, awserrors.ErrorInvalidVolumeNotFound)
|
||||
requireVolumeState(t, svc, id, "creating")
|
||||
|
||||
err = svc.CompleteVolumePreparation(ctx, testVolAccountID, "vol-0000000000000dead")
|
||||
assert.EqualError(t, err, awserrors.ErrorInvalidVolumeNotFound)
|
||||
}
|
||||
|
||||
func TestDiscardVolumePreparation_DeletesTheCreatingVolumeAndItsStorage(t *testing.T) {
|
||||
svc := newTestVolumeService("ap-southeast-2a")
|
||||
id := createPreparingVolume(t, svc)
|
||||
ctx := context.Background()
|
||||
|
||||
require.NoError(t, svc.DiscardVolumePreparation(ctx, testVolAccountID, id))
|
||||
_, err := svc.metadata.GetVolume(ctx, testVolAccountID, id)
|
||||
require.Error(t, err, "the record must be gone")
|
||||
_, err = svc.provider.GetVolume(ctx, ebsprovider.GetVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: id})
|
||||
require.ErrorIs(t, err, ebsprovider.ErrNotFound, "the provider volume must be gone")
|
||||
}
|
||||
|
||||
// Discard is only for a failed preparation, so an ordinary or attached volume,
|
||||
// or one in another account, is never deleted through it.
|
||||
func TestDiscardVolumePreparation_RefusesAnythingButAnUnattachedCreatingVolume(t *testing.T) {
|
||||
svc := newTestVolumeService("ap-southeast-2a")
|
||||
ctx := context.Background()
|
||||
|
||||
normal, err := svc.CreateVolume(ctx, &ec2.CreateVolumeInput{
|
||||
Size: aws.Int64(4), AvailabilityZone: aws.String("ap-southeast-2a"),
|
||||
}, testVolAccountID)
|
||||
require.NoError(t, err)
|
||||
normalID := aws.StringValue(normal.VolumeId)
|
||||
err = svc.DiscardVolumePreparation(ctx, testVolAccountID, normalID)
|
||||
require.ErrorContains(t, err, awserrors.ErrorIncorrectState)
|
||||
requireVolumeState(t, svc, normalID, "available")
|
||||
|
||||
require.NoError(t, svc.UpdateVolumeState(testVolAccountID, normalID, "in-use", "i-0123456789abcdef0", "/dev/sdf"))
|
||||
err = svc.DiscardVolumePreparation(ctx, testVolAccountID, normalID)
|
||||
require.ErrorContains(t, err, awserrors.ErrorIncorrectState)
|
||||
|
||||
id := createPreparingVolume(t, svc)
|
||||
err = svc.DiscardVolumePreparation(ctx, "000000000099", id)
|
||||
assert.EqualError(t, err, awserrors.ErrorInvalidVolumeNotFound)
|
||||
requireVolumeState(t, svc, id, "creating")
|
||||
_, err = svc.provider.GetVolume(ctx, ebsprovider.GetVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: id})
|
||||
require.NoError(t, err, "a refused discard must leave the storage")
|
||||
}
|
||||
@@ -118,6 +118,70 @@ func NewVolumeServiceImplWithStore(cfg *config.Config, store objectstore.ObjectS
|
||||
// CreateVolume allocates a new EBS volume through the EBS provider and records
|
||||
// its control-plane metadata document.
|
||||
func (s *VolumeServiceImpl) CreateVolume(ctx context.Context, input *ec2.CreateVolumeInput, accountID string) (*ec2.Volume, error) {
|
||||
return s.createVolume(ctx, input, accountID, ebsprovider.VolumeStateAvailable)
|
||||
}
|
||||
|
||||
// CreateVolumeForPreparation is CreateVolume for an internal caller that must
|
||||
// write the volume's first contents before anyone else can use it. The record
|
||||
// is born "creating", which attach, delete, modify, force-detach and snapshot
|
||||
// all refuse, until CompleteVolumePreparation makes it available.
|
||||
func (s *VolumeServiceImpl) CreateVolumeForPreparation(ctx context.Context, input *ec2.CreateVolumeInput, accountID string) (*ec2.Volume, error) {
|
||||
return s.createVolume(ctx, input, accountID, ebsprovider.VolumeStateCreating)
|
||||
}
|
||||
|
||||
// CompleteVolumePreparation moves a volume made by CreateVolumeForPreparation
|
||||
// to available. Anything other than an unattached "creating" volume is
|
||||
// refused, so it cannot release a volume some other path is using.
|
||||
func (s *VolumeServiceImpl) CompleteVolumePreparation(ctx context.Context, accountID, volumeID string) error {
|
||||
meta, err := s.getVolumeMetadata(ctx, accountID, volumeID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if meta.TenantID != accountID {
|
||||
return errors.New(awserrors.ErrorInvalidVolumeNotFound)
|
||||
}
|
||||
if meta.State != string(ebsprovider.VolumeStateCreating) || meta.AttachedInstance != "" {
|
||||
return awserrors.Errorf(awserrors.ErrorIncorrectState, "volume %s is %q, not an unattached creating volume", volumeID, meta.State)
|
||||
}
|
||||
meta.State = string(ebsprovider.VolumeStateAvailable)
|
||||
if err := s.metadata.PutVolume(ctx, meta); err != nil {
|
||||
return awserrors.Errorf(awserrors.ErrorServerInternal, "write volume metadata %s: %v", volumeID, err)
|
||||
}
|
||||
slog.InfoContext(ctx, "Volume preparation completed", "volumeId", volumeID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// DiscardVolumePreparation deletes a volume whose preparation failed. Only an
|
||||
// unattached "creating" volume in the caller's account qualifies; the caller
|
||||
// must already know no export of it can still be live.
|
||||
func (s *VolumeServiceImpl) DiscardVolumePreparation(ctx context.Context, accountID, volumeID string) error {
|
||||
if err := s.requireProvider(ctx, "DiscardVolumePreparation"); err != nil {
|
||||
return err
|
||||
}
|
||||
meta, err := s.getVolumeMetadata(ctx, accountID, volumeID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if meta.TenantID != accountID {
|
||||
return errors.New(awserrors.ErrorInvalidVolumeNotFound)
|
||||
}
|
||||
if meta.State != string(ebsprovider.VolumeStateCreating) || meta.AttachedInstance != "" {
|
||||
return awserrors.Errorf(awserrors.ErrorIncorrectState, "volume %s is %q, not an unattached creating volume", volumeID, meta.State)
|
||||
}
|
||||
// Snapshots refuse a creating volume, so none can depend on this one.
|
||||
if err := s.provider.DeleteVolume(ctx, ebsprovider.DeleteVolumeRequest{
|
||||
Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, Handle: meta.ProviderHandle,
|
||||
}); err != nil {
|
||||
return awserrors.Errorf(awserrors.ErrorServerInternal, "delete volume %s: %v", volumeID, err)
|
||||
}
|
||||
if err := s.metadata.DeleteVolume(ctx, accountID, volumeID); err != nil && !objectstore.IsNoSuchKeyError(err) {
|
||||
return awserrors.Errorf(awserrors.ErrorServerInternal, "delete volume metadata %s: %v", volumeID, err)
|
||||
}
|
||||
slog.InfoContext(ctx, "Volume preparation discarded", "volumeId", volumeID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *VolumeServiceImpl) createVolume(ctx context.Context, input *ec2.CreateVolumeInput, accountID string, initialState ebsprovider.VolumeState) (*ec2.Volume, error) {
|
||||
if input == nil {
|
||||
return nil, errors.New(awserrors.ErrorInvalidParameterValue)
|
||||
}
|
||||
@@ -241,7 +305,7 @@ func (s *VolumeServiceImpl) CreateVolume(ctx context.Context, input *ec2.CreateV
|
||||
return nil, errors.New(awserrors.ErrorServerInternal)
|
||||
}
|
||||
if err := s.metadata.PutVolume(ctx, ebsmetadata.Volume{
|
||||
VolumeID: volumeID, TenantID: accountID, CapacityGiB: uint64(size), State: string(ebsprovider.VolumeStateAvailable),
|
||||
VolumeID: volumeID, TenantID: accountID, CapacityGiB: uint64(size), State: string(initialState),
|
||||
CreatedAt: now, AvailabilityZone: *input.AvailabilityZone, VolumeType: volumeType,
|
||||
IOPS: iops, Throughput: throughput, SnapshotID: snapshotID, Tags: tags, ProviderHandle: created.Handle,
|
||||
Encrypted: encrypted,
|
||||
@@ -254,7 +318,7 @@ func (s *VolumeServiceImpl) CreateVolume(ctx context.Context, input *ec2.CreateV
|
||||
slog.InfoContext(ctx, "CreateVolume completed", "volumeId", volumeID, "size", size, "type", volumeType)
|
||||
|
||||
return &ec2.Volume{VolumeId: aws.String(volumeID), Size: aws.Int64(size), VolumeType: aws.String(volumeType),
|
||||
State: aws.String("available"), AvailabilityZone: input.AvailabilityZone, CreateTime: aws.Time(now),
|
||||
State: aws.String(string(initialState)), AvailabilityZone: input.AvailabilityZone, CreateTime: aws.Time(now),
|
||||
Iops: aws.Int64(int64(iops)), Throughput: aws.Int64(int64(throughput)), Encrypted: aws.Bool(encrypted),
|
||||
SnapshotId: snapshotIDOrNil(snapshotID), Tags: awstags.MapToEC2(tags)}, nil
|
||||
}
|
||||
@@ -947,7 +1011,7 @@ func (s *VolumeServiceImpl) ModifyVolume(ctx context.Context, input *ec2.ModifyV
|
||||
if input.Size == nil || *input.Size <= originalSize {
|
||||
return nil, errors.New(awserrors.ErrorInvalidParameterValue)
|
||||
}
|
||||
if meta.AttachedInstance != "" && meta.State == "in-use" {
|
||||
if (meta.AttachedInstance != "" && meta.State == "in-use") || meta.State == string(ebsprovider.VolumeStateCreating) {
|
||||
return nil, errors.New(awserrors.ErrorIncorrectState)
|
||||
}
|
||||
originalType := meta.VolumeType
|
||||
@@ -1070,6 +1134,11 @@ func (s *VolumeServiceImpl) ForceDetachVolume(ctx context.Context, input *ec2.De
|
||||
return nil, errors.New(awserrors.ErrorInvalidVolumeNotFound)
|
||||
}
|
||||
|
||||
// Clearing would also make a volume still being prepared available.
|
||||
if meta.State == string(ebsprovider.VolumeStateCreating) {
|
||||
return nil, errors.New(awserrors.ErrorIncorrectState)
|
||||
}
|
||||
|
||||
previous := meta.AttachedInstance
|
||||
if err := s.UpdateVolumeState(accountID, volumeID, "available", "", ""); err != nil {
|
||||
slog.ErrorContext(ctx, "ForceDetachVolume: failed to clear attachment", "volumeId", volumeID, "err", err)
|
||||
|
||||
@@ -46,6 +46,10 @@ func (l *natsSystemInstanceLauncher) LaunchSystemInstance(input *SystemInstanceI
|
||||
if input.InstanceType == "" {
|
||||
return nil, fmt.Errorf("system instance input missing InstanceType")
|
||||
}
|
||||
// An older receiver would ignore the kind and boot a load balancer.
|
||||
if err := input.RefuseNonLoadBalancerKind(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
payload, err := json.Marshal(input)
|
||||
if err != nil {
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/internal/testkit"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/ec2/systeminstance"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -269,3 +270,44 @@ func TestNATSSystemInstanceLauncher_TerminatePropagatesRemoteError(t *testing.T)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "not found")
|
||||
}
|
||||
|
||||
func TestNATSSystemInstanceLauncher_RefusesNonLoadBalancerKindBeforeSending(t *testing.T) {
|
||||
t.Parallel()
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
defer nc.Close()
|
||||
|
||||
// A responder on the legacy subject stands in for an older daemon that
|
||||
// would ignore the kind and boot a load balancer.
|
||||
var mu sync.Mutex
|
||||
received := 0
|
||||
sub, err := nc.Subscribe("system.LaunchInstance.sys.micro", func(msg *nats.Msg) {
|
||||
mu.Lock()
|
||||
received++
|
||||
mu.Unlock()
|
||||
payload, _ := json.Marshal(struct {
|
||||
Output *SystemInstanceOutput `json:"output,omitempty"`
|
||||
}{Output: &SystemInstanceOutput{InstanceID: "i-0123456789abcdef0"}})
|
||||
_ = msg.Respond(payload)
|
||||
})
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
|
||||
launcher := NewNATSSystemInstanceLauncher(nc, 2*time.Second)
|
||||
for name, input := range map[string]*SystemInstanceInput{
|
||||
"kind": {InstanceType: "sys.micro", Kind: systeminstance.KindSQSStore},
|
||||
"sqs store input": {InstanceType: "sys.micro", SQSStore: &systeminstance.SQSStoreInput{DataVolumeID: "vol-0123456789abcdef0"}},
|
||||
} {
|
||||
out, err := launcher.LaunchSystemInstance(input)
|
||||
require.Error(t, err, name)
|
||||
assert.Nil(t, out, name)
|
||||
}
|
||||
|
||||
// An ownerless load balancer launch still reaches the responder, so the
|
||||
// zero count above is the refusal, not a missing subscriber.
|
||||
out, err := launcher.LaunchSystemInstance(&SystemInstanceInput{InstanceType: "sys.micro"})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, out)
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
assert.Equal(t, 1, received, "only the load balancer launch may be sent")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,301 @@
|
||||
// Package storevolume prepares the data volume an SQS store guest mounts. The
|
||||
// guest never formats a disk, so a new volume gets its ext4 filesystem here,
|
||||
// on the host, before anything else can attach it.
|
||||
package storevolume
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/mulgadc/spinifex/spinifex/providers/ebs"
|
||||
)
|
||||
|
||||
// VolumeService is the slice of the EC2 volume service preparation needs. A
|
||||
// volume it creates stays "creating", which public callers cannot attach,
|
||||
// delete, modify or snapshot, until it is completed or discarded.
|
||||
type VolumeService interface {
|
||||
CreateVolumeForPreparation(ctx context.Context, input *ec2.CreateVolumeInput, accountID string) (*ec2.Volume, error)
|
||||
CompleteVolumePreparation(ctx context.Context, accountID, volumeID string) error
|
||||
DiscardVolumePreparation(ctx context.Context, accountID, volumeID string) error
|
||||
}
|
||||
|
||||
// Publisher is the provider pair used to reach the new volume's blocks.
|
||||
type Publisher interface {
|
||||
PublishVolume(context.Context, ebsprovider.PublishVolumeRequest) (*ebsprovider.PublishedVolume, error)
|
||||
UnpublishVolume(context.Context, ebsprovider.UnpublishVolumeRequest) error
|
||||
}
|
||||
|
||||
// Request names the volume to create.
|
||||
type Request struct {
|
||||
AccountID string
|
||||
AvailabilityZone string
|
||||
SizeGiB int64
|
||||
}
|
||||
|
||||
// Preparer creates fresh SQS store volumes. NodeID must be the node this
|
||||
// process runs on, because the copy reads the export it publishes locally.
|
||||
type Preparer struct {
|
||||
Volumes VolumeService
|
||||
Provider Publisher
|
||||
NodeID string
|
||||
ScratchDir string
|
||||
// WriteImage copies a raw image onto an NBD export. The composition root
|
||||
// supplies it, since a domain does not import the runtime copy helper.
|
||||
WriteImage func(ctx context.Context, sourcePath, nbdURI string, progress io.Writer) error
|
||||
|
||||
lookPath func(string) (string, error)
|
||||
runMkfs func(ctx context.Context, mkfs, path string) error
|
||||
toolPaths []string
|
||||
// cleanupTimeout bounds unpublish and discard, which ignore caller
|
||||
// cancellation so a cancelled caller cannot skip closing the export.
|
||||
cleanupTimeout time.Duration
|
||||
}
|
||||
|
||||
const defaultCleanupTimeout = 30 * time.Second
|
||||
|
||||
// RetainedVolumeError reports a volume kept because deleting it was not safe.
|
||||
// StateUnknown means a failed release may still have made it available.
|
||||
type RetainedVolumeError struct {
|
||||
VolumeID string
|
||||
StateUnknown bool
|
||||
Err error
|
||||
}
|
||||
|
||||
func (e *RetainedVolumeError) Error() string {
|
||||
if e.StateUnknown {
|
||||
return fmt.Sprintf("volume %s retained in unknown state (creating or available) for recovery: %v", e.VolumeID, e.Err)
|
||||
}
|
||||
return fmt.Sprintf("volume %s retained in creating for recovery: %v", e.VolumeID, e.Err)
|
||||
}
|
||||
|
||||
func (e *RetainedVolumeError) Unwrap() error { return e.Err }
|
||||
|
||||
// ErrToolMissing is returned before anything is allocated when the host lacks
|
||||
// a tool preparation needs.
|
||||
var ErrToolMissing = errors.New("required host tool is missing")
|
||||
|
||||
// sbinDirs covers tools a root daemon finds on its PATH but a login shell
|
||||
// may not, since mkfs lives in sbin.
|
||||
var sbinDirs = []string{"/usr/sbin", "/sbin"}
|
||||
|
||||
// Prepare creates a new volume, writes an empty ext4 filesystem to it and
|
||||
// releases it as available. It never writes any volume it did not just create.
|
||||
func (p *Preparer) Prepare(ctx context.Context, req Request) (string, error) {
|
||||
if p.Volumes == nil || p.Provider == nil || p.WriteImage == nil || p.NodeID == "" || p.ScratchDir == "" {
|
||||
return "", errors.New("storevolume: volume service, provider, image writer, node ID and scratch dir are required")
|
||||
}
|
||||
if req.AccountID == "" || req.AvailabilityZone == "" || req.SizeGiB < 1 {
|
||||
return "", fmt.Errorf("storevolume: account, availability zone and a size of at least 1 GiB are required")
|
||||
}
|
||||
mkfs, err := p.findTool("mkfs.ext4")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// The copy runs qemu-img by name, so only a PATH lookup proves it will run.
|
||||
if _, err := p.findOnPath("qemu-img"); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
scratch, err := os.MkdirTemp(p.ScratchDir, "sqs-store-volume-")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("storevolume: scratch dir: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := os.RemoveAll(scratch); err != nil {
|
||||
slog.WarnContext(ctx, "storevolume: scratch cleanup failed", "dir", scratch, "err", err)
|
||||
}
|
||||
}()
|
||||
image := filepath.Join(scratch, "ext4.raw")
|
||||
if err := buildImage(ctx, p.mkfsRunner(), mkfs, image, req.SizeGiB); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
vol, err := p.Volumes.CreateVolumeForPreparation(ctx, &ec2.CreateVolumeInput{
|
||||
AvailabilityZone: aws.String(req.AvailabilityZone),
|
||||
Size: aws.Int64(req.SizeGiB),
|
||||
}, req.AccountID)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("storevolume: create volume: %w", err)
|
||||
}
|
||||
volumeID := aws.StringValue(vol.VolumeId)
|
||||
if volumeID == "" {
|
||||
return "", errors.New("storevolume: create volume returned no volume ID")
|
||||
}
|
||||
// Only a volume held in creating is known to be this call's alone.
|
||||
if state := aws.StringValue(vol.State); state != string(ebsprovider.VolumeStateCreating) {
|
||||
return "", &RetainedVolumeError{VolumeID: volumeID, Err: fmt.Errorf("new volume is %q, not creating; not writing it", state)}
|
||||
}
|
||||
|
||||
if err := p.fill(ctx, volumeID, image); err != nil {
|
||||
return "", p.fail(ctx, req.AccountID, err)
|
||||
}
|
||||
if err := p.Volumes.CompleteVolumePreparation(ctx, req.AccountID, volumeID); err != nil {
|
||||
// The release write may still land, so the volume may already be
|
||||
// available to others and is never discarded here.
|
||||
return "", &RetainedVolumeError{VolumeID: volumeID, StateUnknown: true, Err: fmt.Errorf("complete preparation failed: %w", err)}
|
||||
}
|
||||
slog.InfoContext(ctx, "storevolume: prepared SQS store volume", "volumeId", volumeID, "sizeGiB", req.SizeGiB)
|
||||
return volumeID, nil
|
||||
}
|
||||
|
||||
// prepError records whether the failed volume can still have a live export.
|
||||
type prepError struct {
|
||||
volumeID string
|
||||
err error
|
||||
exportClosed bool
|
||||
}
|
||||
|
||||
func (e *prepError) Error() string { return e.err.Error() }
|
||||
|
||||
// fill publishes the volume, copies the image and unpublishes it. Its error
|
||||
// says whether the export is known to be closed.
|
||||
func (p *Preparer) fill(ctx context.Context, volumeID, image string) error {
|
||||
published, err := p.Provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{
|
||||
Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: p.NodeID,
|
||||
})
|
||||
if err != nil {
|
||||
// These refusals may mean another export exists, which an unpublish
|
||||
// by volume and node would tear down, so it is left alone.
|
||||
if refusedForAnotherExport(err) {
|
||||
return &prepError{volumeID: volumeID, err: fmt.Errorf("publish refused, export not touched: %w", err)}
|
||||
}
|
||||
// Otherwise the publish may still land. Unpublish names only volume
|
||||
// and node, so it is best effort, and the volume is kept regardless.
|
||||
if uerr := p.unpublish(ctx, volumeID); uerr != nil {
|
||||
err = fmt.Errorf("%w (unpublish also failed: %w)", err, uerr)
|
||||
}
|
||||
return &prepError{volumeID: volumeID, err: fmt.Errorf("publish outcome unknown: %w", err)}
|
||||
}
|
||||
|
||||
var writeErr error
|
||||
if published == nil || published.Export.Kind != ebsprovider.ExportKindNBD || published.Export.NBD == nil {
|
||||
writeErr = errors.New("volume was not published as an nbd export")
|
||||
} else {
|
||||
writeErr = p.WriteImage(ctx, image, published.Export.NBD.NBDURI, nil)
|
||||
}
|
||||
|
||||
if err := p.unpublish(ctx, volumeID); err != nil {
|
||||
if writeErr != nil {
|
||||
err = fmt.Errorf("write: %w (unpublish also failed: %w)", writeErr, err)
|
||||
} else {
|
||||
err = fmt.Errorf("unpublish: %w", err)
|
||||
}
|
||||
return &prepError{volumeID: volumeID, err: err}
|
||||
}
|
||||
if writeErr != nil {
|
||||
return &prepError{volumeID: volumeID, err: fmt.Errorf("write: %w", writeErr), exportClosed: true}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Preparer) unpublish(ctx context.Context, volumeID string) error {
|
||||
cctx, cancel := p.cleanupContext(ctx)
|
||||
defer cancel()
|
||||
return p.Provider.UnpublishVolume(cctx, ebsprovider.UnpublishVolumeRequest{
|
||||
Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: p.NodeID,
|
||||
})
|
||||
}
|
||||
|
||||
// fail discards the volume when its export is known closed and otherwise
|
||||
// keeps it in creating, reporting its ID.
|
||||
func (p *Preparer) fail(ctx context.Context, accountID string, err error) error {
|
||||
var pe *prepError
|
||||
if !errors.As(err, &pe) {
|
||||
return err
|
||||
}
|
||||
if !pe.exportClosed {
|
||||
return &RetainedVolumeError{VolumeID: pe.volumeID, Err: pe.err}
|
||||
}
|
||||
cctx, cancel := p.cleanupContext(ctx)
|
||||
defer cancel()
|
||||
if derr := p.Volumes.DiscardVolumePreparation(cctx, accountID, pe.volumeID); derr != nil {
|
||||
return &RetainedVolumeError{VolumeID: pe.volumeID, Err: fmt.Errorf("%w (discard also failed: %w)", pe.err, derr)}
|
||||
}
|
||||
return fmt.Errorf("storevolume: prepare %s failed and the volume was discarded: %w", pe.volumeID, pe.err)
|
||||
}
|
||||
|
||||
func refusedForAnotherExport(err error) bool {
|
||||
return errors.Is(err, ebsprovider.ErrVolumeInUse) || errors.Is(err, ebsprovider.ErrAlreadyExists)
|
||||
}
|
||||
|
||||
// cleanupContext keeps the caller's values but not its cancellation, bounded
|
||||
// so a stuck provider cannot hang the caller.
|
||||
func (p *Preparer) cleanupContext(ctx context.Context) (context.Context, context.CancelFunc) {
|
||||
limit := p.cleanupTimeout
|
||||
if limit <= 0 {
|
||||
limit = defaultCleanupTimeout
|
||||
}
|
||||
return context.WithTimeout(context.WithoutCancel(ctx), limit)
|
||||
}
|
||||
|
||||
// buildImage writes a sparse raw file of the volume's size and formats it.
|
||||
func buildImage(ctx context.Context, run func(context.Context, string, string) error, mkfs, path string, sizeGiB int64) error {
|
||||
f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storevolume: create image: %w", err)
|
||||
}
|
||||
if err := f.Truncate(sizeGiB << 30); err != nil {
|
||||
_ = f.Close()
|
||||
return fmt.Errorf("storevolume: size image: %w", err)
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
return fmt.Errorf("storevolume: close image: %w", err)
|
||||
}
|
||||
if err := run(ctx, mkfs, path); err != nil {
|
||||
return fmt.Errorf("storevolume: format image: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func runMkfsExt4(ctx context.Context, mkfs, path string) error {
|
||||
out, err := exec.CommandContext(ctx, mkfs, "-q", "-F", "-L", "sqsstore", path).CombinedOutput()
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: %w: %s", mkfs, err, out)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Preparer) findOnPath(name string) (string, error) {
|
||||
look := p.lookPath
|
||||
if look == nil {
|
||||
look = exec.LookPath
|
||||
}
|
||||
if path, err := look(name); err == nil {
|
||||
return path, nil
|
||||
}
|
||||
return "", fmt.Errorf("storevolume: %w: %s", ErrToolMissing, name)
|
||||
}
|
||||
|
||||
// findTool also checks sbin, for a tool the caller runs by its resolved path.
|
||||
func (p *Preparer) findTool(name string) (string, error) {
|
||||
if path, err := p.findOnPath(name); err == nil {
|
||||
return path, nil
|
||||
}
|
||||
dirs := p.toolPaths
|
||||
if dirs == nil {
|
||||
dirs = sbinDirs
|
||||
}
|
||||
for _, dir := range dirs {
|
||||
candidate := filepath.Join(dir, name)
|
||||
if info, err := os.Stat(candidate); err == nil && info.Mode().IsRegular() && info.Mode()&0o111 != 0 {
|
||||
return candidate, nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("storevolume: %w: %s", ErrToolMissing, name)
|
||||
}
|
||||
|
||||
func (p *Preparer) mkfsRunner() func(context.Context, string, string) error {
|
||||
if p.runMkfs != nil {
|
||||
return p.runMkfs
|
||||
}
|
||||
return runMkfsExt4
|
||||
}
|
||||
@@ -0,0 +1,436 @@
|
||||
package storevolume
|
||||
|
||||
//test:in-package — the host tools, mkfs and NBD copy are swapped for fakes
|
||||
// through unexported fields so no test formats or writes a real device.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/ec2/volume"
|
||||
"github.com/mulgadc/spinifex/spinifex/providers/ebs"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
var _ VolumeService = (*volume.VolumeServiceImpl)(nil)
|
||||
|
||||
const (
|
||||
testAccount = "000000000000"
|
||||
testVolume = "vol-0123456789abcdef0"
|
||||
testNode = "node-a"
|
||||
testURI = "nbd://127.0.0.1:10809"
|
||||
)
|
||||
|
||||
// harness records every call, in order, across the fakes.
|
||||
type harness struct {
|
||||
mu sync.Mutex
|
||||
calls []string
|
||||
|
||||
createState string
|
||||
createErr error
|
||||
publishErr error
|
||||
exportKind ebsprovider.ExportKind
|
||||
unpublishErr error
|
||||
writeErr error
|
||||
completeErr error
|
||||
discardErr error
|
||||
mkfsErr error
|
||||
missing map[string]bool
|
||||
|
||||
completeApplies bool
|
||||
unpublishBlocks bool
|
||||
state string
|
||||
|
||||
writtenImage string
|
||||
unpublishCtxOK bool
|
||||
}
|
||||
|
||||
func (h *harness) record(s string) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.calls = append(h.calls, s)
|
||||
}
|
||||
|
||||
func (h *harness) CreateVolumeForPreparation(_ context.Context, in *ec2.CreateVolumeInput, accountID string) (*ec2.Volume, error) {
|
||||
h.record(fmt.Sprintf("create %s %dGiB", accountID, aws.Int64Value(in.Size)))
|
||||
if h.createErr != nil {
|
||||
return nil, h.createErr
|
||||
}
|
||||
state := h.createState
|
||||
if state == "" {
|
||||
state = "creating"
|
||||
}
|
||||
return &ec2.Volume{VolumeId: aws.String(testVolume), State: aws.String(state)}, nil
|
||||
}
|
||||
|
||||
func (h *harness) CompleteVolumePreparation(_ context.Context, accountID, volumeID string) error {
|
||||
h.record("complete " + volumeID)
|
||||
if h.completeErr == nil || h.completeApplies {
|
||||
h.state = "available"
|
||||
}
|
||||
return h.completeErr
|
||||
}
|
||||
|
||||
func (h *harness) DiscardVolumePreparation(_ context.Context, accountID, volumeID string) error {
|
||||
h.record("discard " + volumeID)
|
||||
return h.discardErr
|
||||
}
|
||||
|
||||
func (h *harness) PublishVolume(_ context.Context, req ebsprovider.PublishVolumeRequest) (*ebsprovider.PublishedVolume, error) {
|
||||
h.record("publish " + req.VolumeID + " on " + req.NodeID)
|
||||
if h.publishErr != nil {
|
||||
return nil, h.publishErr
|
||||
}
|
||||
kind := h.exportKind
|
||||
if kind == "" {
|
||||
kind = ebsprovider.ExportKindNBD
|
||||
}
|
||||
export := ebsprovider.VolumeExport{Kind: kind}
|
||||
if kind == ebsprovider.ExportKindNBD {
|
||||
export.NBD = &ebsprovider.NBDExport{NBDURI: testURI}
|
||||
}
|
||||
return &ebsprovider.PublishedVolume{VolumeID: req.VolumeID, NodeID: req.NodeID, Export: export}, nil
|
||||
}
|
||||
|
||||
func (h *harness) UnpublishVolume(ctx context.Context, req ebsprovider.UnpublishVolumeRequest) error {
|
||||
h.record("unpublish " + req.VolumeID + " on " + req.NodeID)
|
||||
h.unpublishCtxOK = ctx.Err() == nil
|
||||
if h.unpublishBlocks {
|
||||
<-ctx.Done()
|
||||
return ctx.Err()
|
||||
}
|
||||
return h.unpublishErr
|
||||
}
|
||||
|
||||
func (h *harness) preparer(t *testing.T) *Preparer {
|
||||
t.Helper()
|
||||
return &Preparer{
|
||||
Volumes: h, Provider: h, NodeID: testNode, ScratchDir: t.TempDir(),
|
||||
lookPath: func(name string) (string, error) {
|
||||
if h.missing[name] {
|
||||
return "", errors.New("not found")
|
||||
}
|
||||
return "/fake/" + name, nil
|
||||
},
|
||||
toolPaths: []string{},
|
||||
runMkfs: func(_ context.Context, mkfs, path string) error {
|
||||
h.record("mkfs " + filepath.Base(path))
|
||||
return h.mkfsErr
|
||||
},
|
||||
WriteImage: func(_ context.Context, src, uri string, _ io.Writer) error {
|
||||
h.record("write " + uri)
|
||||
h.writtenImage = src
|
||||
return h.writeErr
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func prepare(t *testing.T, h *harness) (string, error) {
|
||||
t.Helper()
|
||||
return h.preparer(t).Prepare(context.Background(), Request{AccountID: testAccount, AvailabilityZone: "ap-southeast-2a", SizeGiB: 1})
|
||||
}
|
||||
|
||||
func requireRetained(t *testing.T, err error) {
|
||||
t.Helper()
|
||||
var retained *RetainedVolumeError
|
||||
require.ErrorAs(t, err, &retained)
|
||||
assert.Equal(t, testVolume, retained.VolumeID)
|
||||
}
|
||||
|
||||
func TestPrepare_FormatsBeforeAllocatingThenReleasesOnlyAfterUnpublish(t *testing.T) {
|
||||
h := &harness{}
|
||||
p := h.preparer(t)
|
||||
id, err := p.Prepare(context.Background(), Request{AccountID: testAccount, AvailabilityZone: "ap-southeast-2a", SizeGiB: 1})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, testVolume, id)
|
||||
assert.Equal(t, []string{
|
||||
"mkfs ext4.raw",
|
||||
"create 000000000000 1GiB",
|
||||
"publish " + testVolume + " on " + testNode,
|
||||
"write " + testURI,
|
||||
"unpublish " + testVolume + " on " + testNode,
|
||||
"complete " + testVolume,
|
||||
}, h.calls)
|
||||
|
||||
assert.True(t, strings.HasPrefix(h.writtenImage, p.ScratchDir), "the image must come from private scratch")
|
||||
entries, err := os.ReadDir(p.ScratchDir)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, entries, "the scratch image must be removed")
|
||||
}
|
||||
|
||||
func TestPrepare_MissingToolAllocatesNothing(t *testing.T) {
|
||||
for _, tool := range []string{"mkfs.ext4", "qemu-img"} {
|
||||
t.Run(tool, func(t *testing.T) {
|
||||
h := &harness{missing: map[string]bool{tool: true}}
|
||||
_, err := prepare(t, h)
|
||||
require.ErrorIs(t, err, ErrToolMissing)
|
||||
assert.Contains(t, err.Error(), tool)
|
||||
assert.Empty(t, h.calls)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepare_MissingImageWriterAllocatesNothing(t *testing.T) {
|
||||
h := &harness{}
|
||||
p := h.preparer(t)
|
||||
p.WriteImage = nil
|
||||
_, err := p.Prepare(context.Background(), Request{AccountID: testAccount, AvailabilityZone: "ap-southeast-2a", SizeGiB: 1})
|
||||
require.ErrorContains(t, err, "image writer")
|
||||
assert.Empty(t, h.calls)
|
||||
}
|
||||
|
||||
func TestPrepare_FindsSbinToolOffThePath(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "mkfs.ext4"), []byte("#!/bin/sh\n"), 0o755))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "qemu-img"), []byte("not executable"), 0o644))
|
||||
p := &Preparer{lookPath: func(string) (string, error) { return "", errors.New("not found") }, toolPaths: []string{dir}}
|
||||
|
||||
got, err := p.findTool("mkfs.ext4")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, filepath.Join(dir, "mkfs.ext4"), got)
|
||||
_, err = p.findTool("qemu-img")
|
||||
require.ErrorIs(t, err, ErrToolMissing, "a file that is not executable is not the tool")
|
||||
}
|
||||
|
||||
// The copy runs qemu-img by name, so a qemu-img found only in sbin must not
|
||||
// pass the check that runs before anything is allocated.
|
||||
func TestPrepare_QemuImgOnlyInSbinAllocatesNothing(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "qemu-img"), []byte("#!/bin/sh\n"), 0o755))
|
||||
h := &harness{missing: map[string]bool{"qemu-img": true}}
|
||||
p := h.preparer(t)
|
||||
p.toolPaths = []string{dir}
|
||||
_, err := p.Prepare(context.Background(), Request{AccountID: testAccount, AvailabilityZone: "ap-southeast-2a", SizeGiB: 1})
|
||||
require.ErrorIs(t, err, ErrToolMissing)
|
||||
assert.Contains(t, err.Error(), "qemu-img")
|
||||
assert.Empty(t, h.calls)
|
||||
}
|
||||
|
||||
func TestPrepare_FailedFormatAllocatesNothing(t *testing.T) {
|
||||
h := &harness{mkfsErr: errors.New("mkfs exploded")}
|
||||
_, err := prepare(t, h)
|
||||
require.ErrorContains(t, err, "mkfs exploded")
|
||||
assert.Equal(t, []string{"mkfs ext4.raw"}, h.calls)
|
||||
}
|
||||
|
||||
func TestPrepare_FailedCreateTouchesNoVolume(t *testing.T) {
|
||||
h := &harness{createErr: errors.New("no capacity")}
|
||||
_, err := prepare(t, h)
|
||||
require.ErrorContains(t, err, "no capacity")
|
||||
assert.Equal(t, []string{"mkfs ext4.raw", "create 000000000000 1GiB"}, h.calls)
|
||||
}
|
||||
|
||||
// A volume that does not come back creating may not be this call's alone, so
|
||||
// nothing is published, written or deleted.
|
||||
func TestPrepare_VolumeNotBornCreatingIsNeverWritten(t *testing.T) {
|
||||
h := &harness{createState: "available"}
|
||||
_, err := prepare(t, h)
|
||||
requireRetained(t, err)
|
||||
assert.Equal(t, []string{"mkfs ext4.raw", "create 000000000000 1GiB"}, h.calls)
|
||||
}
|
||||
|
||||
func TestPrepare_FailedCopyUnpublishesThenDiscards(t *testing.T) {
|
||||
h := &harness{writeErr: errors.New("qemu-img convert: broken pipe")}
|
||||
_, err := prepare(t, h)
|
||||
require.ErrorContains(t, err, "broken pipe")
|
||||
require.ErrorContains(t, err, "discarded")
|
||||
var retained *RetainedVolumeError
|
||||
assert.NotErrorAs(t, err, &retained)
|
||||
assert.Equal(t, []string{
|
||||
"mkfs ext4.raw", "create 000000000000 1GiB",
|
||||
"publish " + testVolume + " on " + testNode, "write " + testURI,
|
||||
"unpublish " + testVolume + " on " + testNode, "discard " + testVolume,
|
||||
}, h.calls)
|
||||
}
|
||||
|
||||
func TestPrepare_NonNBDExportIsNotWrittenAndIsDiscarded(t *testing.T) {
|
||||
h := &harness{exportKind: "iscsi"}
|
||||
_, err := prepare(t, h)
|
||||
require.ErrorContains(t, err, "not published as an nbd export")
|
||||
assert.NotContains(t, h.calls, "write "+testURI)
|
||||
assert.Equal(t, "discard "+testVolume, h.calls[len(h.calls)-1])
|
||||
}
|
||||
|
||||
// An in-use or already-exists refusal may mean another export of the volume
|
||||
// is live, and unpublish names only volume and node, so it is never sent.
|
||||
func TestPrepare_PublishRefusedForAnotherExportSendsNoUnpublish(t *testing.T) {
|
||||
for name, perr := range map[string]error{
|
||||
"volume in use": &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeVolumeInUse, Message: "in use"},
|
||||
"already exists": &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeAlreadyExists, Message: "exists"},
|
||||
"wrapped": fmt.Errorf("nats provider: %w", &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeVolumeInUse, Message: "in use"}),
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
h := &harness{publishErr: perr}
|
||||
_, err := prepare(t, h)
|
||||
requireRetained(t, err)
|
||||
require.ErrorContains(t, err, "export not touched")
|
||||
assert.Equal(t, []string{
|
||||
"mkfs ext4.raw", "create 000000000000 1GiB",
|
||||
"publish " + testVolume + " on " + testNode,
|
||||
}, h.calls, "no unpublish, write, discard or release")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Any other publish error may still land, so the volume is kept with its ID
|
||||
// reported even when the best-effort unpublish succeeds.
|
||||
func TestPrepare_AmbiguousPublishRetainsTheVolume(t *testing.T) {
|
||||
for name, perr := range map[string]error{
|
||||
"deadline": context.DeadlineExceeded,
|
||||
"unavailable": &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnavailable, Message: "lease busy"},
|
||||
"internal": &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeInternal, Message: "mount half done"},
|
||||
"invalid argument": &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeInvalidArgument, Message: "bad node"},
|
||||
"not found": &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeNotFound, Message: "no volume"},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
h := &harness{publishErr: perr}
|
||||
_, err := prepare(t, h)
|
||||
requireRetained(t, err)
|
||||
assert.Equal(t, []string{
|
||||
"mkfs ext4.raw", "create 000000000000 1GiB",
|
||||
"publish " + testVolume + " on " + testNode,
|
||||
"unpublish " + testVolume + " on " + testNode,
|
||||
}, h.calls, "no write, no discard, no release")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepare_FailedUnpublishRetainsTheVolume(t *testing.T) {
|
||||
for name, writeErr := range map[string]error{"after a good copy": nil, "after a failed copy": errors.New("short write")} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
h := &harness{writeErr: writeErr, unpublishErr: errors.New("nats: timeout")}
|
||||
_, err := prepare(t, h)
|
||||
requireRetained(t, err)
|
||||
require.ErrorContains(t, err, "nats: timeout")
|
||||
require.ErrorContains(t, err, "retained in creating", "the release never ran, so the state is known")
|
||||
last := h.calls[len(h.calls)-1]
|
||||
assert.Equal(t, "unpublish "+testVolume+" on "+testNode, last, "no discard or release after an unconfirmed unpublish")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A failed release write may still have landed, leaving the volume available
|
||||
// to others, so it is never discarded and its state is reported as unknown.
|
||||
func TestPrepare_FailedReleaseRetainsWithUnknownState(t *testing.T) {
|
||||
for name, applied := range map[string]bool{"write not applied": false, "write applied, then error": true} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
h := &harness{completeErr: errors.New("object store: request failed"), completeApplies: applied}
|
||||
_, err := prepare(t, h)
|
||||
requireRetained(t, err)
|
||||
var retained *RetainedVolumeError
|
||||
require.ErrorAs(t, err, &retained)
|
||||
assert.True(t, retained.StateUnknown)
|
||||
require.ErrorContains(t, err, "unknown state (creating or available)")
|
||||
assert.NotContains(t, err.Error(), "retained in creating")
|
||||
assert.Equal(t, "complete "+testVolume, h.calls[len(h.calls)-1], "no discard after an uncertain release")
|
||||
if applied {
|
||||
assert.Equal(t, "available", h.state, "the fake really did release it, which is why discard would be wrong")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Cleanup ignores caller cancellation but is bounded, so a provider that
|
||||
// never answers cannot hang the caller.
|
||||
func TestPrepare_StuckUnpublishIsBounded(t *testing.T) {
|
||||
h := &harness{unpublishBlocks: true}
|
||||
p := h.preparer(t)
|
||||
p.cleanupTimeout = 50 * time.Millisecond
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
p.WriteImage = func(context.Context, string, string, io.Writer) error {
|
||||
cancel()
|
||||
return context.Canceled
|
||||
}
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := p.Prepare(ctx, Request{AccountID: testAccount, AvailabilityZone: "ap-southeast-2a", SizeGiB: 1})
|
||||
done <- err
|
||||
}()
|
||||
select {
|
||||
case err := <-done:
|
||||
requireRetained(t, err)
|
||||
require.ErrorIs(t, err, context.DeadlineExceeded, "unpublish ends on its own bound, not the caller's cancel")
|
||||
assert.NotContains(t, h.calls, "discard "+testVolume)
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("Prepare hung on a stuck unpublish")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepare_FailedDiscardRetainsTheVolume(t *testing.T) {
|
||||
h := &harness{writeErr: errors.New("short write"), discardErr: errors.New("provider unreachable")}
|
||||
_, err := prepare(t, h)
|
||||
requireRetained(t, err)
|
||||
require.ErrorContains(t, err, "provider unreachable")
|
||||
}
|
||||
|
||||
// Unpublish is what closes the export, so a caller cancelling mid-copy must
|
||||
// not make it skip that step.
|
||||
func TestPrepare_CancelledCopyStillUnpublishes(t *testing.T) {
|
||||
h := &harness{}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
p := h.preparer(t)
|
||||
p.WriteImage = func(context.Context, string, string, io.Writer) error {
|
||||
cancel()
|
||||
return context.Canceled
|
||||
}
|
||||
_, err := p.Prepare(ctx, Request{AccountID: testAccount, AvailabilityZone: "ap-southeast-2a", SizeGiB: 1})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, h.calls, "unpublish "+testVolume+" on "+testNode)
|
||||
assert.True(t, h.unpublishCtxOK, "unpublish must run with a live context")
|
||||
assert.Equal(t, "discard "+testVolume, h.calls[len(h.calls)-1])
|
||||
}
|
||||
|
||||
func TestPrepare_RejectsIncompleteRequestBeforeAnything(t *testing.T) {
|
||||
for name, req := range map[string]Request{
|
||||
"no account": {AvailabilityZone: "ap-southeast-2a", SizeGiB: 1},
|
||||
"no zone": {AccountID: testAccount, SizeGiB: 1},
|
||||
"zero size": {AccountID: testAccount, AvailabilityZone: "ap-southeast-2a"},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
h := &harness{}
|
||||
_, err := h.preparer(t).Prepare(context.Background(), req)
|
||||
require.Error(t, err)
|
||||
assert.Empty(t, h.calls)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The real formatter produces an ext4 superblock the guest's e2fsck accepts.
|
||||
func TestBuildImage_RealMkfsWritesExt4(t *testing.T) {
|
||||
mkfs, err := (&Preparer{}).findTool("mkfs.ext4")
|
||||
if err != nil {
|
||||
t.Skip("mkfs.ext4 not installed")
|
||||
}
|
||||
path := filepath.Join(t.TempDir(), "ext4.raw")
|
||||
require.NoError(t, buildImage(context.Background(), runMkfsExt4, mkfs, path, 1))
|
||||
|
||||
info, err := os.Stat(path)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(1<<30), info.Size())
|
||||
f, err := os.Open(path)
|
||||
require.NoError(t, err)
|
||||
defer f.Close()
|
||||
magic := make([]byte, 2)
|
||||
_, err = f.ReadAt(magic, 1024+0x38)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, uint16(0xEF53), binary.LittleEndian.Uint16(magic))
|
||||
if fsck, err := (&Preparer{}).findTool("e2fsck"); err == nil {
|
||||
out, err := exec.Command(fsck, "-n", "-f", path).CombinedOutput()
|
||||
require.NoError(t, err, "e2fsck must find the new filesystem clean: %s", out)
|
||||
}
|
||||
|
||||
require.Error(t, buildImage(context.Background(), runMkfsExt4, mkfs, path, 1), "an existing file is never reused")
|
||||
}
|
||||
@@ -7,22 +7,12 @@ import (
|
||||
"errors"
|
||||
"log/slog"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/iam"
|
||||
"github.com/aws/aws-sdk-go/service/sts"
|
||||
iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
|
||||
stsdomain "github.com/mulgadc/spinifex/spinifex/domains/sts"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
)
|
||||
|
||||
// Principal-type values mirror gateway.principalType{User,AssumedRole,Root},
|
||||
// re-declared here to avoid an import cycle back to the parent gateway package.
|
||||
const (
|
||||
PrincipalTypeUser = "user"
|
||||
PrincipalTypeAssumedRole = "assumed-role"
|
||||
PrincipalTypeRoot = "root"
|
||||
)
|
||||
|
||||
// GetCallerIdentity returns the AWS {Account, Arn, UserId} triple. No policy
|
||||
// gating — every authenticated principal may call this. UserId semantics:
|
||||
//
|
||||
@@ -38,12 +28,12 @@ func GetCallerIdentity(
|
||||
iamSvc iamdomain.IAMService,
|
||||
stsSvc stsdomain.STSService,
|
||||
) (*sts.GetCallerIdentityOutput, error) {
|
||||
userID, _, err := ResolveCallerUser(accountID, callerPrincipalType, identity, assumedRoleID, iamSvc)
|
||||
userID, _, err := stsdomain.ResolveCallerUser(accountID, callerPrincipalType, identity, assumedRoleID, iamSvc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if userID == "" {
|
||||
if callerPrincipalType == PrincipalTypeAssumedRole {
|
||||
if callerPrincipalType == stsdomain.PrincipalTypeAssumedRole {
|
||||
// Session vanished between auth and dispatch (expired record); surface as
|
||||
// InvalidClientTokenId rather than a 500.
|
||||
slog.Warn("GetCallerIdentity: assumed-role session vanished between auth and dispatch")
|
||||
@@ -54,38 +44,3 @@ func GetCallerIdentity(
|
||||
}
|
||||
return stsSvc.GetCallerIdentity(accountID, callerARN, userID, input)
|
||||
}
|
||||
|
||||
// ResolveCallerUser resolves the UserId of an authenticated principal, which is
|
||||
// also aws:userid at the authorization door, and for an IAM user its stored ARN.
|
||||
// An error is a dependency fault; a missing value returns empty.
|
||||
func ResolveCallerUser(
|
||||
accountID, callerPrincipalType, identity, assumedRoleID string,
|
||||
iamSvc iamdomain.IAMService,
|
||||
) (userID, userARN string, err error) {
|
||||
switch callerPrincipalType {
|
||||
case PrincipalTypeRoot:
|
||||
return accountID, "", nil
|
||||
case PrincipalTypeUser:
|
||||
// Root is encoded as principalType=user + identity="root"; UserId is the account ID.
|
||||
if identity == "root" {
|
||||
return accountID, "", nil
|
||||
}
|
||||
if iamSvc == nil {
|
||||
slog.Error("ResolveCallerUser: IAM service not initialized")
|
||||
return "", "", errors.New(awserrors.ErrorInternalError)
|
||||
}
|
||||
out, err := iamSvc.GetUser(accountID, &iam.GetUserInput{UserName: aws.String(identity)})
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if out == nil || out.User == nil {
|
||||
return "", "", nil
|
||||
}
|
||||
return aws.StringValue(out.User.UserId), aws.StringValue(out.User.Arn), nil
|
||||
case PrincipalTypeAssumedRole:
|
||||
return assumedRoleID, "", nil
|
||||
default:
|
||||
slog.Error("ResolveCallerUser: unknown principal type", "principalType", callerPrincipalType)
|
||||
return "", "", errors.New(awserrors.ErrorInternalError)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -195,7 +195,7 @@ func TestGetCallerIdentity_User(t *testing.T) {
|
||||
out, err := GetCallerIdentity(
|
||||
"000000000000",
|
||||
"arn:aws:iam::000000000000:user/alice",
|
||||
PrincipalTypeUser,
|
||||
stsdomain.PrincipalTypeUser,
|
||||
"alice",
|
||||
"",
|
||||
&sts.GetCallerIdentityInput{},
|
||||
@@ -220,7 +220,7 @@ func TestGetCallerIdentity_RootViaUserPath(t *testing.T) {
|
||||
out, err := GetCallerIdentity(
|
||||
"000000000000",
|
||||
"arn:aws:iam::000000000000:root",
|
||||
PrincipalTypeUser,
|
||||
stsdomain.PrincipalTypeUser,
|
||||
"root",
|
||||
"",
|
||||
&sts.GetCallerIdentityInput{},
|
||||
@@ -243,7 +243,7 @@ func TestGetCallerIdentity_AssumedRole(t *testing.T) {
|
||||
out, err := GetCallerIdentity(
|
||||
"000000000000",
|
||||
"arn:aws:sts::000000000000:assumed-role/app/session-1",
|
||||
PrincipalTypeAssumedRole,
|
||||
stsdomain.PrincipalTypeAssumedRole,
|
||||
"session-1",
|
||||
"AROAEXAMPLE:session-1",
|
||||
&sts.GetCallerIdentityInput{},
|
||||
@@ -262,7 +262,7 @@ func TestGetCallerIdentity_AssumedRoleSessionVanished(t *testing.T) {
|
||||
_, err := GetCallerIdentity(
|
||||
"000000000000",
|
||||
"arn:aws:sts::000000000000:assumed-role/app/session-1",
|
||||
PrincipalTypeAssumedRole,
|
||||
stsdomain.PrincipalTypeAssumedRole,
|
||||
"session-1",
|
||||
"",
|
||||
&sts.GetCallerIdentityInput{},
|
||||
@@ -281,7 +281,7 @@ func TestGetCallerIdentity_UserLookupError(t *testing.T) {
|
||||
_, err := GetCallerIdentity(
|
||||
"000000000000",
|
||||
"arn:aws:iam::000000000000:user/alice",
|
||||
PrincipalTypeUser,
|
||||
stsdomain.PrincipalTypeUser,
|
||||
"alice",
|
||||
"",
|
||||
&sts.GetCallerIdentityInput{},
|
||||
@@ -326,13 +326,13 @@ func TestGetSessionToken_ForwardsCallerFields(t *testing.T) {
|
||||
// The wrapper's job is to forward c.identity as the user name, plus
|
||||
// c.principalType and c.accessKey, through to the service so the handler can
|
||||
// enforce its user-only / no-session rule; assert that mapping holds.
|
||||
out, err := GetSessionToken("000000000000", "alice", PrincipalTypeUser, "AKIAEXAMPLE", &sts.GetSessionTokenInput{}, svc)
|
||||
out, err := GetSessionToken("000000000000", "alice", stsdomain.PrincipalTypeUser, "AKIAEXAMPLE", &sts.GetSessionTokenInput{}, svc)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, out)
|
||||
assert.Equal(t, "ASIAEXAMPLE", aws.StringValue(out.Credentials.AccessKeyId))
|
||||
assert.Equal(t, "000000000000", got.accountID)
|
||||
assert.Equal(t, "alice", got.userName)
|
||||
assert.Equal(t, PrincipalTypeUser, got.principalType)
|
||||
assert.Equal(t, stsdomain.PrincipalTypeUser, got.principalType)
|
||||
assert.Equal(t, "AKIAEXAMPLE", got.accessKeyID)
|
||||
}
|
||||
|
||||
@@ -342,7 +342,7 @@ func TestGetSessionToken_PropagatesServiceError(t *testing.T) {
|
||||
return nil, errors.New(awserrors.ErrorAccessDenied)
|
||||
},
|
||||
}
|
||||
_, err := GetSessionToken("000000000000", "s1", PrincipalTypeAssumedRole, "ASIAEXAMPLE", &sts.GetSessionTokenInput{}, svc)
|
||||
_, err := GetSessionToken("000000000000", "s1", stsdomain.PrincipalTypeAssumedRole, "ASIAEXAMPLE", &sts.GetSessionTokenInput{}, svc)
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorAccessDenied, err.Error())
|
||||
}
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
package sts
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"log/slog"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/iam"
|
||||
iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
)
|
||||
|
||||
// Principal-type values an authenticated caller carries.
|
||||
const (
|
||||
PrincipalTypeUser = "user"
|
||||
PrincipalTypeAssumedRole = "assumed-role"
|
||||
PrincipalTypeRoot = "root"
|
||||
)
|
||||
|
||||
// ResolveCallerUser resolves the UserId of an authenticated principal, which is
|
||||
// also aws:userid at the authorization door, and for an IAM user its stored ARN.
|
||||
// An error is a dependency fault; a missing value returns empty.
|
||||
func ResolveCallerUser(
|
||||
accountID, callerPrincipalType, identity, assumedRoleID string,
|
||||
iamSvc iamdomain.IAMService,
|
||||
) (userID, userARN string, err error) {
|
||||
switch callerPrincipalType {
|
||||
case PrincipalTypeRoot:
|
||||
return accountID, "", nil
|
||||
case PrincipalTypeUser:
|
||||
// Root is encoded as principalType=user + identity="root"; UserId is the account ID.
|
||||
if identity == "root" {
|
||||
return accountID, "", nil
|
||||
}
|
||||
if iamSvc == nil {
|
||||
slog.Error("ResolveCallerUser: IAM service not initialized")
|
||||
return "", "", errors.New(awserrors.ErrorInternalError)
|
||||
}
|
||||
out, err := iamSvc.GetUser(accountID, &iam.GetUserInput{UserName: aws.String(identity)})
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if out == nil || out.User == nil {
|
||||
return "", "", nil
|
||||
}
|
||||
return aws.StringValue(out.User.UserId), aws.StringValue(out.User.Arn), nil
|
||||
case PrincipalTypeAssumedRole:
|
||||
return assumedRoleID, "", nil
|
||||
default:
|
||||
slog.Error("ResolveCallerUser: unknown principal type", "principalType", callerPrincipalType)
|
||||
return "", "", errors.New(awserrors.ErrorInternalError)
|
||||
}
|
||||
}
|
||||
@@ -36,6 +36,10 @@ const (
|
||||
// vllm-serving AMI, and the shared Bedrock system VPC.
|
||||
ManagedByBedrock = "bedrock"
|
||||
|
||||
// ManagedBySQS identifies the dedicated SQS store VMs, which live in the
|
||||
// system account on a persistent data volume.
|
||||
ManagedBySQS = "sqs"
|
||||
|
||||
// LBARNKey stores the parent LB ARN on ELBv2-managed ENIs.
|
||||
LBARNKey = "spinifex:lb-arn"
|
||||
|
||||
@@ -64,7 +68,7 @@ const (
|
||||
// VMs bind a system.TerminateInstance.{id} subject so a cluster-wide teardown
|
||||
// invoked on any node can route a terminate to the owning node.
|
||||
func IsSystemManaged(managedBy string) bool {
|
||||
return managedBy == ManagedByELBv2 || managedBy == ManagedByEKS || managedBy == ManagedByRDS || managedBy == ManagedByBedrock
|
||||
return managedBy == ManagedByELBv2 || managedBy == ManagedByEKS || managedBy == ManagedByRDS || managedBy == ManagedByBedrock || managedBy == ManagedBySQS
|
||||
}
|
||||
|
||||
// MapToEC2 converts a tag map to a slice of EC2 Tag pointers. It returns nil
|
||||
|
||||
@@ -17,6 +17,7 @@ func TestIsSystemManaged(t *testing.T) {
|
||||
"elbv2": {ManagedByELBv2, true, "HAProxy VMs are platform-owned"},
|
||||
"eks": {ManagedByEKS, true, "K3s control-plane VMs are platform-owned"},
|
||||
"rds": {ManagedByRDS, true, "DB engine VMs are platform-owned and live in the system account"},
|
||||
"sqs": {ManagedBySQS, true, "SQS store VMs are platform-owned and live in the system account"},
|
||||
// Container instances launched from the ECS node AMI stay customer-owned,
|
||||
// so only the AMI carries the tag — never a VM.
|
||||
"ecs": {ManagedByECS, false, "ECS container instances are customer-owned"},
|
||||
|
||||
@@ -38,7 +38,7 @@ func acmPipelineApp(t *testing.T, docs []iamdomain.PolicyDocument) (*GatewayConf
|
||||
require.NoError(t, err)
|
||||
_, nc := testutil.StartTestNATS(t)
|
||||
|
||||
gw := &GatewayConfig{
|
||||
gw := withVerifier(&GatewayConfig{
|
||||
DisableLogging: true,
|
||||
Region: testRegion,
|
||||
NATSConn: nc,
|
||||
@@ -57,7 +57,7 @@ func acmPipelineApp(t *testing.T, docs []iamdomain.PolicyDocument) (*GatewayConf
|
||||
},
|
||||
getUserPoliciesFn: func(_, _ string) ([]iamdomain.PolicyDocument, error) { return docs, nil },
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
withACM(gw, acmawsapi.Deps{})
|
||||
|
||||
@@ -241,7 +241,7 @@ func TestACMPipeline_AuthzGateBlocksAction(t *testing.T) {
|
||||
func TestRequest_ClusterUnavailableNilConn_ACM(t *testing.T) {
|
||||
for _, action := range []string{"ListCertificates", "DeleteCertificate"} {
|
||||
t.Run(action, func(t *testing.T) {
|
||||
gw := withACM(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}, acmawsapi.Deps{})
|
||||
gw := withACM(withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}), acmawsapi.Deps{})
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/", nil)
|
||||
req.Header.Set("X-Amz-Target", "CertificateManager."+action)
|
||||
|
||||
@@ -23,7 +23,7 @@ func setupACMRequest(target, body string) *http.Request {
|
||||
}
|
||||
|
||||
func TestACMRequest_MissingTarget(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
w := httptest.NewRecorder()
|
||||
err := gw.serveACM(w, setupACMRequest("", ""))
|
||||
require.Error(t, err)
|
||||
@@ -31,7 +31,7 @@ func TestACMRequest_MissingTarget(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestACMRequest_UnknownAction(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
w := httptest.NewRecorder()
|
||||
err := gw.serveACM(w, setupACMRequest("CertificateManager.BogusAction", "{}"))
|
||||
require.Error(t, err)
|
||||
@@ -41,7 +41,7 @@ func TestACMRequest_UnknownAction(t *testing.T) {
|
||||
// A known action with no NATS connection passes routing + policy and fails at
|
||||
// the NATS-availability guard.
|
||||
func TestACMRequest_KnownActionNoNATS(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
w := httptest.NewRecorder()
|
||||
err := gw.serveACM(w, setupACMRequest("CertificateManager.ListCertificates", "{}"))
|
||||
require.Error(t, err)
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/roles/awsgw/credverify"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -100,6 +101,7 @@ func actorProbeGateway(t *testing.T, e *cbEnv) (http.Handler, *[]actorObservatio
|
||||
Region: testRegion,
|
||||
ExpectedNodes: 1,
|
||||
IAMService: e.counter,
|
||||
Credentials: credverify.New(e.counter, e.sts),
|
||||
STSService: e.sts,
|
||||
Services: b.Build(),
|
||||
}
|
||||
|
||||
@@ -52,11 +52,11 @@ func quotaTestGateway(t *testing.T) (*GatewayConfig, string) {
|
||||
}, nil)
|
||||
quota.SetOverrides(kv)
|
||||
|
||||
return &GatewayConfig{
|
||||
return withVerifier(&GatewayConfig{
|
||||
DisableLogging: true,
|
||||
IAMService: "aIAMService{known: account},
|
||||
Quota: quota,
|
||||
}, account
|
||||
}), account
|
||||
}
|
||||
|
||||
func quotaRequestBody(t *testing.T, req AccountQuotaRequest) []byte {
|
||||
@@ -157,7 +157,7 @@ func TestAdminAccountQuotaRejectsBadRequests(t *testing.T) {
|
||||
// A gateway with quotas disabled has no override bucket; the admin surface must
|
||||
// report that rather than pretend the write landed.
|
||||
func TestAdminAccountQuotaWithoutQuotaService(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: "aIAMService{known: "000000000042"}}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: "aIAMService{known: "000000000042"}})
|
||||
|
||||
_, err := gw.adminGetAccountQuota(t.Context(), quotaRequestBody(t, AccountQuotaRequest{AccountID: "000000000042"}))
|
||||
require.Error(t, err)
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
operatorv1 "github.com/mulgadc/spinifex/contracts/operator/v1"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/roles/awsgw/credverify"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -208,6 +209,7 @@ func TestCheckAccountCap(t *testing.T) {
|
||||
gw := &GatewayConfig{
|
||||
DisableLogging: true,
|
||||
IAMService: &capMockIAMService{accounts: tc.accounts},
|
||||
Credentials: credverify.New(&capMockIAMService{accounts: tc.accounts}, nil),
|
||||
SignupMaxAccounts: tc.max,
|
||||
}
|
||||
err := gw.checkAccountCap()
|
||||
@@ -226,6 +228,7 @@ func TestCheckAccountCapFailsClosedOnListError(t *testing.T) {
|
||||
gw := &GatewayConfig{
|
||||
DisableLogging: true,
|
||||
IAMService: &capMockIAMService{err: assert.AnError},
|
||||
Credentials: credverify.New(&capMockIAMService{err: assert.AnError}, nil),
|
||||
SignupMaxAccounts: 128,
|
||||
}
|
||||
|
||||
@@ -254,6 +257,7 @@ func newCreateAccountGateway(t *testing.T) *GatewayConfig {
|
||||
return &GatewayConfig{
|
||||
DisableLogging: true,
|
||||
IAMService: svc,
|
||||
Credentials: credverify.New(svc, nil),
|
||||
NATSConn: nc,
|
||||
SignupMaxAccounts: 128,
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
operatorv1 "github.com/mulgadc/spinifex/contracts/operator/v1"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/roles/awsgw/credverify"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -103,6 +104,7 @@ func newDeleteAccountGateway(t *testing.T) *GatewayConfig {
|
||||
return &GatewayConfig{
|
||||
DisableLogging: true,
|
||||
IAMService: svc,
|
||||
Credentials: credverify.New(svc, nil),
|
||||
NATSConn: nc,
|
||||
ExpectedNodes: 1,
|
||||
NodeIDs: []string{"node-1"},
|
||||
|
||||
@@ -211,7 +211,7 @@ func TestAdminListAccountsFailsClosed(t *testing.T) {
|
||||
// Each admin method is a separate grant. The signup Worker holds
|
||||
// spinifex:CreateAccount and must not reach deletion with it.
|
||||
func TestCreateAccountGrantDoesNotAllowDeleteAccount(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: createAccountPolicy()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: createAccountPolicy()})
|
||||
|
||||
for _, method := range []string{"DeleteAccount", "DescribeAccountDeletion", "ListAccounts"} {
|
||||
t.Run(method, func(t *testing.T) {
|
||||
|
||||
@@ -133,7 +133,7 @@ func TestAuthorizeAdminGates(t *testing.T) {
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: tc.policy}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: tc.policy})
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/CreateAccount", nil)
|
||||
ctx := context.WithValue(req.Context(), ctxService, tc.service)
|
||||
ctx = withPrincipalField(ctx, fieldAccountID, tc.accountID)
|
||||
@@ -155,7 +155,7 @@ func TestAuthorizeAdminGates(t *testing.T) {
|
||||
// An unauthenticated request carries no auth context at all, which must not
|
||||
// read as a permitted principal.
|
||||
func TestAuthorizeAdminDeniesUnauthenticated(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/CreateAccount", nil)
|
||||
|
||||
err := gw.authorizeAdmin(req, "CreateAccount")
|
||||
@@ -185,7 +185,7 @@ func decodeAdminError(t *testing.T, rec *httptest.ResponseRecorder) adminErrorBo
|
||||
// An unknown method is rejected before authorization, so it never discloses
|
||||
// whether the caller would have been permitted.
|
||||
func TestAdminRequestUnknownMethod(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
|
||||
rec := adminRequest(gw, "PurgeEverything", `{}`)
|
||||
|
||||
@@ -194,7 +194,7 @@ func TestAdminRequestUnknownMethod(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestAdminRequestDeniesUnauthorizedCaller(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
|
||||
rec := adminRequest(gw, "CreateAccount", `{}`)
|
||||
|
||||
@@ -206,7 +206,7 @@ func TestAdminRequestDeniesUnauthorizedCaller(t *testing.T) {
|
||||
// The request ID ties a client failure to the gateway log line, so it must be
|
||||
// present on the header and in the body even for a denial.
|
||||
func TestAdminRequestAlwaysCarriesRequestID(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
|
||||
rec := adminRequest(gw, "CreateAccount", `{}`)
|
||||
|
||||
@@ -229,7 +229,7 @@ func TestWriteAdminErrorFallsBackOnUnknownCode(t *testing.T) {
|
||||
// The router serves every verb from one pattern, so the handler is what
|
||||
// refuses a GET — a second route would shadow the POST one.
|
||||
func TestAdminRequestRejectsNonPost(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
req := httptest.NewRequest(http.MethodGet, "/admin/CreateAccount", nil)
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("method", "CreateAccount")
|
||||
@@ -269,7 +269,7 @@ func authorizedAdminRequestForMethod(gw *GatewayConfig, method, body string) *ht
|
||||
// An authorized caller must still get a clear answer when the cluster is not
|
||||
// reachable, rather than a panic or a silent success.
|
||||
func TestAdminRequestReportsClusterUnavailable(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: createAccountPolicy()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: createAccountPolicy()})
|
||||
|
||||
rec := authorizedAdminRequest(gw, `{}`)
|
||||
|
||||
|
||||
+54
-218
@@ -13,23 +13,15 @@ import (
|
||||
"uuid"
|
||||
|
||||
"github.com/mulgadc/bluebottle/pkg/sigv4"
|
||||
iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
|
||||
stsdomain "github.com/mulgadc/spinifex/spinifex/domains/sts"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
telemetry "github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/audit"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/credential"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/envelope"
|
||||
ingresshttp "github.com/mulgadc/spinifex/spinifex/ingress/http"
|
||||
)
|
||||
|
||||
// AKID prefixes. Prefix-first dispatch prevents a misfiled record from being
|
||||
// resolved by the wrong lookup path.
|
||||
const (
|
||||
longLivedAKIDPrefix = "AKIA"
|
||||
sessionAKIDPrefix = "ASIA"
|
||||
)
|
||||
|
||||
// authSource is the caller's address as the auth path uses it. The rate limiter
|
||||
// keys on the connection peer; log lines use the loopback-gated client IP so they
|
||||
// agree with the request audit line for proxied requests.
|
||||
@@ -132,77 +124,74 @@ func (gw *GatewayConfig) SigV4AuthMiddleware() func(http.Handler) http.Handler {
|
||||
return
|
||||
}
|
||||
|
||||
if gw.IAMService == nil {
|
||||
slog.Error("SigV4 auth: IAM service not initialized")
|
||||
if gw.Credentials == nil {
|
||||
// No verifier means no way to authenticate: fail closed, never allow.
|
||||
slog.Error("SigV4 auth: credential verifier not configured")
|
||||
gw.writeSigV4Error(w, r, awserrors.ErrorInternalError, "")
|
||||
return
|
||||
}
|
||||
|
||||
var (
|
||||
secret string
|
||||
principal principalContext
|
||||
lookupCode string
|
||||
)
|
||||
switch {
|
||||
case strings.HasPrefix(sig.Credential.AccessKeyID, longLivedAKIDPrefix):
|
||||
secret, principal, lookupCode = gw.resolveLongLivedAKID(sig.Credential.AccessKeyID, src)
|
||||
case strings.HasPrefix(sig.Credential.AccessKeyID, sessionAKIDPrefix):
|
||||
secret, principal, lookupCode = gw.resolveSessionAKID(r, sig.Credential.AccessKeyID, src)
|
||||
default:
|
||||
slog.Warn("Auth failure: unknown AKID prefix", "accessKeyID", sig.Credential.AccessKeyID, "sourceIP", src.logIP)
|
||||
gw.RateLimiter.RecordFailure(src.limitKey, authlimit.Fingerprint("unknown-prefix", sig.Credential.AccessKeyID))
|
||||
gw.writeSigV4Error(w, r, awserrors.ErrorInvalidClientTokenId, "")
|
||||
return
|
||||
}
|
||||
if lookupCode != "" {
|
||||
gw.writeSigV4Error(w, r, lookupCode, "")
|
||||
return
|
||||
credReq := credential.Request{
|
||||
AccessKeyID: sig.Credential.AccessKeyID,
|
||||
SecurityToken: r.Header.Get("X-Amz-Security-Token"),
|
||||
SourceIP: src.logIP,
|
||||
}
|
||||
|
||||
// Region is the gateway's, or a global service's fixed signing region;
|
||||
// service is the client-claimed scope, gated against supportedServices above.
|
||||
if _, err := sig.Verify(secret, gw.expectedSigningRegion(sig.Credential.Service, sig.Credential.Region), sig.Credential.Service); err != nil {
|
||||
// A mismatch means our canonical request differs from the one the client
|
||||
// signed, so log ours to diff against the SDK's. Warn, not Debug: these
|
||||
// failures are intermittent and never reproduce on demand.
|
||||
slog.Warn("Auth failure: verification failed",
|
||||
"accessKeyID", sig.Credential.AccessKeyID, "sourceIP", src.logIP,
|
||||
"service", sig.Credential.Service,
|
||||
"action", mismatchAction(r, sig.Credential.Service),
|
||||
"canonicalRequest", sig.RedactedCanonicalRequest(),
|
||||
"err", err)
|
||||
// Fingerprinted by the signature as well as the key id: each guess at a
|
||||
// secret produces a different one, while a client retrying an identical
|
||||
// bad request produces the same one.
|
||||
gw.RateLimiter.RecordFailure(src.limitKey, authlimit.Fingerprint("signature", sig.Credential.AccessKeyID, sig.Signature))
|
||||
gw.writeSigV4Error(w, r, awserrors.ErrorSignatureDoesNotMatch, "")
|
||||
var sigErr error
|
||||
authed, f := gw.Credentials.Authenticate(credReq, func(secret string) error {
|
||||
_, sigErr = sig.Verify(secret, gw.expectedSigningRegion(sig.Credential.Service, sig.Credential.Region), sig.Credential.Service)
|
||||
return sigErr
|
||||
})
|
||||
if f != nil {
|
||||
if f.SignatureMismatch {
|
||||
// A mismatch means our canonical request differs from the one the client
|
||||
// signed, so log ours to diff against the SDK's. Warn, not Debug: these
|
||||
// failures are intermittent and never reproduce on demand.
|
||||
slog.Warn("Auth failure: verification failed",
|
||||
"accessKeyID", sig.Credential.AccessKeyID, "sourceIP", src.logIP,
|
||||
"service", sig.Credential.Service,
|
||||
"action", mismatchAction(r, sig.Credential.Service),
|
||||
"canonicalRequest", sig.RedactedCanonicalRequest(),
|
||||
"err", sigErr)
|
||||
// Fingerprinted by the signature as well as the key id: each guess at a
|
||||
// secret produces a different one, while a client retrying an identical
|
||||
// bad request produces the same one.
|
||||
gw.RateLimiter.RecordFailure(src.limitKey, authlimit.Fingerprint("signature", sig.Credential.AccessKeyID, sig.Signature))
|
||||
} else {
|
||||
gw.recordCredentialFailure(src, f)
|
||||
}
|
||||
gw.writeSigV4Error(w, r, f.Code, "")
|
||||
return
|
||||
}
|
||||
|
||||
// Only after the signature holds: an unauthenticated prober must not
|
||||
// be able to use this to tell a suspended account from a live one.
|
||||
if errCode := gw.checkAccountActive(principal.accountID, src); errCode != "" {
|
||||
if errCode := gw.checkAccountActive(authed.AccountID, src); errCode != "" {
|
||||
gw.writeSigV4Error(w, r, errCode, "")
|
||||
return
|
||||
}
|
||||
|
||||
// Principal state, same class as the account check above and deferred for
|
||||
// the same reason: a session name resolves to whichever principal holds it
|
||||
// now, so it has to be checked against the ID the session was minted for.
|
||||
if errCode := gw.checkSessionPrincipal(principal, sig.Credential.AccessKeyID, src); errCode != "" {
|
||||
gw.writeSigV4Error(w, r, errCode, "")
|
||||
// Principal state and the caller's user ID, resolved here once rather than
|
||||
// per policy check: a fault must fail the request rather than authorize it
|
||||
// against a context missing the key.
|
||||
caller, f := gw.Credentials.Resolve(credReq, authed)
|
||||
if f != nil {
|
||||
gw.recordCredentialFailure(src, f)
|
||||
gw.writeSigV4Error(w, r, f.Code, "")
|
||||
return
|
||||
}
|
||||
|
||||
// Resolved here, once, rather than per policy check: a fault must fail
|
||||
// the request rather than authorize it against a context missing the key.
|
||||
userID, userARN, err := gw.principalUser(principal)
|
||||
if err != nil {
|
||||
gw.writeSigV4Error(w, r, err.Error(), "")
|
||||
return
|
||||
principal := principalContext{
|
||||
identity: caller.Identity,
|
||||
accountID: caller.AccountID,
|
||||
principalType: caller.PrincipalType,
|
||||
assumedRoleARN: caller.AssumedRoleARN,
|
||||
assumedRoleID: caller.AssumedRoleID,
|
||||
underlyingRoleARN: caller.UnderlyingRoleARN,
|
||||
userID: caller.UserID,
|
||||
userARN: caller.UserARN,
|
||||
}
|
||||
principal.userID = userID
|
||||
principal.userARN = userARN
|
||||
|
||||
// Parse rewound the body; re-read it for query-arg parsing, then rewind
|
||||
// again for the downstream handler.
|
||||
@@ -344,165 +333,12 @@ func mismatchAction(r *http.Request, service string) string {
|
||||
return args["Action"]
|
||||
}
|
||||
|
||||
// principalContext is the identity envelope set on the request context after
|
||||
// SigV4 verification succeeds.
|
||||
type principalContext struct {
|
||||
identity string
|
||||
accountID string
|
||||
principalType string
|
||||
assumedRoleARN string
|
||||
assumedRoleID string
|
||||
underlyingRoleARN string
|
||||
// userID is aws:userid, resolved where the principal is so the value cannot
|
||||
// differ between two policy checks in the same request.
|
||||
userID string
|
||||
// userARN is an IAM user's stored ARN, path included, from the same read as
|
||||
// userID. Empty for every other principal type.
|
||||
userARN string
|
||||
// sessionCred is the rehydrated STS record on the ASIA path, nil on the
|
||||
// long-lived one. It carries the continuity check past sig.Verify.
|
||||
sessionCred *stsdomain.SessionCredential
|
||||
}
|
||||
|
||||
// resolveLongLivedAKID handles the AKIA path: IAM lookup, status check, secret decrypt.
|
||||
func (gw *GatewayConfig) resolveLongLivedAKID(accessKeyID string, src authSource) (string, principalContext, string) {
|
||||
ak, err := gw.IAMService.LookupAccessKey(accessKeyID)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), awserrors.ErrorIAMNoSuchEntity) {
|
||||
slog.Warn("Auth failure: access key not found", "accessKeyID", accessKeyID, "sourceIP", src.logIP)
|
||||
gw.RateLimiter.RecordFailure(src.limitKey, authlimit.Fingerprint("akid-not-found", accessKeyID))
|
||||
return "", principalContext{}, awserrors.ErrorInvalidClientTokenId
|
||||
}
|
||||
slog.Error("IAM lookup failed", "accessKeyID", accessKeyID, "err", err)
|
||||
return "", principalContext{}, awserrors.ErrorInternalError
|
||||
// recordCredentialFailure counts a verifier rejection toward the source's lockout.
|
||||
func (gw *GatewayConfig) recordCredentialFailure(src authSource, f *credential.Failure) {
|
||||
if f.FingerprintKind == "" {
|
||||
return
|
||||
}
|
||||
if ak.Status != iamdomain.AccessKeyStatusActive {
|
||||
slog.Warn("Auth failure: access key inactive", "accessKeyID", accessKeyID, "sourceIP", src.logIP)
|
||||
gw.RateLimiter.RecordFailure(src.limitKey, authlimit.Fingerprint("akid-inactive", accessKeyID))
|
||||
return "", principalContext{}, awserrors.ErrorInvalidClientTokenId
|
||||
}
|
||||
secret, err := gw.IAMService.DecryptSecret(ak.SecretAccessKey)
|
||||
if err != nil {
|
||||
// Undecryptable secret (e.g. master key rotated): treat as auth failure, not
|
||||
// server fault, so the client re-authenticates instead of retrying a dead request.
|
||||
slog.Error("Failed to decrypt IAM secret", "accessKeyID", accessKeyID, "err", err)
|
||||
gw.RateLimiter.RecordFailure(src.limitKey, authlimit.Fingerprint("akid-secret", accessKeyID))
|
||||
return "", principalContext{}, awserrors.ErrorInvalidClientTokenId
|
||||
}
|
||||
return secret, principalContext{
|
||||
identity: ak.UserName,
|
||||
accountID: ak.AccountID,
|
||||
principalType: principalTypeUser,
|
||||
}, ""
|
||||
}
|
||||
|
||||
// resolveSessionAKID handles the ASIA path: STS lookup, expiry check,
|
||||
// X-Amz-Security-Token HMAC verify, secret decrypt. Nil STSService surfaces
|
||||
// as InternalError so misconfiguration is loud at startup.
|
||||
func (gw *GatewayConfig) resolveSessionAKID(r *http.Request, accessKeyID string, src authSource) (string, principalContext, string) {
|
||||
if gw.STSService == nil {
|
||||
slog.Error("SigV4 auth: STS service not initialized but session AKID presented", "accessKeyID", accessKeyID)
|
||||
return "", principalContext{}, awserrors.ErrorInternalError
|
||||
}
|
||||
cred, err := gw.STSService.LookupSessionCredential(accessKeyID)
|
||||
if err != nil {
|
||||
slog.Error("STS lookup failed", "accessKeyID", accessKeyID, "err", err)
|
||||
return "", principalContext{}, awserrors.ErrorInternalError
|
||||
}
|
||||
if cred == nil {
|
||||
slog.Warn("Auth failure: session credential not found", "accessKeyID", accessKeyID, "sourceIP", src.logIP)
|
||||
gw.RateLimiter.RecordFailure(src.limitKey, authlimit.Fingerprint("session-not-found", accessKeyID))
|
||||
return "", principalContext{}, awserrors.ErrorInvalidClientTokenId
|
||||
}
|
||||
if time.Now().UTC().After(cred.ExpiresAt) {
|
||||
slog.Warn("Auth failure: session credential expired",
|
||||
"accessKeyID", accessKeyID, "sourceIP", src.logIP, "expiresAt", cred.ExpiresAt)
|
||||
gw.RateLimiter.RecordFailure(src.limitKey, authlimit.Fingerprint("session-expired", accessKeyID))
|
||||
return "", principalContext{}, awserrors.ErrorExpiredToken
|
||||
}
|
||||
|
||||
tokenHeader := r.Header.Get("X-Amz-Security-Token")
|
||||
if tokenHeader == "" {
|
||||
slog.Warn("Auth failure: session AKID presented without X-Amz-Security-Token",
|
||||
"accessKeyID", accessKeyID, "sourceIP", src.logIP)
|
||||
gw.RateLimiter.RecordFailure(src.limitKey, authlimit.Fingerprint("session-no-token", accessKeyID))
|
||||
return "", principalContext{}, awserrors.ErrorInvalidClientTokenId
|
||||
}
|
||||
if !gw.STSService.VerifySessionToken(cred, tokenHeader) {
|
||||
slog.Warn("Auth failure: session token HMAC mismatch",
|
||||
"accessKeyID", accessKeyID, "sourceIP", src.logIP)
|
||||
gw.RateLimiter.RecordFailure(src.limitKey, authlimit.Fingerprint("session-token-mismatch", accessKeyID, authlimit.TokenDigest(tokenHeader)))
|
||||
return "", principalContext{}, awserrors.ErrorInvalidClientTokenId
|
||||
}
|
||||
|
||||
secret, err := gw.IAMService.DecryptSecret(cred.SecretEncrypted)
|
||||
if err != nil {
|
||||
// Unverifiable secret: same auth-failure reasoning as resolveLongLivedAKID.
|
||||
slog.Error("Failed to decrypt session secret", "accessKeyID", accessKeyID, "err", err)
|
||||
gw.RateLimiter.RecordFailure(src.limitKey, authlimit.Fingerprint("session-secret", accessKeyID))
|
||||
return "", principalContext{}, awserrors.ErrorInvalidClientTokenId
|
||||
}
|
||||
if cred.PrincipalType == principalTypeUser {
|
||||
// GetSessionToken session: resolve to the user so policy is evaluated as a user.
|
||||
return secret, principalContext{
|
||||
identity: cred.SessionName,
|
||||
accountID: cred.AccountID,
|
||||
principalType: principalTypeUser,
|
||||
sessionCred: cred,
|
||||
}, ""
|
||||
}
|
||||
|
||||
// "assumed-role" or empty (pre-PrincipalType records) — both resolve as role session.
|
||||
return secret, principalContext{
|
||||
identity: cred.SessionName,
|
||||
accountID: cred.AccountID,
|
||||
principalType: principalTypeAssumedRole,
|
||||
assumedRoleARN: cred.AssumedRoleARN,
|
||||
assumedRoleID: cred.AssumedRoleID,
|
||||
underlyingRoleARN: cred.UnderlyingRoleARN,
|
||||
sessionCred: cred,
|
||||
}, ""
|
||||
}
|
||||
|
||||
// checkSessionPrincipal re-verifies that a session's backing principal still
|
||||
// exists with the immutable ID the session was minted for, returning an AWS
|
||||
// error code to fail with or "" to proceed. A nil sessionCred (long-lived
|
||||
// credential) is a no-op.
|
||||
//
|
||||
// Deliberately not folded into resolveSessionAKID: that runs pre-signature
|
||||
// because it must produce the secret, whereas this reads the IAM users and
|
||||
// roles buckets. An AKID travels in cleartext, so running it there would let
|
||||
// anyone holding a copied one drive IAM reads with garbage signatures.
|
||||
func (gw *GatewayConfig) checkSessionPrincipal(principal principalContext, accessKeyID string, src authSource) string {
|
||||
if principal.sessionCred == nil {
|
||||
return ""
|
||||
}
|
||||
if gw.STSService == nil {
|
||||
slog.Error("SigV4 auth: STS service not initialized", "accessKeyID", accessKeyID)
|
||||
return awserrors.ErrorInternalError
|
||||
}
|
||||
|
||||
if _, err := gw.STSService.VerifySessionPrincipal(principal.sessionCred); err != nil {
|
||||
if !stsdomain.IsSessionPrincipalVerdict(err) {
|
||||
slog.Error("Session principal continuity check faulted, refusing to authorize",
|
||||
"accessKeyID", accessKeyID, "err", err)
|
||||
return awserrors.ErrorInternalError
|
||||
}
|
||||
// InvalidClientTokenId, not ExpiredToken: the credential has not expired,
|
||||
// the principal behind it is no longer the one it was minted for.
|
||||
slog.Warn("Auth failure: session principal continuity check failed",
|
||||
"accessKeyID", accessKeyID, "sourceIP", src.logIP,
|
||||
"identity", principal.identity, "principalType", principal.principalType,
|
||||
"reason", err)
|
||||
// Every legacy record fails at once on the deploy that ships this check, so
|
||||
// counting them would lock out a shared egress address on distinct-attempt
|
||||
// volume alone and answer "retry later" to a fleet that cannot act on it.
|
||||
if !errors.Is(err, stsdomain.ErrSessionPrincipalLegacy) {
|
||||
gw.RateLimiter.RecordFailure(src.limitKey, authlimit.Fingerprint("session-principal", accessKeyID))
|
||||
}
|
||||
return awserrors.ErrorInvalidClientTokenId
|
||||
}
|
||||
return ""
|
||||
gw.RateLimiter.RecordFailure(src.limitKey, authlimit.Fingerprint(f.FingerprintKind, f.FingerprintParts...))
|
||||
}
|
||||
|
||||
// writeSigV4Error writes an auth-failure error in the service-appropriate
|
||||
|
||||
@@ -19,7 +19,7 @@ import (
|
||||
func statusGateRouter(t *testing.T, svc iamdomain.IAMService) (chi.Router, *http.Request) {
|
||||
t.Helper()
|
||||
|
||||
gw := &GatewayConfig{DisableLogging: true, Region: testRegion, IAMService: svc}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, Region: testRegion, IAMService: svc})
|
||||
r := chi.NewRouter()
|
||||
r.Use(gw.SigV4AuthMiddleware())
|
||||
r.HandleFunc("/*", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) })
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/roles/awsgw/credverify"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
@@ -136,6 +137,7 @@ func (e *cbEnv) gateway() http.Handler {
|
||||
Region: testRegion,
|
||||
ExpectedNodes: 1,
|
||||
IAMService: e.counter,
|
||||
Credentials: credverify.New(e.counter, e.sts),
|
||||
STSService: e.sts,
|
||||
}
|
||||
return gw.SetupRoutes()
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
authlimit "github.com/mulgadc/spinifex/spinifex/ingress/aws/ratelimit"
|
||||
ingresshttp "github.com/mulgadc/spinifex/spinifex/ingress/http"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/roles/awsgw/credverify"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
@@ -128,6 +129,7 @@ func TestSigV4Auth_FailureLogsLoopbackGatedClientIP(t *testing.T) {
|
||||
DisableLogging: true,
|
||||
Region: testRegion,
|
||||
IAMService: &mockIAMService{masterKey: testMasterKey},
|
||||
Credentials: credverify.New(&mockIAMService{masterKey: testMasterKey}, &mockSTSService{}),
|
||||
STSService: &mockSTSService{},
|
||||
RateLimiter: rl,
|
||||
}
|
||||
|
||||
@@ -12,9 +12,11 @@ import (
|
||||
awsapi "github.com/mulgadc/spinifex/spinifex/domains/ecr/awsapi"
|
||||
taggingawsapi "github.com/mulgadc/spinifex/spinifex/domains/tagging/awsapi"
|
||||
awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/credential"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/envelope"
|
||||
authlimit "github.com/mulgadc/spinifex/spinifex/ingress/aws/ratelimit"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/roles/awsgw/credverify"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
@@ -154,11 +156,11 @@ func setupTestApp(accessKey, secretKey string) http.Handler {
|
||||
},
|
||||
}
|
||||
|
||||
gw := withECR(&GatewayConfig{
|
||||
gw := withECR(withVerifier(&GatewayConfig{
|
||||
DisableLogging: true,
|
||||
Region: testRegion,
|
||||
IAMService: mockSvc,
|
||||
}, awsapi.Deps{})
|
||||
}), awsapi.Deps{})
|
||||
|
||||
r := chi.NewRouter()
|
||||
r.Use(gw.SigV4AuthMiddleware())
|
||||
@@ -186,7 +188,7 @@ func userIDApp(t *testing.T, svc *mockIAMService) (http.Handler, *string) {
|
||||
},
|
||||
}
|
||||
|
||||
gw := &GatewayConfig{DisableLogging: true, Region: testRegion, IAMService: svc}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, Region: testRegion, IAMService: svc})
|
||||
seen := new(string)
|
||||
r := chi.NewRouter()
|
||||
r.Use(gw.SigV4AuthMiddleware())
|
||||
@@ -389,11 +391,11 @@ func TestSigV4Auth_InactiveAccessKey(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
gw := &GatewayConfig{
|
||||
gw := withVerifier(&GatewayConfig{
|
||||
DisableLogging: true,
|
||||
Region: testRegion,
|
||||
IAMService: mockSvc,
|
||||
}
|
||||
})
|
||||
|
||||
r := chi.NewRouter()
|
||||
r.Use(gw.SigV4AuthMiddleware())
|
||||
@@ -418,11 +420,11 @@ func TestSigV4Auth_InactiveAccessKey(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSigV4Auth_NilIAMService(t *testing.T) {
|
||||
gw := &GatewayConfig{
|
||||
gw := withVerifier(&GatewayConfig{
|
||||
DisableLogging: true,
|
||||
Region: testRegion,
|
||||
IAMService: nil,
|
||||
}
|
||||
})
|
||||
|
||||
r := chi.NewRouter()
|
||||
r.Use(gw.SigV4AuthMiddleware())
|
||||
@@ -502,11 +504,11 @@ func TestSigV4Auth_DecryptFailure(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
gw := &GatewayConfig{
|
||||
gw := withVerifier(&GatewayConfig{
|
||||
DisableLogging: true,
|
||||
Region: testRegion,
|
||||
IAMService: mockSvc,
|
||||
}
|
||||
})
|
||||
|
||||
r := chi.NewRouter()
|
||||
r.Use(gw.SigV4AuthMiddleware())
|
||||
@@ -714,7 +716,7 @@ func TestSigV4Auth_MalformedTimestamp(t *testing.T) {
|
||||
// --- writeSigV4Error Response Format Tests ---
|
||||
|
||||
func TestWriteSigV4Error_ResponseFormat(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, Region: testRegion, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, Region: testRegion, IAMService: allowAllIAMService()})
|
||||
|
||||
testCases := []struct {
|
||||
errorCode string
|
||||
@@ -765,7 +767,7 @@ func TestWriteSigV4Error_ResponseFormat(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestWriteSigV4Error_IgnoresClientRequestID(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, Region: testRegion, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, Region: testRegion, IAMService: allowAllIAMService()})
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||
@@ -851,11 +853,11 @@ func TestSigV4Auth_ContextPropagation(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
gw := &GatewayConfig{
|
||||
gw := withVerifier(&GatewayConfig{
|
||||
DisableLogging: true,
|
||||
Region: testRegion,
|
||||
IAMService: mockSvc,
|
||||
}
|
||||
})
|
||||
|
||||
r := chi.NewRouter()
|
||||
r.Use(gw.SigV4AuthMiddleware())
|
||||
@@ -933,11 +935,11 @@ func TestSigV4Auth_ContextDoesNotLeakBetweenRequests(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
gw := &GatewayConfig{
|
||||
gw := withVerifier(&GatewayConfig{
|
||||
DisableLogging: true,
|
||||
Region: testRegion,
|
||||
IAMService: mockSvc,
|
||||
}
|
||||
})
|
||||
|
||||
r := chi.NewRouter()
|
||||
r.Use(gw.SigV4AuthMiddleware())
|
||||
@@ -973,11 +975,11 @@ func TestSigV4Auth_ContextDoesNotLeakBetweenRequests(t *testing.T) {
|
||||
|
||||
func TestSigV4Auth_LookupAccessKeyUnexpectedError(t *testing.T) {
|
||||
// A non-NoSuchEntity error from LookupAccessKey should yield 500 InternalError.
|
||||
gw := &GatewayConfig{
|
||||
gw := withVerifier(&GatewayConfig{
|
||||
DisableLogging: true,
|
||||
Region: testRegion,
|
||||
IAMService: &errorLookupMockIAMService{err: errors.New("nats: connection closed")},
|
||||
}
|
||||
})
|
||||
|
||||
r := chi.NewRouter()
|
||||
r.Use(gw.SigV4AuthMiddleware())
|
||||
@@ -1050,11 +1052,11 @@ func TestSigV4Auth_SignedContentType(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
gw := &GatewayConfig{
|
||||
gw := withVerifier(&GatewayConfig{
|
||||
DisableLogging: true,
|
||||
Region: testRegion,
|
||||
IAMService: mockSvc,
|
||||
}
|
||||
})
|
||||
|
||||
r := chi.NewRouter()
|
||||
r.Use(gw.SigV4AuthMiddleware())
|
||||
@@ -1212,11 +1214,11 @@ func TestCheckPolicy_NonRootNoPolicies_Denied(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
gw := &GatewayConfig{
|
||||
gw := withVerifier(&GatewayConfig{
|
||||
DisableLogging: true,
|
||||
Region: testRegion,
|
||||
IAMService: mockSvc,
|
||||
}
|
||||
})
|
||||
|
||||
handler := setupPolicyTestHandler(gw)
|
||||
|
||||
@@ -1272,11 +1274,11 @@ func TestCheckPolicy_NonRootWithAllow_Passes(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
gw := &GatewayConfig{
|
||||
gw := withVerifier(&GatewayConfig{
|
||||
DisableLogging: true,
|
||||
Region: testRegion,
|
||||
IAMService: mockSvc,
|
||||
}
|
||||
})
|
||||
|
||||
handler := setupPolicyTestHandler(gw)
|
||||
|
||||
@@ -1332,11 +1334,11 @@ func TestCheckPolicy_NonRootWithExplicitDeny_Denied(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
gw := &GatewayConfig{
|
||||
gw := withVerifier(&GatewayConfig{
|
||||
DisableLogging: true,
|
||||
Region: testRegion,
|
||||
IAMService: mockSvc,
|
||||
}
|
||||
})
|
||||
|
||||
handler := setupPolicyTestHandler(gw)
|
||||
|
||||
@@ -1393,11 +1395,11 @@ func TestCheckPolicy_RootGlobalAccount_Bypasses(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
gw := &GatewayConfig{
|
||||
gw := withVerifier(&GatewayConfig{
|
||||
DisableLogging: true,
|
||||
Region: testRegion,
|
||||
IAMService: mockSvc,
|
||||
}
|
||||
})
|
||||
|
||||
handler := setupPolicyTestHandler(gw)
|
||||
|
||||
@@ -1435,11 +1437,11 @@ func TestCheckPolicy_MissingAccountID_InternalError(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
gw := &GatewayConfig{
|
||||
gw := withVerifier(&GatewayConfig{
|
||||
DisableLogging: true,
|
||||
Region: testRegion,
|
||||
IAMService: mockSvc,
|
||||
}
|
||||
})
|
||||
|
||||
handler := setupPolicyTestHandler(gw)
|
||||
|
||||
@@ -1479,11 +1481,11 @@ func TestCheckPolicy_GetUserPoliciesError_InternalError(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
gw := &GatewayConfig{
|
||||
gw := withVerifier(&GatewayConfig{
|
||||
DisableLogging: true,
|
||||
Region: testRegion,
|
||||
IAMService: mockSvc,
|
||||
}
|
||||
})
|
||||
|
||||
handler := setupPolicyTestHandler(gw)
|
||||
|
||||
@@ -1525,11 +1527,11 @@ func TestCheckPolicy_RootNonGlobalAccount_StillEvaluated(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
gw := &GatewayConfig{
|
||||
gw := withVerifier(&GatewayConfig{
|
||||
DisableLogging: true,
|
||||
Region: testRegion,
|
||||
IAMService: mockSvc,
|
||||
}
|
||||
})
|
||||
|
||||
handler := setupPolicyTestHandler(gw)
|
||||
|
||||
@@ -1604,12 +1606,12 @@ func TestSigV4Auth_NATSDisconnectedShortCircuit(t *testing.T) {
|
||||
}
|
||||
nc.Close() // disconnected — IsConnected() returns false
|
||||
|
||||
gw := &GatewayConfig{
|
||||
gw := withVerifier(&GatewayConfig{
|
||||
DisableLogging: true,
|
||||
Region: testRegion,
|
||||
NATSConn: nc,
|
||||
IAMService: nil,
|
||||
}
|
||||
})
|
||||
|
||||
r := chi.NewRouter()
|
||||
r.Use(gw.SigV4AuthMiddleware())
|
||||
@@ -1772,12 +1774,12 @@ func mountSessionGateway(t *testing.T, cred *stsdomain.SessionCredential) (http.
|
||||
},
|
||||
}
|
||||
|
||||
gw := &GatewayConfig{
|
||||
gw := withVerifier(&GatewayConfig{
|
||||
DisableLogging: true,
|
||||
Region: testRegion,
|
||||
IAMService: iamMock,
|
||||
STSService: stsMock,
|
||||
}
|
||||
})
|
||||
|
||||
r := chi.NewRouter()
|
||||
r.Use(gw.SigV4AuthMiddleware())
|
||||
@@ -2048,20 +2050,31 @@ func TestSigV4Auth_SessionPrincipalVerdict_RateLimitRecording(t *testing.T) {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
rl := authlimit.NewAuthRateLimiter()
|
||||
defer rl.Stop()
|
||||
gw := &GatewayConfig{
|
||||
DisableLogging: true,
|
||||
RateLimiter: rl,
|
||||
STSService: &mockSTSService{principalErr: tc.err},
|
||||
gw := &GatewayConfig{DisableLogging: true, RateLimiter: rl}
|
||||
|
||||
// The real verifier decides whether a verdict counts; the gateway records it.
|
||||
encrypted, err := iamdomain.EncryptSecret(testSecretKey, testMasterKey)
|
||||
require.NoError(t, err)
|
||||
cred := &stsdomain.SessionCredential{
|
||||
AccessKeyID: testSessionAKID, SecretEncrypted: encrypted, AccountID: "123456789012",
|
||||
PrincipalType: principalTypeAssumedRole, SessionName: "test-session",
|
||||
ExpiresAt: time.Now().UTC().Add(time.Hour),
|
||||
}
|
||||
principal := principalContext{
|
||||
identity: "test-session",
|
||||
principalType: principalTypeAssumedRole,
|
||||
sessionCred: &stsdomain.SessionCredential{AccessKeyID: testSessionAKID},
|
||||
sts := &mockSTSService{
|
||||
sessions: map[string]*stsdomain.SessionCredential{testSessionAKID: cred},
|
||||
tokens: map[string]string{testSessionAKID: testSessionToken},
|
||||
principalErr: tc.err,
|
||||
}
|
||||
v := credverify.New(&mockIAMService{masterKey: testMasterKey}, sts)
|
||||
req := credential.Request{AccessKeyID: testSessionAKID, SecurityToken: testSessionToken}
|
||||
authed, f := v.Authenticate(req, func(string) error { return nil })
|
||||
require.Nil(t, f)
|
||||
_, f = v.Resolve(req, authed)
|
||||
require.NotNil(t, f)
|
||||
require.Equal(t, awserrors.ErrorInvalidClientTokenId, f.Code)
|
||||
|
||||
const ip = "10.15.8.11"
|
||||
code := gw.checkSessionPrincipal(principal, testSessionAKID, authSource{limitKey: ip, logIP: ip})
|
||||
require.Equal(t, awserrors.ErrorInvalidClientTokenId, code)
|
||||
gw.recordCredentialFailure(authSource{limitKey: ip, logIP: ip}, f)
|
||||
|
||||
// Read at the lockout boundary: with exactly `recorded` failures from
|
||||
// the verdict, n more distinct ones leave the address open and one
|
||||
@@ -2125,7 +2138,7 @@ func TestSigV4Auth_AKIA_PrincipalTypeUser(t *testing.T) {
|
||||
},
|
||||
},
|
||||
}
|
||||
gw := &GatewayConfig{DisableLogging: true, Region: testRegion, IAMService: iamMock}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, Region: testRegion, IAMService: iamMock})
|
||||
|
||||
r := chi.NewRouter()
|
||||
r.Use(gw.SigV4AuthMiddleware())
|
||||
@@ -2155,12 +2168,12 @@ func TestSigV4Auth_Session_STSServiceNil(t *testing.T) {
|
||||
accessKeys: map[string]*iamdomain.AccessKey{},
|
||||
}
|
||||
_ = encryptedSecret
|
||||
gw := &GatewayConfig{
|
||||
gw := withVerifier(&GatewayConfig{
|
||||
DisableLogging: true,
|
||||
Region: testRegion,
|
||||
IAMService: iamMock,
|
||||
STSService: nil,
|
||||
}
|
||||
})
|
||||
|
||||
r := chi.NewRouter()
|
||||
r.Use(gw.SigV4AuthMiddleware())
|
||||
@@ -2229,12 +2242,12 @@ func TestCheckPolicy_SessionNameCollision(t *testing.T) {
|
||||
tokens: map[string]string{testSessionAKID: testSessionToken},
|
||||
}
|
||||
|
||||
gw := &GatewayConfig{
|
||||
gw := withVerifier(&GatewayConfig{
|
||||
DisableLogging: true,
|
||||
Region: testRegion,
|
||||
IAMService: iamMock,
|
||||
STSService: stsMock,
|
||||
}
|
||||
})
|
||||
|
||||
handler := setupPolicyTestHandler(gw)
|
||||
|
||||
@@ -2305,12 +2318,12 @@ func newAssumedRoleEnforcementGateway(t *testing.T, cred *stsdomain.SessionCrede
|
||||
sessions: map[string]*stsdomain.SessionCredential{cred.AccessKeyID: cred},
|
||||
tokens: map[string]string{cred.AccessKeyID: testSessionToken},
|
||||
}
|
||||
return &GatewayConfig{
|
||||
return withVerifier(&GatewayConfig{
|
||||
DisableLogging: true,
|
||||
Region: testRegion,
|
||||
IAMService: iamMock,
|
||||
STSService: stsMock,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// setupPolicyResourceTestHandler is setupPolicyTestHandler for resource-scoped
|
||||
@@ -2751,7 +2764,7 @@ func TestSigV4Auth_ResolvesCtxActionForNonQueryServices(t *testing.T) {
|
||||
},
|
||||
},
|
||||
}
|
||||
gw := &GatewayConfig{DisableLogging: true, Region: testRegion, IAMService: mockSvc}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, Region: testRegion, IAMService: mockSvc})
|
||||
b := dispatch.NewBuilder()
|
||||
require.NoError(t, b.Register(awsapi.NewRegistration(awsapi.Deps{})))
|
||||
require.NoError(t, b.Register(acmawsapi.NewRegistration(acmawsapi.Deps{})))
|
||||
|
||||
@@ -3,6 +3,7 @@ package gateway
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/roles/awsgw/credverify"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
@@ -90,6 +91,7 @@ func newBedrockRequestGateway(t *testing.T, vllmURL string) *GatewayConfig {
|
||||
NATSConn: nc,
|
||||
DisableLogging: true,
|
||||
IAMService: allowAllIAMService(),
|
||||
Credentials: credverify.New(allowAllIAMService(), nil),
|
||||
BedrockEndpoints: map[string]string{
|
||||
bedrockTestLlamaModelID: vllmURL,
|
||||
},
|
||||
|
||||
@@ -29,12 +29,12 @@ import (
|
||||
func newBedrockAgentRequestGateway(t *testing.T, vector ochrevector.VectorService) *GatewayConfig {
|
||||
t.Helper()
|
||||
_, _, js := testutil.StartTestJetStream(t)
|
||||
return &GatewayConfig{
|
||||
return withVerifier(&GatewayConfig{
|
||||
IAMService: allowAllIAMService(),
|
||||
BedrockAgentKB: ochrevector.NewKBStore(js),
|
||||
BedrockAgentDataSources: ochrevector.NewDataSourceStore(js),
|
||||
BedrockAgentVector: vector,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func bedrockAgentRequestWithAccount(method, path, body string) *http.Request {
|
||||
@@ -366,7 +366,7 @@ func TestBedrockAgentRequest_MissingAccountIDReturnsInternalError(t *testing.T)
|
||||
}
|
||||
|
||||
func TestBedrockAgentRequest_NilDependenciesReturnServerInternal(t *testing.T) {
|
||||
gw := &GatewayConfig{IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{IAMService: allowAllIAMService()})
|
||||
|
||||
req := bedrockAgentRequestWithAccount(http.MethodPost, "/knowledgebases/", "")
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
@@ -30,7 +30,7 @@ import (
|
||||
func newBedrockAgentRuntimeRequestGateway(t *testing.T, vector ochrevector.VectorService, vllmURL string) *GatewayConfig {
|
||||
t.Helper()
|
||||
_, _, js := testutil.StartTestJetStream(t)
|
||||
return &GatewayConfig{
|
||||
return withVerifier(&GatewayConfig{
|
||||
IAMService: allowAllIAMService(),
|
||||
BedrockAgentKB: ochrevector.NewKBStore(js),
|
||||
BedrockAgentDataSources: ochrevector.NewDataSourceStore(js),
|
||||
@@ -39,7 +39,7 @@ func newBedrockAgentRuntimeRequestGateway(t *testing.T, vector ochrevector.Vecto
|
||||
bedrockTestLlamaModelID: vllmURL,
|
||||
},
|
||||
BedrockAccess: grantAllModels{},
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func bedrockAgentRuntimeRequestWithAccount(method, path, body string) *http.Request {
|
||||
@@ -217,7 +217,7 @@ func TestBedrockAgentRuntimeRequest_MissingAccountIDReturnsInternalError(t *test
|
||||
}
|
||||
|
||||
func TestBedrockAgentRuntimeRequest_NilDependenciesReturnServerInternal(t *testing.T) {
|
||||
gw := &GatewayConfig{IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{IAMService: allowAllIAMService()})
|
||||
|
||||
req := bedrockAgentRuntimeRequestWithAccount(http.MethodPost, "/knowledgebases/kb-1/retrieve", `{"retrievalQuery":{"text":"q"}}`)
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
@@ -7,6 +7,7 @@ package gateway
|
||||
import (
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
stsdomain "github.com/mulgadc/spinifex/spinifex/domains/sts"
|
||||
"maps"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
@@ -118,7 +119,7 @@ func emittedKeys(t *testing.T) map[string]string {
|
||||
},
|
||||
}
|
||||
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: &mockIAMService{}}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: &mockIAMService{}})
|
||||
union := make(map[string]string)
|
||||
for name, principal := range principals {
|
||||
r := httptest.NewRequest(http.MethodPost, "/?prefix=home/&max-keys=100", nil)
|
||||
@@ -126,7 +127,8 @@ func emittedKeys(t *testing.T) map[string]string {
|
||||
r.RemoteAddr = "10.4.1.9:52344"
|
||||
// Resolved the way the middleware resolves it, so the gate cannot pass on
|
||||
// a value only the test supplies.
|
||||
userID, _, err := gw.principalUser(principal)
|
||||
userID, _, err := stsdomain.ResolveCallerUser(principal.accountID, principal.principalType,
|
||||
principal.identity, principal.assumedRoleID, gw.IAMService)
|
||||
require.NoError(t, err)
|
||||
principal.userID = userID
|
||||
for key := range requestConditionKeys(r, principal) {
|
||||
|
||||
@@ -6,14 +6,11 @@ package gateway
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/iam"
|
||||
"github.com/mulgadc/bluebottle/pkg/iampolicy"
|
||||
iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
@@ -108,116 +105,6 @@ func TestRequestConditionKeys_OmitsUnresolvableUserID(t *testing.T) {
|
||||
assert.NotContains(t, keys, iampolicy.KeyUserID)
|
||||
}
|
||||
|
||||
// principalUser resolves the ID once per request, at the point the principal
|
||||
// is resolved. A user with no record and a session minted before the ID was
|
||||
// recorded both yield an omitted key, not a failed request.
|
||||
func TestPrincipalUserID_ResolvesAndOmits(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: &mockIAMService{}}
|
||||
|
||||
userID, _, err := gw.principalUser(principalContext{
|
||||
identity: "alice", accountID: "000000000001", principalType: principalTypeUser,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "AIDAALICE", userID)
|
||||
|
||||
userID, _, err = gw.principalUser(principalContext{
|
||||
identity: "session", accountID: "000000000001", principalType: principalTypeAssumedRole,
|
||||
assumedRoleID: "AROASHAREDOPS:session",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "AROASHAREDOPS:session", userID)
|
||||
|
||||
// Root's aws:userid is the account ID, matching what GetCallerIdentity
|
||||
// reports for the same principal.
|
||||
userID, _, err = gw.principalUser(principalContext{
|
||||
identity: "root", accountID: "000000000000", principalType: principalTypeUser,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "000000000000", userID)
|
||||
|
||||
missing := &GatewayConfig{DisableLogging: true, IAMService: &mockIAMService{
|
||||
getUserFn: func(string, *iam.GetUserInput) (*iam.GetUserOutput, error) {
|
||||
return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
|
||||
},
|
||||
}}
|
||||
userID, _, err = missing.principalUser(principalContext{
|
||||
identity: "alice", accountID: "000000000001", principalType: principalTypeUser,
|
||||
})
|
||||
require.NoError(t, err, "a deleted user omits the key rather than failing the request")
|
||||
assert.Empty(t, userID)
|
||||
|
||||
// A record predating the field resolves to no ID, which omits the key.
|
||||
legacy := &GatewayConfig{DisableLogging: true, IAMService: &mockIAMService{
|
||||
getUserFn: func(string, *iam.GetUserInput) (*iam.GetUserOutput, error) {
|
||||
return &iam.GetUserOutput{User: &iam.User{UserName: aws.String("alice")}}, nil
|
||||
},
|
||||
}}
|
||||
userID, _, err = legacy.principalUser(principalContext{
|
||||
identity: "alice", accountID: "000000000001", principalType: principalTypeUser,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, userID)
|
||||
|
||||
// A session minted before assumed_role_id was recorded takes the same arm.
|
||||
userID, _, err = gw.principalUser(principalContext{
|
||||
identity: "session", accountID: "000000000001", principalType: principalTypeAssumedRole,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, userID)
|
||||
}
|
||||
|
||||
// A dependency fault must not read as "the principal has no ID": the key would
|
||||
// be omitted, silently narrowing an Allow and widening a Deny to everything.
|
||||
func TestPrincipalUserID_DependencyFaultFailsClosed(t *testing.T) {
|
||||
faulty := &GatewayConfig{DisableLogging: true, IAMService: &mockIAMService{
|
||||
getUserFn: func(string, *iam.GetUserInput) (*iam.GetUserOutput, error) {
|
||||
return nil, errors.New("nats: no responders available for request")
|
||||
},
|
||||
}}
|
||||
|
||||
userID, userARN, err := faulty.principalUser(principalContext{
|
||||
identity: "alice", accountID: "000000000001", principalType: principalTypeUser,
|
||||
})
|
||||
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorInternalError, err.Error())
|
||||
assert.Empty(t, userID)
|
||||
assert.Empty(t, userARN)
|
||||
}
|
||||
|
||||
// An IAM user's ARN comes from its record, path included, in the same read as
|
||||
// its ID. Root and role sessions carry no user ARN.
|
||||
func TestPrincipalUser_ResolvesStoredUserARN(t *testing.T) {
|
||||
const pathed = "arn:aws:iam::000000000001:user/eng/alice"
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: &mockIAMService{
|
||||
getUserFn: func(string, *iam.GetUserInput) (*iam.GetUserOutput, error) {
|
||||
return &iam.GetUserOutput{User: &iam.User{
|
||||
UserName: aws.String("alice"), UserId: aws.String("AIDAALICE"), Arn: aws.String(pathed),
|
||||
}}, nil
|
||||
},
|
||||
}}
|
||||
|
||||
userID, userARN, err := gw.principalUser(principalContext{
|
||||
identity: "alice", accountID: "000000000001", principalType: principalTypeUser,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "AIDAALICE", userID)
|
||||
assert.Equal(t, pathed, userARN)
|
||||
|
||||
_, userARN, err = gw.principalUser(principalContext{
|
||||
identity: "root", accountID: "000000000000", principalType: principalTypeUser,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, userARN)
|
||||
|
||||
_, userARN, err = gw.principalUser(principalContext{
|
||||
identity: "session", accountID: "000000000001", principalType: principalTypeAssumedRole,
|
||||
assumedRoleID: "AROASHAREDOPS:session",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, userARN)
|
||||
}
|
||||
|
||||
// RoleSessionName is chosen by the caller of AssumeRole, so aws:username must
|
||||
// stay absent for a role session — otherwise anyone permitted to assume the
|
||||
// role satisfies an aws:username condition just by naming their session.
|
||||
@@ -304,12 +191,12 @@ func conditionedPolicyIAMService(effect string, cond map[string]map[string]iamdo
|
||||
// nil keys from checkPolicyResources would leave every other test green while
|
||||
// silently killing every conditional Allow on this path.
|
||||
func TestCheckPolicyResources_ConditionalAllowHonoursSourceIP(t *testing.T) {
|
||||
gw := &GatewayConfig{
|
||||
gw := withVerifier(&GatewayConfig{
|
||||
DisableLogging: true,
|
||||
IAMService: conditionedPolicyIAMService("Allow", map[string]map[string]iamdomain.ConditionValue{
|
||||
iampolicy.OpIPAddress: {iampolicy.KeySourceIP: {"10.0.0.0/8"}},
|
||||
}),
|
||||
}
|
||||
})
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
@@ -343,12 +230,12 @@ func TestCheckPolicyResources_ConditionalAllowHonoursSourceIP(t *testing.T) {
|
||||
// the one direction where an absent key widens access. Pin it so the omission
|
||||
// is a decision rather than an accident.
|
||||
func TestCheckPolicyResources_ConditionalDenyHonoursSourceIP(t *testing.T) {
|
||||
gw := &GatewayConfig{
|
||||
gw := withVerifier(&GatewayConfig{
|
||||
DisableLogging: true,
|
||||
IAMService: conditionedPolicyIAMService("Deny", map[string]map[string]iamdomain.ConditionValue{
|
||||
iampolicy.OpIPAddress: {iampolicy.KeySourceIP: {"203.0.113.0/24"}},
|
||||
}),
|
||||
}
|
||||
})
|
||||
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
// Tests that a dispatch failure is logged at the level the status the caller
|
||||
// receives implies, so a client's own 4xx does not read as a fault of ours.
|
||||
|
||||
package gateway
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// loggedLevelOf returns the level of the first record in logs whose msg is
|
||||
// msg, failing the test when no record carries it.
|
||||
func loggedLevelOf(t *testing.T, logs *bytes.Buffer, msg string) string {
|
||||
t.Helper()
|
||||
for line := range strings.SplitSeq(logs.String(), "\n") {
|
||||
var rec struct {
|
||||
Level string `json:"level"`
|
||||
Msg string `json:"msg"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(line), &rec); err != nil {
|
||||
continue
|
||||
}
|
||||
if rec.Msg == msg {
|
||||
return rec.Level
|
||||
}
|
||||
}
|
||||
t.Fatalf("no %q record in:\n%s", msg, logs.String())
|
||||
return ""
|
||||
}
|
||||
|
||||
func TestDispatchSeam_ErrorLevelFollowsTheStatusTheCallerGets(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
code string
|
||||
status int
|
||||
level string
|
||||
}{
|
||||
{"a dry run is the answer AWS documents for one", awserrors.ErrorDryRunOperation, http.StatusPreconditionFailed, "INFO"},
|
||||
{"a volume the caller named does not exist", awserrors.ErrorInvalidVolumeNotFound, http.StatusNotFound, "INFO"},
|
||||
{"a malformed parameter is the caller's request", awserrors.ErrorInvalidParameterValue, http.StatusBadRequest, "INFO"},
|
||||
{"an internal failure is ours", awserrors.ErrorInternalError, http.StatusInternalServerError, "ERROR"},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
logs := &bytes.Buffer{}
|
||||
prev := slog.Default()
|
||||
slog.SetDefault(slog.New(slog.NewJSONHandler(logs, &slog.HandlerOptions{Level: slog.LevelInfo})))
|
||||
t.Cleanup(func() { slog.SetDefault(prev) })
|
||||
|
||||
capt := &dispatchCapture{dispatchErr: errors.New(tc.code)}
|
||||
_, app := dispatchSeamApp(t, allowAllECRDocs(), fakeJSONRegistration(capt))
|
||||
resp := doRequest(app, signDispatchRequest(t, fakeJSONService, []byte("{}")))
|
||||
|
||||
require.Equal(t, tc.status, resp.StatusCode)
|
||||
assert.Equal(t, tc.level, loggedLevelOf(t, logs, "Service request error"))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -147,7 +147,7 @@ func dispatchSeamApp(t *testing.T, docs []iamdomain.PolicyDocument, regs ...disp
|
||||
require.NoError(t, b.Register(reg))
|
||||
}
|
||||
|
||||
gw := &GatewayConfig{
|
||||
gw := withVerifier(&GatewayConfig{
|
||||
DisableLogging: true,
|
||||
Region: testRegion,
|
||||
NATSConn: nc,
|
||||
@@ -167,7 +167,7 @@ func dispatchSeamApp(t *testing.T, docs []iamdomain.PolicyDocument, regs ...disp
|
||||
},
|
||||
getUserPoliciesFn: func(_, _ string) ([]iamdomain.PolicyDocument, error) { return docs, nil },
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
r := chi.NewRouter()
|
||||
r.Use(audit.Middleware)
|
||||
@@ -341,7 +341,7 @@ func TestDispatchSeam_NATSDownGetsSame503AsLegacy(t *testing.T) {
|
||||
b := dispatch.NewBuilder()
|
||||
require.NoError(t, b.Register(fakeJSONRegistration(&dispatchCapture{})))
|
||||
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService(), Services: b.Build()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService(), Services: b.Build()})
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/", nil)
|
||||
ctx := context.WithValue(req.Context(), ctxService, fakeJSONService)
|
||||
@@ -368,7 +368,7 @@ func TestDispatchSeam_UnregisteredUnknownServiceStillInvalidAction(t *testing.T)
|
||||
|
||||
encryptedSecret, err := iamdomain.EncryptSecret(testSecretKey, testMasterKey)
|
||||
require.NoError(t, err)
|
||||
gw := &GatewayConfig{
|
||||
gw := withVerifier(&GatewayConfig{
|
||||
DisableLogging: true,
|
||||
Region: testRegion,
|
||||
Services: b.Build(),
|
||||
@@ -378,7 +378,7 @@ func TestDispatchSeam_UnregisteredUnknownServiceStillInvalidAction(t *testing.T)
|
||||
testAccessKey: {AccessKeyID: testAccessKey, SecretAccessKey: encryptedSecret, UserName: "root", Status: "Active"},
|
||||
},
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
r := chi.NewRouter()
|
||||
r.Use(gw.SigV4AuthMiddleware())
|
||||
|
||||
@@ -26,14 +26,14 @@ const (
|
||||
// scopedPolicyGateway serves one policy document to every principal.
|
||||
func scopedPolicyGateway(statements ...iamdomain.Statement) *GatewayConfig {
|
||||
docs := []iamdomain.PolicyDocument{{Version: "2012-10-17", Statement: statements}}
|
||||
return &GatewayConfig{
|
||||
return withVerifier(&GatewayConfig{
|
||||
DisableLogging: true,
|
||||
Region: authzRegion,
|
||||
IAMService: &policyMockIAMService{
|
||||
getUserPoliciesFn: func(_, _ string) ([]iamdomain.PolicyDocument, error) { return docs, nil },
|
||||
getRolePoliciesFn: func(_, _ string) ([]iamdomain.PolicyDocument, error) { return docs, nil },
|
||||
},
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func statement(effect, action, resource string) iamdomain.Statement {
|
||||
@@ -335,7 +335,7 @@ func passRoleDeniedGateway(t *testing.T, resolveErr error) *GatewayConfig {
|
||||
statement("Allow", "ec2:*", "*"),
|
||||
statement("Deny", "iam:PassRole", "*"),
|
||||
}}}
|
||||
return &GatewayConfig{
|
||||
return withVerifier(&GatewayConfig{
|
||||
DisableLogging: true,
|
||||
Region: authzRegion,
|
||||
NATSConn: startTestNATS(t),
|
||||
@@ -345,7 +345,7 @@ func passRoleDeniedGateway(t *testing.T, resolveErr error) *GatewayConfig {
|
||||
},
|
||||
resolveErr: resolveErr,
|
||||
},
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// An iam:PassRole denial raised during dispatch is EC2's UnauthorizedOperation,
|
||||
|
||||
@@ -5,6 +5,7 @@ package gateway
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/roles/awsgw/credverify"
|
||||
"net/http/httptest"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
@@ -55,6 +56,7 @@ func TestDeleteVolumeAction_MissingConfiguredNodeIsRetryableAndDeletesNothing(t
|
||||
NATSConn: nc,
|
||||
NodeIDs: []string{"node-1", "node-2"},
|
||||
IAMService: allowAllIAMService(),
|
||||
Credentials: credverify.New(allowAllIAMService(), nil),
|
||||
}
|
||||
req := setupEC2Request("Action=DeleteVolume&VolumeId=vol-0123456789abcdef0", "123456789012")
|
||||
ctx, cancel := context.WithTimeout(req.Context(), 30*time.Second)
|
||||
|
||||
@@ -42,7 +42,7 @@ func serveEC2(gw *GatewayConfig, body string) *httptest.ResponseRecorder {
|
||||
// The gateway has no NATS connection, so any request that reached its handler
|
||||
// would fail ServerInternal; DryRunOperation proves nothing was dispatched.
|
||||
func TestEC2Request_DryRunAnswersEveryAction(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, Region: authzRegion, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, Region: authzRegion, IAMService: allowAllIAMService()})
|
||||
|
||||
actions := slices.Sorted(maps.Keys(ec2Actions))
|
||||
|
||||
@@ -61,7 +61,7 @@ func TestEC2Request_DryRunAnswersEveryAction(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestEC2Request_DryRunFalseDispatches(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, Region: authzRegion, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, Region: authzRegion, IAMService: allowAllIAMService()})
|
||||
assertPermitted(t, dispatchEC2(t, gw, "Action=CreateVpc&CidrBlock=10.0.0.0/16&DryRun=false"))
|
||||
}
|
||||
|
||||
|
||||
@@ -36,7 +36,7 @@ func tagSpecBody(action, resourceType string) string {
|
||||
}
|
||||
|
||||
func TestEC2Request_TagSpecificationResourceType(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, Region: authzRegion, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, Region: authzRegion, IAMService: allowAllIAMService()})
|
||||
|
||||
cases := []struct {
|
||||
name, body, code, resourceType string
|
||||
@@ -62,7 +62,7 @@ func TestEC2Request_TagSpecificationResourceType(t *testing.T) {
|
||||
// Each of an action's own types reaches the handler, which fails only on the
|
||||
// absent NATS connection.
|
||||
func TestEC2Request_TagSpecificationAllowedTypes(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, Region: authzRegion, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, Region: authzRegion, IAMService: allowAllIAMService()})
|
||||
|
||||
for _, rt := range []string{"instance", "volume", "network-interface", "spot-instances-request"} {
|
||||
assertPermitted(t, dispatchEC2(t, gw, tagSpecBody("RunInstances", rt)+"&ImageId=ami-1&MinCount=1&MaxCount=1"))
|
||||
@@ -80,7 +80,7 @@ func xmlText(t *testing.T, s string) string {
|
||||
|
||||
// The refusals are AWS's own, captured by scripts/aws-compliance/aws-response-diff/scen_e.py.
|
||||
func TestEC2Request_MaxResultsOutOfRange(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, Region: authzRegion, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, Region: authzRegion, IAMService: allowAllIAMService()})
|
||||
|
||||
cases := []struct {
|
||||
action string
|
||||
@@ -117,7 +117,7 @@ func TestEC2Request_MaxResultsOutOfRange(t *testing.T) {
|
||||
// Captured from AWS: the combination is refused even when MaxResults is also
|
||||
// out of range.
|
||||
func TestEC2Request_DescribeInstancesMaxResultsWithInstanceIds(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, Region: authzRegion, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, Region: authzRegion, IAMService: allowAllIAMService()})
|
||||
|
||||
for _, n := range []string{"2", "5"} {
|
||||
w := serveEC2(gw, "Action=DescribeInstances&InstanceId.1=i-0123456789abcdef0&MaxResults="+n)
|
||||
@@ -128,7 +128,7 @@ func TestEC2Request_DescribeInstancesMaxResultsWithInstanceIds(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestEC2Request_MaxResultsInRange(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, Region: authzRegion, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, Region: authzRegion, IAMService: allowAllIAMService()})
|
||||
|
||||
for _, body := range []string{
|
||||
"Action=DescribeCapacityReservations&MaxResults=1",
|
||||
@@ -149,6 +149,6 @@ func TestEC2Request_MaxResultsInRange(t *testing.T) {
|
||||
|
||||
// AWS answers DryRun ahead of a MaxResults out of range.
|
||||
func TestEC2Request_MaxResultsAfterDryRun(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, Region: authzRegion, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, Region: authzRegion, IAMService: allowAllIAMService()})
|
||||
assertDryRun(t, dispatchEC2(t, gw, "Action=DescribeLaunchTemplates&MaxResults=0&DryRun=true"))
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package gateway
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/roles/awsgw/credverify"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
@@ -52,6 +53,7 @@ func ecrAuthzTestGateway(t *testing.T, policies []iamdomain.PolicyDocument) (*Ga
|
||||
Region: ecrTestRegion,
|
||||
ECRTokenVerifier: verify,
|
||||
IAMService: iamSvc,
|
||||
Credentials: credverify.New(iamSvc, nil),
|
||||
}, iamSvc
|
||||
}
|
||||
|
||||
|
||||
@@ -52,7 +52,7 @@ func ecrPipelineApp(t *testing.T, docs []iamdomain.PolicyDocument) (*GatewayConf
|
||||
require.NoError(t, err)
|
||||
_, nc := testutil.StartTestNATS(t)
|
||||
|
||||
gw := &GatewayConfig{
|
||||
gw := withVerifier(&GatewayConfig{
|
||||
DisableLogging: true,
|
||||
Region: testRegion,
|
||||
InternalSuffix: ecrTestSuffix,
|
||||
@@ -72,7 +72,7 @@ func ecrPipelineApp(t *testing.T, docs []iamdomain.PolicyDocument) (*GatewayConf
|
||||
},
|
||||
getUserPoliciesFn: func(_, _ string) ([]iamdomain.PolicyDocument, error) { return docs, nil },
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
withECR(gw, awsapi.Deps{})
|
||||
|
||||
@@ -219,7 +219,7 @@ func TestECRActionFromTarget_FormsAndTelemetryAgree(t *testing.T) {
|
||||
// any "<prefix>.<Action>" shape (not just the ECR prefix) dispatches on the
|
||||
// bare action name today.
|
||||
func TestECRRequest_UnrelatedPrefixDispatchesByBareAction(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
err := gw.serveECR(httptest.NewRecorder(), setupECRRequest("a.b.ListRepositories", "{}"))
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorNotImplemented, err.Error())
|
||||
@@ -229,7 +229,7 @@ func TestECRRequest_UnrelatedPrefixDispatchesByBareAction(t *testing.T) {
|
||||
// carrying another service's prefix but naming an action outside awsapi.Actions
|
||||
// fails as an unrecognised ECR action, not as a cross-service error.
|
||||
func TestECRRequest_ECSPrefixedNonECRActionIsInvalidAction(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
err := gw.serveECR(httptest.NewRecorder(),
|
||||
setupECRRequest("AmazonEC2ContainerServiceV20141113.ListClusters", "{}"))
|
||||
require.Error(t, err)
|
||||
@@ -239,7 +239,7 @@ func TestECRRequest_ECSPrefixedNonECRActionIsInvalidAction(t *testing.T) {
|
||||
// TestECRRequest_TrailingDotIsMissingAction pins that a target ending in "."
|
||||
// resolves to an empty action, the same as an absent target.
|
||||
func TestECRRequest_TrailingDotIsMissingAction(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
err := gw.serveECR(httptest.NewRecorder(),
|
||||
setupECRRequest(awsapi.TargetPrefix+".", "{}"))
|
||||
require.Error(t, err)
|
||||
@@ -275,7 +275,7 @@ func TestECRInventory_ReachableStubsAnswer501(t *testing.T) {
|
||||
}
|
||||
assert.ElementsMatch(t, want, reachableStubs)
|
||||
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
for _, action := range reachableStubs {
|
||||
t.Run(action, func(t *testing.T) {
|
||||
err := gw.serveECR(httptest.NewRecorder(),
|
||||
@@ -290,7 +290,7 @@ func TestECRInventory_ReachableStubsAnswer501(t *testing.T) {
|
||||
// unsupported scanning action answers OperationNotSupportedException (400),
|
||||
// not NotImplemented (501).
|
||||
func TestECRInventory_ScanningActionsReturnOperationNotSupported(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
for _, action := range awsapi.UnsupportedActionNames() {
|
||||
t.Run(action, func(t *testing.T) {
|
||||
err := gw.serveECR(httptest.NewRecorder(),
|
||||
@@ -395,11 +395,11 @@ func TestECRPipeline_AuthzGateBlocksComposedAction(t *testing.T) {
|
||||
func TestGetAuthorizationToken_IgnoresRequestSuppliedIdentity(t *testing.T) {
|
||||
iss, verify := newECRAuth(t)
|
||||
endpoint := awsapi.RepositoryEndpoint{Region: ecrTestRegion, ServicesDomain: ecrTestSuffix}
|
||||
gw := withECR(&GatewayConfig{
|
||||
gw := withECR(withVerifier(&GatewayConfig{
|
||||
Region: ecrTestRegion, InternalSuffix: ecrTestSuffix,
|
||||
ECRTokenIssuer: iss, ECRTokenVerifier: verify, DisableLogging: true,
|
||||
IAMService: allowAllIAMService(),
|
||||
}, awsapi.Deps{AuthorizationToken: awsapi.NewAuthorizationTokenActionService(iss, endpoint)})
|
||||
}), awsapi.Deps{AuthorizationToken: awsapi.NewAuthorizationTokenActionService(iss, endpoint)})
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(`{"registryIds":["999999999999"]}`))
|
||||
ctx := withPrincipalField(req.Context(), fieldAccountID, ecrTestAccount)
|
||||
@@ -434,7 +434,7 @@ func TestGetAuthorizationToken_IgnoresRequestSuppliedIdentity(t *testing.T) {
|
||||
func TestRequest_ClusterUnavailableNilConn_ECR(t *testing.T) {
|
||||
for _, action := range []string{"ListRepositories", "CreateRepository"} {
|
||||
t.Run(action, func(t *testing.T) {
|
||||
gw := withECR(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}, awsapi.Deps{})
|
||||
gw := withECR(withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}), awsapi.Deps{})
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/", nil)
|
||||
req.Header.Set("X-Amz-Target", awsapi.TargetPrefix+"."+action)
|
||||
|
||||
@@ -30,10 +30,10 @@ func newRepoLifecycleGateway(t *testing.T) (*GatewayConfig, *nats.Conn) {
|
||||
serveECRMeta(t, nc, ecr.SubjectRepoDelete, svc.RepoDelete)
|
||||
serveECRMeta(t, nc, ecr.SubjectManifestPut, svc.ManifestPut)
|
||||
serveECRMeta(t, nc, ecr.SubjectManifestList, svc.ManifestList)
|
||||
gw := withECR(&GatewayConfig{
|
||||
gw := withECR(withVerifier(&GatewayConfig{
|
||||
NATSConn: nc, Region: ecrTestRegion, InternalSuffix: ecrTestSuffix, DisableLogging: true,
|
||||
IAMService: allowAllIAMService(),
|
||||
}, awsapi.Deps{Repository: awsapi.NewRepositoryActionService(ecr.NewNATSMetaStore(nc), awsapi.RepositoryEndpoint{
|
||||
}), awsapi.Deps{Repository: awsapi.NewRepositoryActionService(ecr.NewNATSMetaStore(nc), awsapi.RepositoryEndpoint{
|
||||
Region: ecrTestRegion, ServicesDomain: ecrTestSuffix,
|
||||
})})
|
||||
return gw, nc
|
||||
@@ -300,11 +300,11 @@ func TestECRRequest_CreateDeleteDispatched(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRepositoryActions_MissingComposition(t *testing.T) {
|
||||
gw := &GatewayConfig{
|
||||
gw := withVerifier(&GatewayConfig{
|
||||
Region: ecrTestRegion,
|
||||
DisableLogging: true,
|
||||
IAMService: allowAllIAMService(),
|
||||
}
|
||||
})
|
||||
|
||||
_, err := ecrRepositoryRequest(t, gw, "CreateRepository", `{"repositoryName":"team/app"}`)
|
||||
require.Error(t, err)
|
||||
|
||||
@@ -54,10 +54,10 @@ func newDescribeReposGateway(t *testing.T, repos ...string) *GatewayConfig {
|
||||
for _, r := range repos {
|
||||
require.NoError(t, store.PutRepo(context.Background(), ecrTestAccount, ecr.RepoMeta{Name: r, CreatedAt: time.Now()}))
|
||||
}
|
||||
return withECR(&GatewayConfig{
|
||||
return withECR(withVerifier(&GatewayConfig{
|
||||
NATSConn: nc, Region: ecrTestRegion, InternalSuffix: ecrTestSuffix, DisableLogging: true,
|
||||
IAMService: allowAllIAMService(),
|
||||
}, awsapi.Deps{Repository: awsapi.NewRepositoryActionService(store, awsapi.RepositoryEndpoint{
|
||||
}), awsapi.Deps{Repository: awsapi.NewRepositoryActionService(store, awsapi.RepositoryEndpoint{
|
||||
Region: ecrTestRegion, ServicesDomain: ecrTestSuffix,
|
||||
})})
|
||||
}
|
||||
|
||||
@@ -17,11 +17,11 @@ import (
|
||||
func TestGetAuthorizationToken_MintsUsableToken(t *testing.T) {
|
||||
iss, verify := newECRAuth(t)
|
||||
endpoint := awsapi.RepositoryEndpoint{Region: ecrTestRegion, ServicesDomain: ecrTestSuffix}
|
||||
gw := withECR(&GatewayConfig{
|
||||
gw := withECR(withVerifier(&GatewayConfig{
|
||||
Region: ecrTestRegion, InternalSuffix: ecrTestSuffix,
|
||||
ECRTokenIssuer: iss, ECRTokenVerifier: verify, DisableLogging: true,
|
||||
IAMService: allowAllIAMService(),
|
||||
}, awsapi.Deps{AuthorizationToken: awsapi.NewAuthorizationTokenActionService(iss, endpoint)})
|
||||
}), awsapi.Deps{AuthorizationToken: awsapi.NewAuthorizationTokenActionService(iss, endpoint)})
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader("{}"))
|
||||
ctx := withPrincipalField(req.Context(), fieldAccountID, ecrTestAccount)
|
||||
@@ -71,11 +71,11 @@ func TestGetAuthorizationToken_PathedUserRoundTrips(t *testing.T) {
|
||||
iamSvc.userPolicies[ecrPrincipalTestAccount+"|dev"] = allow
|
||||
iss, verify := newECRAuth(t)
|
||||
endpoint := awsapi.RepositoryEndpoint{Region: ecrTestRegion, ServicesDomain: ecrTestSuffix}
|
||||
gw := withECR(&GatewayConfig{
|
||||
gw := withECR(withVerifier(&GatewayConfig{
|
||||
Region: ecrTestRegion, InternalSuffix: ecrTestSuffix,
|
||||
ECRTokenIssuer: iss, ECRTokenVerifier: verify, DisableLogging: true,
|
||||
IAMService: iamSvc,
|
||||
}, awsapi.Deps{AuthorizationToken: awsapi.NewAuthorizationTokenActionService(iss, endpoint)})
|
||||
}), awsapi.Deps{AuthorizationToken: awsapi.NewAuthorizationTokenActionService(iss, endpoint)})
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader("{}"))
|
||||
ctx := withPrincipalField(req.Context(), fieldAccountID, ecrPrincipalTestAccount)
|
||||
@@ -110,11 +110,11 @@ func TestGetAuthorizationToken_PathedUserRoundTrips(t *testing.T) {
|
||||
func TestGetAuthorizationToken_ProxyEndpointCarriesPort(t *testing.T) {
|
||||
iss, verify := newECRAuth(t)
|
||||
endpoint := awsapi.RepositoryEndpoint{Region: ecrTestRegion, ServicesDomain: ecrTestSuffix, RegistryPort: "9999"}
|
||||
gw := withECR(&GatewayConfig{
|
||||
gw := withECR(withVerifier(&GatewayConfig{
|
||||
Region: ecrTestRegion, InternalSuffix: ecrTestSuffix, RegistryPort: "9999",
|
||||
ECRTokenIssuer: iss, ECRTokenVerifier: verify, DisableLogging: true,
|
||||
IAMService: allowAllIAMService(),
|
||||
}, awsapi.Deps{AuthorizationToken: awsapi.NewAuthorizationTokenActionService(iss, endpoint)})
|
||||
}), awsapi.Deps{AuthorizationToken: awsapi.NewAuthorizationTokenActionService(iss, endpoint)})
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader("{}"))
|
||||
ctx := withPrincipalField(req.Context(), fieldAccountID, ecrTestAccount)
|
||||
@@ -136,10 +136,10 @@ func TestGetAuthorizationToken_ProxyEndpointCarriesPort(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestGetAuthorizationToken_NoIssuerNotImplemented(t *testing.T) {
|
||||
gw := withECR(&GatewayConfig{
|
||||
gw := withECR(withVerifier(&GatewayConfig{
|
||||
DisableLogging: true,
|
||||
IAMService: allowAllIAMService(),
|
||||
}, awsapi.Deps{AuthorizationToken: awsapi.NewAuthorizationTokenActionService(nil, awsapi.RepositoryEndpoint{})})
|
||||
}), awsapi.Deps{AuthorizationToken: awsapi.NewAuthorizationTokenActionService(nil, awsapi.RepositoryEndpoint{})})
|
||||
req := setupECRRequest(awsapi.TargetPrefix+".GetAuthorizationToken", "{}")
|
||||
ctx := withPrincipalField(req.Context(), fieldAccountID, ecrTestAccount)
|
||||
err := gw.serveECR(httptest.NewRecorder(), req.WithContext(ctx))
|
||||
@@ -148,7 +148,7 @@ func TestGetAuthorizationToken_NoIssuerNotImplemented(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestGetAuthorizationToken_MissingComposition(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
req := setupECRRequest(awsapi.TargetPrefix+".GetAuthorizationToken", "{}")
|
||||
err := gw.serveECR(httptest.NewRecorder(), req)
|
||||
require.Error(t, err)
|
||||
@@ -157,11 +157,11 @@ func TestGetAuthorizationToken_MissingComposition(t *testing.T) {
|
||||
|
||||
func TestECRRequest_GetAuthorizationTokenDispatched(t *testing.T) {
|
||||
iss, verify := newECRAuth(t)
|
||||
gw := withECR(&GatewayConfig{
|
||||
gw := withECR(withVerifier(&GatewayConfig{
|
||||
Region: ecrTestRegion, InternalSuffix: ecrTestSuffix,
|
||||
ECRTokenIssuer: iss, ECRTokenVerifier: verify, DisableLogging: true,
|
||||
IAMService: allowAllIAMService(),
|
||||
}, awsapi.Deps{AuthorizationToken: awsapi.NewAuthorizationTokenActionService(iss, awsapi.RepositoryEndpoint{
|
||||
}), awsapi.Deps{AuthorizationToken: awsapi.NewAuthorizationTokenActionService(iss, awsapi.RepositoryEndpoint{
|
||||
Region: ecrTestRegion, ServicesDomain: ecrTestSuffix,
|
||||
})})
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
@@ -26,13 +26,13 @@ func newImageGateway(t *testing.T) *GatewayConfig {
|
||||
t.Helper()
|
||||
meta := ecrdomain.NewMemoryMetaStore()
|
||||
reg := ecrregistry.NewRegistry(objectstore.NewMemoryObjectStore(), meta, ecrTestAccount)
|
||||
return withECR(&GatewayConfig{
|
||||
return withECR(withVerifier(&GatewayConfig{
|
||||
ECRRegistry: reg,
|
||||
Region: ecrTestRegion,
|
||||
InternalSuffix: ecrTestSuffix,
|
||||
DisableLogging: true,
|
||||
IAMService: allowAllIAMService(),
|
||||
}, awsapi.Deps{Registry: awsapi.NewRegistryActionService(reg, reg, reg, reg), LifecyclePreview: awsapi.NewLifecyclePreviewActionService(meta, reg)})
|
||||
}), awsapi.Deps{Registry: awsapi.NewRegistryActionService(reg, reg, reg, reg), LifecyclePreview: awsapi.NewLifecyclePreviewActionService(meta, reg)})
|
||||
}
|
||||
|
||||
// seedGatewayRepo creates the repository metadata so push/PutImage handlers —
|
||||
@@ -300,7 +300,7 @@ func TestImageHandlers_GuardRails(t *testing.T) {
|
||||
assert.Equal(t, "InternalError", err.Error())
|
||||
|
||||
// A missing registry-action composition -> ServerInternal.
|
||||
bare := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
bare := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
err = bare.serveECR(httptest.NewRecorder(), imageReq(t, "ListImages", `{"repositoryName":"team/app"}`))
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, "ServerInternal", err.Error())
|
||||
|
||||
@@ -2,6 +2,7 @@ package gateway
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/roles/awsgw/credverify"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
@@ -94,6 +95,7 @@ func TestLifecyclePreview_MissingComposition(t *testing.T) {
|
||||
Region: ecrTestRegion,
|
||||
DisableLogging: true,
|
||||
IAMService: allowAllIAMService(),
|
||||
Credentials: credverify.New(allowAllIAMService(), nil),
|
||||
}
|
||||
|
||||
_, err := callImage(t, gw, "StartLifecyclePolicyPreview", previewBody("team/app", previewExpireOldest))
|
||||
|
||||
@@ -33,14 +33,14 @@ func setupECRRequest(target, body string) *http.Request {
|
||||
}
|
||||
|
||||
func TestECRRequest_MissingTarget(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
err := gw.serveECR(httptest.NewRecorder(), setupECRRequest("", ""))
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorMissingAction, err.Error())
|
||||
}
|
||||
|
||||
func TestECRRequest_UnknownAction(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
err := gw.serveECR(httptest.NewRecorder(),
|
||||
setupECRRequest("AmazonEC2ContainerRegistry_V20150921.MadeUpAction", "{}"))
|
||||
require.Error(t, err)
|
||||
@@ -52,7 +52,7 @@ func TestECRRequest_UnknownAction(t *testing.T) {
|
||||
// image actions a composed registry capability, so ListRepositories stands in
|
||||
// as a still-stubbed action.
|
||||
func TestECRRequest_KnownActionNotImplemented(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
err := gw.serveECR(httptest.NewRecorder(),
|
||||
setupECRRequest("AmazonEC2ContainerRegistry_V20150921.ListRepositories", "{}"))
|
||||
require.Error(t, err)
|
||||
@@ -60,7 +60,7 @@ func TestECRRequest_KnownActionNotImplemented(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestOCIRegistry_VersionCheck(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
r := chi.NewRouter()
|
||||
gw.mountOCIRegistry(r)
|
||||
|
||||
@@ -72,7 +72,7 @@ func TestOCIRegistry_VersionCheck(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestOCIRegistry_StubReturns501(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
r := chi.NewRouter()
|
||||
gw.mountOCIRegistry(r)
|
||||
|
||||
|
||||
@@ -22,10 +22,10 @@ func decodeTokenResp(t *testing.T, w *httptest.ResponseRecorder) ociTokenRespons
|
||||
|
||||
func TestECRToken_ExchangesBasicForBearer(t *testing.T) {
|
||||
iss, verify := newECRAuth(t)
|
||||
gw := &GatewayConfig{
|
||||
gw := withVerifier(&GatewayConfig{
|
||||
Region: ecrTestRegion, InternalSuffix: ecrTestSuffix, ECRTokenIssuer: iss, ECRTokenVerifier: verify,
|
||||
IAMService: ecrBridgeTestIAM(ecrTestAccount),
|
||||
}
|
||||
})
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/v2/token", nil)
|
||||
req.Header.Set("Authorization", mintBasic(t, iss, ecrTestAccount))
|
||||
@@ -47,10 +47,10 @@ func TestECRToken_ExchangesBasicForBearer(t *testing.T) {
|
||||
|
||||
func TestECRToken_BearerCredentialAccepted(t *testing.T) {
|
||||
iss, verify := newECRAuth(t)
|
||||
gw := &GatewayConfig{
|
||||
gw := withVerifier(&GatewayConfig{
|
||||
Region: ecrTestRegion, InternalSuffix: ecrTestSuffix, ECRTokenIssuer: iss, ECRTokenVerifier: verify,
|
||||
IAMService: ecrBridgeTestIAM(ecrTestAccount),
|
||||
}
|
||||
})
|
||||
|
||||
tok, _, err := iss.Mint(ecrauth.Principal{
|
||||
AccountID: ecrTestAccount,
|
||||
@@ -124,10 +124,10 @@ func TestECRToken_MultipleAuthHeadersRejected(t *testing.T) {
|
||||
// endpoint (200) rather than the bridge-guarded /* catch-all.
|
||||
func TestOCIRegistry_TokenRouted(t *testing.T) {
|
||||
iss, verify := newECRAuth(t)
|
||||
gw := &GatewayConfig{
|
||||
gw := withVerifier(&GatewayConfig{
|
||||
DisableLogging: true, Region: ecrTestRegion, InternalSuffix: ecrTestSuffix, ECRTokenIssuer: iss, ECRTokenVerifier: verify,
|
||||
IAMService: ecrBridgeTestIAM(ecrTestAccount),
|
||||
}
|
||||
})
|
||||
r := chi.NewRouter()
|
||||
gw.mountOCIRegistry(r)
|
||||
|
||||
|
||||
@@ -135,10 +135,10 @@ func TestECRAuthBridge_NoAuthChallenges401(t *testing.T) {
|
||||
|
||||
func TestECRAuthBridge_ValidTokenPassesThrough(t *testing.T) {
|
||||
iss, verify := newECRAuth(t)
|
||||
gw := &GatewayConfig{
|
||||
gw := withVerifier(&GatewayConfig{
|
||||
Region: ecrTestRegion, InternalSuffix: ecrTestSuffix, ECRTokenVerifier: verify,
|
||||
IAMService: ecrBridgeTestIAM(ecrTestAccount),
|
||||
}
|
||||
})
|
||||
next, called := okHandler()
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/v2/", nil)
|
||||
|
||||
@@ -61,7 +61,7 @@ func TestECSRegisterTaskDefinition_DeniedWithoutPassRole(t *testing.T) {
|
||||
return allowRegisterTaskDefAndPassRole("arn:aws:iam::123456789012:role/some-other-role"), nil
|
||||
},
|
||||
}
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: mock}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: mock})
|
||||
req := setupECSUserRequest("AmazonEC2ContainerServiceV20141113.RegisterTaskDefinition",
|
||||
registerTaskDefinitionBody(t, testECSPassRoleTargetARN))
|
||||
|
||||
@@ -76,7 +76,7 @@ func TestECSRegisterTaskDefinition_AllowedWithPassRole(t *testing.T) {
|
||||
return allowRegisterTaskDefAndPassRole(testECSPassRoleTargetARN), nil
|
||||
},
|
||||
}
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: mock}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: mock})
|
||||
req := setupECSUserRequest("AmazonEC2ContainerServiceV20141113.RegisterTaskDefinition",
|
||||
registerTaskDefinitionBody(t, testECSPassRoleTargetARN))
|
||||
|
||||
@@ -96,7 +96,7 @@ func TestECSRegisterTaskDefinition_NoRoleSkipsPassRole(t *testing.T) {
|
||||
return allowPolicyResource("ecs:RegisterTaskDefinition", "*"), nil
|
||||
},
|
||||
}
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: mock}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: mock})
|
||||
req := setupECSUserRequest("AmazonEC2ContainerServiceV20141113.RegisterTaskDefinition",
|
||||
registerTaskDefinitionBody(t, ""))
|
||||
|
||||
@@ -113,7 +113,7 @@ func TestECSRegisterTaskDefinition_DenialRendersAccessDenied403(t *testing.T) {
|
||||
return allowPolicyResource("ecs:RegisterTaskDefinition", "*"), nil
|
||||
},
|
||||
}
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: mock}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: mock})
|
||||
req := setupECSUserRequest("AmazonEC2ContainerServiceV20141113.RegisterTaskDefinition",
|
||||
registerTaskDefinitionBody(t, testECSPassRoleTargetARN))
|
||||
|
||||
|
||||
@@ -62,14 +62,14 @@ func TestECSActionsMap_NamespaceRegistered(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestECSRequest_MissingTarget(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
err := gw.ECS_Request(httptest.NewRecorder(), setupECSRequest("", ""))
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorMissingAction, err.Error())
|
||||
}
|
||||
|
||||
func TestECSRequest_UnknownAction(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
err := gw.ECS_Request(httptest.NewRecorder(),
|
||||
setupECSRequest("AmazonEC2ContainerServiceV20141113.MadeUpAction", "{}"))
|
||||
require.Error(t, err)
|
||||
@@ -78,7 +78,7 @@ func TestECSRequest_UnknownAction(t *testing.T) {
|
||||
|
||||
// An action whose real handler has not yet landed still resolves to the 501 stub.
|
||||
func TestECSRequest_KnownActionNotImplemented(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
err := gw.ECS_Request(httptest.NewRecorder(),
|
||||
setupECSRequest("AmazonEC2ContainerServiceV20141113.UpdateCluster", "{}"))
|
||||
require.Error(t, err)
|
||||
@@ -92,7 +92,7 @@ func TestECSRequest_MissingAccountID(t *testing.T) {
|
||||
req.Header.Set("X-Amz-Target", "AmazonEC2ContainerServiceV20141113.ListClusters")
|
||||
req = withTestIdentity(req.WithContext(context.WithValue(req.Context(), ctxService, "ecs")))
|
||||
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
// Account-less requests are rejected by the policy gate, which runs ahead of
|
||||
// the handler's own account guard.
|
||||
err := gw.ECS_Request(httptest.NewRecorder(), req)
|
||||
|
||||
@@ -55,7 +55,7 @@ func eksPassRoleGateway(statements ...iamdomain.Statement) *GatewayConfig {
|
||||
}
|
||||
|
||||
func eksGatewayWithIAM(iamService *roleStoreIAMService) *GatewayConfig {
|
||||
return &GatewayConfig{DisableLogging: true, Region: authzRegion, IAMService: iamService}
|
||||
return withVerifier(&GatewayConfig{DisableLogging: true, Region: authzRegion, IAMService: iamService})
|
||||
}
|
||||
|
||||
func createNodegroupBody(nodeRole string) string {
|
||||
|
||||
@@ -159,7 +159,7 @@ func TestLookupEKSAction_UnknownReturnsFalse(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestEKSRequest_UnknownActionReturnsInvalidAction(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
req := httptest.NewRequest(http.MethodGet, "/totally-unknown", nil)
|
||||
ctx := context.WithValue(req.Context(), ctxService, "eks")
|
||||
ctx = withPrincipalField(ctx, fieldAccountID, "111122223333")
|
||||
@@ -172,7 +172,7 @@ func TestEKSRequest_UnknownActionReturnsInvalidAction(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestEKSRequest_MissingNATSReturnsServerInternal(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
req := httptest.NewRequest(http.MethodGet, "/clusters", nil)
|
||||
ctx := context.WithValue(req.Context(), ctxService, "eks")
|
||||
ctx = withPrincipalField(ctx, fieldAccountID, "111122223333")
|
||||
@@ -185,7 +185,7 @@ func TestEKSRequest_MissingNATSReturnsServerInternal(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestErrorHandler_EKSEmitsJSONNotXML(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
req := httptest.NewRequest(http.MethodGet, "/clusters", nil)
|
||||
ctx := context.WithValue(req.Context(), ctxService, "eks")
|
||||
req = req.WithContext(ctx)
|
||||
|
||||
@@ -42,7 +42,7 @@ func elbv2PipelineApp(t *testing.T, docs []iamdomain.PolicyDocument, quota *admi
|
||||
require.NoError(t, err)
|
||||
_, nc := testutil.StartTestNATS(t)
|
||||
|
||||
gw := &GatewayConfig{
|
||||
gw := withVerifier(&GatewayConfig{
|
||||
DisableLogging: true,
|
||||
Region: testRegion,
|
||||
NATSConn: nc,
|
||||
@@ -62,7 +62,7 @@ func elbv2PipelineApp(t *testing.T, docs []iamdomain.PolicyDocument, quota *admi
|
||||
},
|
||||
getUserPoliciesFn: func(_, _ string) ([]iamdomain.PolicyDocument, error) { return docs, nil },
|
||||
},
|
||||
}
|
||||
})
|
||||
withELBv2(gw)
|
||||
|
||||
r := chi.NewRouter()
|
||||
@@ -289,7 +289,7 @@ func TestELBv2Pipeline_Authz_UnrelatedGrantDeniedUnknownActionNot(t *testing.T)
|
||||
|
||||
// A nil NATS connection fails at the request-level gate, before dispatch.
|
||||
func TestELBv2Pipeline_NilNATS_503ServiceUnavailable(t *testing.T) {
|
||||
gw := withELBv2(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
gw := withELBv2(withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}))
|
||||
|
||||
req := elbv2Request(authzAccountID, "Action=DescribeLoadBalancers")
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
@@ -20,7 +20,7 @@ func setupELBv2Request(body string) *http.Request {
|
||||
}
|
||||
|
||||
func TestELBv2Request_MissingAction(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
w := httptest.NewRecorder()
|
||||
err := gw.serveELBv2(w, setupELBv2Request(""))
|
||||
require.Error(t, err)
|
||||
@@ -28,7 +28,7 @@ func TestELBv2Request_MissingAction(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestELBv2Request_UnknownAction(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
w := httptest.NewRecorder()
|
||||
err := gw.serveELBv2(w, setupELBv2Request("Action=FakeAction"))
|
||||
require.Error(t, err)
|
||||
|
||||
@@ -236,7 +236,7 @@ func TestErrorEnvelope_ClusterUnavailable(t *testing.T) {
|
||||
b := dispatch.NewBuilder()
|
||||
require.NoError(t, b.Register(envelopeRegistration(restJSONService, dispatch.ErrorEnvelopeRESTJSON, c, nil)))
|
||||
require.NoError(t, b.Register(envelopeRegistration(json10Service, dispatch.ErrorEnvelopeJSON10Query, c, nil)))
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService(), Services: b.Build()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService(), Services: b.Build()})
|
||||
|
||||
call := func(svc string) *http.Response {
|
||||
req := httptest.NewRequest(http.MethodPost, "/", nil)
|
||||
|
||||
+22
-35
@@ -34,11 +34,11 @@ import (
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/ochre"
|
||||
ochrevector "github.com/mulgadc/spinifex/spinifex/domains/ochre/vector"
|
||||
stsdomain "github.com/mulgadc/spinifex/spinifex/domains/sts"
|
||||
stsawsapi "github.com/mulgadc/spinifex/spinifex/domains/sts/awsapi"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/policy"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/audit"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/credential"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/envelope"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/query"
|
||||
@@ -138,6 +138,9 @@ type GatewayConfig struct {
|
||||
// Nil makes DeleteAccount refuse rather than tear down around the data.
|
||||
BucketStore accountteardown.BucketStore
|
||||
STSService stsdomain.STSService
|
||||
|
||||
// Credentials verifies SigV4 credentials. Nil fails every request closed.
|
||||
Credentials credential.Verifier
|
||||
RateLimiter *authlimit.AuthRateLimiter // Per-IP auth failure rate limiter
|
||||
Throttler *ratelimit.Throttler // Per-account+action API request throttler
|
||||
// accountStatus caches which accounts are ACTIVE, so enforcing account
|
||||
@@ -549,13 +552,30 @@ func (gw *GatewayConfig) Request(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
slog.Error("Service request error", "service", svc, "action", action, "error", err)
|
||||
slog.Log(r.Context(), serviceErrorLevel(svc, err), "Service request error",
|
||||
"service", svc, "action", action, "error", err)
|
||||
gw.ErrorHandler(w, r, err)
|
||||
} else {
|
||||
slog.Info("Service request completed", "service", svc, "action", action)
|
||||
}
|
||||
}
|
||||
|
||||
// serviceErrorLevel reports the level a dispatch failure belongs at, taken from
|
||||
// the status the caller is about to be given. A 4xx is the client's own
|
||||
// request, and DryRunOperation is a 412 that AWS documents as the success
|
||||
// answer to a dry run, so only a 5xx is a fault of ours.
|
||||
func serviceErrorLevel(svc string, err error) slog.Level {
|
||||
code, _, ok := awserrors.ResolveErrorDetail(err)
|
||||
if !ok {
|
||||
return slog.LevelError
|
||||
}
|
||||
status := awserrors.LookupErrorMessage(svc, code).HTTPCode
|
||||
if status >= 400 && status < 500 {
|
||||
return slog.LevelInfo
|
||||
}
|
||||
return slog.LevelError
|
||||
}
|
||||
|
||||
// dispatchRegistered hands the authenticated request to a registered service
|
||||
// with the caller's account, the gateway Region and ingress-owned
|
||||
// authorization and actor capabilities.
|
||||
@@ -754,39 +774,6 @@ func principalTypeCondition(principalType string) (string, bool) {
|
||||
}
|
||||
}
|
||||
|
||||
// principalUser resolves aws:userid: an IAM user's unique ID, the role ID and
|
||||
// session name STS minted for a role session, or the account ID for root. Both
|
||||
// halves of a session's ID come from the resolved role, so unlike aws:username
|
||||
// it is not caller-chosen. An IAM user's stored ARN comes from the same read.
|
||||
//
|
||||
// A principal with no ID on record returns empty and the door omits the key. A
|
||||
// dependency fault returns InternalError instead: authorizing against a context
|
||||
// missing the key silently narrows an Allow and widens a Deny.
|
||||
func (gw *GatewayConfig) principalUser(principal principalContext) (userID, userARN string, err error) {
|
||||
if principal.identity == "" || principal.accountID == "" {
|
||||
return "", "", nil
|
||||
}
|
||||
userID, userARN, err = stsawsapi.ResolveCallerUser(principal.accountID, principal.principalType,
|
||||
principal.identity, principal.assumedRoleID, gw.IAMService)
|
||||
switch {
|
||||
case err == nil:
|
||||
if userID == "" {
|
||||
slog.Warn("aws:userid unavailable: principal record carries no user ID",
|
||||
"accountID", principal.accountID, "identity", principal.identity,
|
||||
"principalType", principal.principalType)
|
||||
}
|
||||
return userID, userARN, nil
|
||||
case strings.Contains(err.Error(), awserrors.ErrorIAMNoSuchEntity):
|
||||
slog.Warn("aws:userid unavailable: no such IAM user",
|
||||
"accountID", principal.accountID, "user", principal.identity)
|
||||
return "", "", nil
|
||||
default:
|
||||
slog.Error("aws:userid: IAM dependency fault, refusing to authorize on a degraded context",
|
||||
"accountID", principal.accountID, "identity", principal.identity, "err", err)
|
||||
return "", "", errors.New(awserrors.ErrorInternalError)
|
||||
}
|
||||
}
|
||||
|
||||
// policyResolveRetryBase scales the backoff between policy-resolve attempts
|
||||
// after a transient NATS error. A var only so tests can shorten it.
|
||||
var policyResolveRetryBase = 200 * time.Millisecond
|
||||
|
||||
@@ -84,7 +84,7 @@ func TestXMLErrorBody_EnvelopePerService(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestErrorHandler_IAMService(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
|
||||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := context.WithValue(r.Context(), ctxService, "iam")
|
||||
@@ -106,7 +106,7 @@ func TestErrorHandler_IAMService(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestErrorHandler_UnknownError(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
|
||||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := context.WithValue(r.Context(), ctxService, "ec2")
|
||||
@@ -127,7 +127,7 @@ func TestErrorHandler_UnknownError(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestErrorHandler_WrappedErrorCode(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
|
||||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := context.WithValue(r.Context(), ctxService, "ec2")
|
||||
@@ -151,7 +151,7 @@ func TestErrorHandler_WrappedErrorCode(t *testing.T) {
|
||||
// fidelity gap: a call site that names the blocking resource via awserrors.Errorf
|
||||
// must have that wording reach the client instead of the generic ErrorLookup text.
|
||||
func TestErrorHandler_PrefersCallSiteMessage(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
|
||||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := context.WithValue(r.Context(), ctxService, "ec2")
|
||||
@@ -176,7 +176,7 @@ func TestErrorHandler_PrefersCallSiteMessage(t *testing.T) {
|
||||
// ResourceInUseException collision check: ACM's DeleteCertificate must not
|
||||
// surface EKS's "cluster already exists" wording for the shared wire code.
|
||||
func TestErrorHandler_ACMResourceInUse_UsesACMWording(t *testing.T) {
|
||||
gw := withACM(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}, acmawsapi.Deps{})
|
||||
gw := withACM(withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}), acmawsapi.Deps{})
|
||||
req := httptest.NewRequest(http.MethodPost, "/", nil)
|
||||
ctx := context.WithValue(req.Context(), ctxService, "acm")
|
||||
req = req.WithContext(ctx)
|
||||
@@ -197,7 +197,7 @@ func TestErrorHandler_ACMResourceInUse_UsesACMWording(t *testing.T) {
|
||||
// own ResourceInUseException must keep its existing wording, unaffected by the
|
||||
// ACM-specific override.
|
||||
func TestErrorHandler_EKSResourceInUse_UsesEKSWording(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
req := httptest.NewRequest(http.MethodPost, "/", nil)
|
||||
ctx := context.WithValue(req.Context(), ctxService, "eks")
|
||||
req = req.WithContext(ctx)
|
||||
@@ -217,7 +217,7 @@ func TestErrorHandler_EKSResourceInUse_UsesEKSWording(t *testing.T) {
|
||||
// case: an error path that supplies no message (the vast majority of call
|
||||
// sites) must keep rendering exactly today's ErrorLookup text, unchanged.
|
||||
func TestErrorHandler_NoMessageSupplied_MatchesErrorLookup(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
|
||||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := context.WithValue(r.Context(), ctxService, "ec2")
|
||||
@@ -239,7 +239,7 @@ func TestErrorHandler_NoMessageSupplied_MatchesErrorLookup(t *testing.T) {
|
||||
// validation code outside EC2: the default must not be EC2's Reserved Instance
|
||||
// or resource-ID wording, whichever envelope the service uses.
|
||||
func TestErrorHandler_NonEC2ValidationDefaults_DropEC2Wording(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
b := dispatch.NewBuilder()
|
||||
require.NoError(t, b.Register(awsapi.NewRegistration(awsapi.Deps{})))
|
||||
require.NoError(t, b.Register(acmawsapi.NewRegistration(acmawsapi.Deps{})))
|
||||
@@ -269,7 +269,7 @@ func TestErrorHandler_NonEC2ValidationDefaults_DropEC2Wording(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestErrorHandler_ELBv2Service(t *testing.T) {
|
||||
gw := withELBv2(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
gw := withELBv2(withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}))
|
||||
|
||||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := context.WithValue(r.Context(), ctxService, "elasticloadbalancing")
|
||||
@@ -291,7 +291,7 @@ func TestErrorHandler_ELBv2Service(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestErrorHandler_EC2Service(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
|
||||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := context.WithValue(r.Context(), ctxService, "ec2")
|
||||
@@ -311,7 +311,7 @@ func TestErrorHandler_EC2Service(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestErrorHandler_IgnoresClientRequestID(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
|
||||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := context.WithValue(r.Context(), ctxService, "ec2")
|
||||
@@ -329,7 +329,7 @@ func TestErrorHandler_IgnoresClientRequestID(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestErrorHandler_ContentTypeXML(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
|
||||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := context.WithValue(r.Context(), ctxService, "ec2")
|
||||
@@ -658,7 +658,7 @@ func TestParseAWSQueryArgs_CapturedClientBodies(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestGetService(t *testing.T) {
|
||||
gw := withTagging(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}, nil)
|
||||
gw := withTagging(withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}), nil)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
@@ -817,7 +817,7 @@ func TestGetService(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRequest_NoServiceContext(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
req := httptest.NewRequest(http.MethodPost, "/", nil)
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
@@ -830,7 +830,7 @@ func TestRequest_NoServiceContext(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRequest_UnsupportedService(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
req := httptest.NewRequest(http.MethodPost, "/", nil)
|
||||
ctx := context.WithValue(req.Context(), ctxService, "s3")
|
||||
req = req.WithContext(ctx)
|
||||
@@ -858,7 +858,7 @@ func TestRequest_MalformedQueryString_EndToEnd(t *testing.T) {
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
// A live NATS connection bypasses the cluster-unavailable gate.
|
||||
gw := &GatewayConfig{DisableLogging: true, NATSConn: connectedNATS(t), IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, NATSConn: connectedNATS(t), IAMService: allowAllIAMService()})
|
||||
if tc.service == "elasticloadbalancing" {
|
||||
withELBv2(gw)
|
||||
}
|
||||
@@ -882,7 +882,7 @@ func TestRequest_MalformedQueryString_EndToEnd(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRequest_EC2MissingAction(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, NATSConn: connectedNATS(t), IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, NATSConn: connectedNATS(t), IAMService: allowAllIAMService()})
|
||||
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(""))
|
||||
ctx := context.WithValue(req.Context(), ctxService, "ec2")
|
||||
ctx = withPrincipalField(ctx, fieldAccountID, "123456789012")
|
||||
@@ -899,7 +899,7 @@ func TestRequest_EC2MissingAction(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRequest_IAMNilService(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: nil, NATSConn: connectedNATS(t)}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: nil, NATSConn: connectedNATS(t)})
|
||||
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader("Action=CreateUser&UserName=test"))
|
||||
ctx := context.WithValue(req.Context(), ctxService, "iam")
|
||||
ctx = withPrincipalField(ctx, fieldAccountID, "123456789012")
|
||||
@@ -937,7 +937,7 @@ func setupEC2Request(body string, accountID string) *http.Request {
|
||||
}
|
||||
|
||||
func TestEC2Request_MissingAction(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
req := setupEC2Request("", "123456789012")
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
@@ -947,7 +947,7 @@ func TestEC2Request_MissingAction(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestEC2Request_UnknownAction(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
req := setupEC2Request("Action=FakeAction", "123456789012")
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
@@ -957,7 +957,7 @@ func TestEC2Request_UnknownAction(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestEC2Request_NilNATSNonLocalAction(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, NATSConn: nil, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, NATSConn: nil, IAMService: allowAllIAMService()})
|
||||
req := setupEC2Request("Action=DescribeInstances", "123456789012")
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
@@ -967,13 +967,13 @@ func TestEC2Request_NilNATSNonLocalAction(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestEC2Request_NilNATSLocalAction(t *testing.T) {
|
||||
gw := &GatewayConfig{
|
||||
gw := withVerifier(&GatewayConfig{
|
||||
DisableLogging: true,
|
||||
NATSConn: nil,
|
||||
Region: "us-east-1",
|
||||
AZ: "us-east-1a",
|
||||
IAMService: allowAllIAMService(),
|
||||
}
|
||||
})
|
||||
req := setupEC2Request("Action=DescribeRegions", "123456789012")
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
@@ -989,7 +989,7 @@ func TestEC2Request_NilNATSLocalAction(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestEC2Request_MissingAccountID(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, NATSConn: nil, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, NATSConn: nil, IAMService: allowAllIAMService()})
|
||||
// Use a local action so we don't fail on nil NATS first
|
||||
req := setupEC2Request("Action=DescribeRegions", "")
|
||||
w := httptest.NewRecorder()
|
||||
@@ -1002,7 +1002,7 @@ func TestEC2Request_MissingAccountID(t *testing.T) {
|
||||
|
||||
// Static attributes are served without NATS; only default-vpc needs a lookup.
|
||||
func TestEC2Request_DescribeAccountAttributes(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, NATSConn: nil, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, NATSConn: nil, IAMService: allowAllIAMService()})
|
||||
req := setupEC2Request("Action=DescribeAccountAttributes&AttributeName.1=max-instances", "123456789012")
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
@@ -1034,7 +1034,7 @@ func TestEC2Request_DescribeAccountAttributes_DefaultVPC(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
|
||||
gw := &GatewayConfig{DisableLogging: true, NATSConn: nc, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, NATSConn: nc, IAMService: allowAllIAMService()})
|
||||
req := setupEC2Request("Action=DescribeAccountAttributes&AttributeName.1=default-vpc", "123456789012")
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
@@ -1044,13 +1044,13 @@ func TestEC2Request_DescribeAccountAttributes_DefaultVPC(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestEC2Request_DescribeAvailabilityZones(t *testing.T) {
|
||||
gw := &GatewayConfig{
|
||||
gw := withVerifier(&GatewayConfig{
|
||||
DisableLogging: true,
|
||||
NATSConn: nil,
|
||||
Region: "us-east-1",
|
||||
AZ: "us-east-1a",
|
||||
IAMService: allowAllIAMService(),
|
||||
}
|
||||
})
|
||||
req := setupEC2Request("Action=DescribeAvailabilityZones", "123456789012")
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
@@ -1085,19 +1085,19 @@ func TestCheckPolicy_FailsClosed(t *testing.T) {
|
||||
}{
|
||||
{
|
||||
name: "nil IAM service",
|
||||
gw: &GatewayConfig{DisableLogging: true, IAMService: nil},
|
||||
gw: withVerifier(&GatewayConfig{DisableLogging: true, IAMService: nil}),
|
||||
req: authed("alice"),
|
||||
wantErr: awserrors.ErrorInternalError,
|
||||
},
|
||||
{
|
||||
name: "no auth context",
|
||||
gw: &GatewayConfig{DisableLogging: true, IAMService: &mockIAMService{}},
|
||||
gw: withVerifier(&GatewayConfig{DisableLogging: true, IAMService: &mockIAMService{}}),
|
||||
req: httptest.NewRequest(http.MethodPost, "/", nil),
|
||||
wantErr: awserrors.ErrorAccessDenied,
|
||||
},
|
||||
{
|
||||
name: "empty identity",
|
||||
gw: &GatewayConfig{DisableLogging: true, IAMService: &mockIAMService{}},
|
||||
gw: withVerifier(&GatewayConfig{DisableLogging: true, IAMService: &mockIAMService{}}),
|
||||
req: authed(""),
|
||||
wantErr: awserrors.ErrorAccessDenied,
|
||||
},
|
||||
@@ -1120,7 +1120,7 @@ func TestCheckPolicyResources_DenialIsOpaque(t *testing.T) {
|
||||
return nil, nil
|
||||
},
|
||||
}
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: mock}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: mock})
|
||||
req := httptest.NewRequest(http.MethodPost, "/", nil)
|
||||
ctx := withPrincipalField(req.Context(), fieldIdentity, "alice")
|
||||
ctx = withPrincipalField(ctx, fieldAccountID, "123456789012")
|
||||
@@ -1143,7 +1143,7 @@ func TestCheckPolicyResources_DenialIsOpaque(t *testing.T) {
|
||||
// call site: an unauthenticated RunInstances carrying an instance profile is
|
||||
// denied rather than passing the iam:PassRole check.
|
||||
func TestCheckPolicy_PassRoleFailsClosed(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
req := httptest.NewRequest(http.MethodPost, "/", nil)
|
||||
|
||||
err := gw.checkPolicyResources(req, "iam", "PassRole",
|
||||
@@ -1153,7 +1153,7 @@ func TestCheckPolicy_PassRoleFailsClosed(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCheckPolicy_RootUserGlobalAccount(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: &mockIAMService{}}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: &mockIAMService{}})
|
||||
req := httptest.NewRequest(http.MethodPost, "/", nil)
|
||||
ctx := withPrincipalField(req.Context(), fieldIdentity, "root")
|
||||
ctx = withPrincipalField(ctx, fieldAccountID, "000000000000") // GlobalAccountID
|
||||
@@ -1168,7 +1168,7 @@ func TestCheckPolicy_RootUserGlobalAccount(t *testing.T) {
|
||||
// fires BEFORE the identity-string root short-circuit. A session whose
|
||||
// SessionName is "root" must not inherit root privileges.
|
||||
func TestCheckPolicy_AssumedRoleSessionNamedRoot(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: &mockIAMService{}}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: &mockIAMService{}})
|
||||
req := httptest.NewRequest(http.MethodPost, "/", nil)
|
||||
ctx := withPrincipalField(req.Context(), fieldIdentity, "root")
|
||||
ctx = withPrincipalField(ctx, fieldAccountID, "000000000000")
|
||||
@@ -1194,7 +1194,7 @@ func TestCheckPolicy_NonRootAllowPolicy(t *testing.T) {
|
||||
}, nil
|
||||
},
|
||||
}
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: mock}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: mock})
|
||||
req := httptest.NewRequest(http.MethodPost, "/", nil)
|
||||
ctx := withPrincipalField(req.Context(), fieldIdentity, "alice")
|
||||
ctx = withPrincipalField(ctx, fieldAccountID, "123456789012")
|
||||
@@ -1218,7 +1218,7 @@ func TestCheckPolicy_NonRootDenyPolicy(t *testing.T) {
|
||||
}, nil
|
||||
},
|
||||
}
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: mock}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: mock})
|
||||
req := httptest.NewRequest(http.MethodPost, "/", nil)
|
||||
ctx := withPrincipalField(req.Context(), fieldIdentity, "alice")
|
||||
ctx = withPrincipalField(ctx, fieldAccountID, "123456789012")
|
||||
@@ -1236,7 +1236,7 @@ func TestCheckPolicy_NonRootNoPolicies(t *testing.T) {
|
||||
return nil, nil
|
||||
},
|
||||
}
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: mock}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: mock})
|
||||
req := httptest.NewRequest(http.MethodPost, "/", nil)
|
||||
ctx := withPrincipalField(req.Context(), fieldIdentity, "alice")
|
||||
ctx = withPrincipalField(ctx, fieldAccountID, "123456789012")
|
||||
@@ -1254,7 +1254,7 @@ func TestCheckPolicy_GetUserPoliciesError(t *testing.T) {
|
||||
return nil, errors.New("db connection failed")
|
||||
},
|
||||
}
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: mock}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: mock})
|
||||
req := httptest.NewRequest(http.MethodPost, "/", nil)
|
||||
ctx := withPrincipalField(req.Context(), fieldIdentity, "alice")
|
||||
ctx = withPrincipalField(ctx, fieldAccountID, "123456789012")
|
||||
@@ -1267,7 +1267,7 @@ func TestCheckPolicy_GetUserPoliciesError(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCheckPolicy_MissingAccountID(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: &mockIAMService{}}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: &mockIAMService{}})
|
||||
req := httptest.NewRequest(http.MethodPost, "/", nil)
|
||||
ctx := withPrincipalField(req.Context(), fieldIdentity, "alice")
|
||||
// No account ID
|
||||
@@ -1296,7 +1296,7 @@ func TestCheckPolicy_NATSTransientRetriesAllAttempts(t *testing.T) {
|
||||
return nil, fmt.Errorf("get user: %w", nats.ErrNoResponders)
|
||||
},
|
||||
}
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: mock}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: mock})
|
||||
req := httptest.NewRequest(http.MethodPost, "/", nil)
|
||||
ctx := withPrincipalField(req.Context(), fieldIdentity, "alice")
|
||||
ctx = withPrincipalField(ctx, fieldAccountID, "123456789012")
|
||||
@@ -1328,7 +1328,7 @@ func TestCheckPolicy_NATSTransientRetriesThenSucceeds(t *testing.T) {
|
||||
}, nil
|
||||
},
|
||||
}
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: mock}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: mock})
|
||||
req := httptest.NewRequest(http.MethodPost, "/", nil)
|
||||
ctx := withPrincipalField(req.Context(), fieldIdentity, "alice")
|
||||
ctx = withPrincipalField(ctx, fieldAccountID, "123456789012")
|
||||
@@ -1346,7 +1346,7 @@ func TestCheckPolicy_NonTransientErrorStillFails(t *testing.T) {
|
||||
return nil, errors.New("database corruption")
|
||||
},
|
||||
}
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: mock}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: mock})
|
||||
req := httptest.NewRequest(http.MethodPost, "/", nil)
|
||||
ctx := withPrincipalField(req.Context(), fieldIdentity, "alice")
|
||||
ctx = withPrincipalField(ctx, fieldAccountID, "123456789012")
|
||||
@@ -1390,7 +1390,7 @@ func TestImportKeyPair_Base64PaddingWorkaround(t *testing.T) {
|
||||
// --- Throttle middleware integration tests ---
|
||||
|
||||
func TestThrottleKeyFuncs_ExtractsAccountAndAction(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
keyFuncs := gw.throttleKeyFuncs()
|
||||
require.Len(t, keyFuncs, 2)
|
||||
|
||||
@@ -1409,7 +1409,7 @@ func TestThrottleKeyFuncs_ExtractsAccountAndAction(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestThrottleKeyFuncs_UnknownAction(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
keyFuncs := gw.throttleKeyFuncs()
|
||||
|
||||
// No ctxAction in context — should return "unknown".
|
||||
@@ -1423,7 +1423,7 @@ func TestThrottleKeyFuncs_UnknownAction(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestThrottleKeyFuncs_MissingAccountID(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
keyFuncs := gw.throttleKeyFuncs()
|
||||
|
||||
// No ctxAccountID in context — should return an error.
|
||||
@@ -1435,7 +1435,7 @@ func TestThrottleKeyFuncs_MissingAccountID(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestWriteThrottleError_EC2(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/", nil)
|
||||
ctx := context.WithValue(req.Context(), ctxService, "ec2")
|
||||
@@ -1453,7 +1453,7 @@ func TestWriteThrottleError_EC2(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestWriteThrottleError_IAM(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/", nil)
|
||||
ctx := context.WithValue(req.Context(), ctxService, "iam")
|
||||
@@ -1533,11 +1533,11 @@ func TestThrottleMiddleware_Integration(t *testing.T) {
|
||||
throttler := ratelimit.New(cfg)
|
||||
defer throttler.Stop()
|
||||
|
||||
gw := &GatewayConfig{
|
||||
gw := withVerifier(&GatewayConfig{
|
||||
DisableLogging: true,
|
||||
Throttler: throttler,
|
||||
IAMService: allowAllIAMService(),
|
||||
}
|
||||
})
|
||||
|
||||
inner := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
@@ -1574,7 +1574,7 @@ func TestThrottleMiddleware_Integration(t *testing.T) {
|
||||
|
||||
func TestThrottleMiddleware_DisabledConfig(t *testing.T) {
|
||||
// When Throttler is nil, SetupRoutes skips middleware — no panic.
|
||||
gw := &GatewayConfig{DisableLogging: true, Throttler: nil, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, Throttler: nil, IAMService: allowAllIAMService()})
|
||||
handler := gw.SetupRoutes()
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader("Action=DescribeInstances"))
|
||||
@@ -1598,7 +1598,7 @@ func TestThrottleMiddleware_PerActionIsolation(t *testing.T) {
|
||||
throttler := ratelimit.New(cfg)
|
||||
defer throttler.Stop()
|
||||
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
|
||||
inner := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
@@ -1629,7 +1629,7 @@ func TestThrottleMiddleware_PerActionIsolation(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRequest_ClusterUnavailableNilConn_EC2(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/", nil)
|
||||
ctx := context.WithValue(req.Context(), ctxService, "ec2")
|
||||
@@ -1649,7 +1649,7 @@ func TestRequest_ClusterUnavailableNilConn_EC2(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRequest_ClusterUnavailableNilConn_IAM(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/", nil)
|
||||
ctx := context.WithValue(req.Context(), ctxService, "iam")
|
||||
@@ -1672,7 +1672,7 @@ func TestRequest_ClusterUnavailableClosedConn(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
nc.Close()
|
||||
|
||||
gw := &GatewayConfig{DisableLogging: true, NATSConn: nc, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, NATSConn: nc, IAMService: allowAllIAMService()})
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/", nil)
|
||||
ctx := context.WithValue(req.Context(), ctxService, "ec2")
|
||||
|
||||
@@ -55,7 +55,7 @@ func iamScopedGateway(resources map[string]string, statements ...iamdomain.State
|
||||
},
|
||||
arns: resources,
|
||||
}
|
||||
return &GatewayConfig{DisableLogging: true, IAMService: svc}, svc
|
||||
return withVerifier(&GatewayConfig{DisableLogging: true, IAMService: svc}), svc
|
||||
}
|
||||
|
||||
func dispatchIAM(t *testing.T, gw *GatewayConfig, body string) error {
|
||||
|
||||
@@ -3,6 +3,7 @@ package gateway
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/roles/awsgw/credverify"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
@@ -191,6 +192,7 @@ func setupIAMRequestHandler(svc iamdomain.IAMService) http.Handler {
|
||||
gw := &GatewayConfig{
|
||||
DisableLogging: true,
|
||||
IAMService: svc,
|
||||
Credentials: credverify.New(svc, nil),
|
||||
}
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
@@ -376,6 +378,7 @@ func TestIAMRequest_NilService(t *testing.T) {
|
||||
gw := &GatewayConfig{
|
||||
DisableLogging: true,
|
||||
IAMService: nil,
|
||||
Credentials: credverify.New(nil, nil),
|
||||
}
|
||||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
@@ -423,6 +426,7 @@ func TestIAMRequest_MissingAccountID(t *testing.T) {
|
||||
gw := &GatewayConfig{
|
||||
DisableLogging: true,
|
||||
IAMService: &flexMockIAMService{},
|
||||
Credentials: credverify.New(&flexMockIAMService{}, nil),
|
||||
}
|
||||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := context.WithValue(r.Context(), ctxService, "iam")
|
||||
|
||||
@@ -47,7 +47,7 @@ func newDiscoveryEnv(t *testing.T) *discoveryEnv {
|
||||
js := testutil.NewJetStream(t, nc)
|
||||
kv, err := js.CreateOrUpdateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: "eks-account-" + testDiscAccount})
|
||||
require.NoError(t, err)
|
||||
gw := &GatewayConfig{DisableLogging: true, NATSConn: nc, IAMService: allowAllIAMService(), OIDC: ekscluster.NewOIDCPublisher(js)}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, NATSConn: nc, IAMService: allowAllIAMService(), OIDC: ekscluster.NewOIDCPublisher(js)})
|
||||
return &discoveryEnv{nc: nc, kv: kv, h: gw.SetupRoutes()}
|
||||
}
|
||||
|
||||
@@ -149,7 +149,7 @@ func TestCurrentBehaviour_OIDCDiscoveryKVUnavailable(t *testing.T) {
|
||||
}
|
||||
})
|
||||
t.Run("no_connection", func(t *testing.T) {
|
||||
h := (&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}).SetupRoutes()
|
||||
h := (withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})).SetupRoutes()
|
||||
for _, p := range []string{cbDiscPath, cbKeysPath} {
|
||||
rec := getOIDC(t, h, p)
|
||||
assert.Equal(t, http.StatusInternalServerError, rec.Code, p)
|
||||
@@ -206,7 +206,7 @@ func TestCurrentBehaviour_WebIdentityJWKSMissingWireResponse(t *testing.T) {
|
||||
signed, err := tok.SignedString(priv)
|
||||
require.NoError(t, err)
|
||||
|
||||
gw := &GatewayConfig{DisableLogging: true, NATSConn: nc, IAMService: iamSvc, STSService: stsSvc}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, NATSConn: nc, IAMService: iamSvc, STSService: stsSvc})
|
||||
form := url.Values{
|
||||
"Action": {"AssumeRoleWithWebIdentity"}, "Version": {"2011-06-15"},
|
||||
"RoleArn": {aws.StringValue(role.Role.Arn)}, "RoleSessionName": {"s1"}, "WebIdentityToken": {signed},
|
||||
|
||||
@@ -38,7 +38,7 @@ func seedDiscoveryCluster(t *testing.T) http.Handler {
|
||||
t.Fatalf("put jwks: %v", err)
|
||||
}
|
||||
|
||||
gw := &GatewayConfig{DisableLogging: true, NATSConn: nc, IAMService: allowAllIAMService(), OIDC: ekscluster.NewOIDCPublisher(js)}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, NATSConn: nc, IAMService: allowAllIAMService(), OIDC: ekscluster.NewOIDCPublisher(js)})
|
||||
return gw.SetupRoutes()
|
||||
}
|
||||
|
||||
|
||||
@@ -48,7 +48,7 @@ func operatorPipelineApp(t *testing.T, accountID string, docs []iamdomain.Policy
|
||||
require.NoError(t, err)
|
||||
_, nc := testutil.StartTestNATS(t)
|
||||
|
||||
gw := &GatewayConfig{
|
||||
gw := withVerifier(&GatewayConfig{
|
||||
DisableLogging: true,
|
||||
Region: testRegion,
|
||||
NATSConn: nc,
|
||||
@@ -69,7 +69,7 @@ func operatorPipelineApp(t *testing.T, accountID string, docs []iamdomain.Policy
|
||||
},
|
||||
getUserPoliciesFn: func(_, _ string) ([]iamdomain.PolicyDocument, error) { return docs, nil },
|
||||
},
|
||||
}
|
||||
})
|
||||
gw.IAMService = operatorIAM{gw.IAMService.(*policyMockIAMService)}
|
||||
if mutate != nil {
|
||||
mutate(gw)
|
||||
|
||||
@@ -25,7 +25,7 @@ func passRoleTo(effect, operator, service string) iamdomain.Statement {
|
||||
func passRoleToGateway(t *testing.T, statements ...iamdomain.Statement) *GatewayConfig {
|
||||
t.Helper()
|
||||
docs := []iamdomain.PolicyDocument{{Version: "2012-10-17", Statement: statements}}
|
||||
return &GatewayConfig{
|
||||
return withVerifier(&GatewayConfig{
|
||||
DisableLogging: true,
|
||||
Region: authzRegion,
|
||||
NATSConn: startTestNATS(t),
|
||||
@@ -34,7 +34,7 @@ func passRoleToGateway(t *testing.T, statements ...iamdomain.Statement) *Gateway
|
||||
getUserPoliciesFn: func(_, _ string) ([]iamdomain.PolicyDocument, error) { return docs, nil },
|
||||
},
|
||||
},
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func associateProfile(t *testing.T, gw *GatewayConfig) error {
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
package gateway
|
||||
|
||||
// AKID prefixes. Prefix-first dispatch prevents a misfiled record from being
|
||||
// resolved by the wrong lookup path.
|
||||
const (
|
||||
longLivedAKIDPrefix = "AKIA"
|
||||
sessionAKIDPrefix = "ASIA"
|
||||
)
|
||||
|
||||
// principalContext is the identity envelope set on the request context after
|
||||
// SigV4 verification succeeds.
|
||||
type principalContext struct {
|
||||
identity string
|
||||
accountID string
|
||||
principalType string
|
||||
assumedRoleARN string
|
||||
assumedRoleID string
|
||||
underlyingRoleARN string
|
||||
// userID is aws:userid, resolved where the principal is so the value cannot
|
||||
// differ between two policy checks in the same request.
|
||||
userID string
|
||||
// userARN is an IAM user's stored ARN, path included, from the same read as
|
||||
// userID. Empty for every other principal type.
|
||||
userARN string
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package gateway
|
||||
|
||||
import (
|
||||
authlimit "github.com/mulgadc/spinifex/spinifex/ingress/aws/ratelimit"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/roles/awsgw/credverify"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
@@ -51,6 +52,7 @@ func setupTestAppWithRateLimiter(t *testing.T, accessKey, secretKey string, rl *
|
||||
DisableLogging: true,
|
||||
Region: testRegion,
|
||||
IAMService: mockSvc,
|
||||
Credentials: credverify.New(mockSvc, nil),
|
||||
RateLimiter: rl,
|
||||
NATSConn: nc,
|
||||
}
|
||||
|
||||
@@ -31,7 +31,7 @@ func rdsPipelineApp(t *testing.T, docs []iamdomain.PolicyDocument) (*GatewayConf
|
||||
require.NoError(t, err)
|
||||
_, nc := testutil.StartTestNATS(t)
|
||||
|
||||
gw := &GatewayConfig{
|
||||
gw := withVerifier(&GatewayConfig{
|
||||
DisableLogging: true,
|
||||
Region: testRegion,
|
||||
NATSConn: nc,
|
||||
@@ -51,7 +51,7 @@ func rdsPipelineApp(t *testing.T, docs []iamdomain.PolicyDocument) (*GatewayConf
|
||||
},
|
||||
getUserPoliciesFn: func(_, _ string) ([]iamdomain.PolicyDocument, error) { return docs, nil },
|
||||
},
|
||||
}
|
||||
})
|
||||
withRDS(gw)
|
||||
|
||||
r := chi.NewRouter()
|
||||
@@ -214,7 +214,7 @@ func TestRDSPipeline_Authz_UnrelatedGrantDeniedUnknownActionNot(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRDSPipeline_NilNATS_503ServiceUnavailable(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
withRDS(gw)
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
@@ -75,25 +75,25 @@ func newRDSGatewayWithResolver(
|
||||
}
|
||||
// Both branches are wired: an assumed-role principal resolves by role, and a
|
||||
// test that only stubbed users would miss it entirely.
|
||||
return &GatewayConfig{
|
||||
return withVerifier(&GatewayConfig{
|
||||
DisableLogging: true,
|
||||
Region: "ap-southeast-2",
|
||||
IAMService: &policyMockIAMService{
|
||||
getUserPoliciesFn: counted,
|
||||
getRolePoliciesFn: counted,
|
||||
},
|
||||
}, probe
|
||||
}), probe
|
||||
}
|
||||
|
||||
func TestRDSRequest_MissingAction(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
err := gw.RDS_Request(httptest.NewRecorder(), setupRDSRequest(""))
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorMissingAction, err.Error())
|
||||
}
|
||||
|
||||
func TestRDSRequest_UnknownAction(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
err := gw.RDS_Request(httptest.NewRecorder(), setupRDSRequest("Action=FakeAction"))
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorInvalidAction, err.Error())
|
||||
@@ -101,7 +101,7 @@ func TestRDSRequest_UnknownAction(t *testing.T) {
|
||||
|
||||
// A malformed percent-encoding is a client error, not an internal one.
|
||||
func TestRDSRequest_MalformedQueryString(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
err := gw.RDS_Request(httptest.NewRecorder(), setupRDSRequest("Action=%zz"))
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorMalformedQueryString, err.Error())
|
||||
@@ -109,7 +109,7 @@ func TestRDSRequest_MalformedQueryString(t *testing.T) {
|
||||
|
||||
func TestRDSRequest_MalformedScalarReturnsHTTP400(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
gw := &GatewayConfig{DisableLogging: true, NATSConn: nc, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, NATSConn: nc, IAMService: allowAllIAMService()})
|
||||
for _, body := range []string{
|
||||
"Action=CreateDBInstance&AllocatedStorage=abc",
|
||||
"Action=CreateDBInstance&MultiAZ=yes",
|
||||
@@ -131,7 +131,7 @@ func TestRDSRequest_MalformedScalarReturnsHTTP400(t *testing.T) {
|
||||
// A known action still needs a NATS connection to reach the control plane, so
|
||||
// the disconnected case must fail rather than answer from the gateway alone.
|
||||
func TestRDSRequest_NoNATSConn(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
err := gw.RDS_Request(httptest.NewRecorder(), setupRDSRequest("Action=DescribeDBInstances"))
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
|
||||
@@ -341,7 +341,7 @@ func TestRDSRequest_NoIdentityDenied(t *testing.T) {
|
||||
// RDS errors must use the IAM-style <ErrorResponse> envelope: the aws-sdk-go
|
||||
// query unmarshaler rejects the EC2 <Response><Errors> shape for this service.
|
||||
func TestRDSErrorHandler_UsesIAMEnvelope(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
w := httptest.NewRecorder()
|
||||
gw.ErrorHandler(w, setupRDSRequest("Action=CreateDBInstance"), errNotImplementedForTest)
|
||||
|
||||
@@ -351,7 +351,7 @@ func TestRDSErrorHandler_UsesIAMEnvelope(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRDSErrorHandler_UsesRDSUnsupportedActionWording(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
w := httptest.NewRecorder()
|
||||
gw.ErrorHandler(w, setupRDSRequest("Action=CreateDBCluster"),
|
||||
errors.New(awserrors.ErrorOperationNotSupported))
|
||||
@@ -380,7 +380,7 @@ func TestRDSRequest_CreateDBInstance_EmptyMasterUserPasswordReturns400(t *testin
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
|
||||
gw := &GatewayConfig{DisableLogging: true, NATSConn: nc, IAMService: allowAllIAMService()}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, NATSConn: nc, IAMService: allowAllIAMService()})
|
||||
req := setupRDSRequest("Action=CreateDBInstance&DBInstanceIdentifier=orders-db&Engine=postgres&" +
|
||||
"DBInstanceClass=db.t3.medium&AllocatedStorage=20&MasterUsername=appuser&DBName=orders")
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ package gateway
|
||||
import (
|
||||
"fmt"
|
||||
authlimit "github.com/mulgadc/spinifex/spinifex/ingress/aws/ratelimit"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/roles/awsgw/credverify"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
@@ -31,6 +32,7 @@ func auditRouter(t *testing.T, keys map[string]*iamdomain.AccessKey) (http.Handl
|
||||
DisableLogging: true,
|
||||
Region: testRegion,
|
||||
IAMService: &mockIAMService{masterKey: testMasterKey, accessKeys: keys},
|
||||
Credentials: credverify.New(&mockIAMService{masterKey: testMasterKey, accessKeys: keys}, nil),
|
||||
RateLimiter: rl,
|
||||
}
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ import (
|
||||
// AssumeRoleWithWebIdentity is routed to the STS dispatcher before SigV4, not rejected.
|
||||
func TestAnonymousSTS_WebIdentityNoAuthReachesHandler(t *testing.T) {
|
||||
var seenToken string
|
||||
gw := &GatewayConfig{
|
||||
gw := withVerifier(&GatewayConfig{
|
||||
DisableLogging: true,
|
||||
IAMService: &flexMockIAMService{},
|
||||
STSService: &flexMockSTSService{
|
||||
@@ -32,7 +32,7 @@ func TestAnonymousSTS_WebIdentityNoAuthReachesHandler(t *testing.T) {
|
||||
}, nil
|
||||
},
|
||||
},
|
||||
}
|
||||
})
|
||||
handler := gw.SetupRoutes()
|
||||
|
||||
body := "Action=AssumeRoleWithWebIdentity&Version=2011-06-15" +
|
||||
|
||||
@@ -94,11 +94,11 @@ func setupSTSRequestHandler(p stsRequestParams) http.Handler {
|
||||
if p.iamSvc == nil {
|
||||
p.iamSvc = &flexMockIAMService{}
|
||||
}
|
||||
gw := &GatewayConfig{
|
||||
gw := withVerifier(&GatewayConfig{
|
||||
DisableLogging: true,
|
||||
STSService: p.stsSvc,
|
||||
IAMService: p.iamSvc,
|
||||
}
|
||||
})
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
ctx = context.WithValue(ctx, ctxService, "sts")
|
||||
@@ -335,7 +335,7 @@ func TestSTSRequest_AssumeRole_MissingRoleArn(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSTSRequest_NilService_InternalError(t *testing.T) {
|
||||
gw := &GatewayConfig{DisableLogging: true, STSService: nil, IAMService: &flexMockIAMService{}}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, STSService: nil, IAMService: &flexMockIAMService{}})
|
||||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := context.WithValue(r.Context(), ctxService, "sts")
|
||||
ctx = withPrincipalField(ctx, fieldAccountID, awsidentifiers.GlobalAccountID)
|
||||
@@ -367,7 +367,7 @@ func TestSTSRequest_AssumeRole_NilIAMService_InternalError(t *testing.T) {
|
||||
return nil, nil
|
||||
},
|
||||
}
|
||||
gw := &GatewayConfig{DisableLogging: true, STSService: svc, IAMService: nil}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, STSService: svc, IAMService: nil})
|
||||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := context.WithValue(r.Context(), ctxService, "sts")
|
||||
ctx = withPrincipalField(ctx, fieldAccountID, awsidentifiers.GlobalAccountID)
|
||||
@@ -451,7 +451,7 @@ func TestSTSRequest_GetCallerIdentity_PathedUserThroughMiddleware(t *testing.T)
|
||||
const pathed = "arn:aws:iam::123456789012:user/eng/alice"
|
||||
encryptedSecret, err := iamdomain.EncryptSecret(testSecretKey, testMasterKey)
|
||||
require.NoError(t, err)
|
||||
gw := &GatewayConfig{
|
||||
gw := withVerifier(&GatewayConfig{
|
||||
DisableLogging: true,
|
||||
Region: testRegion,
|
||||
STSService: &flexMockSTSService{},
|
||||
@@ -467,7 +467,7 @@ func TestSTSRequest_GetCallerIdentity_PathedUserThroughMiddleware(t *testing.T)
|
||||
}}, nil
|
||||
},
|
||||
},
|
||||
}
|
||||
})
|
||||
handler := gw.SigV4AuthMiddleware()(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if err := gw.STS_Request(w, r); err != nil {
|
||||
gw.ErrorHandler(w, r, err)
|
||||
|
||||
@@ -55,7 +55,7 @@ func taggingPipelineApp(t *testing.T, docs []iamdomain.PolicyDocument, agg *tagg
|
||||
require.NoError(t, err)
|
||||
_, nc := testutil.StartTestNATS(t)
|
||||
|
||||
gw := &GatewayConfig{
|
||||
gw := withVerifier(&GatewayConfig{
|
||||
DisableLogging: true,
|
||||
Region: testRegion,
|
||||
NATSConn: nc,
|
||||
@@ -74,7 +74,7 @@ func taggingPipelineApp(t *testing.T, docs []iamdomain.PolicyDocument, agg *tagg
|
||||
},
|
||||
getUserPoliciesFn: func(_, _ string) ([]iamdomain.PolicyDocument, error) { return docs, nil },
|
||||
},
|
||||
}
|
||||
})
|
||||
withTagging(gw, agg)
|
||||
|
||||
r := chi.NewRouter()
|
||||
@@ -309,7 +309,7 @@ func TestTaggingPipeline_OversizedBody_413OverHTTP(t *testing.T) {
|
||||
|
||||
// A nil NATS connection fails at the request-level gate, before dispatch.
|
||||
func TestRequest_ClusterUnavailableNilConn_Tagging(t *testing.T) {
|
||||
gw := withTagging(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}, tagging.NewAggregator())
|
||||
gw := withTagging(withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}), tagging.NewAggregator())
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/", nil)
|
||||
req.Header.Set("X-Amz-Target", taggingTargetPrefix+"GetResources")
|
||||
|
||||
@@ -50,7 +50,7 @@ func newTaggingCBGateway(t *testing.T) (*GatewayConfig, *ec2tags.TagsServiceImpl
|
||||
})
|
||||
})
|
||||
|
||||
gw := &GatewayConfig{DisableLogging: true, Region: authzRegion, IAMService: allowAllIAMService(), NATSConn: nc}
|
||||
gw := withVerifier(&GatewayConfig{DisableLogging: true, Region: authzRegion, IAMService: allowAllIAMService(), NATSConn: nc})
|
||||
return gw, svc
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
package gateway
|
||||
|
||||
import "github.com/mulgadc/spinifex/spinifex/runtime/roles/awsgw/credverify"
|
||||
|
||||
// withVerifier authenticates through the same IAM and STS instances the config
|
||||
// holds, so a test that changes one of them changes authentication too.
|
||||
func withVerifier(gw *GatewayConfig) *GatewayConfig {
|
||||
gw.Credentials = credverify.New(gw.IAMService, gw.STSService)
|
||||
return gw
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
// Package credential holds the vocabulary a request authenticator shares with the
|
||||
// adapter that verifies credentials against the identity stores. It imports no
|
||||
// domain, so the adapter can live beside IAM and STS without importing the gateway.
|
||||
package credential
|
||||
|
||||
// Request is what the gateway knows about a credential before it is verified.
|
||||
type Request struct {
|
||||
AccessKeyID string
|
||||
// SecurityToken is the X-Amz-Security-Token header, empty when absent.
|
||||
SecurityToken string
|
||||
// SourceIP is the client address for log lines only.
|
||||
SourceIP string
|
||||
}
|
||||
|
||||
// Failure is a rejected credential. The verifier logs the cause; the gateway owns
|
||||
// the response and the rate limiter, so the failure only names what to record.
|
||||
type Failure struct {
|
||||
// Code is the AWS error code to answer with.
|
||||
Code string
|
||||
// SignatureMismatch marks the verifySig callback rejecting the signature.
|
||||
// The gateway holds the signature, so it logs and fingerprints that case.
|
||||
SignatureMismatch bool
|
||||
// FingerprintKind and FingerprintParts identify the rate-limit entry to
|
||||
// record. An empty kind records nothing.
|
||||
FingerprintKind string
|
||||
FingerprintParts []string
|
||||
}
|
||||
|
||||
// Authenticated is a credential whose signature has verified. AccountID is exposed
|
||||
// so the gateway can run its account-active check before Resolve reads anything else.
|
||||
type Authenticated struct {
|
||||
AccountID string
|
||||
// Handle is the verifier's own state for Resolve; the gateway never reads it.
|
||||
Handle any
|
||||
}
|
||||
|
||||
// Principal is the caller a verified credential resolves to.
|
||||
type Principal struct {
|
||||
Identity string
|
||||
AccountID string
|
||||
PrincipalType string
|
||||
AssumedRoleARN string
|
||||
AssumedRoleID string
|
||||
UnderlyingRoleARN string
|
||||
// UserID is aws:userid; UserARN is an IAM user's stored ARN, empty otherwise.
|
||||
UserID string
|
||||
UserARN string
|
||||
}
|
||||
|
||||
// Verifier authenticates a request's credential. The secret never leaves it.
|
||||
type Verifier interface {
|
||||
// Authenticate looks the credential up and passes its secret to verifySig. The
|
||||
// secret lookup is the only identity-store read before verifySig succeeds.
|
||||
Authenticate(req Request, verifySig func(secret string) error) (Authenticated, *Failure)
|
||||
// Resolve runs the checks that must follow a verified signature and the
|
||||
// gateway's account-active check: session-principal continuity and the
|
||||
// caller's user ID.
|
||||
Resolve(req Request, auth Authenticated) (Principal, *Failure)
|
||||
}
|
||||
@@ -5,10 +5,6 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/providers/ebs"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/compute/nbd"
|
||||
@@ -109,58 +105,4 @@ func ImportImage(ctx context.Context, provider ebsprovider.EBSProvider, opts Imp
|
||||
|
||||
// writeImage is the bulk copy step, indirected so a test can drive the verb
|
||||
// sequence around it without a live NBD export to write to.
|
||||
var writeImage = writeImageToNBD
|
||||
|
||||
// writeImageToNBD copies a raw image onto a published export with qemu-img,
|
||||
// which speaks NBD as a target and is already a runtime dependency. Passing
|
||||
// the export as the target rather than a file is what keeps the bulk data off
|
||||
// NATS, whose request size the provider contract caps far below an image.
|
||||
func writeImageToNBD(ctx context.Context, sourcePath, nbdURI string, progress io.Writer) error {
|
||||
target, err := qemuNBDTarget(nbdURI)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// -n keeps qemu-img from trying to create a target that already exists,
|
||||
// and --target-is-zero lets it skip the source's holes: the volume was
|
||||
// created moments ago and has never been written.
|
||||
cmd := exec.CommandContext(ctx, "qemu-img", "convert",
|
||||
"-f", "raw", "-O", "raw", "-n", "--target-is-zero", "-p",
|
||||
sourcePath, target)
|
||||
if progress != nil {
|
||||
cmd.Stdout = progress
|
||||
}
|
||||
var stderr []byte
|
||||
stderrPipe, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
return fmt.Errorf("qemu-img stderr: %w", err)
|
||||
}
|
||||
if err := cmd.Start(); err != nil {
|
||||
return fmt.Errorf("start qemu-img: %w", err)
|
||||
}
|
||||
stderr, _ = io.ReadAll(stderrPipe)
|
||||
if err := cmd.Wait(); err != nil {
|
||||
return fmt.Errorf("qemu-img convert: %w: %s", err, stderr)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// qemuNBDTarget renders a provider NBD URI in the form qemu-img accepts. The
|
||||
// contract's unix form is not qemu's, so it is rebuilt rather than passed on.
|
||||
func qemuNBDTarget(nbdURI string) (string, error) {
|
||||
serverType, socketPath, host, port, err := nbd.ParseNBDURI(nbdURI)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("parse NBD URI %q: %w", nbdURI, err)
|
||||
}
|
||||
switch serverType {
|
||||
case "unix":
|
||||
if _, err := os.Stat(socketPath); err != nil {
|
||||
return "", fmt.Errorf("NBD socket %s: %w", socketPath, err)
|
||||
}
|
||||
return "nbd+unix:///?socket=" + socketPath, nil
|
||||
case "inet":
|
||||
return "nbd://" + net.JoinHostPort(host, strconv.Itoa(port)), nil
|
||||
default:
|
||||
return "", fmt.Errorf("unsupported NBD server type %q in %q", serverType, nbdURI)
|
||||
}
|
||||
}
|
||||
var writeImage = nbd.WriteRawImage
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/providers/ebs"
|
||||
@@ -185,22 +184,3 @@ func TestImportImage_RejectsIncompleteOpts(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestQemuNBDTarget_RewritesUnixForm locks the translation: the contract's
|
||||
// unix URI is not a form qemu accepts, so passing it through unchanged would
|
||||
// fail at the copy rather than here.
|
||||
func TestQemuNBDTarget_RewritesUnixForm(t *testing.T) {
|
||||
socket := t.TempDir() + "/export.sock"
|
||||
require.NoError(t, os.WriteFile(socket, nil, 0o600))
|
||||
|
||||
got, err := qemuNBDTarget("nbd:unix:" + socket)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "nbd+unix:///?socket="+socket, got)
|
||||
|
||||
got, err = qemuNBDTarget("nbd://10.0.0.5:10809")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "nbd://10.0.0.5:10809", got)
|
||||
|
||||
_, err = qemuNBDTarget("nbd:unix:" + t.TempDir() + "/absent.sock")
|
||||
require.Error(t, err, "a socket that is not there must fail before qemu-img is run")
|
||||
}
|
||||
|
||||
@@ -179,6 +179,7 @@ type CapacityRange struct {
|
||||
type VolumeState string
|
||||
|
||||
const (
|
||||
VolumeStateCreating VolumeState = "creating"
|
||||
VolumeStateAvailable VolumeState = "available"
|
||||
VolumeStateInUse VolumeState = "in-use"
|
||||
)
|
||||
|
||||
@@ -72,8 +72,8 @@ const volumeLeaseRenewTimeout = 12 * time.Second
|
||||
const volumeLeaseValidity = 40 * time.Second
|
||||
|
||||
// volumeLeaseAcquireMargin is the gap left between validity and the server TTL.
|
||||
// It covers scheduling delay and request latency on this side, and the entry
|
||||
// still has to age out before a peer can claim it.
|
||||
// Validity counts from when a write is sent, so request latency cannot use it
|
||||
// up; it covers scheduling delay in fencing the export on this side.
|
||||
const volumeLeaseAcquireMargin = 5 * time.Second
|
||||
|
||||
// deviceStallBound is the longest one stalled I/O lasts under JetStream, from
|
||||
@@ -143,10 +143,22 @@ type volumeLeases struct {
|
||||
// production interval; tests shorten it to reach a deadline quickly.
|
||||
checkEvery time.Duration
|
||||
|
||||
// now overrides time.Now when set, so a test can stand in for an
|
||||
// acknowledgement that arrives long after its write was applied.
|
||||
now func() time.Time
|
||||
|
||||
mu sync.Mutex
|
||||
held map[string]*volumeLease
|
||||
}
|
||||
|
||||
// clock is the time validity is measured on.
|
||||
func (l *volumeLeases) clock() time.Time {
|
||||
if l.now != nil {
|
||||
return l.now()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
// checkInterval is how often a renew loop tests validity.
|
||||
func (l *volumeLeases) checkInterval() time.Duration {
|
||||
if l.checkEvery > 0 {
|
||||
@@ -201,9 +213,9 @@ type volumeLease struct {
|
||||
// lost records that the entry moved out from under this holder, which
|
||||
// means somebody else may now be writing the volume.
|
||||
lost bool
|
||||
// confirmed is when the last conditional update was acknowledged by the
|
||||
// server. An attempt still in flight does not count: only a write the
|
||||
// server accepted proves the entry is still ours.
|
||||
// confirmed is when the last acknowledged write was sent. The server starts
|
||||
// its TTL when it applies the write, never earlier, so counting validity
|
||||
// from the send keeps a slow acknowledgement from stretching it.
|
||||
confirmed time.Time
|
||||
// refs counts opens on this node sharing the lease. The lease is released
|
||||
// when the last one lets go.
|
||||
@@ -226,9 +238,17 @@ func (l *volumeLeases) acquire(ctx context.Context, volumeName string) (*volumeL
|
||||
l.mu.Lock()
|
||||
if lease, ok := l.held[volumeName]; ok {
|
||||
lease.mu.Lock()
|
||||
lease.refs++
|
||||
// A lost or lapsed lease stays held until its fence releases it, and
|
||||
// sharing it then would authorize an open the lease no longer covers.
|
||||
usable := !lease.lost && l.clock().Sub(lease.confirmed) < volumeLeaseValidity
|
||||
if usable {
|
||||
lease.refs++
|
||||
}
|
||||
lease.mu.Unlock()
|
||||
l.mu.Unlock()
|
||||
if !usable {
|
||||
return nil, fmt.Errorf("lease for %s is lost or lapsed and still being fenced: %w", volumeName, ErrLeaseStoreUnavailable)
|
||||
}
|
||||
return lease, nil
|
||||
}
|
||||
l.mu.Unlock()
|
||||
@@ -239,6 +259,7 @@ func (l *volumeLeases) acquire(ctx context.Context, volumeName string) (*volumeL
|
||||
return nil, fmt.Errorf("marshal lease: %w", err)
|
||||
}
|
||||
|
||||
sent := l.clock()
|
||||
revision, err := l.kv.Create(ctx, volumeName, payload)
|
||||
switch {
|
||||
case errors.Is(err, jetstream.ErrKeyExists):
|
||||
@@ -258,7 +279,7 @@ func (l *volumeLeases) acquire(ctx context.Context, volumeName string) (*volumeL
|
||||
volume: volumeName,
|
||||
generation: revision,
|
||||
revision: revision,
|
||||
confirmed: time.Now(),
|
||||
confirmed: sent,
|
||||
done: make(chan struct{}),
|
||||
refs: 1,
|
||||
}
|
||||
@@ -267,13 +288,22 @@ func (l *volumeLeases) acquire(ctx context.Context, volumeName string) (*volumeL
|
||||
// published afterwards. A failure here costs observability, not exclusion.
|
||||
record.Generation = revision
|
||||
if published, perr := json.Marshal(record); perr == nil {
|
||||
publishSent := l.clock()
|
||||
if rev, uerr := l.kv.Update(ctx, volumeName, published, revision); uerr == nil {
|
||||
lease.revision = rev
|
||||
lease.confirmed = publishSent
|
||||
} else {
|
||||
slog.Warn("volume lease: could not publish generation", "volume", volumeName, "err", uerr)
|
||||
}
|
||||
}
|
||||
|
||||
// An acknowledgement that took the whole validity window proves nothing
|
||||
// about now: a peer may already hold the entry. It is left to expire,
|
||||
// since a reclaimed one can still cover exports from this node's last run.
|
||||
if l.clock().Sub(lease.confirmed) >= volumeLeaseValidity {
|
||||
return nil, fmt.Errorf("lease for %s was acknowledged after its validity lapsed: %w", volumeName, ErrLeaseStoreUnavailable)
|
||||
}
|
||||
|
||||
l.mu.Lock()
|
||||
l.held[volumeName] = lease
|
||||
l.mu.Unlock()
|
||||
@@ -393,6 +423,11 @@ func (lease *volumeLease) renewLoop(ctx context.Context) {
|
||||
outcome := make(chan bool, 1)
|
||||
inFlight := false
|
||||
|
||||
// Fires at the validity deadline itself, so a lapse is fenced then rather
|
||||
// than up to a check interval later, which would eat the margin.
|
||||
deadline := time.NewTimer(lease.untilLapse())
|
||||
defer deadline.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
@@ -402,12 +437,19 @@ func (lease *volumeLease) renewLoop(ctx context.Context) {
|
||||
if !stillOurs {
|
||||
return
|
||||
}
|
||||
deadline.Reset(lease.untilLapse())
|
||||
case <-deadline.C:
|
||||
if lease.expiredLocally() {
|
||||
lease.surrender(ctx)
|
||||
return
|
||||
}
|
||||
deadline.Reset(lease.untilLapse())
|
||||
case <-ticker.C:
|
||||
if lease.expiredLocally() {
|
||||
lease.surrender(ctx)
|
||||
return
|
||||
}
|
||||
if inFlight || time.Since(lease.lastConfirmed()) < volumeLeaseRenewInterval {
|
||||
if inFlight || lease.leases.clock().Sub(lease.lastConfirmed()) < volumeLeaseRenewInterval {
|
||||
continue
|
||||
}
|
||||
inFlight = true
|
||||
@@ -422,7 +464,12 @@ func (lease *volumeLease) renewLoop(ctx context.Context) {
|
||||
// volumeLeaseValidity without a confirmed renewal, so the server may be about
|
||||
// to grant the entry to somebody else.
|
||||
func (lease *volumeLease) expiredLocally() bool {
|
||||
return time.Since(lease.lastConfirmed()) >= volumeLeaseValidity
|
||||
return lease.leases.clock().Sub(lease.lastConfirmed()) >= volumeLeaseValidity
|
||||
}
|
||||
|
||||
// untilLapse is how long this holder may still write, and never negative.
|
||||
func (lease *volumeLease) untilLapse() time.Duration {
|
||||
return max(lease.lastConfirmed().Add(volumeLeaseValidity).Sub(lease.leases.clock()), 0)
|
||||
}
|
||||
|
||||
func (lease *volumeLease) lastConfirmed() time.Time {
|
||||
@@ -436,8 +483,14 @@ func (lease *volumeLease) lastConfirmed() time.Time {
|
||||
// belong to another node.
|
||||
func (lease *volumeLease) surrender(ctx context.Context) {
|
||||
lease.mu.Lock()
|
||||
// Already given up by another path, which fenced it; fencing twice would
|
||||
// report a second loss that never happened.
|
||||
if lease.lost {
|
||||
lease.mu.Unlock()
|
||||
return
|
||||
}
|
||||
lease.lost = true
|
||||
since := time.Since(lease.confirmed)
|
||||
since := lease.leases.clock().Sub(lease.confirmed)
|
||||
attempts := lease.failures
|
||||
failingFor := time.Duration(0)
|
||||
if attempts > 0 {
|
||||
@@ -481,12 +534,27 @@ func (lease *volumeLease) renew(ctx context.Context) bool {
|
||||
updateCtx, cancel := context.WithTimeout(ctx, volumeLeaseRenewTimeout)
|
||||
defer cancel()
|
||||
|
||||
sent := lease.leases.clock()
|
||||
renewed, err := lease.leases.kv.Update(updateCtx, lease.key, payload, revision)
|
||||
switch {
|
||||
case err == nil:
|
||||
lease.mu.Lock()
|
||||
// Once validity has lapsed the lease is over, even if the fence has not
|
||||
// run yet; a late success must not revive it. Decided under the same
|
||||
// lock as the update, so nothing sees a lapsed lease become valid.
|
||||
if lease.lost {
|
||||
lease.mu.Unlock()
|
||||
slog.Warn("volume lease: a renewal succeeded after the lease was given up, ignoring it", "volume", lease.volume, "generation", lease.generation)
|
||||
return false
|
||||
}
|
||||
if lease.leases.clock().Sub(lease.confirmed) >= volumeLeaseValidity {
|
||||
lease.mu.Unlock()
|
||||
slog.Warn("volume lease: a renewal succeeded after validity lapsed, giving the volume up", "volume", lease.volume, "generation", lease.generation)
|
||||
lease.surrender(ctx)
|
||||
return false
|
||||
}
|
||||
lease.revision = renewed
|
||||
lease.confirmed = time.Now()
|
||||
lease.confirmed = sent
|
||||
attempts, outage := lease.failures, time.Since(lease.failingSince)
|
||||
lease.failures, lease.failingSince = 0, time.Time{}
|
||||
lease.mu.Unlock()
|
||||
@@ -576,6 +644,10 @@ func (lease *volumeLease) readopt(ctx context.Context, cause error) bool {
|
||||
// that starts.
|
||||
func (lease *volumeLease) loseToPeer(ctx context.Context, cause error, why string) {
|
||||
lease.mu.Lock()
|
||||
if lease.lost {
|
||||
lease.mu.Unlock()
|
||||
return
|
||||
}
|
||||
lease.lost = true
|
||||
lease.mu.Unlock()
|
||||
|
||||
|
||||
@@ -0,0 +1,339 @@
|
||||
package viperblock
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// testClock runs at wall speed from an offset a test can jump forward, which
|
||||
// is how a reply that took tens of seconds is staged without waiting for it.
|
||||
type testClock struct {
|
||||
mu sync.Mutex
|
||||
offset time.Duration
|
||||
}
|
||||
|
||||
func (c *testClock) now() time.Time {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return time.Now().Add(c.offset)
|
||||
}
|
||||
|
||||
func (c *testClock) advance(d time.Duration) {
|
||||
c.mu.Lock()
|
||||
c.offset += d
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
// slowAckKV applies each write on the server, then moves the clock on by the
|
||||
// write's delay before acknowledging it. failUpdates refuses Updates unapplied,
|
||||
// and gate, when set, holds an applied Update's reply until it is closed.
|
||||
type slowAckKV struct {
|
||||
jetstream.KeyValue
|
||||
|
||||
clock *testClock
|
||||
createDelay time.Duration
|
||||
updateDelay time.Duration
|
||||
failUpdates bool
|
||||
gate chan struct{}
|
||||
entered chan struct{}
|
||||
|
||||
mu sync.Mutex
|
||||
applied []time.Time
|
||||
}
|
||||
|
||||
func (k *slowAckKV) record() {
|
||||
k.mu.Lock()
|
||||
k.applied = append(k.applied, k.clock.now())
|
||||
k.mu.Unlock()
|
||||
}
|
||||
|
||||
func (k *slowAckKV) lastApplied(t *testing.T) time.Time {
|
||||
t.Helper()
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
require.NotEmpty(t, k.applied, "no write reached the server")
|
||||
return k.applied[len(k.applied)-1]
|
||||
}
|
||||
|
||||
func (k *slowAckKV) Create(ctx context.Context, key string, value []byte, opts ...jetstream.KVCreateOpt) (uint64, error) {
|
||||
rev, err := k.KeyValue.Create(ctx, key, value, opts...)
|
||||
if err == nil {
|
||||
k.record()
|
||||
k.clock.advance(k.createDelay)
|
||||
}
|
||||
return rev, err
|
||||
}
|
||||
|
||||
func (k *slowAckKV) Update(ctx context.Context, key string, value []byte, last uint64) (uint64, error) {
|
||||
if k.failUpdates {
|
||||
return 0, errors.New("store unwritable")
|
||||
}
|
||||
rev, err := k.KeyValue.Update(ctx, key, value, last)
|
||||
if err != nil {
|
||||
return rev, err
|
||||
}
|
||||
k.record()
|
||||
k.clock.advance(k.updateDelay)
|
||||
if k.gate != nil {
|
||||
close(k.entered)
|
||||
<-k.gate
|
||||
}
|
||||
return rev, nil
|
||||
}
|
||||
|
||||
// newSlowAckLeases is a lease store whose writes go through a slowAckKV.
|
||||
func newSlowAckLeases(t *testing.T, owner string) (*volumeLeases, *slowAckKV) {
|
||||
t.Helper()
|
||||
_, natsURL := setupEmbeddedNATS(t)
|
||||
leases := newTestLeases(t, natsURL, owner)
|
||||
clock := &testClock{}
|
||||
kv := &slowAckKV{KeyValue: leases.kv, clock: clock}
|
||||
leases.kv = kv
|
||||
leases.now = clock.now
|
||||
return leases, kv
|
||||
}
|
||||
|
||||
// stopRenewals ends the loop acquire started, so a test drives renewal itself.
|
||||
func stopRenewals(lease *volumeLease) {
|
||||
lease.stop()
|
||||
<-lease.done
|
||||
}
|
||||
|
||||
// TestVolumeLease_SlowAcquireAckDoesNotExtendValidity: the server's TTL starts
|
||||
// when it applies the create, so validity measured from a late reply outlives
|
||||
// the entry and lets a holder keep writing after a peer has taken the volume.
|
||||
func TestVolumeLease_SlowAcquireAckDoesNotExtendValidity(t *testing.T) {
|
||||
leases, kv := newSlowAckLeases(t, "node-a")
|
||||
kv.createDelay = 30 * time.Second
|
||||
kv.failUpdates = true // the generation publish never lands, so only the create proves the lease
|
||||
|
||||
lease, err := leases.acquire(t.Context(), "vol-slowacquireack")
|
||||
require.NoError(t, err, "a lease acknowledged inside its validity is still usable")
|
||||
stopRenewals(lease)
|
||||
|
||||
assert.False(t, lease.lastConfirmed().After(kv.lastApplied(t)),
|
||||
"validity counted from the reply rather than the send outlasts the server's entry by the reply's delay")
|
||||
}
|
||||
|
||||
// TestVolumeLease_AcquireAckedAfterValidityIsRefused: a reply that took the
|
||||
// whole validity window cannot authorize an open, because the server may
|
||||
// already have expired the entry and granted it to a peer.
|
||||
func TestVolumeLease_AcquireAckedAfterValidityIsRefused(t *testing.T) {
|
||||
leases, kv := newSlowAckLeases(t, "node-a")
|
||||
kv.createDelay = volumeLeaseValidity + time.Second
|
||||
kv.failUpdates = true
|
||||
|
||||
lease, err := leases.acquire(t.Context(), "vol-lateacquireack")
|
||||
require.Error(t, err)
|
||||
assert.Nil(t, lease)
|
||||
assert.ErrorIs(t, err, ErrLeaseStoreUnavailable, "the claim may succeed on a retry, so it must not read as held by a peer")
|
||||
|
||||
leases.mu.Lock()
|
||||
defer leases.mu.Unlock()
|
||||
assert.NotContains(t, leases.held, "vol-lateacquireack", "a refused acquisition must not be shared with a later open")
|
||||
}
|
||||
|
||||
// TestVolumeLease_GenerationPublishRenewsALateAcquisition: the publish is
|
||||
// conditioned on the create's revision, so its success proves the entry is
|
||||
// still ours as of when it was sent, however late the create's reply was.
|
||||
func TestVolumeLease_GenerationPublishRenewsALateAcquisition(t *testing.T) {
|
||||
leases, kv := newSlowAckLeases(t, "node-a")
|
||||
kv.createDelay = volumeLeaseValidity + time.Second
|
||||
|
||||
lease, err := leases.acquire(t.Context(), "vol-latepublished")
|
||||
require.NoError(t, err)
|
||||
stopRenewals(lease)
|
||||
|
||||
assert.False(t, lease.expiredLocally(), "a confirmed publish is a fresh proof of ownership")
|
||||
assert.False(t, lease.lastConfirmed().After(kv.lastApplied(t)),
|
||||
"validity must still start no later than the server applied the publish")
|
||||
}
|
||||
|
||||
// TestVolumeLease_SlowRenewalAckDoesNotExtendValidity is the renewal half of
|
||||
// the slow-reply overlap.
|
||||
func TestVolumeLease_SlowRenewalAckDoesNotExtendValidity(t *testing.T) {
|
||||
leases, kv := newSlowAckLeases(t, "node-a")
|
||||
|
||||
lease, err := leases.acquire(t.Context(), "vol-slowrenewalack")
|
||||
require.NoError(t, err)
|
||||
stopRenewals(lease)
|
||||
|
||||
kv.updateDelay = 10 * time.Second
|
||||
require.True(t, lease.renew(t.Context()))
|
||||
|
||||
assert.False(t, lease.lastConfirmed().After(kv.lastApplied(t)),
|
||||
"validity counted from the reply rather than the send outlasts the server's entry by the reply's delay")
|
||||
}
|
||||
|
||||
// TestVolumeLease_RenewalSucceedingAfterSurrenderDoesNotReviveTheLease: the
|
||||
// export is already being fenced and a peer may hold the volume, so a renewal
|
||||
// that lands afterwards must not make the holder valid again.
|
||||
func TestVolumeLease_RenewalSucceedingAfterSurrenderDoesNotReviveTheLease(t *testing.T) {
|
||||
leases, kv := newSlowAckLeases(t, "node-a")
|
||||
|
||||
lease, err := leases.acquire(t.Context(), "vol-renewalaftersurrender")
|
||||
require.NoError(t, err)
|
||||
stopRenewals(lease)
|
||||
|
||||
kv.gate = make(chan struct{})
|
||||
kv.entered = make(chan struct{})
|
||||
result := make(chan bool, 1)
|
||||
go func() { result <- lease.renew(t.Context()) }()
|
||||
<-kv.entered
|
||||
|
||||
// The renewal is applied but unanswered; meanwhile validity runs out.
|
||||
lease.mu.Lock()
|
||||
lease.confirmed = leases.clock().Add(-volumeLeaseValidity - time.Second)
|
||||
lease.mu.Unlock()
|
||||
lease.surrender(t.Context())
|
||||
close(kv.gate)
|
||||
|
||||
assert.False(t, <-result, "a renewal answered after surrender must not report the lease as ours")
|
||||
assert.True(t, lease.expiredLocally(), "a late renewal revived a lease whose export is being fenced")
|
||||
lease.mu.Lock()
|
||||
defer lease.mu.Unlock()
|
||||
assert.True(t, lease.lost)
|
||||
}
|
||||
|
||||
// TestVolumeLease_LapseIsFencedAtTheDeadlineNotTheNextCheck: the check
|
||||
// interval equals the margin, so waiting for the next tick to notice a lapse
|
||||
// can reach the server's expiry and overlap the next holder.
|
||||
func TestVolumeLease_LapseIsFencedAtTheDeadlineNotTheNextCheck(t *testing.T) {
|
||||
_, natsURL := setupEmbeddedNATS(t)
|
||||
leases := newTestLeases(t, natsURL, "node-a")
|
||||
leases.checkEvery = time.Hour
|
||||
|
||||
lost := make(chan leaseLossKind, 1)
|
||||
leases.onLost = func(_ context.Context, _ string, kind leaseLossKind) { lost <- kind }
|
||||
|
||||
lease, err := leases.acquire(t.Context(), "vol-lapsedeadline")
|
||||
require.NoError(t, err)
|
||||
stopRenewals(lease)
|
||||
|
||||
// Store unwritable, validity due to lapse shortly, and no tick before then.
|
||||
leases.kv = &slowAckKV{KeyValue: leases.kv, clock: &testClock{}, failUpdates: true}
|
||||
lease.mu.Lock()
|
||||
lease.confirmed = time.Now().Add(-volumeLeaseValidity + 100*time.Millisecond)
|
||||
lease.done = make(chan struct{})
|
||||
lease.mu.Unlock()
|
||||
go lease.renewLoop(t.Context())
|
||||
|
||||
select {
|
||||
case kind := <-lost:
|
||||
assert.Equal(t, leaseLostStalled, kind)
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("validity lapsed and nothing fenced the volume until the next check")
|
||||
}
|
||||
}
|
||||
|
||||
// TestVolumeLease_LostOrLapsedLeaseIsNotShared: a lease stays held until its
|
||||
// fence releases it, and handing it to another open in between would
|
||||
// authorize writes the lease no longer covers.
|
||||
func TestVolumeLease_LostOrLapsedLeaseIsNotShared(t *testing.T) {
|
||||
t.Run("lost", func(t *testing.T) {
|
||||
leases, _ := newSlowAckLeases(t, "node-a")
|
||||
lease, err := leases.acquire(t.Context(), "vol-sharelost")
|
||||
require.NoError(t, err)
|
||||
stopRenewals(lease)
|
||||
lease.surrender(t.Context())
|
||||
|
||||
shared, err := leases.acquire(t.Context(), "vol-sharelost")
|
||||
require.ErrorIs(t, err, ErrLeaseStoreUnavailable)
|
||||
assert.Nil(t, shared)
|
||||
lease.mu.Lock()
|
||||
defer lease.mu.Unlock()
|
||||
assert.Equal(t, 1, lease.refs)
|
||||
})
|
||||
t.Run("lapsed", func(t *testing.T) {
|
||||
leases, kv := newSlowAckLeases(t, "node-a")
|
||||
lease, err := leases.acquire(t.Context(), "vol-sharelapsed")
|
||||
require.NoError(t, err)
|
||||
stopRenewals(lease)
|
||||
kv.clock.advance(volumeLeaseValidity + time.Second)
|
||||
|
||||
shared, err := leases.acquire(t.Context(), "vol-sharelapsed")
|
||||
require.ErrorIs(t, err, ErrLeaseStoreUnavailable)
|
||||
assert.Nil(t, shared)
|
||||
lease.mu.Lock()
|
||||
defer lease.mu.Unlock()
|
||||
assert.Equal(t, 1, lease.refs)
|
||||
})
|
||||
}
|
||||
|
||||
// TestVolumeLease_RenewalAckedAfterTheDeadlineDoesNotRevive: the renewal was
|
||||
// sent inside validity, but its reply lands after the deadline and before the
|
||||
// deadline timer has run. The holder must give the volume up rather than let
|
||||
// the new confirmation hide the lapse from the timer.
|
||||
func TestVolumeLease_RenewalAckedAfterTheDeadlineDoesNotRevive(t *testing.T) {
|
||||
leases, kv := newSlowAckLeases(t, "node-a")
|
||||
var fences []leaseLossKind
|
||||
var fenceMu sync.Mutex
|
||||
fenced := make(chan struct{}, 4)
|
||||
leases.onLost = func(_ context.Context, _ string, kind leaseLossKind) {
|
||||
fenceMu.Lock()
|
||||
fences = append(fences, kind)
|
||||
fenceMu.Unlock()
|
||||
fenced <- struct{}{}
|
||||
}
|
||||
|
||||
lease, err := leases.acquire(t.Context(), "vol-renewalpastdeadline")
|
||||
require.NoError(t, err)
|
||||
stopRenewals(lease)
|
||||
|
||||
// One second of validity left when the renewal is sent.
|
||||
lease.mu.Lock()
|
||||
lease.confirmed = leases.clock().Add(-volumeLeaseValidity + time.Second)
|
||||
lease.mu.Unlock()
|
||||
|
||||
kv.gate = make(chan struct{})
|
||||
kv.entered = make(chan struct{})
|
||||
result := make(chan bool, 1)
|
||||
go func() { result <- lease.renew(t.Context()) }()
|
||||
<-kv.entered
|
||||
kv.clock.advance(2 * time.Second) // the old deadline passes while the reply is held
|
||||
close(kv.gate)
|
||||
|
||||
assert.False(t, <-result, "a renewal answered after the deadline must not keep the lease")
|
||||
assert.True(t, lease.expiredLocally(), "a reply after the deadline revived the lease")
|
||||
select {
|
||||
case <-fenced:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("a renewal answered after the deadline left the volume unfenced")
|
||||
}
|
||||
|
||||
// The deadline timer and a peer conflict arriving afterwards fence nothing more.
|
||||
lease.surrender(t.Context())
|
||||
lease.loseToPeer(t.Context(), errors.New("conflict"), "late")
|
||||
select {
|
||||
case <-fenced:
|
||||
t.Fatal("a lease already given up was fenced a second time")
|
||||
case <-time.After(200 * time.Millisecond):
|
||||
}
|
||||
fenceMu.Lock()
|
||||
defer fenceMu.Unlock()
|
||||
assert.Equal(t, []leaseLossKind{leaseLostStalled}, fences)
|
||||
}
|
||||
|
||||
// TestVolumeLease_RenewalAckedJustBeforeTheDeadlineKeepsTheLease is the
|
||||
// boundary: a reply that lands while validity still holds is a real renewal.
|
||||
func TestVolumeLease_RenewalAckedJustBeforeTheDeadlineKeepsTheLease(t *testing.T) {
|
||||
leases, kv := newSlowAckLeases(t, "node-a")
|
||||
lease, err := leases.acquire(t.Context(), "vol-renewalbeforedeadline")
|
||||
require.NoError(t, err)
|
||||
stopRenewals(lease)
|
||||
|
||||
lease.mu.Lock()
|
||||
lease.confirmed = leases.clock().Add(-volumeLeaseValidity + 2*time.Second)
|
||||
lease.mu.Unlock()
|
||||
kv.updateDelay = time.Second
|
||||
|
||||
require.True(t, lease.renew(t.Context()))
|
||||
assert.False(t, lease.expiredLocally())
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user