mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-24 05:04:03 +00:00
Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 98b864249c | |||
| aebc2f0efb | |||
| 0c488d9470 | |||
| 9a3ab0468d | |||
| 520478dc37 |
@@ -28,9 +28,6 @@
|
||||
# that runs identically on either, and because a quota left switched on would
|
||||
# cap every suite sharing the environment — so it must be proved to restore
|
||||
# itself wherever it runs, not only on one node.
|
||||
# route53 belongs to both topologies because every node serves every hosted
|
||||
# zone from one shared bucket: on one node it proves the zone loads at all, on
|
||||
# three that each node's sync reaches the same set.
|
||||
# storagefault runs last in both lists and belongs to both topologies, for
|
||||
# different reasons. On one node it is RS(1,0), so freezing the only predastore
|
||||
# is a total outage and the guest-survival assertion is unambiguous. On three it
|
||||
@@ -40,7 +37,7 @@
|
||||
# received. It is last because it SIGSTOPs a cluster-wide
|
||||
# service: anything sharing the environment would see the same outage and report
|
||||
# a failure that says nothing about itself.
|
||||
E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds quota route53 storagefault"
|
||||
E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds quota storagefault"
|
||||
|
||||
# Suites runnable against a multi-node environment.
|
||||
#
|
||||
@@ -60,7 +57,7 @@ E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds quota
|
||||
# replicated rather than sitting on one node, so it runs before the pair that
|
||||
# take a node away — a cluster whose buckets are already wrong should report
|
||||
# that, not report a recovery failing.
|
||||
E2E_SUITES_MULTI="multinode lb cert quota route53 kvquorum lbrecovery instancerecovery storagefault"
|
||||
E2E_SUITES_MULTI="multinode lb cert quota kvquorum lbrecovery instancerecovery storagefault"
|
||||
|
||||
# grep -xE alternation form, for narrowing a requested set to the eligible one.
|
||||
E2E_SUITES_SINGLE_RE="$(printf '%s' "${E2E_SUITES_SINGLE}" | tr ' ' '|')"
|
||||
|
||||
@@ -2,7 +2,8 @@ name: Apt Packages
|
||||
|
||||
# Path-filtered like image-scripts.yml: this resolves setup.sh's runtime package
|
||||
# list against each supported suite's real indices, so a renamed, dropped, or
|
||||
# ambiguously virtual package fails here rather than on a user's machine.
|
||||
# ambiguously virtual package fails here rather than on a user's machine. The
|
||||
# dnf job does the same for Oracle Linux, through setup.sh's own repo setup.
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
@@ -11,6 +12,7 @@ on:
|
||||
paths:
|
||||
- scripts/setup.sh
|
||||
- scripts/lint-apt-packages.sh
|
||||
- scripts/lint-dnf-packages.sh
|
||||
- .github/workflows/apt-packages.yml
|
||||
pull_request:
|
||||
branches:
|
||||
@@ -19,6 +21,7 @@ on:
|
||||
paths:
|
||||
- scripts/setup.sh
|
||||
- scripts/lint-apt-packages.sh
|
||||
- scripts/lint-dnf-packages.sh
|
||||
- .github/workflows/apt-packages.yml
|
||||
workflow_dispatch:
|
||||
|
||||
@@ -44,3 +47,14 @@ jobs:
|
||||
|
||||
- name: Resolve runtime package list
|
||||
run: ./scripts/lint-apt-packages.sh ${{ matrix.image }}
|
||||
|
||||
dnf_packages:
|
||||
name: Resolve dnf list (oraclelinux:10)
|
||||
runs-on: ubuntu-26.04
|
||||
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v7.0.1
|
||||
|
||||
- name: Resolve runtime package list
|
||||
run: ./scripts/lint-dnf-packages.sh oraclelinux:10
|
||||
|
||||
@@ -102,7 +102,7 @@ jobs:
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
|
||||
- name: Build microVM artifacts (lb-agent + kernel + initramfs)
|
||||
- name: Build microVM artifacts (lb-agent + kernel + initramfs + qboot)
|
||||
if: steps.version.outputs.exists != 'true'
|
||||
run: cd spinifex && make build-microvm-image
|
||||
|
||||
@@ -144,6 +144,7 @@ jobs:
|
||||
--notes-file "$RUNNER_TEMP/notes.md" \
|
||||
"dist/spinifex-$TAG-linux-amd64.tar.gz" \
|
||||
"dist/spinifex-$TAG-linux-amd64.tar.gz.sha256" \
|
||||
dist/qboot-*.tar.gz \
|
||||
"scripts/setup.sh"
|
||||
|
||||
# The assertion that matters. A prerelease must never become the latest
|
||||
|
||||
@@ -57,7 +57,7 @@ jobs:
|
||||
go-version-file: spinifex/go.mod
|
||||
cache: false
|
||||
|
||||
- name: Build microVM artifacts (lb-agent + kernel + initramfs)
|
||||
- name: Build microVM artifacts (lb-agent + kernel + initramfs + qboot)
|
||||
run: cd spinifex && make build-microvm-image
|
||||
|
||||
- name: Build amd64 Tarball
|
||||
@@ -72,6 +72,7 @@ jobs:
|
||||
path: |
|
||||
spinifex/dist/spinifex-${{ github.ref_name }}-linux-amd64.tar.gz
|
||||
spinifex/dist/spinifex-${{ github.ref_name }}-linux-amd64.tar.gz.sha256
|
||||
spinifex/dist/qboot-*.tar.gz
|
||||
spinifex/scripts/setup.sh
|
||||
|
||||
distro-arm64:
|
||||
|
||||
@@ -19,6 +19,9 @@ bin/
|
||||
dist/
|
||||
build/microvm/vmlinuz
|
||||
build/microvm/initramfs.cpio.gz
|
||||
build/microvm/qboot.rom
|
||||
build/microvm/qboot.LICENSE
|
||||
build/qboot/
|
||||
!spinifex/runtime/roles/spinifexui/frontend/dist/
|
||||
tmp/
|
||||
*.test
|
||||
|
||||
@@ -140,12 +140,23 @@ $(MICROVM_ARTIFACTS) &: $(MICROVM_INPUTS)
|
||||
bin/lb-agent:
|
||||
$(MAKE) build-lb-agent
|
||||
|
||||
build-microvm-image: $(MICROVM_ARTIFACTS) ## Build microVM kernel + initramfs (incremental — skips when up to date)
|
||||
# qboot is the microvm firmware. The source tarball ships with each release
|
||||
# because qboot is GPL-2.0.
|
||||
QBOOT_COMMIT := $(shell sed -n 's/^QBOOT_COMMIT="\(.*\)"$$/\1/p' scripts/build-qboot.sh)
|
||||
QBOOT_SOURCE := build/qboot/qboot-$(QBOOT_COMMIT).tar.gz
|
||||
QBOOT_ARTIFACTS := $(MICROVM_OUT_DIR)/qboot.rom $(MICROVM_OUT_DIR)/qboot.LICENSE $(QBOOT_SOURCE)
|
||||
|
||||
install-microvm: $(MICROVM_ARTIFACTS) ## Install microVM artifacts to /usr/share/spinifex/microvm/
|
||||
$(QBOOT_ARTIFACTS) &: scripts/build-qboot.sh
|
||||
./scripts/build-qboot.sh
|
||||
|
||||
build-microvm-image: $(MICROVM_ARTIFACTS) $(QBOOT_ARTIFACTS) ## Build microVM kernel, initramfs and qboot firmware (incremental — skips when up to date)
|
||||
|
||||
install-microvm: $(MICROVM_ARTIFACTS) $(QBOOT_ARTIFACTS) ## Install microVM artifacts to /usr/share/spinifex/microvm/
|
||||
sudo install -d /usr/share/spinifex/microvm
|
||||
sudo install -m 0644 $(MICROVM_OUT_DIR)/vmlinuz /usr/share/spinifex/microvm/vmlinuz
|
||||
sudo install -m 0644 $(MICROVM_OUT_DIR)/initramfs.cpio.gz /usr/share/spinifex/microvm/initramfs.cpio.gz
|
||||
sudo install -m 0644 $(MICROVM_OUT_DIR)/qboot.rom /usr/share/spinifex/microvm/qboot.rom
|
||||
sudo install -m 0644 $(MICROVM_OUT_DIR)/qboot.LICENSE /usr/share/spinifex/microvm/qboot.LICENSE
|
||||
|
||||
# the pre-commit gate: manifest checks, lint, vuln, and the unit and e2e-harness tiers.
|
||||
# integration and race tests skipped to keep quick, they run in CI.
|
||||
@@ -167,7 +178,7 @@ preflight:
|
||||
# reason: between them they install the software, wire OVN, erase the state and
|
||||
# remove the install, so a shellcheck warning in one is an outage or a wrong
|
||||
# deletion, not a style note.
|
||||
NODE_SCRIPTS := scripts/setup.sh scripts/setup-ovn.sh scripts/install-node.sh scripts/smoke-test.sh scripts/node-reset.sh scripts/uninstall-spx.sh
|
||||
NODE_SCRIPTS := scripts/setup.sh scripts/setup-ovn.sh scripts/install-node.sh scripts/smoke-test.sh scripts/node-reset.sh scripts/uninstall-spx.sh scripts/lib/distro.sh
|
||||
|
||||
test-build-scripts:
|
||||
@echo -e "\n....Running build/scripts/**/*_test.sh...."
|
||||
@@ -175,8 +186,8 @@ test-build-scripts:
|
||||
echo "-- $$t"; \
|
||||
sh "$$t" || exit 1; \
|
||||
done
|
||||
@echo -e "\n....Running scripts/*_test.sh...."
|
||||
@for t in $$(find scripts -maxdepth 1 -name '*_test.sh' | sort); do \
|
||||
@echo -e "\n....Running scripts/*_test.sh and scripts/lib/*_test.sh...."
|
||||
@for t in $$( (find scripts -maxdepth 1 -name '*_test.sh'; find scripts/lib -name '*_test.sh') | sort); do \
|
||||
echo "-- $$t"; \
|
||||
sh "$$t" || exit 1; \
|
||||
done
|
||||
@@ -333,14 +344,20 @@ fuzz:
|
||||
# Microvm artifacts are reinstalled when they already exist on disk — the rule's
|
||||
# input timestamps drive a rebuild only if anything actually changed. On a fresh
|
||||
# checkout (no build/microvm/vmlinuz yet) the install step is skipped; run
|
||||
# `make install-microvm` explicitly the first time.
|
||||
# `make install-microvm` explicitly the first time. They go in before the
|
||||
# preflight, which checks the microvm firmware.
|
||||
deploy: build
|
||||
sudo install -m 755 bin/spx /usr/local/bin/spx
|
||||
@if [ -f $(MICROVM_OUT_DIR)/vmlinuz ]; then \
|
||||
$(MAKE) install-microvm; \
|
||||
else \
|
||||
echo "[deploy] microvm artifacts absent — run 'make install-microvm' for first-time setup"; \
|
||||
fi
|
||||
@if [ "$${SKIP_ASSET_PREFLIGHT:-}" != "1" ]; then \
|
||||
if ! sudo /usr/local/bin/spx admin preflight; then \
|
||||
echo ""; \
|
||||
echo "[deploy] host-asset preflight failed: this node's helper scripts or sudoers"; \
|
||||
echo " grants don't match the binary just installed. Remediate with:"; \
|
||||
echo "[deploy] host-asset preflight failed: this node's helper scripts, sudoers"; \
|
||||
echo " grants, QEMU or microvm firmware don't match the binary just installed. Remediate with:"; \
|
||||
echo " scripts/update-nodes.sh (from the mulga root)"; \
|
||||
echo " or:"; \
|
||||
echo " make reinstall"; \
|
||||
@@ -350,11 +367,6 @@ deploy: build
|
||||
else \
|
||||
echo "[deploy] SKIP_ASSET_PREFLIGHT=1 — skipping host-asset preflight"; \
|
||||
fi
|
||||
@if [ -f $(MICROVM_OUT_DIR)/vmlinuz ]; then \
|
||||
$(MAKE) install-microvm; \
|
||||
else \
|
||||
echo "[deploy] microvm artifacts absent — run 'make install-microvm' for first-time setup"; \
|
||||
fi
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl restart spinifex.target
|
||||
|
||||
@@ -367,16 +379,25 @@ clean:
|
||||
rm -f ./bin/$(GO_PROJECT_NAME)
|
||||
rm -rf spinifex/runtime/roles/spinifexui/frontend/dist
|
||||
|
||||
# On dnf hosts the runtime packages and repositories come from setup.sh, so the
|
||||
# two cannot drift; nbdkit-devel, for building the plugin, is only in CodeReady.
|
||||
install-system:
|
||||
@echo -e "\n....Installing system dependencies for $(ARCH)...."
|
||||
sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \
|
||||
-o Dpkg::Options::="--force-confdef" \
|
||||
-o Dpkg::Options::="--force-confold" \
|
||||
nbdkit nbdkit-plugin-dev pkg-config qemu-system-x86 qemu-system-arm qemu-utils \
|
||||
ovmf qemu-efi-aarch64 \
|
||||
libvirt-daemon-system libvirt-clients libvirt-dev make gcc jq curl \
|
||||
iproute2 netcat-openbsd openssh-client wget git unzip sudo xz-utils file \
|
||||
ovn-central ovn-host openvswitch-switch dhcpcd-base
|
||||
@pm=$$(INSTALL_SPINIFEX_LIB_ONLY=1 bash -c '. scripts/setup.sh && distro_pkg_manager') || exit 1; \
|
||||
if [ "$$pm" = dnf ]; then \
|
||||
INSTALL_SPINIFEX_LIB_ONLY=1 bash -c '. scripts/setup.sh && setup_sudo && detect_os && install_dnf_deps' && \
|
||||
sudo dnf config-manager --set-enabled ol10_codeready_builder && \
|
||||
sudo dnf install -y nbdkit-devel pkgconf-pkg-config make gcc openssh-clients git sudo; \
|
||||
else \
|
||||
sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \
|
||||
-o Dpkg::Options::="--force-confdef" \
|
||||
-o Dpkg::Options::="--force-confold" \
|
||||
nbdkit nbdkit-plugin-dev pkg-config qemu-system-x86 qemu-system-arm qemu-utils \
|
||||
ovmf qemu-efi-aarch64 \
|
||||
libvirt-daemon-system libvirt-clients libvirt-dev make gcc jq curl \
|
||||
iproute2 netcat-openbsd openssh-client wget git unzip sudo xz-utils file \
|
||||
ovn-central ovn-host openvswitch-switch dhcpcd-base; \
|
||||
fi
|
||||
|
||||
install-go:
|
||||
@echo -e "\n....Installing Go 1.27.3 for $(ARCH) ($(GO_ARCH))...."
|
||||
@@ -438,10 +459,11 @@ distro-amd64:
|
||||
-f build/Dockerfile.distro \
|
||||
--output type=local,dest=dist/amd64/ \
|
||||
../
|
||||
@if [ -f $(MICROVM_OUT_DIR)/vmlinuz ] && [ -f $(MICROVM_OUT_DIR)/initramfs.cpio.gz ]; then \
|
||||
@if [ -f $(MICROVM_OUT_DIR)/vmlinuz ] && [ -f $(MICROVM_OUT_DIR)/initramfs.cpio.gz ] && [ -f $(MICROVM_OUT_DIR)/qboot.rom ] && [ -f $(QBOOT_SOURCE) ]; then \
|
||||
echo "[distro-amd64] staging microvm artifacts into tarball"; \
|
||||
mkdir -p dist/amd64/microvm; \
|
||||
cp $(MICROVM_OUT_DIR)/vmlinuz $(MICROVM_OUT_DIR)/initramfs.cpio.gz dist/amd64/microvm/; \
|
||||
cp $(MICROVM_OUT_DIR)/vmlinuz $(MICROVM_OUT_DIR)/initramfs.cpio.gz $(MICROVM_OUT_DIR)/qboot.rom $(MICROVM_OUT_DIR)/qboot.LICENSE dist/amd64/microvm/; \
|
||||
cp $(QBOOT_SOURCE) dist/; \
|
||||
else \
|
||||
echo "[distro-amd64] WARNING: microvm artifacts missing — tarball will not include them"; \
|
||||
echo "[distro-amd64] run 'make build-microvm-image' before 'make distro-amd64'"; \
|
||||
|
||||
@@ -45,6 +45,7 @@ RUN cd viperblock && GOFIPS140=v1.0.0 go build -o /out/nbdkit-viperblock-plugin.
|
||||
# Copy setup scripts and systemd units
|
||||
RUN cp spinifex/scripts/setup.sh /out/setup.sh 2>/dev/null || true
|
||||
RUN cp spinifex/scripts/setup-ovn.sh /out/setup-ovn.sh
|
||||
RUN mkdir -p /out/lib && cp spinifex/scripts/lib/distro.sh /out/lib/distro.sh
|
||||
RUN mkdir -p /out/systemd && cp spinifex/build/systemd/* /out/systemd/
|
||||
RUN mkdir -p /out/scripts && cp spinifex/build/scripts/* /out/scripts/
|
||||
RUN cp spinifex/build/logrotate/spinifex /out/logrotate-spinifex
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
# spinifex-unit-version: 3
|
||||
# spinifex-unit-version: 5
|
||||
[Unit]
|
||||
Description=Spinifex Daemon (VM Orchestration)
|
||||
PartOf=spinifex.target
|
||||
After=spinifex-nats.service spinifex-viperblock.service openvswitch-switch.service ovn-controller.service openvswitch-ipsec.service
|
||||
# Wants (not Requires): stay bootable when openvswitch-ipsec is absent
|
||||
Wants=openvswitch-ipsec.service
|
||||
After=spinifex-nats.service spinifex-viperblock.service ovsdb-server.service ovs-vswitchd.service ovn-controller.service openvswitch-ipsec.service spinifex-hostnet.service
|
||||
# Wants (not Requires): boot without openvswitch-ipsec, survive OVS restarts via hostnet
|
||||
Wants=openvswitch-ipsec.service spinifex-hostnet.service
|
||||
|
||||
[Service]
|
||||
EnvironmentFile=-/etc/spinifex/telemetry.env
|
||||
@@ -14,6 +14,7 @@ Type=simple
|
||||
Slice=spinifex-system.slice
|
||||
User=spinifex-daemon
|
||||
Group=spinifex
|
||||
ExecStartPre=/usr/local/bin/spx admin hostnet check
|
||||
ExecStart=/usr/local/bin/spx service spinifex start
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
# spinifex-unit-version: 1
|
||||
[Unit]
|
||||
Description=Spinifex host network wiring (veth pairs, mgmt address, encap route)
|
||||
# PartOf/WantedBy ovs-vswitchd: re-applied whenever OVS starts; daemon and vpcd only Want it
|
||||
Requires=ovs-vswitchd.service
|
||||
After=ovs-vswitchd.service NetworkManager-wait-online.service
|
||||
PartOf=ovs-vswitchd.service
|
||||
Before=spinifex.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
ExecStart=/usr/local/bin/spx admin hostnet apply
|
||||
# Covers the 120s wait behind setup-ovn.sh plus the 60s wait for links
|
||||
TimeoutStartSec=240
|
||||
TimeoutStopSec=10
|
||||
ProtectSystem=full
|
||||
ProtectHome=yes
|
||||
PrivateTmp=yes
|
||||
|
||||
[Install]
|
||||
WantedBy=spinifex.target ovs-vswitchd.service
|
||||
@@ -1,9 +1,9 @@
|
||||
# spinifex-unit-version: 1
|
||||
# spinifex-unit-version: 3
|
||||
[Unit]
|
||||
Description=Spinifex VPC Daemon (OVN Networking)
|
||||
PartOf=spinifex.target
|
||||
After=spinifex-nats.service spinifex-daemon.service systemd-networkd.service openvswitch-switch.service ovn-controller.service
|
||||
Wants=spinifex-nats.service ovn-controller.service
|
||||
After=spinifex-nats.service spinifex-daemon.service systemd-networkd.service ovsdb-server.service ovs-vswitchd.service ovn-controller.service spinifex-hostnet.service
|
||||
Wants=spinifex-nats.service ovn-controller.service spinifex-hostnet.service
|
||||
|
||||
[Service]
|
||||
EnvironmentFile=-/etc/spinifex/telemetry.env
|
||||
@@ -11,6 +11,7 @@ Type=simple
|
||||
Slice=spinifex-system.slice
|
||||
User=spinifex-vpcd
|
||||
Group=spinifex
|
||||
ExecStartPre=/usr/local/bin/spx admin hostnet check
|
||||
ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
|
||||
ExecStart=/usr/local/bin/spx service vpcd start
|
||||
Restart=on-failure
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
# spinifex-unit-version: 1
|
||||
# spinifex-unit-version: 2
|
||||
[Unit]
|
||||
Description=Spinifex Infrastructure Platform
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
Wants=spinifex-nats.service spinifex-predastore.service spinifex-northstar.service spinifex-viperblock.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service spinifex-ui.service spinifex-qmp-collector.service spinifex-shutdown.service spinifex-nats-watchdog.timer openvswitch-switch.service ovn-controller.service
|
||||
Wants=spinifex-nats.service spinifex-predastore.service spinifex-northstar.service spinifex-viperblock.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service spinifex-ui.service spinifex-qmp-collector.service spinifex-shutdown.service spinifex-nats-watchdog.timer ovsdb-server.service ovs-vswitchd.service ovn-controller.service
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
||||
@@ -1,42 +0,0 @@
|
||||
# Admission contract, version 1
|
||||
|
||||
This package owns the cross-process contract for the admission domain's
|
||||
quota override. It is not an in-process service interface or the owner of
|
||||
quota state.
|
||||
|
||||
## Seal account quota override
|
||||
|
||||
`SealAccountQuotaOverrideSubject` (`admission.v1.SealAccountQuotaOverride`) is
|
||||
a request/reply route on which a gateway, in the `awsgw-quota-seal` queue
|
||||
group, irreversibly seals one account's quota override record.
|
||||
`domains/admission/quota` owns it: `awsgw` serves it.
|
||||
`accountteardown` is the consumer: it calls this subject to fence every
|
||||
override writer at the quota owner before the account record is removed.
|
||||
|
||||
The request names the account in its body (`account_id`) and may carry an
|
||||
opaque `correlation_ref` that is echoed for logs.
|
||||
`awsgw` ignores `X-Account-ID` for this subject: the account sealed is the one
|
||||
named in the body.
|
||||
A seal is refused with `IncorrectState` unless IAM already holds the account
|
||||
as `TERMINATING`; a precondition read failure, or no termination checker
|
||||
wired, answers `InternalError`.
|
||||
Both refusals write nothing and never open the override bucket.
|
||||
|
||||
A reply is sent only on success: `disposition` is always `sealed` and
|
||||
`revision` is the stream leader's accepted write for that seal.
|
||||
A failure is the standard awserrors error payload, not this reply shape.
|
||||
Sealing is idempotent: a seal on an already-sealed record re-writes the same
|
||||
value and returns a fresh, higher revision.
|
||||
Any current gateway may answer; the leader-accepted revision is the evidence
|
||||
whichever one does, and the override bucket is opened at seal time even on a
|
||||
gateway whose quotas were disabled at startup.
|
||||
|
||||
### Compatibility boundary
|
||||
|
||||
The subject name carries the contract major; `account_quota_seal_test.go`
|
||||
preserves the version-1 JSON shape.
|
||||
Producer and consumer are in the coordinated Spinifex release, so the window
|
||||
is N and N-1.
|
||||
A gateway that predates the subject has no subscription, so the request gets
|
||||
no responder.
|
||||
An incompatible change to the subject or fields needs a new major.
|
||||
@@ -1,31 +0,0 @@
|
||||
// Package admissionv1 holds the admission domain's cross-process contract:
|
||||
// the subjects, request/reply shapes and wire vocabulary the quota owner and
|
||||
// its consumers share.
|
||||
package admissionv1
|
||||
|
||||
// SealAccountQuotaOverrideSubject is the request/reply subject on which a
|
||||
// gateway seals one account's quota override record. Sealing is irreversible
|
||||
// and refused unless IAM already holds the account as TERMINATING.
|
||||
const SealAccountQuotaOverrideSubject = "admission.v1.SealAccountQuotaOverride"
|
||||
|
||||
// AccountQuotaOverrideDispositionSealed is the only disposition a successful
|
||||
// reply carries: a reply is sent only on success, so there is no other value.
|
||||
const AccountQuotaOverrideDispositionSealed = "sealed"
|
||||
|
||||
// SealAccountQuotaOverrideRequest asks for one account's quota override to be
|
||||
// sealed. CorrelationRef is an opaque caller reference echoed for logs; it
|
||||
// grants nothing and is never a fence or authority to write.
|
||||
type SealAccountQuotaOverrideRequest struct {
|
||||
AccountID string `json:"account_id"`
|
||||
CorrelationRef string `json:"correlation_ref,omitempty"`
|
||||
}
|
||||
|
||||
// SealAccountQuotaOverrideReply confirms accountID's quota override record
|
||||
// was sealed at Revision, the stream leader's accepted write for that seal.
|
||||
// A failure is the standard awserrors error payload, not this type.
|
||||
type SealAccountQuotaOverrideReply struct {
|
||||
AccountID string `json:"account_id"`
|
||||
CorrelationRef string `json:"correlation_ref,omitempty"`
|
||||
Disposition string `json:"disposition"`
|
||||
Revision uint64 `json:"revision"`
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
package admissionv1
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSealAccountQuotaOverrideSubject(t *testing.T) {
|
||||
require.Equal(t, "admission.v1.SealAccountQuotaOverride", SealAccountQuotaOverrideSubject)
|
||||
}
|
||||
|
||||
func TestSealAccountQuotaOverrideRequestJSONContract(t *testing.T) {
|
||||
data, err := json.Marshal(SealAccountQuotaOverrideRequest{AccountID: "111122223333", CorrelationRef: "req-1"})
|
||||
require.NoError(t, err)
|
||||
require.JSONEq(t, `{"account_id":"111122223333","correlation_ref":"req-1"}`, string(data))
|
||||
|
||||
data, err = json.Marshal(SealAccountQuotaOverrideRequest{AccountID: "111122223333"})
|
||||
require.NoError(t, err)
|
||||
require.JSONEq(t, `{"account_id":"111122223333"}`, string(data))
|
||||
}
|
||||
|
||||
func TestSealAccountQuotaOverrideReplyJSONContract(t *testing.T) {
|
||||
reply := SealAccountQuotaOverrideReply{
|
||||
AccountID: "111122223333",
|
||||
CorrelationRef: "req-1",
|
||||
Disposition: AccountQuotaOverrideDispositionSealed,
|
||||
Revision: 42,
|
||||
}
|
||||
data, err := json.Marshal(reply)
|
||||
require.NoError(t, err)
|
||||
require.JSONEq(t, `{
|
||||
"account_id": "111122223333",
|
||||
"correlation_ref": "req-1",
|
||||
"disposition": "sealed",
|
||||
"revision": 42
|
||||
}`, string(data))
|
||||
|
||||
data, err = json.Marshal(SealAccountQuotaOverrideReply{AccountID: "111122223333", Disposition: AccountQuotaOverrideDispositionSealed})
|
||||
require.NoError(t, err)
|
||||
require.JSONEq(t, `{"account_id":"111122223333","disposition":"sealed","revision":0}`, string(data))
|
||||
}
|
||||
@@ -60,42 +60,3 @@ The subject name carries the contract major; `platform_inventory_test.go` preser
|
||||
Producer and consumer are in the coordinated Spinifex release, so the window is N and N-1.
|
||||
A daemon that predates the subject has no subscription, so the request gets no responder; the consumer treats no responder, a timeout, an error reply, an undecodable reply, a reply for another account, and a missing, repeated or unknown-status kind as `scan-incomplete`, never clean.
|
||||
An incompatible change to the subject, fields or vocabulary needs a new major.
|
||||
|
||||
## Seal account settings
|
||||
|
||||
`SealAccountSettingsSubject` (`ec2.v1.SealAccountSettings`) is a request/reply route on which a daemon in the `spinifex-workers` queue group irreversibly seals one account's EC2 settings record.
|
||||
EC2 owns it: `domains/ec2/account` serves it through the daemon.
|
||||
`accountteardown`'s StageAccount seal reaper is the consumer: it calls this subject only after the seal-capability probe below reports every configured node compatible, and completes only once it holds a successful sealed reply.
|
||||
|
||||
The request names the account in its body (`account_id`) and may carry an opaque `correlation_ref` that is echoed for logs.
|
||||
The daemon ignores `X-Account-ID` for this subject: the account sealed is the one named in the body.
|
||||
A seal is refused with `IncorrectState` unless IAM already holds the account as `TERMINATING`; a precondition read failure answers `InternalError`.
|
||||
Both refusals write nothing.
|
||||
|
||||
A reply is sent only on success: `disposition` is always `sealed` and `revision` is the stream leader's accepted write for that seal.
|
||||
A failure is the standard awserrors error payload, not this reply shape.
|
||||
Sealing is idempotent: a seal on an already-sealed record re-writes the same value and returns a fresh, higher revision.
|
||||
|
||||
### Compatibility boundary
|
||||
|
||||
The subject name carries the contract major; `account_settings_seal_test.go` preserves the version-1 JSON shape.
|
||||
Producer and consumer are in the coordinated Spinifex release, so the window is N and N-1.
|
||||
A daemon that predates the subject has no subscription, so the request gets no responder.
|
||||
An incompatible change to the subject or fields needs a new major.
|
||||
|
||||
## Seal-account-settings capability
|
||||
|
||||
`AccountSettingsSealCapabilitySubject(node)` (`ec2.v1.AccountSettingsSealCapability.<node>`) is a per-node, read-only request/reply route on which `accountteardown` asks one configured daemon whether it supports the account-settings seal, immediately before every seal attempt.
|
||||
EC2 owns it: the daemon serves it with no queue group, so a request addressed to one node reaches only subscribers configured as that node, and two daemons sharing a node ID both answer.
|
||||
`accountteardown` is the consumer: its StageAccount seal reaper probes every configured node and sends no seal request unless every one answers compatible.
|
||||
|
||||
`AccountSettingsSealCapabilityVersion` is a claim about the whole binary's writers, not about the responder: version 1 means every EC2 account-settings writer in the binary is revision-guarded and refuses a sealed record, and the binary serves the seal subject.
|
||||
A daemon binary without this responder — predating the capability, or carrying only the seal subject without it — has no subscriber, so the probe for that node reads as incomplete (`no-reply`), the same as a stopped or partitioned node.
|
||||
The reply proves nothing beyond the moment it was sent: a daemon downgraded or replaced after a successful probe is outside the guarantee.
|
||||
|
||||
### Compatibility boundary
|
||||
|
||||
The subject name carries the contract major; `account_settings_seal_capability_test.go` preserves the version-1 JSON shape and the subject builder.
|
||||
Producer and consumer are in the coordinated Spinifex release, so the window is N and N-1.
|
||||
A daemon that predates the capability has no subscription, so the probe gets no responder for that node.
|
||||
An incompatible change to the subject or fields needs a new major.
|
||||
|
||||
@@ -1,28 +0,0 @@
|
||||
package ec2v1
|
||||
|
||||
// SealAccountSettingsSubject is the request/reply subject on which a daemon
|
||||
// seals one account's EC2 settings record. Sealing is irreversible and refused
|
||||
// unless IAM already holds the account as TERMINATING.
|
||||
const SealAccountSettingsSubject = "ec2.v1.SealAccountSettings"
|
||||
|
||||
// AccountSettingsDispositionSealed is the only disposition a successful reply
|
||||
// carries: a reply is sent only on success, so there is no other value.
|
||||
const AccountSettingsDispositionSealed = "sealed"
|
||||
|
||||
// SealAccountSettingsRequest asks for one account's EC2 settings to be
|
||||
// sealed. CorrelationRef is an opaque caller reference echoed for logs; it
|
||||
// grants nothing and is never a fence or authority to write.
|
||||
type SealAccountSettingsRequest struct {
|
||||
AccountID string `json:"account_id"`
|
||||
CorrelationRef string `json:"correlation_ref,omitempty"`
|
||||
}
|
||||
|
||||
// SealAccountSettingsReply confirms accountID's settings record was sealed at
|
||||
// Revision, the stream leader's accepted write for that seal. A failure is
|
||||
// the standard awserrors error payload, not this type.
|
||||
type SealAccountSettingsReply struct {
|
||||
AccountID string `json:"account_id"`
|
||||
CorrelationRef string `json:"correlation_ref,omitempty"`
|
||||
Disposition string `json:"disposition"`
|
||||
Revision uint64 `json:"revision"`
|
||||
}
|
||||
@@ -1,39 +0,0 @@
|
||||
package ec2v1
|
||||
|
||||
import "strings"
|
||||
|
||||
// AccountSettingsSealCapabilityVersion is the version this binary reports.
|
||||
// Version 1: every EC2 account-settings writer in the binary is
|
||||
// revision-guarded and refuses a sealed record, and the seal subject is served.
|
||||
const AccountSettingsSealCapabilityVersion = 1
|
||||
|
||||
// AccountSettingsSealCapabilitySubject is one daemon's read-only
|
||||
// seal-capability route, addressed per node so a probe reaches exactly the
|
||||
// node it asks about.
|
||||
func AccountSettingsSealCapabilitySubject(node string) string {
|
||||
return "ec2.v1.AccountSettingsSealCapability." + node
|
||||
}
|
||||
|
||||
// ValidSealCapabilityNode reports whether node is exactly one NATS subject
|
||||
// token. Responder and probe both check it before building a subject, so
|
||||
// neither can address a wildcard or another node's route.
|
||||
func ValidSealCapabilityNode(node string) bool {
|
||||
return node != "" && !strings.ContainsAny(node, ".*>\t\n\r ")
|
||||
}
|
||||
|
||||
// AccountSettingsSealCapabilityRequest asks node whether it supports the
|
||||
// account-settings seal. CorrelationRef is an opaque caller reference echoed
|
||||
// for logs; it grants nothing and is never a fence or authority to write.
|
||||
type AccountSettingsSealCapabilityRequest struct {
|
||||
Node string `json:"node"`
|
||||
CorrelationRef string `json:"correlation_ref,omitempty"`
|
||||
}
|
||||
|
||||
// AccountSettingsSealCapabilityReply reports the responding node's capability
|
||||
// version. It carries no account data and proves nothing beyond the moment it
|
||||
// was sent.
|
||||
type AccountSettingsSealCapabilityReply struct {
|
||||
Node string `json:"node"`
|
||||
CapabilityVersion int `json:"capability_version"`
|
||||
CorrelationRef string `json:"correlation_ref,omitempty"`
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
package ec2v1
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestAccountSettingsSealCapabilitySubject(t *testing.T) {
|
||||
require.Equal(t, "ec2.v1.AccountSettingsSealCapability.node-1", AccountSettingsSealCapabilitySubject("node-1"))
|
||||
require.Equal(t, "ec2.v1.AccountSettingsSealCapability.node-2", AccountSettingsSealCapabilitySubject("node-2"))
|
||||
}
|
||||
|
||||
func TestAccountSettingsSealCapabilityRequestJSONContract(t *testing.T) {
|
||||
data, err := json.Marshal(AccountSettingsSealCapabilityRequest{Node: "node-1", CorrelationRef: "req-1"})
|
||||
require.NoError(t, err)
|
||||
require.JSONEq(t, `{"node":"node-1","correlation_ref":"req-1"}`, string(data))
|
||||
|
||||
data, err = json.Marshal(AccountSettingsSealCapabilityRequest{Node: "node-1"})
|
||||
require.NoError(t, err)
|
||||
require.JSONEq(t, `{"node":"node-1"}`, string(data))
|
||||
}
|
||||
|
||||
func TestAccountSettingsSealCapabilityReplyJSONContract(t *testing.T) {
|
||||
reply := AccountSettingsSealCapabilityReply{Node: "node-1", CapabilityVersion: 1, CorrelationRef: "req-1"}
|
||||
data, err := json.Marshal(reply)
|
||||
require.NoError(t, err)
|
||||
require.JSONEq(t, `{"node":"node-1","capability_version":1,"correlation_ref":"req-1"}`, string(data))
|
||||
|
||||
data, err = json.Marshal(AccountSettingsSealCapabilityReply{Node: "node-1", CapabilityVersion: 1})
|
||||
require.NoError(t, err)
|
||||
require.JSONEq(t, `{"node":"node-1","capability_version":1}`, string(data))
|
||||
}
|
||||
|
||||
func TestValidSealCapabilityNode(t *testing.T) {
|
||||
for _, node := range []string{"node-1", "node_2", "n3"} {
|
||||
require.True(t, ValidSealCapabilityNode(node), node)
|
||||
}
|
||||
for _, node := range []string{"", "a.b", "*", ">", "node-*", "node >", "node\t1", "node\n1"} {
|
||||
require.False(t, ValidSealCapabilityNode(node), "%q", node)
|
||||
}
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
package ec2v1
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSealAccountSettingsSubject(t *testing.T) {
|
||||
require.Equal(t, "ec2.v1.SealAccountSettings", SealAccountSettingsSubject)
|
||||
}
|
||||
|
||||
func TestSealAccountSettingsRequestJSONContract(t *testing.T) {
|
||||
data, err := json.Marshal(SealAccountSettingsRequest{AccountID: "111122223333", CorrelationRef: "req-1"})
|
||||
require.NoError(t, err)
|
||||
require.JSONEq(t, `{"account_id":"111122223333","correlation_ref":"req-1"}`, string(data))
|
||||
|
||||
data, err = json.Marshal(SealAccountSettingsRequest{AccountID: "111122223333"})
|
||||
require.NoError(t, err)
|
||||
require.JSONEq(t, `{"account_id":"111122223333"}`, string(data))
|
||||
}
|
||||
|
||||
func TestSealAccountSettingsReplyJSONContract(t *testing.T) {
|
||||
reply := SealAccountSettingsReply{
|
||||
AccountID: "111122223333",
|
||||
CorrelationRef: "req-1",
|
||||
Disposition: AccountSettingsDispositionSealed,
|
||||
Revision: 42,
|
||||
}
|
||||
data, err := json.Marshal(reply)
|
||||
require.NoError(t, err)
|
||||
require.JSONEq(t, `{
|
||||
"account_id": "111122223333",
|
||||
"correlation_ref": "req-1",
|
||||
"disposition": "sealed",
|
||||
"revision": 42
|
||||
}`, string(data))
|
||||
|
||||
data, err = json.Marshal(SealAccountSettingsReply{AccountID: "111122223333", Disposition: AccountSettingsDispositionSealed})
|
||||
require.NoError(t, err)
|
||||
require.JSONEq(t, `{"account_id":"111122223333","disposition":"sealed","revision":0}`, string(data))
|
||||
}
|
||||
+1
-2
@@ -110,7 +110,6 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
|
||||
|---------|-------|-------------|
|
||||
| `spx admin kv digest` | `--store-dir` (default `<spinifex-dir>/nats/jetstream`), `--work-dir` (holds the temporary copy, default the system temp dir), `--stream` (repeatable filter), `--seqs` (also print a hash per sequence), `--json` (print one JSON document, the input `spx admin kv compare` reads) | Read-only per-replica consistency check. Copies this node's `$G` streams tree (without consumer state) to a temporary directory, opens the copy in a private in-process NATS server with no listener, and prints one line per stream with first/last sequence, message count and a SHA-256 over every stored message (sequence, subject, timestamp, headers, data). Run it on every node with `sudo` and diff the output with the `#` lines removed: replicas of one stream must match. A direct get over NATS is not used because any replica may answer it. Writes landing during the copy can change a replica's tail, so gate API writes or re-run before calling a tail difference divergence. Prefer `--json --seqs` on every node followed by `spx admin kv compare`, which tells a diverged replica from one that is merely behind. |
|
||||
| `spx admin kv compare <digest.json>...` | — (`-` reads any number of digests from stdin) | Reads the `spx admin kv digest --json --seqs` output of every node and checks that each stream's replicas hold the same content. A replica that is merely behind is not divergence; these are: the same sequence holding different content on two replicas, a message missing from one replica that nothing on it explains (no later write to the same subject, no expiry under `max_age` or a message TTL, no stream limit dropping its head), a stream held by fewer or more nodes than its replica count, replicas disagreeing on that count, or no sequence range common to every replica. A stream with a `max_age` whose replicas were copied at different times and share no sequence range is reported as `skipped` (volatile) rather than diverged, since a short-lived bucket such as a leader lease turns over between copies; any sequence two of its replicas share must still match, and each message a replica lacks must be superseded, expired, or written after that replica was copied. Sequences past the slowest replica's last are not compared. Exits 0 when every stream is consistent, 1 on divergence, 2 on bad input. Digests are taken from live stores seconds apart, so re-run before trusting a failure that a write landing between two copies could explain. `install-node.sh` gates cluster formation on it. |
|
||||
| `spx admin dns nameservers` | `--check` (compare against the parent zone's delegation), `--timeout` (per DNS query, default 5s) | Prints the `nsN` nameservers this cluster publishes for its base domain, with the node and address behind each. At most four are published however many nodes run northstar; `[northstar] nameserver_count` (1–4) and `nameserver_nodes` (ordered pin) in `spinifex.toml` set the count and order, and every northstar node keeps serving every zone either way. `--check` asks a server of the parent zone (a `.net` server for `spx3.net`) for the delegation without recursion and reports each name as `ok`, `not delegated`, `no glue`, `glue mismatch` or `not published here`, exiting non-zero unless all are `ok`. |
|
||||
| `spx admin cluster shutdown` | `--force` (shutdown even if nodes don't respond), `--timeout` (max wait per phase, default 120s), `--dry-run` (print phase plan without executing) | Performs coordinated, phased shutdown of entire cluster. Phases execute in order: GATE (stop API/UI) → DRAIN (stop VMs) → STORAGE (stop viperblock) → PERSIST (stop predastore) → INFRA (stop NATS/daemon). Each phase waits for all nodes to ACK before proceeding. Uses JetStream state tracking. |
|
||||
| `spx admin cluster drain-dhcp` | `--timeout` (reply-collection window, default 30s) | Asks each vpcd to DHCPRELEASE every external-pool DHCP lease it currently holds, returning them to the upstream DHCP server. Run on teardown before stopping services — an env reset otherwise strands held leases upstream until their TTL expires, eventually exhausting the upstream scope. Best-effort: warns and exits 0 if the cluster is already down. |
|
||||
| `spx admin node drain --local` | `--local` (drain the local node only, required), `--timeout` (max wait per phase, default 120s), `--unless-restarting` (drain on a real host shutdown or a plain target stop, and skip only when a restart of the stack is already queued; unset, it always drains) | Runs the GATE and DRAIN phases against the local node only: powers down its guests via QMP and unmounts their volumes (flushing the viperblock WAL) while every service is still up. STORAGE/PERSIST/INFRA are left to systemd's ordered teardown. Wired as the `ExecStop` of `spinifex-shutdown.service`, so `systemctl stop spinifex.target` and host reboot/poweroff drain guests before any storage service stops. |
|
||||
@@ -127,7 +126,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
|
||||
|
||||
| Command | Flags | Description |
|
||||
|---------|-------|-------------|
|
||||
| `spx admin preflight` | `--quiet` (print only non-OK rows) | Reports whether this host has the privileged helper scripts and sudoers grants the installed build depends on. `spx version` knows the build but not the host: `setup.sh` and `setup-ovn.sh` install those assets, and a binary-swap deploy (`make deploy`, or a manual copy) skips them. It also reports any running service still executing a replaced binary — overwriting `/usr/local/bin/spx` while `spinifex.target` is up does not move services onto it, so each keeps executing the replaced inode until it restarts and a node can serve a mixture of two builds indefinitely. Checks compare content, not existence, so a present-but-stale asset is caught. Exits non-zero if any asset is Missing, Stale or Ungranted. |
|
||||
| `spx admin preflight` | `--quiet` (print only non-OK rows) | Reports whether this host has the privileged helper scripts and sudoers grants the installed build depends on. `spx version` knows the build but not the host: `setup.sh` and `setup-ovn.sh` install those assets, and a binary-swap deploy (`make deploy`, or a manual copy) skips them. It also reports any running service still executing a replaced binary — overwriting `/usr/local/bin/spx` while `spinifex.target` is up does not move services onto it, so each keeps executing the replaced inode until it restarts and a node can serve a mixture of two builds indefinitely. Checks compare content, not existence, so a present-but-stale asset is caught. On x86_64 it also checks that the QEMU binary the daemon will run lists the `microvm` machine type and that the microvm firmware `/usr/share/spinifex/microvm/qboot.rom` is installed, since load balancers boot on both. Exits non-zero if any asset is Missing, Stale, Ungranted or Unsupported. |
|
||||
| `spx admin node js-probe` | `--timeout` (default: 10s) | Round-trips a canary key through the cluster-state KV bucket and exits non-zero if the write or the read-back fails. JetStream can latch a write error per message block and keep serving reads — nats-server's filestore records the failure on the block and returns it on every later write, and nothing clears it short of a restart. The HTTP health endpoint reports the server-wide JetStream state, which that latch never touches, so only an actual write observes it. |
|
||||
| `spx admin banner` | `--boot-check` | Writes the node information banner to `/etc/issue` (shown before login on the physical/serial console) and `/etc/motd` (shown after SSH login). With `--boot-check` it also detects a management IP that has changed since last boot and updates `/etc/spinifex/node.conf` accordingly. |
|
||||
|
||||
|
||||
@@ -446,7 +446,7 @@ These residuals were recorded by individual slices and must be resolved once the
|
||||
- `providers/ebs/viperblock` → `runtime/compute/nbd` and `runtime/host/process` (dependency, moved unchanged from `services/viperblockd`): the adapter formats NBD endpoints and manages nbdkit processes through runtime packages. ADR-0001 does not list providers → runtime as forbidden and the layering test does not enforce it (same category as `providers/ebs/conformance` → `runtime/compute/nbd`), but a provider should not depend on node runtime. Clears when NBD endpoint naming has a provider-facing home (a candidate for `contracts/ebs/v1`) and process supervision is injected by the role. The adapter also imports `providers/objectstore`, a provider-to-provider edge that is allowed.
|
||||
- `gateway/ec2.go` (registration): the EC2 dispatch map stays a `GatewayConfig` method in `gateway` and imports every `domains/ec2/awsapi/*` package; `gateway/ec2_idempotency.go` also imports `domains/ec2/awsapi/idem` (`KVBucket`, `TTL`, `TokenAndParams`) for the dispatch-time idempotency wrapper. ADR-0001 makes the domain own its action registration; moving the map waits on the generic ingress registration seam and must not be done as a package move.
|
||||
- `gateway/requestaudit.go`: the outermost request-audit middleware records AWS fields and is updated from four places in the gateway's authentication flow through the gateway's own context key. Moving it would export about seven names used only by that flow, so it stays in `gateway` until the generic ingress registration seam defines how ingress middleware receives authentication results.
|
||||
- `accountteardown` (dependency): its EC2 reapers call `domains/ec2/awsapi/*` action adapters directly, as they did under `gateway/ec2`. The ADR's `accountteardown` current-to-target map row replaces this with domain-owned cleanup capabilities before the package moves. It also holds a read-only consumer of the EC2 platform-inventory contract (`contracts/ec2/v1`, `ec2.v1.PlatformInventory`), served by `domains/ec2/accountinventory`; only the explicit dry run (`Engine.DryRun`) calls it, to log a shadow comparison, and the key-pair, placement-group and launch-template reapers still use the `awsapi` adapters. A domain-owned cleanup capability for the proven-safe subset of those three kinds (key pairs and empty placement groups no instance, spot request or live launch-template version names, and launch-template residue; never live launch templates) exists as `domains/ec2/accountcleanup` (a Go interface with injected owners and reference readers, no NATS subject); nothing calls it yet. It also holds the StageAccount settings-seal reaper, a consumer of `contracts/ec2/v1`'s `ec2.v1.SealAccountSettings` and per-node `ec2.v1.AccountSettingsSealCapability.<node>` subjects, served by `domains/ec2/account` through the daemon; the reaper reads no KV and imports no domain package, reaching both subjects only through NATS and the configured node set `NewClusterEngine` already receives.
|
||||
- `accountteardown` (dependency): its EC2 reapers call `domains/ec2/awsapi/*` action adapters directly, as they did under `gateway/ec2`. The ADR's `accountteardown` current-to-target map row replaces this with domain-owned cleanup capabilities before the package moves. It also holds a read-only consumer of the EC2 platform-inventory contract (`contracts/ec2/v1`, `ec2.v1.PlatformInventory`), served by `domains/ec2/accountinventory`; only the explicit dry run (`Engine.DryRun`) calls it, to log a shadow comparison, and the key-pair, placement-group and launch-template reapers still use the `awsapi` adapters. A domain-owned cleanup capability for the proven-safe subset of those three kinds (key pairs and empty placement groups no instance, spot request or live launch-template version names, and launch-template residue; never live launch templates) exists as `domains/ec2/accountcleanup` (a Go interface with injected owners and reference readers, no NATS subject); nothing calls it yet.
|
||||
- `domains/admission/quota` (dependency, moved from `handlers/quota` unchanged): `live.go` imports `gateway/elbv2` and `handlers/rds` for its live ELBv2 and RDS counters; `records.go` imports `runtime/compute/vm` for the instance-record vCPU sweep; `live.go`, `vcpu.go` and `storage.go` call `domains/ec2/awsapi/{vpc,eip,volume}` describe adapters to count VPCs, subnets, addresses and volumes, as they did under `gateway/ec2`. The ADR's `handlers/quota` current-to-target map row replaces these with reader capabilities once their final locations exist.
|
||||
- `domains/ec2/awsapi/instance` (dependency): imports `handlers/iam` (`IAMService` in `DescribeInstances.go`, `RunInstances.go` and `IamInstanceProfileAssociation.go`; `InstanceProfile` in the latter two), `runtime/compute/cache` (`NodeState`, `NodeUnknown`, `NodeLive`) and `runtime/compute/vm` (`VM`), both in `status_synthesis.go`. The IAM edge needs an IAM capability; the runtime edges are domain-to-runtime and need the instance record and node liveness exposed through an EC2-owned projection.
|
||||
- `domains/ec2/awsapi/spotinstance` (dependency): imports `handlers/iam.IAMService` and `domains/admission/quota.Service` in `RequestSpotInstances.go`.
|
||||
|
||||
@@ -31,7 +31,6 @@ services:
|
||||
- "ec2.*"
|
||||
- "elbv2.*"
|
||||
- "acm.*"
|
||||
- "route53.*"
|
||||
- "iam.*"
|
||||
- "rds.*"
|
||||
- "ecs.*"
|
||||
@@ -40,7 +39,6 @@ services:
|
||||
- "ec2.*"
|
||||
- "elbv2.*"
|
||||
- "acm.*"
|
||||
- "route53.*"
|
||||
- "iam.*"
|
||||
- "rds.*"
|
||||
- "ecs.*"
|
||||
@@ -78,7 +76,6 @@ services:
|
||||
- spinifex/domains/ec2/vpc
|
||||
- spinifex/handlers/elbv2
|
||||
- spinifex/domains/acm
|
||||
- spinifex/domains/route53
|
||||
- spinifex/handlers/iam
|
||||
- spinifex/handlers/ecs
|
||||
- spinifex/handlers/rds
|
||||
@@ -157,10 +154,6 @@ services:
|
||||
- "acm.DescribeCertificate"
|
||||
- "acm.ListCertificates"
|
||||
- "acm.DeleteCertificate"
|
||||
# Route 53 hosted zones (domains/route53; subscribed in daemon.go).
|
||||
- "route53.*"
|
||||
# Operator-only zone activation and delegation, sent by spx admin dns.
|
||||
- "route53.admin.*"
|
||||
- "elbv2.AddListenerCertificates"
|
||||
- "elbv2.RemoveListenerCertificates"
|
||||
- "elbv2.DescribeListenerCertificates"
|
||||
@@ -473,12 +466,6 @@ suites:
|
||||
# Exercises the full daemon stack under network failure injection;
|
||||
# any service touching the wire path is in scope.
|
||||
|
||||
e2e-route53:
|
||||
path: tests/e2e/route53
|
||||
covers: [awsgw, spinifex_daemon, predastore]
|
||||
# Hosted zones end to end: delegation check, publish to the northstar zone
|
||||
# store in predastore, and every node answering before a change is INSYNC.
|
||||
|
||||
# Suites below are bash-only today. Entries activate as the Go ports
|
||||
# land (Bead 2 fixtures unblock new-shape ports for multinode etc.).
|
||||
#
|
||||
|
||||
@@ -6,8 +6,8 @@ Suites live in `tests/e2e/<name>/` and build to `tests/e2e/_bin/<name>.test`. Ev
|
||||
|
||||
| Set | Suites |
|
||||
| --- | --- |
|
||||
| `E2E_SUITES_SINGLE` | `single iam cert eks ecs storagegrowth partialblock rds quota route53 storagefault` |
|
||||
| `E2E_SUITES_MULTI` | `multinode lb cert quota route53 kvquorum lbrecovery instancerecovery storagefault` |
|
||||
| `E2E_SUITES_SINGLE` | `single iam cert eks ecs storagegrowth partialblock rds quota storagefault` |
|
||||
| `E2E_SUITES_MULTI` | `multinode lb cert quota kvquorum lbrecovery instancerecovery storagefault` |
|
||||
| `E2E_SUITES_NIGHTLY_SINGLE` | `single cert iam` |
|
||||
| `E2E_SUITES_NIGHTLY_MULTI` | `multinode cert lb` |
|
||||
|
||||
@@ -120,7 +120,6 @@ The create/describe/connect path against a live cluster: a DB VM that actually b
|
||||
### Other
|
||||
|
||||
- **`quota`** — every enforced dimension refuses with the same `quotaExceeded` code; a launch is charged for every instance it asks for; one instance wider than the cap is refused.
|
||||
- **`route53`** — hosted zones as northstar serves them, on every node. `TestNorthstarServesZonesBeyondFirstListPage` writes 1,100 zone objects (more than one `ListObjectsV2` page), requires the last one to resolve on every node and the platform base zone to survive the sync, then requires the deleted zones to stop resolving.
|
||||
- **`gpu`** — device visible in the guest, driver access, launch.
|
||||
- **`bedrock`** — `ListFoundationModels`/`GetFoundationModel` against a running cluster. Wire-format parity is unit-tested, not here.
|
||||
- **`ddil`** — Denied, Disrupted, Intermittent, Limited network scenarios.
|
||||
|
||||
@@ -9,7 +9,6 @@ require (
|
||||
github.com/aws/aws-sdk-go-v2/config v1.33.9
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.20.9
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.3
|
||||
github.com/cenkalti/backoff/v4 v4.3.0
|
||||
github.com/charmbracelet/bubbles v1.0.0
|
||||
github.com/charmbracelet/bubbletea v1.3.10
|
||||
@@ -25,7 +24,7 @@ require (
|
||||
github.com/miekg/dns v1.1.73
|
||||
github.com/minio/highwayhash v1.0.4
|
||||
github.com/mulgadc/bluebottle v1.22.0
|
||||
github.com/mulgadc/northstar v1.22.1-0.20261023224547-8c5555ef1d1d
|
||||
github.com/mulgadc/northstar v1.22.0
|
||||
github.com/mulgadc/predastore v1.22.0
|
||||
github.com/mulgadc/viperblock v1.22.0
|
||||
github.com/nats-io/nats-server/v2 v2.15.1
|
||||
@@ -68,9 +67,10 @@ require (
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 // indirect
|
||||
|
||||
@@ -40,14 +40,14 @@ github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 h1:Qmn60RWuk9+bd2ddvJJqwf3Cpac3
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6/go.mod h1:h/YJhe4AVyQxEUMluph10iqV7mjajH7A9L+BHF5ppis=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 h1:e6YW5uME30TkrElR+4/xcY7NB6CCj2dcv7vtmGXhfhQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21/go.mod h1:obpHAMP693+f3/A8PkLhIqfK1C+NWWn8zXzqD3OH520=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7 h1:CwvFlKw0uUScfgOMflXSRZa+ayXgOf6wk7lhyMemQYU=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7/go.mod h1:4SbD8BB+p+QEnG0aX6DoWwrtk9SufyYXRsUv0nve8LE=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 h1:/TYsZXdA8UTa+WCtCYSAJIr1vwl0+eho6TUgJGwFFO8=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5/go.mod h1:qPqp1Uwd/BqdhPufv6oem9j5J7HNsgc2V22dUiDPn+s=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 h1:LHHdobjo5le1sjJdolitV7uVvYg3wkaeM1Ju2fDgeZQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6/go.mod h1:CBGB0evAiaMD3Lk4wPO7LQCcCZLuhzL5Qf/wfEUHQ5w=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6 h1:mYViDr2azcgxOpRrObZOl631bvavqILoE/L+Kxgbwys=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6/go.mod h1:OXVdAo9jNhggB+hJRGOyy7WolGTKB6Rg7IIulDeSzPM=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.3 h1:NyeuyXkZL476uoAowNActm4XJO1QPKf7aassgXHpiqA=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.3/go.mod h1:GfoBeKMDJecgRUsGf/0RGdx0NJAh81ObTd922h8pQys=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 h1:pPiWfgeNxqluKEph7hvU88kuGKBPOWzO+Dk9t2zqqNs=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4/go.mod h1:YlwGoIUDG/3kBQbdNOVs/xKZ9J01G8e/6D1mRBj9uTk=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0 h1:VMAdYqr4Jn/8ATs9BHC5riwrs0d6m1Z2ohFriSwZwm0=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0/go.mod h1:9APRWGLFITKD+xzWSIyT9V7QV4bNlEuIieWlzXgGFlI=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 h1:tvpDjuUvgKURf3Yk4hOaGbZ4QdY+070OWcA5d6LUv2A=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.10.4/go.mod h1:3ibVKg5Ng82Zby8++jELTD2nHxiPk2DJRLEeW/LKp8M=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 h1:ChCtagixSFOxltv0o6E3NT8F7ksvZtZGPwUBynO5Nmk=
|
||||
@@ -295,12 +295,8 @@ github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc
|
||||
github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
|
||||
github.com/mulgadc/bluebottle v1.22.0 h1:pGZ5/Ly3CYem62pPu2zKBRpvMpAG0LHMIMw9Yt/lkIg=
|
||||
github.com/mulgadc/bluebottle v1.22.0/go.mod h1:DwT91bJyUsrpg1gCJmcCOTG+tgw5J124LloHYvGAQu0=
|
||||
github.com/mulgadc/northstar v1.22.1-0.20261023220040-11ff69dd571a h1:juPJ0jubDYy6Sdk14BGnYB4QF7N0PhAmV522FdBA2yI=
|
||||
github.com/mulgadc/northstar v1.22.1-0.20261023220040-11ff69dd571a/go.mod h1:Halhinpgo5rzUhYemGN8uYTWu1pJsL5e/7l1eEKc7nM=
|
||||
github.com/mulgadc/northstar v1.22.1-0.20261023224401-31cf7e48e4db h1:OrXatiBIUEv6InRzh0zHIPnfp+CrIxf2+rX4T3fo7cI=
|
||||
github.com/mulgadc/northstar v1.22.1-0.20261023224401-31cf7e48e4db/go.mod h1:Halhinpgo5rzUhYemGN8uYTWu1pJsL5e/7l1eEKc7nM=
|
||||
github.com/mulgadc/northstar v1.22.1-0.20261023224547-8c5555ef1d1d h1:FTch26kEKKv+c4X3L0cQKMXwBJ4D1SCFlV4Mfgh7nEU=
|
||||
github.com/mulgadc/northstar v1.22.1-0.20261023224547-8c5555ef1d1d/go.mod h1:Halhinpgo5rzUhYemGN8uYTWu1pJsL5e/7l1eEKc7nM=
|
||||
github.com/mulgadc/northstar v1.22.0 h1:fTndlk47hK76fcNr2MqsyeKhBNVEDt8+BC6Z30ohokA=
|
||||
github.com/mulgadc/northstar v1.22.0/go.mod h1:yLH8waqfHZx177Ae6dGQGTDt/02rPE9alnUKyxDNhVs=
|
||||
github.com/mulgadc/predastore v1.22.0 h1:mUQJMTxMiRDVvemMP+34BP/+r1U0iCptc9B++u4NJ8o=
|
||||
github.com/mulgadc/predastore v1.22.0/go.mod h1:/cMIoKtnQG/DWyeX/0matEQrG2/abY60tNHWcBKS9Kc=
|
||||
github.com/mulgadc/viperblock v1.22.0 h1:8vS2AtpIDkIhGNOazQPicKXV2y/8PAy8BNrVWIOCP6I=
|
||||
|
||||
+39
-1
@@ -7,11 +7,45 @@ Node installation, dev-environment lifecycle, guest image builds, and verificati
|
||||
| Script | What it does |
|
||||
|---|---|
|
||||
| `setup.sh` | The production binary installer, served from `install.mulgadc.com`. Installs deps, users, sudoers, files, systemd units, logrotate and udev rules. `SETUP_STAGES` runs a subset; `ISO_BUILD=1` runs it inside the ISO builder's chroot. |
|
||||
| `setup-ovn.sh` | Bootstraps a compute node for OVN VPC networking: installs OVN/OVS, creates `br-int`, configures the WAN bridge (bridged, DHCP, or routed-NAT uplink), sets the chassis identity, and applies overlay sysctls. `--management` also starts the OVN central services. |
|
||||
| `setup-ovn.sh` | Bootstraps a compute node for OVN VPC networking: installs OVN/OVS, creates `br-int`, configures the WAN bridge (bridged, DHCP, or routed-NAT uplink), sets the chassis identity, and applies overlay sysctls. It runs OVS and OVN as root, setting `OVS_USER_ID` and `OVN_USER_ID` where `/etc/sysconfig` holds them, and hooks the socket-permission helper onto `ovs-vswitchd`, `ovn-controller`, `openvswitch-ipsec` and, on EL, `ovn-northd`. `--management` also starts the OVN central services. The links it creates persist through systemd-networkd where that is enabled, and otherwise through `/etc/spinifex/hostnet.json`, which `spinifex-hostnet.service` (`spx admin hostnet apply`) replays at boot and on every OVS start; the first run records the choice in `/etc/spinifex/hostnet-backend`. Where NetworkManager runs, it leaves the links Spinifex owns unmanaged. |
|
||||
| `install-node.sh` | Forms a multi-node cluster across hosts that are already running Spinifex: rebuilds the OVN database in clustered form, runs `spx admin init` on the first host and `spx admin join` on the rest, then verifies. The scripted form of [install-multi-node](../docs/install/install-multi-node/README.md). Destroys the joining nodes' logical network state and crypto — see its `--help`. |
|
||||
| `dev-install.sh` | Full local dev setup through the production installer: builds from source, packs a tarball, runs `setup.sh`, initialises the cluster, and starts services under systemd. |
|
||||
| `clone-deps.sh` | Clones or updates the viperblock and predastore checkouts alongside spinifex for cross-repo development. |
|
||||
|
||||
### `lib/distro.sh`: host-OS capability probes
|
||||
|
||||
The node scripts select host-OS behaviour by probing for the capability, not by reading `/etc/os-release`; only `setup.sh`'s supported-OS gate keys on the OS ID.
|
||||
Each probe prints its answer, or names every candidate it tried on stderr and returns 1, which it also does when it finds both candidates of a pair.
|
||||
|
||||
| Probe | Answers |
|
||||
|---|---|
|
||||
| `distro_pkg_manager` | `apt` or `dnf` |
|
||||
| `distro_ovs_unit` | `openvswitch-switch.service` (Debian, Ubuntu) or `openvswitch.service` (EL) |
|
||||
| `distro_ovn_central_layout` | `split` (Debian's `ovn-central` plus separate NB and SB units) or `lone-northd` (EL's single `ovn-northd.service`) |
|
||||
| `distro_ca_trust` | the trust-store update command and its anchor directory |
|
||||
| `distro_ovn_logrotate` | `/etc/logrotate.d/ovn-common` or `/etc/logrotate.d/ovn` |
|
||||
|
||||
`distro_missing_ovs_deps <unit-file>...` is a check rather than a probe: it prints each OVS or OVN unit the given unit files depend on or order against that is not installed, and fails only when a file cannot be read.
|
||||
`setup-ovn.sh` runs it against the installed Spinifex units and stops when one is missing, because systemd silently drops ordering against a unit that does not exist; a missing `openvswitch-ipsec.service` is only a warning.
|
||||
|
||||
`setup-ovn.sh`, `uninstall-spx.sh` and `dev-install.sh` source it from `lib/` beside themselves, then from `/usr/local/share/spinifex/lib/distro.sh`, where `setup.sh` installs it, and stop naming both paths when neither exists.
|
||||
`setup.sh` is piped from the install endpoint before the tarball is downloaded, and `install-node.sh --wipe` copies `node-reset.sh` alone to each host, so those two carry inline copies of the probes they need between `# --- BEGIN distro.sh probes ---` markers.
|
||||
`lib/distro_test.sh` fails when an inline copy differs from the library.
|
||||
Go code uses `spinifex/runtime/host/hostos` for the same purpose.
|
||||
|
||||
### System packages
|
||||
|
||||
`setup.sh` installs `APT_RUNTIME_PACKAGES` with apt, or on Oracle Linux `OL_RUNTIME_PACKAGES` with dnf, choosing by `distro_pkg_manager`.
|
||||
Before the dnf install it enables `ol10_UEKR8` and `ol10_kvm_utils`, excludes `qemu-kvm*` and `qemu-img*` from `ol10_appstream` so QEMU always comes from KVM Utilities, and writes `/etc/yum.repos.d/spinifex-nfv-sig.repo` for OVS 3.5 and OVN 25.03 from the CentOS NFV SIG.
|
||||
That repo carries an `includepkgs=` allowlist, and its key is refused unless its fingerprint matches `NFV_SIG_KEY_FPR`.
|
||||
It then installs `kernel-uek-modules` for the running kernel, stopping with a reboot instruction if that version has left the repositories, and loads `openvswitch`, `geneve`, `vhost_net` and `tun`.
|
||||
`make install-system` takes the same dnf path, adding the build packages from CodeReady Builder.
|
||||
|
||||
| Script | What it does |
|
||||
|---|---|
|
||||
| `lint-apt-packages.sh <image>` | Resolves the apt list against a Debian or Ubuntu image's real indices with `apt-get install -s`. |
|
||||
| `lint-dnf-packages.sh [image]` | Runs `setup.sh`'s Oracle Linux repository setup in an `oraclelinux:10` container, resolves the dnf list, and fails unless QEMU comes from KVM Utilities and OVS and OVN from the NFV SIG. |
|
||||
|
||||
## Teardown
|
||||
|
||||
Both ship to `/usr/local/share/spinifex/` on every node, so they are available on a host installed with `curl … | bash` and not only from a checkout. They differ in what they take away:
|
||||
@@ -21,6 +55,9 @@ Both ship to `/usr/local/share/spinifex/` on every node, so they are available o
|
||||
| `node-reset.sh` | Clears **state** and leaves the node installed: stops services, destroys guests, wipes `/etc/spinifex`, `/var/lib/spinifex`, `/var/log/spinifex` and the OVN databases, and removes the OVS bridges. The node is then ready for a fresh `spx admin init`. `--keep-data` preserves `/var/lib/spinifex`; `SPX_RESET_CONFIRM=reset` runs it unattended. |
|
||||
| `uninstall-spx.sh` | Removes the **software** — the inverse of `setup.sh`. Delegates the state teardown above to `node-reset.sh` rather than reimplementing it, then removes the binary, the nbdkit plugin, the systemd units and slices, the service users, the sudoers grant, the sysctl/logrotate/chrony/udev drop-ins, the `systemd-networkd` units, the host firewall policy and its loaded nft table, and the Spinifex CA from the host trust store. Verifies afterwards and exits non-zero naming anything that survived. |
|
||||
|
||||
`node-reset.sh` finds guests by process name, `qemu-system-*` or `qemu-kvm`, owned by the daemon's service user, and waits for them to exit before it deletes any state.
|
||||
`node-reset_test.sh` checks that match against real processes, and that `reset-dev-env.sh` uses the same pattern.
|
||||
|
||||
`uninstall-spx.sh` keeps `/var/lib/spinifex` and every apt package by default — uninstalling software should not destroy data, and a `curl … | bash` install lands on hosts that were already using openvswitch, libvirt and chrony. `--purge-data` and `--purge-deps` opt into each, and both confirm separately even under `--yes` (`SPX_PURGE_CONFIRM=destroy` for unattended data removal). `--dry-run` prints the whole plan and touches nothing.
|
||||
|
||||
**It refuses on a multi-node cluster, and `--yes` does not override that.** Uninstalling one node of a running cluster strands its guests and leaves the survivors reconciling against a chassis that will never answer. Run `spx admin cluster shutdown` first, then the uninstaller on each node.
|
||||
@@ -52,6 +89,7 @@ Both ship to `/usr/local/share/spinifex/` on every node, so they are available o
|
||||
| `build-system-image.sh` | Builds a minimal system AMI (Alpine or Ubuntu) from a manifest. All customisation happens inside the libguestfs appliance, so the build touches no host block device, no host mount, and needs no sudo. `--import` also registers it as an AMI. |
|
||||
| `publish-system-image.sh` | Converts a built raw image to compressed qcow2, writes the `.sha256` sidecar the catalog verifier expects, and uploads both to the R2 bucket behind `iso.mulgadc.com/system-ami/`. |
|
||||
| `build-microvm-image.sh` | Builds the minimal Alpine kernel + initramfs used by the QEMU microvm machine type and the lb-agent, carrying haproxy (ALB) and nginx-stream (NLB) data planes. |
|
||||
| `build-qboot.sh` | Builds qboot, the firmware the daemon passes to every microvm with `-bios`, from a pinned upstream commit in a digest-pinned Debian container. Fails unless the output matches the recorded sha256, which is also Debian 13's `qboot.rom`. Writes `qboot.rom` and its GPL-2.0 licence beside the microvm kernel, and the matching source tarball, which releases publish, under `build/qboot/`. `make build-microvm-image` runs it. |
|
||||
| `mkosi-build.sh` | Runs an mkosi system-image build inside the pinned builder container, so a runner needs only Docker and the toolchain cannot drift. `--image` names an output and expands to the right profile list. |
|
||||
| `mkosi-publish.sh` | Publishes an mkosi-built image to R2 from inside the same pinned container — qcow2 conversion, `.sha256` sidecar, and upload. |
|
||||
| `mkosi-common.sh` | Shared plumbing sourced by both mkosi scripts. Builds the toolchain image once, with the builder account matched to the invoking user so bind-mounted output is not written back as root. |
|
||||
|
||||
Executable
+75
@@ -0,0 +1,75 @@
|
||||
#!/bin/bash
|
||||
# build-qboot.sh — build qboot, the firmware the QEMU microvm machine boots
|
||||
# system instances (ALB/NLB) with, from a pinned upstream commit.
|
||||
#
|
||||
# Outputs:
|
||||
# $REPO_ROOT/build/microvm/qboot.rom firmware, shipped beside the microvm kernel
|
||||
# $REPO_ROOT/build/microvm/qboot.LICENSE qboot's GPL-2.0 licence text
|
||||
# $REPO_ROOT/build/qboot/qboot-<commit>.tar.gz the matching source, published with releases
|
||||
#
|
||||
# The build runs in a digest-pinned Debian container, so its output is
|
||||
# reproducible and is checked against a recorded hash. That hash is also the
|
||||
# sha256 of Debian 13's /usr/share/qemu/qboot.rom, which is built from the same commit.
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
OUT_DIR="$REPO_ROOT/build/microvm"
|
||||
SRC_DIR="$REPO_ROOT/build/qboot"
|
||||
|
||||
QBOOT_REPO="${QBOOT_REPO:-https://gitlab.com/qemu-project/qboot.git}"
|
||||
QBOOT_COMMIT="8ca302e86d685fa05b16e2b208888243da319941"
|
||||
QBOOT_SHA256="5c9c4b12c4438e3cf11d087263afe6519c8c05dd5896727b8b169b8f5dbf9877"
|
||||
BUILD_IMAGE="debian:trixie@sha256:4c25275446ca371766b8b093c696782eabce06263fccacb8d5d813b3dc44b308"
|
||||
|
||||
log() { echo "[build-qboot] $*"; }
|
||||
fail() { echo "ERROR: $*" >&2; exit 1; }
|
||||
|
||||
CONTAINER_TOOL=""
|
||||
for tool in docker podman; do
|
||||
if command -v "$tool" >/dev/null 2>&1 && "$tool" info >/dev/null 2>&1; then
|
||||
CONTAINER_TOOL=$tool
|
||||
break
|
||||
fi
|
||||
done
|
||||
[ -n "$CONTAINER_TOOL" ] || fail "qboot builds in a container; install docker or podman"
|
||||
command -v git >/dev/null 2>&1 || fail "required tool not found: git"
|
||||
|
||||
WORK=$(mktemp -d)
|
||||
trap 'rm -rf "$WORK"' EXIT
|
||||
|
||||
log "fetching qboot $QBOOT_COMMIT from $QBOOT_REPO"
|
||||
git init -q "$WORK/src"
|
||||
git -C "$WORK/src" fetch -q --depth 1 "$QBOOT_REPO" "$QBOOT_COMMIT"
|
||||
git -C "$WORK/src" checkout -q FETCH_HEAD
|
||||
got_commit=$(git -C "$WORK/src" rev-parse HEAD)
|
||||
[ "$got_commit" = "$QBOOT_COMMIT" ] || fail "fetched qboot $got_commit, not the pinned $QBOOT_COMMIT"
|
||||
|
||||
mkdir -p "$SRC_DIR"
|
||||
src_tarball="$SRC_DIR/qboot-${QBOOT_COMMIT}.tar.gz"
|
||||
git -C "$WORK/src" archive --format=tar.gz --prefix="qboot-${QBOOT_COMMIT}/" -o "$src_tarball" "$QBOOT_COMMIT"
|
||||
|
||||
log "building in $BUILD_IMAGE with $CONTAINER_TOOL"
|
||||
# The firmware comes back on stdout, so nothing the container writes is left
|
||||
# root-owned on the host.
|
||||
"$CONTAINER_TOOL" run --rm --platform linux/amd64 -v "$WORK/src:/src:ro" \
|
||||
-e DEBIAN_FRONTEND=noninteractive "$BUILD_IMAGE" sh -euc '
|
||||
{
|
||||
apt-get update -qq
|
||||
apt-get install -y -qq --no-install-recommends gcc libc6-dev meson ninja-build binutils
|
||||
meson setup /tmp/build /src
|
||||
ninja -C /tmp/build
|
||||
} >/dev/null
|
||||
cat /tmp/build/bios.bin
|
||||
' > "$WORK/qboot.rom"
|
||||
|
||||
got_sha=$(sha256sum "$WORK/qboot.rom" | cut -d' ' -f1)
|
||||
if [ "$got_sha" != "$QBOOT_SHA256" ]; then
|
||||
fail "qboot.rom has sha256 $got_sha, not the recorded $QBOOT_SHA256.
|
||||
The pinned commit or build image changed; rebuild, boot an ALB on it, then update QBOOT_SHA256."
|
||||
fi
|
||||
|
||||
mkdir -p "$OUT_DIR"
|
||||
install -m 0644 "$WORK/qboot.rom" "$OUT_DIR/qboot.rom"
|
||||
install -m 0644 "$WORK/src/LICENSE" "$OUT_DIR/qboot.LICENSE"
|
||||
log "qboot.rom sha256 $got_sha"
|
||||
log "source: $src_tarball"
|
||||
@@ -19,6 +19,19 @@ SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
MULGA_ROOT="$(cd "$PROJECT_ROOT/.." && pwd)"
|
||||
|
||||
# Host-OS capability probes: beside this script in a checkout and in the
|
||||
# installed share dir, else where setup.sh installs them.
|
||||
DISTRO_LIB=""
|
||||
for candidate in "$(dirname "${BASH_SOURCE[0]}")/lib/distro.sh" /usr/local/share/spinifex/lib/distro.sh; do
|
||||
if [ -f "$candidate" ]; then DISTRO_LIB="$candidate"; break; fi
|
||||
done
|
||||
if [ -z "$DISTRO_LIB" ]; then
|
||||
echo "ERROR: distro.sh not found at $(dirname "${BASH_SOURCE[0]}")/lib/distro.sh or /usr/local/share/spinifex/lib/distro.sh; re-run setup.sh to install it" >&2
|
||||
exit 1
|
||||
fi
|
||||
# shellcheck source=lib/distro.sh
|
||||
. "$DISTRO_LIB"
|
||||
|
||||
echo "=== Building ==="
|
||||
cd "$PROJECT_ROOT" && make build
|
||||
cd "$MULGA_ROOT/viperblock" && make go_build_nbd
|
||||
@@ -31,6 +44,8 @@ cp "$MULGA_ROOT/viperblock/lib/nbdkit-viperblock-plugin.so" "$STAGING/"
|
||||
cp "$PROJECT_ROOT/scripts/setup-ovn.sh" "$STAGING/"
|
||||
cp "$PROJECT_ROOT/scripts/node-reset.sh" "$STAGING/"
|
||||
cp "$PROJECT_ROOT/scripts/uninstall-spx.sh" "$STAGING/"
|
||||
mkdir -p "$STAGING/lib"
|
||||
cp "$PROJECT_ROOT/scripts/lib/distro.sh" "$STAGING/lib/"
|
||||
mkdir -p "$STAGING/systemd"
|
||||
cp "$PROJECT_ROOT/build/systemd/"* "$STAGING/systemd/"
|
||||
mkdir -p "$STAGING/scripts"
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
# shellcheck shell=bash
|
||||
# distro.sh — host-OS capability probes. Each prints its answer, or names every
|
||||
# candidate it tried on stderr and returns 1, also when it finds both of a pair.
|
||||
# setup.sh and node-reset.sh carry inline copies; distro_test.sh keeps them equal.
|
||||
|
||||
# Reads unit files off disk, not through systemctl, so it answers in a chroot.
|
||||
# DISTRO_ROOT prefixes every host path the probes read; tests set it.
|
||||
distro_unit_exists() {
|
||||
local dir
|
||||
for dir in /etc/systemd/system /run/systemd/system /usr/lib/systemd/system /lib/systemd/system; do
|
||||
if [ -e "${DISTRO_ROOT:-}$dir/$1" ] || [ -L "${DISTRO_ROOT:-}$dir/$1" ]; then
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
distro_pkg_manager() {
|
||||
local apt=false dnf=false
|
||||
command -v apt-get >/dev/null 2>&1 && apt=true
|
||||
command -v dnf >/dev/null 2>&1 && dnf=true
|
||||
if $apt && $dnf; then
|
||||
echo "distro.sh: package manager: found both apt-get and dnf; cannot tell which one manages this host" >&2
|
||||
return 1
|
||||
fi
|
||||
if $apt; then echo apt; return 0; fi
|
||||
if $dnf; then echo dnf; return 0; fi
|
||||
echo "distro.sh: package manager: found neither apt-get nor dnf on PATH" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
# The OVS umbrella unit: openvswitch-switch.service on Debian, openvswitch.service on EL.
|
||||
distro_ovs_unit() {
|
||||
local deb=false el=false
|
||||
distro_unit_exists openvswitch-switch.service && deb=true
|
||||
distro_unit_exists openvswitch.service && el=true
|
||||
if $deb && $el; then
|
||||
echo "distro.sh: OVS unit: found both openvswitch-switch.service and openvswitch.service; cannot tell which one owns OVS" >&2
|
||||
return 1
|
||||
fi
|
||||
if $deb; then echo openvswitch-switch.service; return 0; fi
|
||||
if $el; then echo openvswitch.service; return 0; fi
|
||||
echo "distro.sh: OVS unit: found neither openvswitch-switch.service nor openvswitch.service; is Open vSwitch installed?" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
# "split": Debian's ovn-central plus separate NB and SB DB units.
|
||||
# "lone-northd": EL's single ovn-northd.service, which starts both DB servers.
|
||||
distro_ovn_central_layout() {
|
||||
local split=false lone=false unit missing=""
|
||||
for unit in ovn-central.service ovn-ovsdb-server-nb.service ovn-ovsdb-server-sb.service; do
|
||||
if distro_unit_exists "$unit"; then
|
||||
split=true
|
||||
else
|
||||
missing="${missing:+$missing, }$unit"
|
||||
fi
|
||||
done
|
||||
if distro_unit_exists ovn-northd.service && [ -f "${DISTRO_ROOT:-}/etc/sysconfig/ovn" ]; then
|
||||
lone=true
|
||||
fi
|
||||
if $split && $lone; then
|
||||
echo "distro.sh: OVN central layout: found both the split units (ovn-central.service) and a lone ovn-northd.service with /etc/sysconfig/ovn; cannot tell which one owns the OVN databases" >&2
|
||||
return 1
|
||||
fi
|
||||
if $split; then
|
||||
if [ -n "$missing" ]; then
|
||||
echo "distro.sh: OVN central layout: ovn-central is installed but $missing is missing" >&2
|
||||
return 1
|
||||
fi
|
||||
echo split
|
||||
return 0
|
||||
fi
|
||||
if $lone; then echo lone-northd; return 0; fi
|
||||
echo "distro.sh: OVN central layout: found neither ovn-central.service nor ovn-northd.service with /etc/sysconfig/ovn; is OVN central installed?" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
# The trust-store update command and its local anchor directory, space-separated.
|
||||
distro_ca_trust() {
|
||||
local deb=false el=false
|
||||
command -v update-ca-certificates >/dev/null 2>&1 && deb=true
|
||||
command -v update-ca-trust >/dev/null 2>&1 && el=true
|
||||
if $deb && $el; then
|
||||
echo "distro.sh: CA trust: found both update-ca-certificates and update-ca-trust; cannot tell which trust store TLS clients read" >&2
|
||||
return 1
|
||||
fi
|
||||
if $deb; then echo "update-ca-certificates /usr/local/share/ca-certificates"; return 0; fi
|
||||
if $el; then echo "update-ca-trust /etc/pki/ca-trust/source/anchors"; return 0; fi
|
||||
echo "distro.sh: CA trust: found neither update-ca-certificates nor update-ca-trust on PATH" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
distro_ovn_logrotate() {
|
||||
local deb=false el=false
|
||||
[ -f "${DISTRO_ROOT:-}/etc/logrotate.d/ovn-common" ] && deb=true
|
||||
[ -f "${DISTRO_ROOT:-}/etc/logrotate.d/ovn" ] && el=true
|
||||
if $deb && $el; then
|
||||
echo "distro.sh: OVN logrotate: found both /etc/logrotate.d/ovn-common and /etc/logrotate.d/ovn; cannot tell which one the OVN packages own" >&2
|
||||
return 1
|
||||
fi
|
||||
if $deb; then echo /etc/logrotate.d/ovn-common; return 0; fi
|
||||
if $el; then echo /etc/logrotate.d/ovn; return 0; fi
|
||||
echo "distro.sh: OVN logrotate: found neither /etc/logrotate.d/ovn-common nor /etc/logrotate.d/ovn; is OVN installed?" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
# The OVS and OVN units the given unit files depend on or order against that
|
||||
# are not installed, one per line; prints nothing when all are. Fails only
|
||||
# when a unit file cannot be read.
|
||||
distro_missing_ovs_deps() {
|
||||
local file line dep missing=""
|
||||
for file in "$@"; do
|
||||
if [ ! -r "${DISTRO_ROOT:-}$file" ]; then
|
||||
echo "distro.sh: OVS dependencies: cannot read $file" >&2
|
||||
return 1
|
||||
fi
|
||||
while IFS= read -r line || [ -n "$line" ]; do
|
||||
case "$line" in
|
||||
After=* | Before=* | Wants=* | Requires=* | Requisite=* | BindsTo=* | PartOf=* | Upholds=*) ;;
|
||||
*) continue ;;
|
||||
esac
|
||||
for dep in ${line#*=}; do
|
||||
case "$dep" in
|
||||
ovs* | ovn* | openvswitch*) ;;
|
||||
*) continue ;;
|
||||
esac
|
||||
case " $missing " in
|
||||
*" $dep "*) continue ;;
|
||||
esac
|
||||
distro_unit_exists "$dep" || missing="${missing:+$missing }$dep"
|
||||
done
|
||||
done <"${DISTRO_ROOT:-}$file"
|
||||
done
|
||||
for dep in $missing; do echo "$dep"; done
|
||||
}
|
||||
@@ -0,0 +1,233 @@
|
||||
#!/bin/sh
|
||||
# distro_test.sh — tests for the host-OS probes and the inline copies of them.
|
||||
#
|
||||
# Every probe runs against a fixture tree (DISTRO_ROOT) and a PATH holding only
|
||||
# stub commands, so the result never depends on the distro running the test.
|
||||
set -eu
|
||||
|
||||
LIB_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
|
||||
SCRIPTS_DIR=$(dirname -- "$LIB_DIR")
|
||||
DISTRO="$LIB_DIR/distro.sh"
|
||||
TMP=$(mktemp -d)
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
|
||||
fails=0
|
||||
pass() { echo " ok - $1"; }
|
||||
fail() { echo " FAIL - $1"; fails=$((fails + 1)); }
|
||||
|
||||
echo "-- lib/distro.sh"
|
||||
|
||||
# fixture <name> — a fresh empty host root and stub bin dir.
|
||||
fixture() {
|
||||
ROOT="$TMP/$1/root"
|
||||
BIN="$TMP/$1/bin"
|
||||
mkdir -p "$ROOT" "$BIN"
|
||||
}
|
||||
unit() { mkdir -p "$ROOT/usr/lib/systemd/system" && : > "$ROOT/usr/lib/systemd/system/$1"; }
|
||||
file() { mkdir -p "$ROOT$(dirname "$1")" && : > "$ROOT$1"; }
|
||||
cmd() { printf '#!/bin/sh\nexit 0\n' > "$BIN/$1" && chmod +x "$BIN/$1"; }
|
||||
|
||||
# probe <fn> [arg...] — runs one probe in bash against the current fixture,
|
||||
# with only the stub dir on PATH. Sets OUT, ERR and RC.
|
||||
probe() {
|
||||
RC=0
|
||||
OUT=$(DISTRO_ROOT="$ROOT" PATH="$BIN" /bin/bash -c '. "$1"; shift; "$@"' _ "$DISTRO" "$@" 2>"$TMP/err") || RC=$?
|
||||
ERR=$(cat "$TMP/err")
|
||||
}
|
||||
|
||||
# expect_ok <desc> <want>; expect_fail <desc> <word>... — every word must appear
|
||||
# in the error, which is how "names every candidate" is checked.
|
||||
expect_ok() {
|
||||
if [ "$RC" = 0 ] && [ "$OUT" = "$2" ]; then pass "$1"; else fail "$1 (rc=$RC out='$OUT' err='$ERR')"; fi
|
||||
}
|
||||
expect_fail() {
|
||||
desc=$1; shift
|
||||
if [ "$RC" = 0 ]; then fail "$desc (succeeded with '$OUT')"; return; fi
|
||||
for word in "$@"; do
|
||||
case "$ERR" in
|
||||
*"$word"*) ;;
|
||||
*) fail "$desc (error does not name $word: '$ERR')"; return ;;
|
||||
esac
|
||||
done
|
||||
pass "$desc"
|
||||
}
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Package manager.
|
||||
# --------------------------------------------------------------------------
|
||||
fixture pm-apt; cmd apt-get; probe distro_pkg_manager
|
||||
expect_ok "package manager: apt-get alone is apt" apt
|
||||
fixture pm-dnf; cmd dnf; probe distro_pkg_manager
|
||||
expect_ok "package manager: dnf alone is dnf" dnf
|
||||
fixture pm-both; cmd apt-get; cmd dnf; probe distro_pkg_manager
|
||||
expect_fail "package manager: both fails naming both" apt-get dnf
|
||||
fixture pm-none; probe distro_pkg_manager
|
||||
expect_fail "package manager: neither fails naming both" apt-get dnf
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# OVS umbrella unit. Unit files are found under any systemd unit directory, and
|
||||
# a masked unit (a symlink to /dev/null) still counts as installed.
|
||||
# --------------------------------------------------------------------------
|
||||
fixture ovs-deb; unit openvswitch-switch.service; probe distro_ovs_unit
|
||||
expect_ok "OVS unit: Debian name" openvswitch-switch.service
|
||||
fixture ovs-el; unit openvswitch.service; probe distro_ovs_unit
|
||||
expect_ok "OVS unit: EL name" openvswitch.service
|
||||
fixture ovs-masked; mkdir -p "$ROOT/etc/systemd/system"
|
||||
ln -s /dev/null "$ROOT/etc/systemd/system/openvswitch.service"; probe distro_ovs_unit
|
||||
expect_ok "OVS unit: a masked unit in /etc counts" openvswitch.service
|
||||
fixture ovs-lib; mkdir -p "$ROOT/lib/systemd/system"
|
||||
: > "$ROOT/lib/systemd/system/openvswitch-switch.service"; probe distro_ovs_unit
|
||||
expect_ok "OVS unit: found under /lib on an unmerged-/usr host" openvswitch-switch.service
|
||||
fixture ovs-both; unit openvswitch-switch.service; unit openvswitch.service; probe distro_ovs_unit
|
||||
expect_fail "OVS unit: both fails naming both" openvswitch-switch.service openvswitch.service
|
||||
fixture ovs-none; probe distro_ovs_unit
|
||||
expect_fail "OVS unit: neither fails naming both" openvswitch-switch.service openvswitch.service
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# OVN central layout. Both distros ship ovn-northd.service, so the unit alone
|
||||
# must not decide it: Debian's split layout carries one too.
|
||||
# --------------------------------------------------------------------------
|
||||
split_units() { unit ovn-central.service; unit ovn-ovsdb-server-nb.service; unit ovn-ovsdb-server-sb.service; unit ovn-northd.service; }
|
||||
|
||||
fixture ovn-split; split_units; probe distro_ovn_central_layout
|
||||
expect_ok "OVN layout: Debian split units" split
|
||||
fixture ovn-lone; unit ovn-northd.service; file /etc/sysconfig/ovn; probe distro_ovn_central_layout
|
||||
expect_ok "OVN layout: EL lone ovn-northd" lone-northd
|
||||
fixture ovn-both; split_units; file /etc/sysconfig/ovn; probe distro_ovn_central_layout
|
||||
expect_fail "OVN layout: both fails naming both" ovn-central.service ovn-northd.service /etc/sysconfig/ovn
|
||||
fixture ovn-partial; unit ovn-central.service; unit ovn-ovsdb-server-nb.service; probe distro_ovn_central_layout
|
||||
expect_fail "OVN layout: a partial split install names the missing unit" ovn-ovsdb-server-sb.service
|
||||
fixture ovn-northd-only; unit ovn-northd.service; probe distro_ovn_central_layout
|
||||
expect_fail "OVN layout: ovn-northd without /etc/sysconfig/ovn is not EL" ovn-central.service ovn-northd.service /etc/sysconfig/ovn
|
||||
fixture ovn-none; probe distro_ovn_central_layout
|
||||
expect_fail "OVN layout: neither fails naming both" ovn-central.service ovn-northd.service
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# CA trust.
|
||||
# --------------------------------------------------------------------------
|
||||
fixture ca-deb; cmd update-ca-certificates; probe distro_ca_trust
|
||||
expect_ok "CA trust: Debian command and anchor dir" "update-ca-certificates /usr/local/share/ca-certificates"
|
||||
fixture ca-el; cmd update-ca-trust; probe distro_ca_trust
|
||||
expect_ok "CA trust: EL command and anchor dir" "update-ca-trust /etc/pki/ca-trust/source/anchors"
|
||||
fixture ca-both; cmd update-ca-certificates; cmd update-ca-trust; probe distro_ca_trust
|
||||
expect_fail "CA trust: both fails naming both" update-ca-certificates update-ca-trust
|
||||
fixture ca-none; probe distro_ca_trust
|
||||
expect_fail "CA trust: neither fails naming both" update-ca-certificates update-ca-trust
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# OVN logrotate file.
|
||||
# --------------------------------------------------------------------------
|
||||
fixture lr-deb; file /etc/logrotate.d/ovn-common; probe distro_ovn_logrotate
|
||||
expect_ok "OVN logrotate: Debian file" /etc/logrotate.d/ovn-common
|
||||
fixture lr-el; file /etc/logrotate.d/ovn; probe distro_ovn_logrotate
|
||||
expect_ok "OVN logrotate: EL file" /etc/logrotate.d/ovn
|
||||
fixture lr-both; file /etc/logrotate.d/ovn-common; file /etc/logrotate.d/ovn; probe distro_ovn_logrotate
|
||||
expect_fail "OVN logrotate: both fails naming both" /etc/logrotate.d/ovn-common /etc/logrotate.d/ovn
|
||||
fixture lr-none; probe distro_ovn_logrotate
|
||||
expect_fail "OVN logrotate: neither fails naming both" /etc/logrotate.d/ovn-common /etc/logrotate.d/ovn
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# OVS and OVN units the shipped Spinifex units name. Run against the real unit
|
||||
# files, so a unit that names a Debian-only OVS unit fails on the EL fixture.
|
||||
# --------------------------------------------------------------------------
|
||||
UNITS_SRC="$(dirname -- "$SCRIPTS_DIR")/build/systemd"
|
||||
spinifex_units() {
|
||||
mkdir -p "$ROOT/etc/systemd/system"
|
||||
cp "$UNITS_SRC"/spinifex.target "$UNITS_SRC"/spinifex-*.service "$ROOT/etc/systemd/system/"
|
||||
}
|
||||
deps_probe() {
|
||||
set --
|
||||
for f in "$ROOT"/etc/systemd/system/spinifex*; do set -- "$@" "${f#"$ROOT"}"; done
|
||||
probe distro_missing_ovs_deps "$@"
|
||||
}
|
||||
el_ovs_units() {
|
||||
for u in openvswitch.service ovsdb-server.service ovs-vswitchd.service ovn-controller.service ovn-northd.service openvswitch-ipsec.service; do unit "$u"; done
|
||||
}
|
||||
|
||||
fixture deps-el; spinifex_units; el_ovs_units; deps_probe
|
||||
expect_ok "OVS dependencies: every one the shipped units name is installed on EL" ""
|
||||
fixture deps-no-ipsec; spinifex_units; el_ovs_units; rm "$ROOT/usr/lib/systemd/system/openvswitch-ipsec.service"; deps_probe
|
||||
expect_ok "OVS dependencies: a missing unit is printed once" openvswitch-ipsec.service
|
||||
fixture deps-debian-only; el_ovs_units
|
||||
mkdir -p "$ROOT/etc/systemd/system"
|
||||
printf '[Unit]\n# After=ovs-old.service\nAfter=openvswitch-switch.service ovn-controller.service\nWants=openvswitch-switch.service\n' >"$ROOT/etc/systemd/system/spinifex-x.service"
|
||||
printf '[Unit]\nRequires=ovn-central.service' >"$ROOT/etc/systemd/system/spinifex-y.service"
|
||||
deps_probe
|
||||
expect_ok "OVS dependencies: Debian-only names are missing on EL, comments are not read" "openvswitch-switch.service
|
||||
ovn-central.service"
|
||||
fixture deps-unreadable; probe distro_missing_ovs_deps /etc/systemd/system/spinifex.target
|
||||
expect_fail "OVS dependencies: an unreadable unit file fails naming it" "cannot read /etc/systemd/system/spinifex.target"
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Inline copies. setup.sh and node-reset.sh cannot source the library, so they
|
||||
# carry copies between markers; each copied function must match the library's
|
||||
# byte for byte, or a fix to one silently misses the other.
|
||||
# --------------------------------------------------------------------------
|
||||
# fn_body <file> <name> — the function from "name() {" to the first "}" line.
|
||||
fn_body() {
|
||||
awk -v name="$2" '$0 == name "() {" { on = 1 } on { print } on && $0 == "}" { exit }' "$1"
|
||||
}
|
||||
|
||||
for copy in setup.sh node-reset.sh; do
|
||||
path="$SCRIPTS_DIR/$copy"
|
||||
block=$(sed -n '/^# --- BEGIN distro.sh probes ---$/,/^# --- END distro.sh probes ---$/p' "$path")
|
||||
if [ -z "$block" ]; then
|
||||
fail "$copy: no distro.sh probe block"
|
||||
continue
|
||||
fi
|
||||
names=$(printf '%s\n' "$block" | sed -n 's/^\([a-z_][a-z0-9_]*\)() {$/\1/p')
|
||||
if [ -z "$names" ]; then
|
||||
fail "$copy: the distro.sh probe block holds no functions"
|
||||
continue
|
||||
fi
|
||||
for name in $names; do
|
||||
want=$(fn_body "$DISTRO" "$name")
|
||||
got=$(fn_body "$path" "$name")
|
||||
if [ -z "$want" ]; then
|
||||
fail "$copy: $name is not in lib/distro.sh"
|
||||
elif [ "$got" = "$want" ]; then
|
||||
pass "$copy: $name matches lib/distro.sh"
|
||||
else
|
||||
fail "$copy: $name differs from lib/distro.sh"
|
||||
fi
|
||||
done
|
||||
|
||||
# A copied probe calling a probe helper the block lacks fails only when run.
|
||||
for called in $(printf '%s\n' "$block" | grep -v '^ *#' | grep -o 'distro_[a-z_]*' | sort -u); do
|
||||
case " $(echo $names) " in
|
||||
*" $called "*) ;;
|
||||
*) fail "$copy: copies a probe that calls $called without copying it" ;;
|
||||
esac
|
||||
done
|
||||
done
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# A consumer with no library beside it and none installed must stop naming
|
||||
# both paths, not run on with the probes undefined. Skipped on a host where
|
||||
# setup.sh has installed the library, since the fallback then finds it.
|
||||
# --------------------------------------------------------------------------
|
||||
if [ -e /usr/local/share/spinifex/lib/distro.sh ]; then
|
||||
echo " skip - consumers without distro.sh: /usr/local/share/spinifex/lib/distro.sh is installed here"
|
||||
else
|
||||
# uninstall-spx.sh sources the library after its --help, so it is run into
|
||||
# its multi-node refusal, which exits 1 without touching the host either way.
|
||||
mkdir -p "$TMP/alone/bin" "$TMP/alone/etc"
|
||||
printf '#!/bin/sh\nexec "$@"\n' > "$TMP/alone/bin/sudo"
|
||||
chmod +x "$TMP/alone/bin/sudo"
|
||||
printf '[nodes.node1]\n[nodes.node2]\n' > "$TMP/alone/etc/spinifex.toml"
|
||||
for consumer in "setup-ovn.sh --help" "uninstall-spx.sh --dry-run"; do
|
||||
script=${consumer%% *}
|
||||
cp "$SCRIPTS_DIR/$script" "$TMP/alone/"
|
||||
PATH="$TMP/alone/bin:$PATH" SPX_ETC_DIR="$TMP/alone/etc" "$TMP/alone/$script" "${consumer#* }" >"$TMP/out" 2>&1 && rc=0 || rc=$?
|
||||
if [ "$rc" != 0 ] && grep -q "distro.sh not found at $TMP/alone/lib/distro.sh or /usr/local/share/spinifex/lib/distro.sh" "$TMP/out"; then
|
||||
pass "$script: stops naming both paths when distro.sh is missing"
|
||||
else
|
||||
fail "$script: without distro.sh exited $rc: $(head -3 "$TMP/out")"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
if [ "$fails" -gt 0 ]; then
|
||||
echo "$fails failure(s)"
|
||||
exit 1
|
||||
fi
|
||||
Executable
+45
@@ -0,0 +1,45 @@
|
||||
#!/usr/bin/env bash
|
||||
# Runs setup.sh's Oracle Linux repository setup in a container, then resolves
|
||||
# its runtime dnf list against the real repositories with dnf install --assumeno.
|
||||
# Catches a renamed or dropped package, a SIG dependency the includepkgs
|
||||
# allowlist misses, a changed SIG key, and QEMU or OVS resolving from the wrong
|
||||
# repository.
|
||||
set -euo pipefail
|
||||
|
||||
IMAGE="${1:-oraclelinux:10}"
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
|
||||
echo "Resolving ${IMAGE}..."
|
||||
# The repository setup and the list both come from setup.sh itself, so the lint
|
||||
# cannot drift from what the installer does.
|
||||
docker run --rm -v "${SCRIPT_DIR}/setup.sh:/setup.sh:ro" "${IMAGE}" bash -c '
|
||||
set -euo pipefail
|
||||
INSTALL_SPINIFEX_LIB_ONLY=1 . /setup.sh
|
||||
SUDO=""
|
||||
ensure_ol_repos
|
||||
|
||||
# --assumeno always exits 1, so the transaction is judged from its output.
|
||||
# shellcheck disable=SC2086
|
||||
out=$(dnf install --assumeno $OL_RUNTIME_PACKAGES 2>&1) || true
|
||||
if ! grep -q "^Operation aborted" <<< "$out" || grep -qE "^(Error|Problem)|No match for argument" <<< "$out"; then
|
||||
echo "$out"
|
||||
echo "FAIL: the Oracle Linux package list does not resolve" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# repo_of <package> — the repository dnf would install the package from.
|
||||
repo_of() { awk -v p="$1" "\$1 == p { print \$4; exit }" <<< "$out"; }
|
||||
for want in "qemu-kvm $OL_KVM_UTILS_REPO" "qemu-img $OL_KVM_UTILS_REPO" \
|
||||
"openvswitch3.5 $NFV_SIG_REPO_ID" "ovn25.03-central $NFV_SIG_REPO_ID"; do
|
||||
pkg=${want% *} repo=${want#* }
|
||||
got=$(repo_of "$pkg")
|
||||
if [ "$got" != "$repo" ]; then
|
||||
echo "$out"
|
||||
echo "FAIL: $pkg resolves from ${got:-nowhere}, not $repo" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
'
|
||||
|
||||
echo "OK: every package resolves on ${IMAGE}, QEMU from KVM Utilities and OVS/OVN from the NFV SIG"
|
||||
+78
-3
@@ -57,6 +57,77 @@ run() {
|
||||
"$@"
|
||||
}
|
||||
|
||||
# --- BEGIN distro.sh probes ---
|
||||
# Copied from scripts/lib/distro.sh: install-node.sh --wipe copies this script
|
||||
# alone to each host, so it cannot source the library. distro_test.sh keeps them equal.
|
||||
distro_unit_exists() {
|
||||
local dir
|
||||
for dir in /etc/systemd/system /run/systemd/system /usr/lib/systemd/system /lib/systemd/system; do
|
||||
if [ -e "${DISTRO_ROOT:-}$dir/$1" ] || [ -L "${DISTRO_ROOT:-}$dir/$1" ]; then
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
distro_ovs_unit() {
|
||||
local deb=false el=false
|
||||
distro_unit_exists openvswitch-switch.service && deb=true
|
||||
distro_unit_exists openvswitch.service && el=true
|
||||
if $deb && $el; then
|
||||
echo "distro.sh: OVS unit: found both openvswitch-switch.service and openvswitch.service; cannot tell which one owns OVS" >&2
|
||||
return 1
|
||||
fi
|
||||
if $deb; then echo openvswitch-switch.service; return 0; fi
|
||||
if $el; then echo openvswitch.service; return 0; fi
|
||||
echo "distro.sh: OVS unit: found neither openvswitch-switch.service nor openvswitch.service; is Open vSwitch installed?" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
distro_ovn_central_layout() {
|
||||
local split=false lone=false unit missing=""
|
||||
for unit in ovn-central.service ovn-ovsdb-server-nb.service ovn-ovsdb-server-sb.service; do
|
||||
if distro_unit_exists "$unit"; then
|
||||
split=true
|
||||
else
|
||||
missing="${missing:+$missing, }$unit"
|
||||
fi
|
||||
done
|
||||
if distro_unit_exists ovn-northd.service && [ -f "${DISTRO_ROOT:-}/etc/sysconfig/ovn" ]; then
|
||||
lone=true
|
||||
fi
|
||||
if $split && $lone; then
|
||||
echo "distro.sh: OVN central layout: found both the split units (ovn-central.service) and a lone ovn-northd.service with /etc/sysconfig/ovn; cannot tell which one owns the OVN databases" >&2
|
||||
return 1
|
||||
fi
|
||||
if $split; then
|
||||
if [ -n "$missing" ]; then
|
||||
echo "distro.sh: OVN central layout: ovn-central is installed but $missing is missing" >&2
|
||||
return 1
|
||||
fi
|
||||
echo split
|
||||
return 0
|
||||
fi
|
||||
if $lone; then echo lone-northd; return 0; fi
|
||||
echo "distro.sh: OVN central layout: found neither ovn-central.service nor ovn-northd.service with /etc/sysconfig/ovn; is OVN central installed?" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
distro_ca_trust() {
|
||||
local deb=false el=false
|
||||
command -v update-ca-certificates >/dev/null 2>&1 && deb=true
|
||||
command -v update-ca-trust >/dev/null 2>&1 && el=true
|
||||
if $deb && $el; then
|
||||
echo "distro.sh: CA trust: found both update-ca-certificates and update-ca-trust; cannot tell which trust store TLS clients read" >&2
|
||||
return 1
|
||||
fi
|
||||
if $deb; then echo "update-ca-certificates /usr/local/share/ca-certificates"; return 0; fi
|
||||
if $el; then echo "update-ca-trust /etc/pki/ca-trust/source/anchors"; return 0; fi
|
||||
echo "distro.sh: CA trust: found neither update-ca-certificates nor update-ca-trust on PATH" >&2
|
||||
return 1
|
||||
}
|
||||
# --- END distro.sh probes ---
|
||||
|
||||
WIPE_DIRS=("$ETC_DIR" "$LOG_DIR" "$RUN_DIR")
|
||||
$KEEP_DATA || WIPE_DIRS+=("$DATA_DIR")
|
||||
|
||||
@@ -102,6 +173,9 @@ fi
|
||||
|
||||
log "stopping services"
|
||||
run sudo systemctl stop spinifex.target 2>/dev/null || true
|
||||
# spinifex-hostnet starts with OVS, which is started below to delete bridges; it
|
||||
# would rebuild the old veth pairs. setup-ovn.sh enables it again.
|
||||
run sudo systemctl disable --now spinifex-hostnet.service 2>/dev/null || true
|
||||
run sudo systemctl reset-failed 'spinifex-*' 2>/dev/null || true
|
||||
|
||||
# Stopping the target is not the same as its services being stopped.
|
||||
@@ -134,8 +208,9 @@ fi
|
||||
# `pgrep -x` and `pkill -x` match against comm, which the kernel truncates to 15
|
||||
# characters, so `qemu-system-x86_64` is stored as `qemu-system-x86` and an -x
|
||||
# match on the full name can never fire. Match the prefix instead: it is 11
|
||||
# characters, well under the limit, and covers both x86_64 and aarch64. This is
|
||||
# the same rule qemuProcessPrefix applies in spinifex/runtime/compute/vm/orphan_scan.go.
|
||||
# characters, well under the limit, and covers both x86_64 and aarch64. EL-family
|
||||
# hosts run /usr/libexec/qemu-kvm, whose comm is exactly `qemu-kvm`. This is the
|
||||
# rule hostos.IsQEMUProcess applies in spinifex/runtime/host/hostos.
|
||||
#
|
||||
# Deliberately not -f. That matches the whole command line, where argv[0] may
|
||||
# carry a path, so an anchored pattern misses the process it was written for.
|
||||
@@ -144,7 +219,7 @@ fi
|
||||
# ones has both under the same comm and an unscoped kill would take the wrong
|
||||
# set down. The daemon runs guests as its own service user on an installed node;
|
||||
# a dev install runs them as whoever invoked spx.
|
||||
QEMU_PGREP=('^qemu-system')
|
||||
QEMU_PGREP=('^(qemu-system|qemu-kvm$)')
|
||||
for qemu_user in spinifex-daemon spinifex "${SUDO_USER:-$(id -un)}"; do
|
||||
if id -u "$qemu_user" >/dev/null 2>&1; then
|
||||
QEMU_PGREP=(-u "$qemu_user" "${QEMU_PGREP[@]}")
|
||||
|
||||
Executable
+90
@@ -0,0 +1,90 @@
|
||||
#!/bin/sh
|
||||
# node-reset_test.sh — tests the QEMU process match node-reset.sh uses to stop
|
||||
# guests before it deletes Viperblock state, against real processes.
|
||||
#
|
||||
# Each fake guest is a copy of sleep named like a QEMU binary, so the kernel
|
||||
# gives it the same comm, including the 15-character truncation.
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
|
||||
TMP=$(mktemp -d)
|
||||
pids=""
|
||||
cleanup() {
|
||||
for pid in $pids; do kill "$pid" 2>/dev/null || true; done
|
||||
rm -rf "$TMP"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
fails=0
|
||||
pass() { echo " ok - $1"; }
|
||||
fail() { echo " FAIL - $1"; fails=$((fails + 1)); }
|
||||
|
||||
echo "-- node-reset.sh QEMU process match"
|
||||
|
||||
# pattern_in <script> — the pgrep pattern the script assigns to QEMU_PGREP.
|
||||
pattern_in() {
|
||||
sed -n "s/^QEMU_PGREP=('\(.*\)')$/\1/p" "$1"
|
||||
}
|
||||
|
||||
PATTERN=$(pattern_in "$SCRIPT_DIR/node-reset.sh")
|
||||
if [ -z "$PATTERN" ]; then
|
||||
echo " FAIL - no QEMU_PGREP=('...') line in node-reset.sh"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# reset-dev-env.sh stops guests the same way and must not drift from it.
|
||||
dev_pattern=$(pattern_in "$SCRIPT_DIR/reset-dev-env.sh")
|
||||
if [ "$dev_pattern" = "$PATTERN" ]; then
|
||||
pass "reset-dev-env.sh uses node-reset.sh's pattern"
|
||||
else
|
||||
fail "reset-dev-env.sh pattern '$dev_pattern' differs from node-reset.sh's '$PATTERN'"
|
||||
fi
|
||||
|
||||
# start <name> — runs a process whose comm is <name>, truncated to 15
|
||||
# characters. Its PID is left in $!.
|
||||
start() {
|
||||
cp "$(command -v sleep)" "$TMP/$1"
|
||||
"$TMP/$1" 300 &
|
||||
pids="$pids $!"
|
||||
}
|
||||
start qemu-system-x86_64; pid_x86=$!
|
||||
start qemu-system-aarch64; pid_arm=$!
|
||||
start qemu-kvm; pid_kvm=$!
|
||||
start qemu-img; pid_img=$!
|
||||
start qemu-kvm-helper; pid_helper=$!
|
||||
|
||||
me=$(id -un)
|
||||
found=" $(pgrep -u "$me" "$PATTERN" | tr '\n' ' ' || true) "
|
||||
|
||||
check_found() {
|
||||
case "$found" in
|
||||
*" $2 "*) pass "finds a $1 guest owned by the service user" ;;
|
||||
*) fail "misses a $1 guest (pid $2) owned by the service user" ;;
|
||||
esac
|
||||
}
|
||||
check_found qemu-system-x86_64 "$pid_x86"
|
||||
check_found qemu-system-aarch64 "$pid_arm"
|
||||
check_found qemu-kvm "$pid_kvm"
|
||||
|
||||
check_ignored() {
|
||||
case "$found" in
|
||||
*" $2 "*) fail "matches $1 (pid $2), which is not a guest" ;;
|
||||
*) pass "ignores $1" ;;
|
||||
esac
|
||||
}
|
||||
check_ignored qemu-img "$pid_img"
|
||||
check_ignored qemu-kvm-helper "$pid_helper"
|
||||
|
||||
# The owner scope: the same guests, asked for as another user, are not found.
|
||||
other=root
|
||||
[ "$me" = root ] && other=nobody
|
||||
other_found=" $(pgrep -u "$other" "$PATTERN" | tr '\n' ' ' || true) "
|
||||
case "$other_found" in
|
||||
*" $pid_kvm "* | *" $pid_x86 "*) fail "a scope of -u $other still matches guests owned by $me" ;;
|
||||
*) pass "ignores guests owned by another user" ;;
|
||||
esac
|
||||
|
||||
if [ "$fails" -gt 0 ]; then
|
||||
echo "$fails failure(s)"
|
||||
exit 1
|
||||
fi
|
||||
@@ -359,17 +359,29 @@ fi
|
||||
# --- Shutdown services ---
|
||||
echo "==> Stopping services"
|
||||
sudo systemctl stop spinifex.target 2>/dev/null || true
|
||||
# spinifex-hostnet starts with OVS, which is started below to delete bridges; it
|
||||
# would rebuild the old veth pairs. setup-ovn.sh enables it again.
|
||||
sudo systemctl disable --now spinifex-hostnet.service 2>/dev/null || true
|
||||
sudo systemctl reset-failed 'spinifex-*' 2>/dev/null || true
|
||||
sudo pkill -x qemu-system-x86_64 2>/dev/null || true
|
||||
sudo pkill -x qemu-system-aarch64 2>/dev/null || true
|
||||
# comm truncates to 15 characters, so match qemu-system-* by prefix; EL-family
|
||||
# hosts run qemu-kvm. Scoped to the user the daemon runs guests as, as in
|
||||
# node-reset.sh, so libvirt guests on the same host are left alone.
|
||||
QEMU_PGREP=('^(qemu-system|qemu-kvm$)')
|
||||
for qemu_user in spinifex-daemon spinifex "$INVOKING_USER"; do
|
||||
if id -u "$qemu_user" >/dev/null 2>&1; then
|
||||
QEMU_PGREP=(-u "$qemu_user" "${QEMU_PGREP[@]}")
|
||||
break
|
||||
fi
|
||||
done
|
||||
sudo pkill "${QEMU_PGREP[@]}" 2>/dev/null || true
|
||||
|
||||
# Wait for QEMU to fully exit before tearing down viperblock state.
|
||||
timeout=30
|
||||
elapsed=0
|
||||
while pgrep -x 'qemu-system-x86_64|qemu-system-aarch64' > /dev/null 2>&1; do
|
||||
while pgrep "${QEMU_PGREP[@]}" > /dev/null 2>&1; do
|
||||
if [ "$elapsed" -ge "$timeout" ]; then
|
||||
echo "❌ QEMU still running after ${timeout}s:"
|
||||
pgrep -af 'qemu-system-' || true
|
||||
pgrep -a "${QEMU_PGREP[@]}" || true
|
||||
echo " Kill them manually and re-run this script."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -66,7 +66,7 @@ MAX_QEMU_ATTEMPTS=60
|
||||
QEMU_CMD=""
|
||||
|
||||
while [ $QEMU_ATTEMPTS -lt $MAX_QEMU_ATTEMPTS ]; do
|
||||
QEMU_CMD=$(ps auxw | grep "$INSTANCE_ID" | grep qemu-system | grep -v grep)
|
||||
QEMU_CMD=$(ps auxw | grep "$INSTANCE_ID" | grep -E "qemu-system|qemu-kvm" | grep -v grep)
|
||||
|
||||
if [ -n "$QEMU_CMD" ]; then
|
||||
break
|
||||
|
||||
+376
-74
@@ -4,12 +4,20 @@
|
||||
#
|
||||
# This script bootstraps a compute node for OVN-based VPC networking:
|
||||
# 1. Installs OVN/OVS packages (if not present)
|
||||
# 2. Enables required services (openvswitch-switch, ovn-controller)
|
||||
# 2. Enables required services (Open vSwitch, ovn-controller)
|
||||
# 3. Creates br-int with secure fail-mode
|
||||
# 4. Configures WAN bridge for public subnet uplink (auto-detected or manual)
|
||||
# 5. Configures OVS external_ids for OVN chassis identity
|
||||
# 6. Applies sysctl tuning for overlay networking
|
||||
#
|
||||
# Host network persistence: the links this script creates are kernel-only. On
|
||||
# a host where systemd-networkd is enabled, or that already has Spinifex
|
||||
# networkd files, they persist in /etc/systemd/network. Otherwise (for example
|
||||
# a NetworkManager host) the topology goes to /etc/spinifex/hostnet.json and
|
||||
# spinifex-hostnet.service re-creates it at boot and on every OVS start. The
|
||||
# first run records which in /etc/spinifex/hostnet-backend; it never changes.
|
||||
# Where NetworkManager runs, the links Spinifex owns are made unmanaged.
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/setup-ovn.sh [options]
|
||||
#
|
||||
@@ -121,6 +129,19 @@
|
||||
|
||||
set -e
|
||||
|
||||
# Host-OS capability probes: beside this script in a checkout and in the
|
||||
# installed share dir, else where setup.sh installs them.
|
||||
DISTRO_LIB=""
|
||||
for candidate in "$(dirname "${BASH_SOURCE[0]}")/lib/distro.sh" /usr/local/share/spinifex/lib/distro.sh; do
|
||||
if [ -f "$candidate" ]; then DISTRO_LIB="$candidate"; break; fi
|
||||
done
|
||||
if [ -z "$DISTRO_LIB" ]; then
|
||||
echo "ERROR: distro.sh not found at $(dirname "${BASH_SOURCE[0]}")/lib/distro.sh or /usr/local/share/spinifex/lib/distro.sh; re-run setup.sh to install it" >&2
|
||||
exit 1
|
||||
fi
|
||||
# shellcheck source=lib/distro.sh
|
||||
. "$DISTRO_LIB"
|
||||
|
||||
# Defaults
|
||||
MANAGEMENT=false
|
||||
WAN_BRIDGE=""
|
||||
@@ -222,6 +243,152 @@ iface_master() {
|
||||
printf '%s' "$m"
|
||||
}
|
||||
|
||||
# --- Host network persistence ---
|
||||
#
|
||||
# Links this script creates are kernel-only and vanish on reboot. The first run
|
||||
# picks how they persist and records it: networkd where systemd-networkd is
|
||||
# enabled, else hostnet (spinifex-hostnet.service replays HOSTNET_STATE at boot
|
||||
# and on every ovs-vswitchd start). There is no switching; reset the node.
|
||||
HOSTNET_BACKEND_FILE=/etc/spinifex/hostnet-backend
|
||||
HOSTNET_STATE=/etc/spinifex/hostnet.json
|
||||
HOSTNET_LOCK=/run/spinifex/hostnet.lock
|
||||
NETWORKD_DIR=/etc/systemd/network
|
||||
NM_UNMANAGED_CONF=/etc/NetworkManager/conf.d/00-spinifex-unmanaged.conf
|
||||
NM_UNMANAGED_RULE=/etc/udev/rules.d/90-spinifex-nm-unmanaged.rules
|
||||
# System-instance management TAPs: "mg" plus 13 hex digits of the instance ID.
|
||||
# NetworkManager globs have no character classes, so this match goes to udev.
|
||||
MGMT_TAP_GLOB='mg[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]'
|
||||
|
||||
# Prints the recorded backend, or picks one and records it. A node with
|
||||
# Spinifex networkd files and no record was set up before the record existed.
|
||||
select_hostnet_backend() {
|
||||
local backend
|
||||
if sudo test -e "$HOSTNET_BACKEND_FILE"; then
|
||||
backend=$(sudo cat "$HOSTNET_BACKEND_FILE" | tr -d '[:space:]')
|
||||
case "$backend" in
|
||||
networkd|hostnet) printf '%s\n' "$backend"; return 0 ;;
|
||||
esac
|
||||
echo "ERROR: $HOSTNET_BACKEND_FILE says '$backend'; expected networkd or hostnet" >&2
|
||||
return 1
|
||||
fi
|
||||
if compgen -G "$NETWORKD_DIR/*-spinifex-*.network" >/dev/null || compgen -G "$NETWORKD_DIR/*-spinifex-*.netdev" >/dev/null; then
|
||||
backend=networkd
|
||||
elif systemctl is-enabled --quiet systemd-networkd.service 2>/dev/null; then
|
||||
backend=networkd
|
||||
else
|
||||
backend=hostnet
|
||||
fi
|
||||
sudo mkdir -p "$(dirname "$HOSTNET_BACKEND_FILE")"
|
||||
printf '%s\n' "$backend" | sudo tee "$HOSTNET_BACKEND_FILE" >/dev/null
|
||||
# Read by `spx admin hostnet check`, which the daemon and vpcd run unprivileged.
|
||||
sudo chmod 0644 "$HOSTNET_BACKEND_FILE"
|
||||
printf '%s\n' "$backend"
|
||||
}
|
||||
|
||||
# write_hostnet_state <true|false> — records the topology for
|
||||
# `spx admin hostnet apply`, atomically. true marks it pending: apply refuses
|
||||
# it until the run that wrote it finishes.
|
||||
write_hostnet_state() {
|
||||
local pending="$1" body
|
||||
body=$(printf '{\n "version": 1,\n "pending": %s,\n "int_bridge": "br-int"' "$pending")
|
||||
case "$WAN_BRIDGE_MODE" in
|
||||
existing)
|
||||
body="$body"$(printf ',\n "wan": {"mode": "existing", "ovs_bridge": "%s"}' "$WAN_BRIDGE") ;;
|
||||
veth)
|
||||
body="$body"$(printf ',\n "wan": {"mode": "veth", "ovs_bridge": "%s", "linux_bridge": "%s", "host_end": "veth-wan-br", "ovs_end": "veth-wan-ovs"}' "$WAN_BRIDGE" "$LINUX_BRIDGE") ;;
|
||||
direct)
|
||||
body="$body"$(printf ',\n "wan": {"mode": "direct", "ovs_bridge": "%s", "nic": "%s"}' "$WAN_BRIDGE" "$WAN_IFACE") ;;
|
||||
nat)
|
||||
body="$body"$(printf ',\n "wan": {"mode": "nat", "ovs_bridge": "%s", "host_end": "spx-nat-host", "ovs_end": "spx-nat-ovs", "host_mac": "%s", "host_cidr": "%s"}' "$WAN_BRIDGE" "$NAT_TRANSIT_HOST_MAC" "$NAT_TRANSIT_GW_CIDR") ;;
|
||||
esac
|
||||
if [ "$MGMT_BRIDGE_ENABLED" = true ]; then
|
||||
body="$body"$(printf ',\n "mgmt": {"bridge": "%s", "cidr": "%s", "nic": "%s"}' "$MGMT_BRIDGE" "$MGMT_CIDR" "$MGMT_IFACE")
|
||||
fi
|
||||
if [ -n "${ENCAP_ROUTE_DST:-}" ]; then
|
||||
body="$body"$(printf ',\n "encap_route": {"dst": "%s", "dev": "%s", "src": "%s", "metric": %s}' \
|
||||
"$ENCAP_ROUTE_DST" "$ENCAP_ROUTE_DEV" "$ENCAP_IP" "$ENCAP_ROUTE_METRIC")
|
||||
fi
|
||||
sudo mkdir -p "$(dirname "$HOSTNET_STATE")"
|
||||
printf '%s\n}\n' "$body" | sudo tee "$HOSTNET_STATE.tmp" >/dev/null
|
||||
sudo chmod 0644 "$HOSTNET_STATE.tmp"
|
||||
sudo mv -f "$HOSTNET_STATE.tmp" "$HOSTNET_STATE"
|
||||
}
|
||||
|
||||
# Holds the lock `spx admin hostnet apply` takes, from a root flock process fed
|
||||
# by a pipe, so an apply started by an OVS restart waits for this run's links.
|
||||
# Never run systemctl against OVS or spinifex-hostnet while holding it.
|
||||
hostnet_lock() {
|
||||
local reply=""
|
||||
sudo mkdir -p "$(dirname "$HOSTNET_LOCK")"
|
||||
coproc HOSTNET_LOCK_HOLDER { exec sudo flock -w 120 "$HOSTNET_LOCK" sh -c 'echo locked; read -r _'; }
|
||||
read -r reply <&"${HOSTNET_LOCK_HOLDER[0]}" || true
|
||||
if [ "$reply" != locked ]; then
|
||||
echo "ERROR: $HOSTNET_LOCK still held after 120 s; is spx admin hostnet apply stuck? (systemctl status spinifex-hostnet)" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
hostnet_unlock() {
|
||||
local fd="${HOSTNET_LOCK_HOLDER[1]}" pid="$HOSTNET_LOCK_HOLDER_PID"
|
||||
exec {fd}>&-
|
||||
wait "$pid" 2>/dev/null || true
|
||||
}
|
||||
|
||||
nm_running() {
|
||||
systemctl is-active --quiet NetworkManager.service 2>/dev/null
|
||||
}
|
||||
|
||||
# nm_release_nic <nic> — before a NIC moves into OVS, refuse the one carrying
|
||||
# the default route or this SSH session, then take it from NetworkManager. Its
|
||||
# profiles are left alone: once unmanaged, none can activate on it.
|
||||
nm_release_nic() {
|
||||
local nic="$1" ssh_ip state
|
||||
if sudo ovs-vsctl port-to-br "$nic" >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
if ip -4 route show default 2>/dev/null | awk '{for(i=1;i<NF;i++) if($i=="dev") print $(i+1)}' | grep -qx "$nic"; then
|
||||
echo "ERROR: $nic carries the default route; moving it into OVS would cut this host off" >&2
|
||||
exit 1
|
||||
fi
|
||||
ssh_ip=$(printf '%s\n' "${SSH_CONNECTION:-}" | awk '{print $3}')
|
||||
if [ -n "$ssh_ip" ] && ip -o -4 addr show dev "$nic" 2>/dev/null | awk '{print $4}' | cut -d/ -f1 | grep -qxF "$ssh_ip"; then
|
||||
echo "ERROR: this SSH session arrived on $nic ($ssh_ip); moving it into OVS would drop it" >&2
|
||||
exit 1
|
||||
fi
|
||||
state=$(nmcli -g GENERAL.STATE device show "$nic" 2>/dev/null || true)
|
||||
case "$state" in
|
||||
*"(connect"*)
|
||||
sudo nmcli device disconnect "$nic" >/dev/null
|
||||
echo " $nic: disconnected from NetworkManager"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Writes the NetworkManager config and udev rule that leave every link Spinifex
|
||||
# owns unmanaged, then has NetworkManager and udev reread them. The host uplink
|
||||
# (a Linux bridge such as br-wan, and its NIC) stays NetworkManager's.
|
||||
write_nm_unmanaged() {
|
||||
local names=(br-int "$WAN_BRIDGE" 'veth-wan-*' 'spx-nat-*' 'tap*' 'ime-*' 'imi-*' 'vhp-*' br-imds 'genev_sys_*')
|
||||
local list="" name
|
||||
if [ "$MGMT_BRIDGE_ENABLED" = true ]; then
|
||||
names+=("$MGMT_BRIDGE" "$MGMT_IFACE")
|
||||
fi
|
||||
if [ "$WAN_BRIDGE_MODE" = direct ]; then
|
||||
names+=("$WAN_IFACE")
|
||||
fi
|
||||
for name in "${names[@]}"; do
|
||||
[ -n "$name" ] && list="${list:+$list;}interface-name:$name"
|
||||
done
|
||||
sudo mkdir -p "$(dirname "$NM_UNMANAGED_CONF")" "$(dirname "$NM_UNMANAGED_RULE")"
|
||||
printf '# Written by setup-ovn.sh on every run: links Spinifex owns.\n[keyfile]\nunmanaged-devices+=%s\n' "$list" \
|
||||
| sudo tee "$NM_UNMANAGED_CONF" >/dev/null
|
||||
printf '# Written by setup-ovn.sh: system-instance management TAPs.\nSUBSYSTEM=="net", KERNEL=="%s", ENV{NM_UNMANAGED}="1"\n' "$MGMT_TAP_GLOB" \
|
||||
| sudo tee "$NM_UNMANAGED_RULE" >/dev/null
|
||||
sudo udevadm control --reload
|
||||
sudo udevadm trigger --action=change --subsystem-match=net --sysname-match='mg*'
|
||||
sudo nmcli general reload conf
|
||||
}
|
||||
|
||||
teardown() {
|
||||
echo "Tearing down the OVN host wiring built by this script"
|
||||
[ "$DRY_RUN" = true ] && echo " (dry run — nothing is changed)"
|
||||
@@ -469,6 +636,15 @@ else
|
||||
detect_wan_bridge
|
||||
fi
|
||||
|
||||
# Routed NAT transit, used by nat mode and recorded for spinifex-hostnet.
|
||||
NAT_TRANSIT_CIDR="100.127.0.0/24"
|
||||
NAT_TRANSIT_GW_CIDR="100.127.0.1/24"
|
||||
# Same on every node. OVN keeps one MAC binding for the transit nexthop
|
||||
# cluster-wide, so a random per-node MAC sends every chassis's egress to
|
||||
# whichever node seeded it; the pair is node-local, so one address everywhere
|
||||
# is unambiguous.
|
||||
NAT_TRANSIT_HOST_MAC="02:00:64:7f:00:01"
|
||||
|
||||
# Auto-detect encap IP if not specified
|
||||
if [ -z "$ENCAP_IP" ]; then
|
||||
# Prefer br-vpc IP if it exists (dedicated VPC data plane)
|
||||
@@ -575,19 +751,70 @@ ensure_clustered_db_storage() {
|
||||
# fresh SB and re-registers under whatever system-id it started with — which
|
||||
# on a renamed node is the old one, and that row then holds this node's encap
|
||||
# IP so the correctly-named chassis can never commit. Step 5 restarts it.
|
||||
sudo systemctl stop ovn-northd ovn-ovsdb-server-nb ovn-ovsdb-server-sb ovn-central ovn-controller 2>/dev/null || true
|
||||
sudo systemctl stop "${OVN_CENTRAL_UNITS[@]}" ovn-controller 2>/dev/null || true
|
||||
sudo rm -f "${standalone[@]}"
|
||||
}
|
||||
|
||||
# set_root_service_user <file> <var> — sets <var>="root:root" in <file> when
|
||||
# the file exists. Returns 0 only when it changed the file.
|
||||
set_root_service_user() {
|
||||
local file=$1 var=$2
|
||||
[ -f "$file" ] || return 1
|
||||
grep -qx "$var=\"root:root\"" "$file" && return 1
|
||||
if grep -q "^$var=" "$file"; then
|
||||
sudo sed -i "s|^$var=.*|$var=\"root:root\"|" "$file"
|
||||
else
|
||||
echo "$var=\"root:root\"" | sudo tee -a "$file" >/dev/null
|
||||
fi
|
||||
echo " $file: $var=\"root:root\""
|
||||
}
|
||||
|
||||
echo ""
|
||||
echo "Step 2: Enabling services..."
|
||||
|
||||
sudo systemctl enable openvswitch-switch
|
||||
sudo systemctl start openvswitch-switch
|
||||
echo " openvswitch-switch: started"
|
||||
# Unit names differ by distro: the OVS umbrella, and whether the OVN databases
|
||||
# have units of their own (Debian) or are started by ovn-northd.service (EL).
|
||||
OVS_UNIT=$(distro_ovs_unit) || exit 1
|
||||
OVN_CENTRAL_LAYOUT=""
|
||||
if [ "$MANAGEMENT" = true ] || distro_unit_exists ovn-central.service || distro_unit_exists ovn-northd.service; then
|
||||
OVN_CENTRAL_LAYOUT=$(distro_ovn_central_layout) || exit 1
|
||||
fi
|
||||
case "$OVN_CENTRAL_LAYOUT" in
|
||||
split)
|
||||
OVN_CENTRAL_UNIT=ovn-central.service
|
||||
OVN_DB_UNITS=(ovn-ovsdb-server-nb.service ovn-ovsdb-server-sb.service)
|
||||
OVN_SERVER_UNITS=("${OVN_DB_UNITS[@]}" ovn-northd.service)
|
||||
OVN_CENTRAL_UNITS=(ovn-central.service "${OVN_SERVER_UNITS[@]}")
|
||||
;;
|
||||
lone-northd)
|
||||
OVN_CENTRAL_UNIT=ovn-northd.service
|
||||
OVN_DB_UNITS=(ovn-northd.service)
|
||||
OVN_SERVER_UNITS=(ovn-northd.service)
|
||||
OVN_CENTRAL_UNITS=(ovn-northd.service)
|
||||
;;
|
||||
esac
|
||||
|
||||
# OVS and OVN run as root, as on Debian: the service users reach the sockets
|
||||
# through the spinifex group, a model tested only against root-owned daemons.
|
||||
# EL ships non-root users and re-chowns the run dirs to them on every start.
|
||||
OVS_USER_CHANGED=false
|
||||
if set_root_service_user /etc/sysconfig/openvswitch OVS_USER_ID; then
|
||||
OVS_USER_CHANGED=true
|
||||
fi
|
||||
# Takes effect below: ovn-northd and ovn-controller are restarted either way.
|
||||
set_root_service_user /etc/sysconfig/ovn OVN_USER_ID || true
|
||||
|
||||
sudo systemctl enable "$OVS_UNIT"
|
||||
if [ "$OVS_USER_CHANGED" = true ]; then
|
||||
# start is a no-op on a running OVS, which would keep its old user.
|
||||
sudo systemctl restart "$OVS_UNIT"
|
||||
else
|
||||
sudo systemctl start "$OVS_UNIT"
|
||||
fi
|
||||
echo " ${OVS_UNIT%.service}: started"
|
||||
|
||||
if [ "$MANAGEMENT" = true ]; then
|
||||
sudo systemctl enable ovn-central
|
||||
sudo systemctl enable "$OVN_CENTRAL_UNIT"
|
||||
|
||||
# LAN-plane NB/SB client listener, added to the loopback one set below.
|
||||
# Guard is load-bearing: --db-nb-addr defaults to 0.0.0.0, so
|
||||
@@ -651,10 +878,10 @@ ExecStop=/usr/share/ovn/scripts/ovn-ctl stop_northd --no-monitor --ovn-manage-ov
|
||||
EOF
|
||||
sudo systemctl daemon-reload
|
||||
|
||||
# The ovn-central aggregator is ExecStart=/bin/true, so restarting it
|
||||
# won't restart the children — restart the per-DB units directly to
|
||||
# pick up the new OVN_CTL_OPTS.
|
||||
sudo systemctl restart ovn-ovsdb-server-nb ovn-ovsdb-server-sb ovn-northd
|
||||
# Debian's ovn-central aggregator is ExecStart=/bin/true, so restarting
|
||||
# it won't restart the children — restart the DB and northd units
|
||||
# directly to pick up the new OVN_CTL_OPTS.
|
||||
sudo systemctl restart "${OVN_SERVER_UNITS[@]}"
|
||||
echo " ovn-central: started clustered (NB DB + SB DB + ovn-northd)"
|
||||
else
|
||||
# Always write the file, even when there is nothing to put in it. This
|
||||
@@ -664,12 +891,12 @@ EOF
|
||||
# peer that no longer exists, which looks like a hung leader election.
|
||||
echo "OVN_CTL_OPTS=\"$OVN_DB_LISTEN_OPTS\"" | sudo tee /etc/default/ovn-central >/dev/null
|
||||
echo " wrote /etc/default/ovn-central"
|
||||
sudo systemctl start ovn-central
|
||||
sudo systemctl start "$OVN_CENTRAL_UNIT"
|
||||
|
||||
# ovn-central is ExecStart=/bin/true; restarting it does not restart
|
||||
# its children, so a re-run with changed options needs the per-DB units
|
||||
# restarted directly to pick them up.
|
||||
sudo systemctl restart ovn-ovsdb-server-nb ovn-ovsdb-server-sb
|
||||
# Debian's ovn-central is ExecStart=/bin/true; restarting it does not
|
||||
# restart its children, so a re-run with changed options needs the DB
|
||||
# units restarted directly to pick them up.
|
||||
sudo systemctl restart "${OVN_DB_UNITS[@]}"
|
||||
echo " ovn-central: started (NB DB + SB DB + ovn-northd)"
|
||||
if [ -n "$LAN_ADDR" ]; then
|
||||
echo " NB/SB client listen: 127.0.0.1, $LAN_ADDR"
|
||||
@@ -812,10 +1039,10 @@ else
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if systemctl is-enabled ovn-central >/dev/null 2>&1 ||
|
||||
systemctl is-active ovn-central >/dev/null 2>&1; then
|
||||
sudo systemctl stop ovn-central ovn-northd ovn-ovsdb-server-nb ovn-ovsdb-server-sb 2>/dev/null || true
|
||||
sudo systemctl disable ovn-central ovn-northd ovn-ovsdb-server-nb ovn-ovsdb-server-sb 2>/dev/null || true
|
||||
if [ -n "$OVN_CENTRAL_LAYOUT" ] && { systemctl is-enabled "$OVN_CENTRAL_UNIT" >/dev/null 2>&1 ||
|
||||
systemctl is-active "$OVN_CENTRAL_UNIT" >/dev/null 2>&1; }; then
|
||||
sudo systemctl stop "${OVN_CENTRAL_UNITS[@]}" 2>/dev/null || true
|
||||
sudo systemctl disable "${OVN_CENTRAL_UNITS[@]}" 2>/dev/null || true
|
||||
echo " ovn-central: stopped and disabled (compute node runs no database)"
|
||||
fi
|
||||
for db in "$OVN_DBDIR/ovnnb_db.db" "$OVN_DBDIR/ovnsb_db.db"; do
|
||||
@@ -825,6 +1052,34 @@ else
|
||||
done
|
||||
fi
|
||||
|
||||
# --- Step 2b: Host network persistence and NetworkManager ---
|
||||
echo ""
|
||||
echo "Step 2b: Host network persistence..."
|
||||
HOSTNET_BACKEND=$(select_hostnet_backend)
|
||||
echo " backend: $HOSTNET_BACKEND (recorded in $HOSTNET_BACKEND_FILE)"
|
||||
|
||||
if nm_running; then
|
||||
if [ "$MGMT_BRIDGE_ENABLED" = true ] && [ -n "$MGMT_IFACE" ]; then
|
||||
nm_release_nic "$MGMT_IFACE"
|
||||
fi
|
||||
if [ "$WAN_BRIDGE_MODE" = direct ]; then
|
||||
nm_release_nic "$WAN_IFACE"
|
||||
fi
|
||||
write_nm_unmanaged
|
||||
echo " NetworkManager: Spinifex links unmanaged ($NM_UNMANAGED_CONF, $NM_UNMANAGED_RULE)"
|
||||
fi
|
||||
|
||||
# Held until the wiring below is complete; a run that dies before then leaves
|
||||
# the state pending, and spinifex-hostnet refuses it until this script is rerun.
|
||||
ENCAP_ROUTE_DST=""
|
||||
ENCAP_ROUTE_DEV=""
|
||||
ENCAP_ROUTE_METRIC=50
|
||||
if [ "$HOSTNET_BACKEND" = hostnet ]; then
|
||||
hostnet_lock
|
||||
write_hostnet_state true
|
||||
echo " $HOSTNET_STATE: pending until this run finishes wiring"
|
||||
fi
|
||||
|
||||
# --- Step 3: Create and configure br-int ---
|
||||
echo ""
|
||||
echo "Step 3: Configuring br-int..."
|
||||
@@ -842,7 +1097,7 @@ echo " br-int: created, fail-mode=secure, up"
|
||||
# still forwards, but ovn-controller flow programming and any tooling that
|
||||
# probes link state misbehave. Unmanaged=yes keeps OVS in sole control.
|
||||
OVS_INTERNAL_NET=/etc/systemd/network/05-spinifex-ovs-internal.network
|
||||
if [ ! -f "$OVS_INTERNAL_NET" ]; then
|
||||
if [ "$HOSTNET_BACKEND" = networkd ] && [ ! -f "$OVS_INTERNAL_NET" ]; then
|
||||
sudo tee "$OVS_INTERNAL_NET" >/dev/null <<'NETWORK'
|
||||
[Match]
|
||||
Name=br-int br-ext
|
||||
@@ -956,23 +1211,24 @@ if [ -n "$WAN_BRIDGE" ]; then
|
||||
# existing networkd config. networkd searches /etc, /run, /usr/lib;
|
||||
# check all three. Our own file is excluded so idempotent re-runs
|
||||
# still rewrite when needed.
|
||||
BR_WAN_NETDEV="/etc/systemd/network/14-spinifex-br-wan.netdev"
|
||||
VETH_NETDEV="/etc/systemd/network/15-spinifex-veth-wan.netdev"
|
||||
VETH_NETWORK="/etc/systemd/network/15-spinifex-veth-wan.network"
|
||||
VETH_OVS_NETWORK="/etc/systemd/network/16-spinifex-veth-wan-ovs.network"
|
||||
EXISTING_BR_NETDEV=$(grep -rls --include="*.netdev" -E "^\s*Name=$LINUX_BRIDGE\s*$" \
|
||||
/etc/systemd/network /run/systemd/network /usr/lib/systemd/network 2>/dev/null \
|
||||
| grep -v "^$BR_WAN_NETDEV$" || true)
|
||||
if [ -n "$EXISTING_BR_NETDEV" ]; then
|
||||
echo " skipping $BR_WAN_NETDEV — operator-managed NetDev already declares $LINUX_BRIDGE: $EXISTING_BR_NETDEV"
|
||||
else
|
||||
sudo tee "$BR_WAN_NETDEV" >/dev/null <<NETDEV
|
||||
if [ "$HOSTNET_BACKEND" = networkd ]; then
|
||||
BR_WAN_NETDEV="/etc/systemd/network/14-spinifex-br-wan.netdev"
|
||||
VETH_NETDEV="/etc/systemd/network/15-spinifex-veth-wan.netdev"
|
||||
VETH_NETWORK="/etc/systemd/network/15-spinifex-veth-wan.network"
|
||||
VETH_OVS_NETWORK="/etc/systemd/network/16-spinifex-veth-wan-ovs.network"
|
||||
EXISTING_BR_NETDEV=$(grep -rls --include="*.netdev" -E "^\s*Name=$LINUX_BRIDGE\s*$" \
|
||||
/etc/systemd/network /run/systemd/network /usr/lib/systemd/network 2>/dev/null \
|
||||
| grep -v "^$BR_WAN_NETDEV$" || true)
|
||||
if [ -n "$EXISTING_BR_NETDEV" ]; then
|
||||
echo " skipping $BR_WAN_NETDEV — operator-managed NetDev already declares $LINUX_BRIDGE: $EXISTING_BR_NETDEV"
|
||||
else
|
||||
sudo tee "$BR_WAN_NETDEV" >/dev/null <<NETDEV
|
||||
[NetDev]
|
||||
Name=$LINUX_BRIDGE
|
||||
Kind=bridge
|
||||
NETDEV
|
||||
fi
|
||||
sudo tee "$VETH_NETDEV" >/dev/null <<NETDEV
|
||||
fi
|
||||
sudo tee "$VETH_NETDEV" >/dev/null <<NETDEV
|
||||
[NetDev]
|
||||
Name=veth-wan-br
|
||||
Kind=veth
|
||||
@@ -980,7 +1236,7 @@ Kind=veth
|
||||
[Peer]
|
||||
Name=veth-wan-ovs
|
||||
NETDEV
|
||||
sudo tee "$VETH_NETWORK" >/dev/null <<NETWORK
|
||||
sudo tee "$VETH_NETWORK" >/dev/null <<NETWORK
|
||||
[Match]
|
||||
Name=veth-wan-br
|
||||
|
||||
@@ -988,12 +1244,12 @@ Name=veth-wan-br
|
||||
Bridge=$LINUX_BRIDGE
|
||||
ConfigureWithoutCarrier=yes
|
||||
NETWORK
|
||||
# Second unit admin-ups the OVS end of the pair. OVS owns the port
|
||||
# (enslaved via ovs-vsctl add-port above) but does not flip admin
|
||||
# state on external ports — that's networkd's job. Without this,
|
||||
# veth-wan-ovs stays DOWN after reboot, peer goes LOWERLAYERDOWN,
|
||||
# br-wan loses carrier.
|
||||
sudo tee "$VETH_OVS_NETWORK" >/dev/null <<NETWORK
|
||||
# Second unit admin-ups the OVS end of the pair. OVS owns the port
|
||||
# (enslaved via ovs-vsctl add-port above) but does not flip admin
|
||||
# state on external ports — that's networkd's job. Without this,
|
||||
# veth-wan-ovs stays DOWN after reboot, peer goes LOWERLAYERDOWN,
|
||||
# br-wan loses carrier.
|
||||
sudo tee "$VETH_OVS_NETWORK" >/dev/null <<NETWORK
|
||||
[Match]
|
||||
Name=veth-wan-ovs
|
||||
|
||||
@@ -1003,8 +1259,9 @@ RequiredForOnline=no
|
||||
[Network]
|
||||
ConfigureWithoutCarrier=yes
|
||||
NETWORK
|
||||
sudo networkctl reload 2>/dev/null || true
|
||||
echo " wrote $VETH_NETDEV + $VETH_NETWORK + $VETH_OVS_NETWORK (veth persists on reboot)"
|
||||
sudo networkctl reload 2>/dev/null || true
|
||||
echo " wrote $VETH_NETDEV + $VETH_NETWORK + $VETH_OVS_NETWORK (veth persists on reboot)"
|
||||
fi
|
||||
;;
|
||||
|
||||
direct)
|
||||
@@ -1038,14 +1295,6 @@ NETWORK
|
||||
# forwards and masquerades the transit /24 out whatever uplink it
|
||||
# has. OVN SNATs each VPC CIDR to its gateway LRP transit IP, so
|
||||
# the masquerade rule below is the only host-side NAT state.
|
||||
NAT_TRANSIT_CIDR="100.127.0.0/24"
|
||||
NAT_TRANSIT_GW_CIDR="100.127.0.1/24"
|
||||
# Same on every node. OVN keeps one MAC binding for the transit
|
||||
# nexthop cluster-wide, so a random per-node MAC sends every
|
||||
# chassis's egress to whichever node seeded it; the pair is
|
||||
# node-local, so one address everywhere is unambiguous.
|
||||
NAT_TRANSIT_HOST_MAC="02:00:64:7f:00:01"
|
||||
|
||||
if ! sudo ovs-vsctl br-exists "$WAN_BRIDGE" 2>/dev/null; then
|
||||
sudo ovs-vsctl --may-exist add-br "$WAN_BRIDGE"
|
||||
echo " created OVS bridge: $WAN_BRIDGE"
|
||||
@@ -1073,10 +1322,11 @@ NETWORK
|
||||
|
||||
# Persist the veth pair + transit IP across reboot (veths are
|
||||
# kernel-only; same rationale as veth mode above).
|
||||
NAT_NETDEV="/etc/systemd/network/17-spinifex-nat.netdev"
|
||||
NAT_NETWORK="/etc/systemd/network/17-spinifex-nat.network"
|
||||
NAT_OVS_NETWORK="/etc/systemd/network/18-spinifex-nat-ovs.network"
|
||||
sudo tee "$NAT_NETDEV" >/dev/null <<NETDEV
|
||||
if [ "$HOSTNET_BACKEND" = networkd ]; then
|
||||
NAT_NETDEV="/etc/systemd/network/17-spinifex-nat.netdev"
|
||||
NAT_NETWORK="/etc/systemd/network/17-spinifex-nat.network"
|
||||
NAT_OVS_NETWORK="/etc/systemd/network/18-spinifex-nat-ovs.network"
|
||||
sudo tee "$NAT_NETDEV" >/dev/null <<NETDEV
|
||||
[NetDev]
|
||||
Name=spx-nat-host
|
||||
Kind=veth
|
||||
@@ -1085,7 +1335,7 @@ MACAddress=$NAT_TRANSIT_HOST_MAC
|
||||
[Peer]
|
||||
Name=spx-nat-ovs
|
||||
NETDEV
|
||||
sudo tee "$NAT_NETWORK" >/dev/null <<NETWORK
|
||||
sudo tee "$NAT_NETWORK" >/dev/null <<NETWORK
|
||||
[Match]
|
||||
Name=spx-nat-host
|
||||
|
||||
@@ -1093,9 +1343,9 @@ Name=spx-nat-host
|
||||
Address=$NAT_TRANSIT_GW_CIDR
|
||||
ConfigureWithoutCarrier=yes
|
||||
NETWORK
|
||||
# Admin-up the OVS end after reboot (OVS owns the port but does
|
||||
# not flip admin state on external ports — same as veth mode).
|
||||
sudo tee "$NAT_OVS_NETWORK" >/dev/null <<NETWORK
|
||||
# Admin-up the OVS end after reboot (OVS owns the port but does
|
||||
# not flip admin state on external ports — same as veth mode).
|
||||
sudo tee "$NAT_OVS_NETWORK" >/dev/null <<NETWORK
|
||||
[Match]
|
||||
Name=spx-nat-ovs
|
||||
|
||||
@@ -1105,8 +1355,9 @@ RequiredForOnline=no
|
||||
[Network]
|
||||
ConfigureWithoutCarrier=yes
|
||||
NETWORK
|
||||
sudo networkctl reload 2>/dev/null || true
|
||||
echo " wrote $NAT_NETDEV + $NAT_NETWORK + $NAT_OVS_NETWORK (transit veth persists on reboot)"
|
||||
sudo networkctl reload 2>/dev/null || true
|
||||
echo " wrote $NAT_NETDEV + $NAT_NETWORK + $NAT_OVS_NETWORK (transit veth persists on reboot)"
|
||||
fi
|
||||
|
||||
# Kernel egress: masquerade the transit /24 out any uplink and
|
||||
# accept forwarded transit traffic. vpcd re-ensures these on every
|
||||
@@ -1261,8 +1512,9 @@ if [ "$MGMT_BRIDGE_ENABLED" = true ]; then
|
||||
# Persist the IP across reboots via a systemd-networkd drop-in. The
|
||||
# OVS bridge definition itself survives in ovsdb; only the L3 IP
|
||||
# needs re-applying on boot.
|
||||
MGMT_NETD_UNIT="/etc/systemd/network/10-spinifex-mgmt.network"
|
||||
sudo tee "$MGMT_NETD_UNIT" >/dev/null <<NETD
|
||||
if [ "$HOSTNET_BACKEND" = networkd ]; then
|
||||
MGMT_NETD_UNIT="/etc/systemd/network/10-spinifex-mgmt.network"
|
||||
sudo tee "$MGMT_NETD_UNIT" >/dev/null <<NETD
|
||||
[Match]
|
||||
Name=$MGMT_BRIDGE
|
||||
|
||||
@@ -1270,7 +1522,8 @@ Name=$MGMT_BRIDGE
|
||||
Address=$MGMT_CIDR
|
||||
ConfigureWithoutCarrier=yes
|
||||
NETD
|
||||
echo " wrote $MGMT_NETD_UNIT (IP persists on reboot via systemd-networkd)"
|
||||
echo " wrote $MGMT_NETD_UNIT (IP persists on reboot via systemd-networkd)"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
@@ -1401,8 +1654,10 @@ DATA_IFACE=$(ip -o -4 addr show | awk -v ip="$ENCAP_IP" '$0 ~ ip"/" {print $2}')
|
||||
if [ -n "$DATA_IFACE" ]; then
|
||||
SUBNET=$(ip -o -4 route show dev "$DATA_IFACE" proto kernel scope link | awk '{print $1}' | head -1)
|
||||
if [ -n "$SUBNET" ]; then
|
||||
sudo ip route replace "$SUBNET" dev "$DATA_IFACE" src "$ENCAP_IP" metric 50
|
||||
echo " data route: $SUBNET via $DATA_IFACE src $ENCAP_IP (metric 50)"
|
||||
sudo ip route replace "$SUBNET" dev "$DATA_IFACE" src "$ENCAP_IP" metric "$ENCAP_ROUTE_METRIC"
|
||||
echo " data route: $SUBNET via $DATA_IFACE src $ENCAP_IP (metric $ENCAP_ROUTE_METRIC)"
|
||||
ENCAP_ROUTE_DST="$SUBNET"
|
||||
ENCAP_ROUTE_DEV="$DATA_IFACE"
|
||||
else
|
||||
echo " skipped: no kernel route found for $DATA_IFACE"
|
||||
fi
|
||||
@@ -1410,6 +1665,12 @@ else
|
||||
echo " skipped: could not find interface for $ENCAP_IP"
|
||||
fi
|
||||
|
||||
if [ "$HOSTNET_BACKEND" = hostnet ]; then
|
||||
write_hostnet_state false
|
||||
hostnet_unlock
|
||||
echo " $HOSTNET_STATE: recorded for spinifex-hostnet"
|
||||
fi
|
||||
|
||||
# --- Step 7: Verify Geneve kernel support ---
|
||||
echo ""
|
||||
echo "Step 7: Verifying Geneve kernel module..."
|
||||
@@ -1509,17 +1770,24 @@ echo " OVS/OVN sockets: 0660 root:spinifex (group-scoped, no world access)"
|
||||
# file is the old 0666 override, and skipping would leave that exposure in place.
|
||||
# openvswitch-ipsec is included because Step 10 restarts it after this sweep,
|
||||
# so its ctl socket would otherwise be the one file left at the shipped mode.
|
||||
for unit in openvswitch-switch:/var/run/openvswitch/db.sock \
|
||||
"ovn-controller:/var/run/ovn/*.ctl" \
|
||||
"openvswitch-ipsec:/var/run/openvswitch/ovs-monitor-ipsec.*.ctl"; do
|
||||
# ovs-vswitchd starts after ovsdb-server and restarts with it (Requires=), so
|
||||
# its hook sees both daemons' sockets and fires on a restart of either.
|
||||
PERMS_UNITS=("ovs-vswitchd:/var/run/openvswitch/ovs-vswitchd.*.ctl"
|
||||
"ovn-controller:/var/run/ovn/*.ctl"
|
||||
"openvswitch-ipsec:/var/run/openvswitch/ovs-monitor-ipsec.*.ctl")
|
||||
# EL's ovn-northd.service starts the NB and SB databases as well as northd.
|
||||
if [ "$OVN_CENTRAL_LAYOUT" = lone-northd ]; then
|
||||
PERMS_UNITS+=("ovn-northd:/var/run/ovn/ovn-northd.*.ctl")
|
||||
fi
|
||||
for unit in "${PERMS_UNITS[@]}"; do
|
||||
UNIT="${unit%%:*}"
|
||||
WAIT_GLOB="${unit#*:}"
|
||||
OVERRIDE_DIR="/etc/systemd/system/${UNIT}.service.d"
|
||||
sudo mkdir -p "$OVERRIDE_DIR"
|
||||
# `ExecStartPost=-` so a failure here cannot stop the unit. This drop-in goes
|
||||
# into another package's service, and tightening a socket mode is hardening,
|
||||
# not a liveness requirement — it must never be the reason openvswitch-switch
|
||||
# or ovn-controller will not start. Without the dash, removing the helper
|
||||
# not a liveness requirement — it must never be the reason OVS or
|
||||
# ovn-controller will not start. Without the dash, removing the helper
|
||||
# (an uninstall, a partial re-run that has not reached this step yet) takes
|
||||
# OVS down across the whole host with `Failed at step EXEC`, which reads as
|
||||
# an OVS fault and is nothing of the sort.
|
||||
@@ -1570,7 +1838,7 @@ fi
|
||||
# to prevent log spam during those retries.
|
||||
echo ""
|
||||
echo "Step 10: Enabling OVN auto-start on boot..."
|
||||
sudo systemctl enable openvswitch-switch 2>/dev/null || true
|
||||
sudo systemctl enable "$OVS_UNIT" 2>/dev/null || true
|
||||
sudo systemctl enable ovn-controller 2>/dev/null || true
|
||||
# ovs-monitor-ipsec drives strongSwan from OVS DB cert pointers. The daemon's
|
||||
# enableOVNIPSec() flips ipsec_encapsulation=true at runtime and silently drops
|
||||
@@ -1583,9 +1851,43 @@ sudo systemctl enable openvswitch-ipsec.service 2>/dev/null || true
|
||||
# now that strongswan-starter is present.
|
||||
sudo systemctl reset-failed openvswitch-ipsec.service 2>/dev/null || true
|
||||
sudo systemctl restart openvswitch-ipsec.service 2>/dev/null || true
|
||||
echo " openvswitch-switch: enabled on boot"
|
||||
echo " ovn-controller: enabled on boot"
|
||||
echo " openvswitch-ipsec: enabled on boot"
|
||||
printf ' %-21s enabled on boot\n' "${OVS_UNIT%.service}:" ovn-controller: openvswitch-ipsec:
|
||||
|
||||
# Applied now, outside the hostnet lock, so the daemon and vpcd find its
|
||||
# readiness marker; on the networkd backend the apply does nothing.
|
||||
if [ -f /etc/systemd/system/spinifex-hostnet.service ]; then
|
||||
sudo systemctl enable spinifex-hostnet.service >/dev/null 2>&1
|
||||
if ! sudo systemctl restart spinifex-hostnet.service; then
|
||||
echo "ERROR: spinifex-hostnet could not apply the host network: journalctl -u spinifex-hostnet" >&2
|
||||
exit 1
|
||||
fi
|
||||
printf ' %-21s enabled on boot, applied (%s backend)\n' spinifex-hostnet: "$HOSTNET_BACKEND"
|
||||
else
|
||||
echo " spinifex-hostnet: not installed yet — run setup.sh, then re-run this script"
|
||||
fi
|
||||
|
||||
# systemd silently drops ordering against a unit that is not installed, so a
|
||||
# missing OVS or OVN unit would leave Spinifex starting unordered against OVS.
|
||||
SPINIFEX_UNITS=()
|
||||
for f in /etc/systemd/system/spinifex.target /etc/systemd/system/spinifex-*.service; do
|
||||
[ -f "$f" ] && SPINIFEX_UNITS+=("$f")
|
||||
done
|
||||
if [ ${#SPINIFEX_UNITS[@]} -eq 0 ]; then
|
||||
echo " Spinifex units: not installed yet — run setup.sh, then re-run this script"
|
||||
else
|
||||
MISSING_DEPS=$(distro_missing_ovs_deps "${SPINIFEX_UNITS[@]}") || exit 1
|
||||
DEPS_OK=true
|
||||
for dep in $MISSING_DEPS; do
|
||||
if [ "$dep" = openvswitch-ipsec.service ]; then
|
||||
echo " WARNING: $dep is not installed; IPsec cannot be turned on"
|
||||
else
|
||||
echo "ERROR: the Spinifex units order against $dep, which is not installed" >&2
|
||||
DEPS_OK=false
|
||||
fi
|
||||
done
|
||||
[ "$DEPS_OK" = true ] || exit 1
|
||||
echo " OVS/OVN units the Spinifex units name: installed"
|
||||
fi
|
||||
|
||||
# --- Step 11: Health check ---
|
||||
echo ""
|
||||
|
||||
+121
-1
@@ -131,7 +131,7 @@ esac
|
||||
|
||||
# The script and the Go constant must agree: vpcd repairs a veth made before
|
||||
# this existed, and a disagreement would have the two fighting every pass.
|
||||
script_mac=$(printf '%s\n' "$nat_mode_body" | sed -n 's/.*NAT_TRANSIT_HOST_MAC="\([^"]*\)".*/\1/p' | head -1)
|
||||
script_mac=$(sed -n 's/^NAT_TRANSIT_HOST_MAC="\([^"]*\)".*/\1/p' "$SETUP" | head -1)
|
||||
go_mac=$(sed -n 's/.*NATTransitHostMAC = "\([^"]*\)".*/\1/p' "$SCRIPT_DIR/../spinifex/domains/network/host/routed.go" | head -1)
|
||||
if [ -n "$script_mac" ] && [ "$script_mac" = "$go_mac" ]; then
|
||||
pass "the transit MAC matches host.NATTransitHostMAC"
|
||||
@@ -139,6 +139,126 @@ else
|
||||
fail "transit MAC disagrees: script=$script_mac go=$go_mac"
|
||||
fi
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# OVS and OVN service users.
|
||||
# --------------------------------------------------------------------------
|
||||
# EL runs OVS and OVN as non-root users and re-chowns their sockets on every
|
||||
# start. The function is run on its own against scratch files, through a sudo
|
||||
# that only runs its arguments.
|
||||
awk '$0 == "set_root_service_user() {" { on = 1 } on { print } on && $0 == "}" { exit }' "$SETUP" >"$TMP/user_fn.sh"
|
||||
mkdir -p "$TMP/bin"
|
||||
printf '#!/bin/sh\nexec "$@"\n' >"$TMP/bin/sudo"
|
||||
chmod +x "$TMP/bin/sudo"
|
||||
|
||||
# set_user <file> — runs the function on <file> for OVS_USER_ID; sets RC.
|
||||
set_user() {
|
||||
RC=0
|
||||
PATH="$TMP/bin:$PATH" bash -c '. "$1"; set_root_service_user "$2" OVS_USER_ID' _ "$TMP/user_fn.sh" "$1" >/dev/null || RC=$?
|
||||
}
|
||||
|
||||
set_user "$TMP/absent"
|
||||
if [ "$RC" != 0 ] && [ ! -e "$TMP/absent" ]; then pass "service user: no file, no change"; else fail "service user: absent file gave rc=$RC or was created"; fi
|
||||
|
||||
printf '# OVS_USER_ID="openvswitch:openvswitch"\nOPTIONS=""\nOVS_USER_ID="openvswitch:hugetlbfs"\n' >"$TMP/el"
|
||||
set_user "$TMP/el"
|
||||
want='# OVS_USER_ID="openvswitch:openvswitch"
|
||||
OPTIONS=""
|
||||
OVS_USER_ID="root:root"'
|
||||
if [ "$RC" = 0 ] && [ "$(cat "$TMP/el")" = "$want" ]; then pass "service user: EL default replaced in place, comments and other lines kept"; else fail "service user: EL file rc=$RC: $(cat "$TMP/el")"; fi
|
||||
|
||||
set_user "$TMP/el"
|
||||
if [ "$RC" != 0 ] && [ "$(cat "$TMP/el")" = "$want" ]; then pass "service user: already root reports no change"; else fail "service user: second run rc=$RC: $(cat "$TMP/el")"; fi
|
||||
|
||||
printf 'OPTIONS=""\n' >"$TMP/unset"
|
||||
set_user "$TMP/unset"
|
||||
if [ "$RC" = 0 ] && [ "$(cat "$TMP/unset")" = 'OPTIONS=""
|
||||
OVS_USER_ID="root:root"' ]; then pass "service user: appended when the file does not set it"; else fail "service user: unset file rc=$RC: $(cat "$TMP/unset")"; fi
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Host network persistence backend.
|
||||
# --------------------------------------------------------------------------
|
||||
# The backend is picked once and recorded; every later run reads the record.
|
||||
# The functions run on their own against scratch paths, with systemctl stubbed.
|
||||
for fn in select_hostnet_backend write_nm_unmanaged; do
|
||||
awk -v f="$fn() {" '$0 == f { on = 1 } on { print } on && $0 == "}" { exit }' "$SETUP" >>"$TMP/hostnet_fn.sh"
|
||||
done
|
||||
|
||||
# backend <networkd-enabled:yes|no> — runs select_hostnet_backend; sets RC, OUT.
|
||||
backend() {
|
||||
printf '#!/bin/sh
|
||||
[ "$1 $2" = "is-enabled --quiet" ] && [ "%s" = yes ]
|
||||
' "$1" >"$TMP/bin/systemctl"
|
||||
chmod +x "$TMP/bin/systemctl"
|
||||
RC=0
|
||||
OUT=$(PATH="$TMP/bin:$PATH" HOSTNET_BACKEND_FILE="$TMP/hn/etc/hostnet-backend" NETWORKD_DIR="$TMP/hn/network" \
|
||||
bash -c '. "$1"; select_hostnet_backend' _ "$TMP/hostnet_fn.sh" 2>&1) || RC=$?
|
||||
}
|
||||
|
||||
reset_hn() { rm -rf "$TMP/hn"; mkdir -p "$TMP/hn/network"; }
|
||||
|
||||
reset_hn
|
||||
backend no
|
||||
if [ "$RC" = 0 ] && [ "$OUT" = hostnet ] && [ "$(cat "$TMP/hn/etc/hostnet-backend")" = hostnet ]; then pass "backend: no networkd picks and records hostnet"; else fail "backend: no networkd gave rc=$RC out=$OUT"; fi
|
||||
|
||||
reset_hn
|
||||
backend yes
|
||||
if [ "$RC" = 0 ] && [ "$OUT" = networkd ] && [ "$(cat "$TMP/hn/etc/hostnet-backend")" = networkd ]; then pass "backend: networkd enabled picks networkd"; else fail "backend: networkd enabled gave rc=$RC out=$OUT"; fi
|
||||
|
||||
reset_hn
|
||||
: >"$TMP/hn/network/10-spinifex-mgmt.network"
|
||||
backend no
|
||||
if [ "$RC" = 0 ] && [ "$OUT" = networkd ]; then pass "backend: legacy Spinifex networkd files keep networkd"; else fail "backend: legacy files gave rc=$RC out=$OUT"; fi
|
||||
|
||||
reset_hn
|
||||
mkdir -p "$TMP/hn/etc"
|
||||
echo hostnet >"$TMP/hn/etc/hostnet-backend"
|
||||
backend yes
|
||||
if [ "$RC" = 0 ] && [ "$OUT" = hostnet ]; then pass "backend: the record wins over networkd being enabled"; else fail "backend: recorded hostnet gave rc=$RC out=$OUT"; fi
|
||||
|
||||
echo ifupdown >"$TMP/hn/etc/hostnet-backend"
|
||||
backend yes
|
||||
case "$RC:$OUT" in
|
||||
0:*) fail "backend: an unknown record was accepted: $OUT" ;;
|
||||
*"expected networkd or hostnet"*) pass "backend: an unknown record is refused" ;;
|
||||
*) fail "backend: an unknown record gave rc=$RC out=$OUT" ;;
|
||||
esac
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# NetworkManager unmanaged config.
|
||||
# --------------------------------------------------------------------------
|
||||
# NetworkManager globs have no character classes, so the management TAPs go to
|
||||
# a udev rule; the keyfile lists the rest. The host uplink bridge stays managed.
|
||||
printf '#!/bin/sh
|
||||
exit 0
|
||||
' >"$TMP/bin/udevadm"
|
||||
cp "$TMP/bin/udevadm" "$TMP/bin/nmcli"
|
||||
chmod +x "$TMP/bin/udevadm" "$TMP/bin/nmcli"
|
||||
glob=$(sed -n "s/^MGMT_TAP_GLOB='\([^']*\)'$/\1/p" "$SETUP")
|
||||
RC=0
|
||||
PATH="$TMP/bin:$PATH" NM_UNMANAGED_CONF="$TMP/nm/conf.d/00-spinifex-unmanaged.conf" NM_UNMANAGED_RULE="$TMP/nm/rules.d/90.rules" \
|
||||
MGMT_TAP_GLOB="$glob" WAN_BRIDGE=br-ext WAN_BRIDGE_MODE=direct WAN_IFACE=eth3 LINUX_BRIDGE=br-wan \
|
||||
MGMT_BRIDGE_ENABLED=true MGMT_BRIDGE=br-mgmt MGMT_IFACE=eth2 \
|
||||
bash -c '. "$1"; write_nm_unmanaged' _ "$TMP/hostnet_fn.sh" >/dev/null 2>&1 || RC=$?
|
||||
conf=$(cat "$TMP/nm/conf.d/00-spinifex-unmanaged.conf" 2>/dev/null || true)
|
||||
line=$(printf '%s\n' "$conf" | grep '^unmanaged-devices+=' || true)
|
||||
if [ "$RC" = 0 ] && printf '%s\n' "$conf" | grep -qx '\[keyfile\]' && [ -n "$line" ]; then pass "nm: keyfile section appends to unmanaged-devices"; else fail "nm: rc=$RC conf=$conf"; fi
|
||||
for want in br-int br-ext 'veth-wan-*' 'spx-nat-*' 'tap*' br-mgmt eth2 eth3; do
|
||||
case ";${line#unmanaged-devices+=};" in
|
||||
*";interface-name:$want;"*) pass "nm: $want unmanaged" ;;
|
||||
*) fail "nm: $want missing from $line" ;;
|
||||
esac
|
||||
done
|
||||
case "$line" in
|
||||
*br-wan*) fail "nm: the host uplink bridge br-wan was made unmanaged" ;;
|
||||
*'['*) fail "nm: keyfile uses a character class NetworkManager cannot match: $line" ;;
|
||||
*) pass "nm: host uplink left managed and no character classes" ;;
|
||||
esac
|
||||
rule=$(cat "$TMP/nm/rules.d/90.rules" 2>/dev/null || true)
|
||||
case "$rule" in
|
||||
*"KERNEL==\"$glob\", ENV{NM_UNMANAGED}=\"1\""*) pass "nm: udev rule unmanages management TAPs" ;;
|
||||
*) fail "nm: udev rule is $rule" ;;
|
||||
esac
|
||||
|
||||
if [ "$fails" -ne 0 ]; then
|
||||
echo " setup-ovn.sh: $fails failure(s)"
|
||||
exit 1
|
||||
|
||||
+288
-30
@@ -7,7 +7,7 @@
|
||||
# INSTALL_SPINIFEX_CHANNEL Release channel: latest (default), dev
|
||||
# INSTALL_SPINIFEX_VERSION Pin to specific version (overrides channel)
|
||||
# INSTALL_SPINIFEX_TARBALL Path to local tarball (skips download, for testing/air-gapped)
|
||||
# INSTALL_SPINIFEX_SKIP_APT Set to 1 to skip apt dependency install
|
||||
# INSTALL_SPINIFEX_SKIP_APT Set to 1 to skip the system package install (apt or dnf)
|
||||
# INSTALL_SPINIFEX_SKIP_AWS Set to 1 to skip AWS CLI install
|
||||
# INSTALL_SPINIFEX_SKIP_NEWGRP Set to 1 to skip newgrp exec at end (for callers like dev-install.sh)
|
||||
# ISO_BUILD Set to 1 when running inside a debootstrap chroot from the ISO
|
||||
@@ -58,6 +58,24 @@ info() { echo -e "${GREEN}[INFO]${NC} $*"; }
|
||||
warn() { echo -e "${YELLOW}[WARN]${NC} $*"; }
|
||||
fatal() { echo -e "${RED}[ERROR]${NC} $*"; exit 1; }
|
||||
|
||||
# --- BEGIN distro.sh probes ---
|
||||
# Copied from scripts/lib/distro.sh, which this script installs but cannot source
|
||||
# when piped from the install endpoint. distro_test.sh keeps the copies equal.
|
||||
distro_pkg_manager() {
|
||||
local apt=false dnf=false
|
||||
command -v apt-get >/dev/null 2>&1 && apt=true
|
||||
command -v dnf >/dev/null 2>&1 && dnf=true
|
||||
if $apt && $dnf; then
|
||||
echo "distro.sh: package manager: found both apt-get and dnf; cannot tell which one manages this host" >&2
|
||||
return 1
|
||||
fi
|
||||
if $apt; then echo apt; return 0; fi
|
||||
if $dnf; then echo dnf; return 0; fi
|
||||
echo "distro.sh: package manager: found neither apt-get nor dnf on PATH" >&2
|
||||
return 1
|
||||
}
|
||||
# --- END distro.sh probes ---
|
||||
|
||||
stage() {
|
||||
[ "${VERBOSE:-0}" = "1" ] && echo "[setup] $*"
|
||||
return 0
|
||||
@@ -92,14 +110,25 @@ setup_sudo() {
|
||||
}
|
||||
|
||||
# --- OS detection ---
|
||||
# os_release_field <key> <file> — the value of KEY=value, with one level of quotes removed.
|
||||
os_release_field() {
|
||||
sed -n "s/^$1=//p" "$2" | head -n 1 | sed -e 's/^"\(.*\)"$/\1/' -e "s/^'\(.*\)'$/\1/"
|
||||
}
|
||||
|
||||
# The only place that keys on the OS ID: the gate and, later, the package table.
|
||||
# Everything else probes for the capability (scripts/lib/distro.sh).
|
||||
detect_os() {
|
||||
if [ ! -f /etc/os-release ]; then
|
||||
fatal "Cannot detect OS: /etc/os-release not found"
|
||||
local os_release="${SPINIFEX_OS_RELEASE:-/etc/os-release}"
|
||||
if [ ! -f "$os_release" ]; then
|
||||
fatal "Cannot detect OS: $os_release not found"
|
||||
fi
|
||||
|
||||
. /etc/os-release
|
||||
# Parsed, not sourced, so a malformed file is refused instead of executed.
|
||||
ID=$(os_release_field ID "$os_release")
|
||||
VERSION_ID=$(os_release_field VERSION_ID "$os_release")
|
||||
PRETTY_NAME=$(os_release_field PRETTY_NAME "$os_release")
|
||||
|
||||
case "$ID" in
|
||||
case "${ID:-}" in
|
||||
debian)
|
||||
if [ "${VERSION_ID%%.*}" -lt 13 ] 2>/dev/null; then
|
||||
fatal "Debian $VERSION_ID is not supported. Minimum: Debian 13"
|
||||
@@ -111,12 +140,77 @@ detect_os() {
|
||||
fatal "Ubuntu $VERSION_ID is not supported. Minimum: Ubuntu 24.04"
|
||||
fi
|
||||
;;
|
||||
ol)
|
||||
check_oracle_linux
|
||||
;;
|
||||
*)
|
||||
fatal "Unsupported OS: $ID $VERSION_ID. Spinifex requires Debian 13+ or Ubuntu 24.04+"
|
||||
fatal "Unsupported OS: ${ID:-unknown} ${VERSION_ID:-}. Spinifex requires Debian 13+, Ubuntu 24.04+ or Oracle Linux 10.2+"
|
||||
;;
|
||||
esac
|
||||
|
||||
info "Detected OS: $PRETTY_NAME"
|
||||
info "Detected OS: ${PRETTY_NAME:-$ID $VERSION_ID}"
|
||||
}
|
||||
|
||||
# Oracle Linux 10.2+ on x86_64 only, running UEK R8: the KVM Utilities QEMU
|
||||
# needs UEK, and Oracle ships kernel modules per kernel, so installing a newer
|
||||
# kernel-uek alone leaves the running one without openvswitch.ko.
|
||||
check_oracle_linux() {
|
||||
local unsupported="Oracle Linux ${VERSION_ID:-<no VERSION_ID>} is not supported. Spinifex requires Oracle Linux 10.2 or a later 10.x"
|
||||
local major="${VERSION_ID%%.*}" minor="${VERSION_ID#*.}" kernel
|
||||
minor="${minor%%.*}"
|
||||
case "${VERSION_ID:-}" in *.*) ;; *) fatal "$unsupported" ;; esac
|
||||
case "$major" in "" | *[!0-9]*) fatal "$unsupported" ;; esac
|
||||
case "$minor" in "" | *[!0-9]*) fatal "$unsupported" ;; esac
|
||||
if [ "$major" -ne 10 ] || [ "$minor" -lt 2 ]; then
|
||||
fatal "$unsupported"
|
||||
fi
|
||||
|
||||
if [ "$(uname -m)" != "x86_64" ]; then
|
||||
fatal "Oracle Linux on $(uname -m) is not supported. Spinifex supports Oracle Linux on x86_64 only"
|
||||
fi
|
||||
|
||||
kernel=$(uname -r)
|
||||
case "$kernel" in
|
||||
6.12.0-*uek*) ;;
|
||||
*) fatal "The running kernel $kernel is not UEK R8 (6.12.0-*uek). Install kernel-uek, reboot into it, then re-run this installer" ;;
|
||||
esac
|
||||
|
||||
check_x86_64_v3
|
||||
check_selinux_not_enforcing
|
||||
}
|
||||
|
||||
# Spinifex ships no SELinux policy yet. Oracle's templates boot enforcing, and
|
||||
# setenforce alone reverts on reboot, so the config file is checked as well.
|
||||
check_selinux_not_enforcing() {
|
||||
local config="${SPINIFEX_SELINUX_CONFIG:-/etc/selinux/config}" now="" boot=""
|
||||
if command -v getenforce >/dev/null 2>&1; then
|
||||
now=$(getenforce 2>/dev/null | tr '[:upper:]' '[:lower:]')
|
||||
fi
|
||||
if [ -f "$config" ]; then
|
||||
boot=$(os_release_field SELINUX "$config" | tr '[:upper:]' '[:lower:]')
|
||||
fi
|
||||
if [ "$now" = "enforcing" ]; then
|
||||
fatal "SELinux is enforcing, and Spinifex has no SELinux policy. Run 'sudo setenforce 0', set SELINUX=permissive in $config, then re-run this installer"
|
||||
fi
|
||||
if [ "$boot" = "enforcing" ]; then
|
||||
fatal "SELinux is set to enforcing at next boot (SELINUX=enforcing in $config). Set SELINUX=permissive there, then re-run this installer"
|
||||
fi
|
||||
}
|
||||
|
||||
# The NFV SIG OVS/OVN and the KVM Utilities QEMU are built for x86-64-v3, so an
|
||||
# older CPU installs cleanly and then dies on an illegal instruction.
|
||||
check_x86_64_v3() {
|
||||
local cpuinfo="${SPINIFEX_CPUINFO:-/proc/cpuinfo}" flags flag missing=""
|
||||
flags=" $(grep -m1 '^flags' "$cpuinfo" 2>/dev/null | cut -d: -f2) "
|
||||
for flag in cx16 lahf_lm popcnt sse4_1 sse4_2 ssse3 avx avx2 bmi1 bmi2 f16c fma abm movbe xsave; do
|
||||
case "$flags" in
|
||||
*" $flag "*) ;;
|
||||
*) missing="${missing:+$missing }$flag" ;;
|
||||
esac
|
||||
done
|
||||
if [ -n "$missing" ]; then
|
||||
fatal "This CPU is below x86-64-v3 (missing: $missing). Oracle Linux's OVS, OVN and QEMU builds require x86-64-v3"
|
||||
fi
|
||||
}
|
||||
|
||||
# --- Architecture detection ---
|
||||
@@ -733,7 +827,7 @@ SUDOERS
|
||||
info "Scoped sudoers rules installed for spinifex-daemon (spinifex-vpcd needs none)"
|
||||
}
|
||||
|
||||
# --- Install apt dependencies ---
|
||||
# --- Install system dependencies ---
|
||||
# NOTE: Runtime deps must stay in sync with the ISO package list at
|
||||
# scripts/iso-builder/build/packages.list in the mulga repo. When you add,
|
||||
# remove, or change a runtime package here, review that file too — drift means
|
||||
@@ -750,29 +844,43 @@ jq curl iproute2 ethtool netcat-openbsd wget unzip xz-utils file
|
||||
ovn-central ovn-host openvswitch-switch openvswitch-ipsec strongswan-charon dhcpcd-base
|
||||
chrony nftables"
|
||||
|
||||
install_apt_deps() {
|
||||
stage "installing apt dependencies"
|
||||
# The Oracle Linux equivalents, plus what a minimal OL install lacks. OVS 3.5
|
||||
# and OVN 25.03 come from the CentOS NFV SIG; 25.03 is Debian 13's OVN, which
|
||||
# the hand-written OVN NB models are written against.
|
||||
OL_RUNTIME_PACKAGES="nbdkit nbdkit-basic-filters
|
||||
qemu-kvm qemu-img edk2-ovmf seabios-bin less
|
||||
pciutils
|
||||
jq curl iproute ethtool nmap-ncat wget unzip xz file
|
||||
ovn25.03-central ovn25.03-host openvswitch3.5 openvswitch3.5-ipsec libreswan dhcpcd
|
||||
chrony nftables iptables-nft systemd-resolved"
|
||||
|
||||
# Oracle ships no OVS or OVN for OL10. The allowlist stops the SIG supplying
|
||||
# anything but OVS, OVN and the two SIG-only packages they depend on.
|
||||
NFV_SIG_REPO_ID="spinifex-nfv-sig"
|
||||
NFV_SIG_KEY_URL="https://www.centos.org/keys/RPM-GPG-KEY-CentOS-SIG-NFV"
|
||||
NFV_SIG_KEY_FILE="/etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-NFV"
|
||||
NFV_SIG_KEY_FPR="35154228174901BEFA8E69A621465E289D2A76A7"
|
||||
NFV_SIG_INCLUDEPKGS="openvswitch3.5* ovn25.03* python3-openvswitch3.5 openvswitch-selinux-extra-policy"
|
||||
|
||||
# Only the KVM Utilities QEMU has the microvm machine. Excluding QEMU from
|
||||
# AppStream stops a later AppStream build ever replacing part of that stack.
|
||||
OL_KVM_UTILS_REPO="ol10_kvm_utils"
|
||||
OL_APPSTREAM_REPO="ol10_appstream"
|
||||
OL_APPSTREAM_EXCLUDES="qemu-kvm* qemu-img*"
|
||||
# The running kernel's modules come from here, and KVM Utilities requires kernel-uek.
|
||||
OL_UEK_REPO="ol10_UEKR8"
|
||||
|
||||
install_deps() {
|
||||
stage "installing system dependencies"
|
||||
if [ "${INSTALL_SPINIFEX_SKIP_APT}" = "1" ]; then
|
||||
info "Skipping apt dependencies (INSTALL_SPINIFEX_SKIP_APT=1)"
|
||||
info "Skipping system dependencies (INSTALL_SPINIFEX_SKIP_APT=1)"
|
||||
else
|
||||
info "Installing system dependencies..."
|
||||
$SUDO apt-get update -qq
|
||||
|
||||
# Unquoted on purpose: both variables are whitespace-separated package lists.
|
||||
# shellcheck disable=SC2086
|
||||
DEBIAN_FRONTEND=noninteractive $SUDO apt-get install -y -qq \
|
||||
$QEMU_PACKAGES $APT_RUNTIME_PACKAGES \
|
||||
> /dev/null
|
||||
|
||||
info "System dependencies installed"
|
||||
|
||||
# Only on a machine that already boots from ZFS. Pulling zfsutils-linux
|
||||
# in unconditionally would drag zfs-dkms onto every dev box and rebuild
|
||||
# the module on each kernel upgrade for no benefit.
|
||||
if [ "$(findmnt -no FSTYPE /)" = "zfs" ] && ! command -v zpool >/dev/null 2>&1; then
|
||||
info "ZFS root detected, installing zfsutils-linux..."
|
||||
DEBIAN_FRONTEND=noninteractive $SUDO apt-get install -y -qq zfsutils-linux > /dev/null
|
||||
fi
|
||||
local pm
|
||||
pm=$(distro_pkg_manager) || fatal "Cannot tell which package manager this host uses"
|
||||
case "$pm" in
|
||||
apt) install_apt_deps ;;
|
||||
dnf) install_dnf_deps ;;
|
||||
esac
|
||||
fi
|
||||
|
||||
# Mask the standalone dhcpcd.service auto-enabled on Debian Trixie when
|
||||
@@ -786,6 +894,149 @@ install_apt_deps() {
|
||||
$SUDO systemctl mask dhcpcd.service 2>/dev/null || true
|
||||
}
|
||||
|
||||
install_apt_deps() {
|
||||
info "Installing system dependencies..."
|
||||
$SUDO apt-get update -qq
|
||||
|
||||
# Unquoted on purpose: both variables are whitespace-separated package lists.
|
||||
# shellcheck disable=SC2086
|
||||
DEBIAN_FRONTEND=noninteractive $SUDO apt-get install -y -qq \
|
||||
$QEMU_PACKAGES $APT_RUNTIME_PACKAGES \
|
||||
> /dev/null
|
||||
|
||||
info "System dependencies installed"
|
||||
|
||||
# Only on a machine that already boots from ZFS. Pulling zfsutils-linux
|
||||
# in unconditionally would drag zfs-dkms onto every dev box and rebuild
|
||||
# the module on each kernel upgrade for no benefit.
|
||||
if [ "$(findmnt -no FSTYPE /)" = "zfs" ] && ! command -v zpool >/dev/null 2>&1; then
|
||||
info "ZFS root detected, installing zfsutils-linux..."
|
||||
DEBIAN_FRONTEND=noninteractive $SUDO apt-get install -y -qq zfsutils-linux > /dev/null
|
||||
fi
|
||||
}
|
||||
|
||||
# The package table is the other place keyed on the OS ID: dnf alone does not
|
||||
# say which repositories or package names a host has.
|
||||
install_dnf_deps() {
|
||||
case "${ID:-}" in
|
||||
ol) ;;
|
||||
*) fatal "No dnf package list for ${ID:-an unknown OS}. Spinifex installs with dnf on Oracle Linux only" ;;
|
||||
esac
|
||||
if [ "$(findmnt -no FSTYPE /)" = "zfs" ]; then
|
||||
fatal "A ZFS root is not supported on Oracle Linux"
|
||||
fi
|
||||
|
||||
ensure_ol_repos
|
||||
install_running_kernel_modules
|
||||
|
||||
info "Installing system dependencies..."
|
||||
# shellcheck disable=SC2086 # a whitespace-separated package list
|
||||
$SUDO dnf install -y -q $OL_RUNTIME_PACKAGES > /dev/null
|
||||
info "System dependencies installed"
|
||||
|
||||
load_kernel_modules openvswitch geneve vhost_net tun
|
||||
}
|
||||
|
||||
# Enables KVM Utilities, pins QEMU to it, and adds the NFV SIG repository with
|
||||
# a key whose fingerprint must match the one recorded above.
|
||||
ensure_ol_repos() {
|
||||
info "Configuring Oracle Linux repositories..."
|
||||
local repo
|
||||
$SUDO dnf install -y -q dnf-plugins-core gnupg2 curl > /dev/null
|
||||
|
||||
for repo in "$OL_UEK_REPO" "$OL_KVM_UTILS_REPO" "$OL_APPSTREAM_REPO"; do
|
||||
dnf config-manager --dump "$repo" > /dev/null 2>&1 \
|
||||
|| fatal "Repository $repo is not defined on this host. Oracle Linux's release packages define it; reinstall oraclelinux-release-el10"
|
||||
done
|
||||
$SUDO dnf config-manager --set-enabled "$OL_UEK_REPO" "$OL_KVM_UTILS_REPO" > /dev/null
|
||||
exclude_from_appstream
|
||||
|
||||
install_nfv_sig_key
|
||||
$SUDO tee "/etc/yum.repos.d/${NFV_SIG_REPO_ID}.repo" > /dev/null <<EOF
|
||||
# Written by the Spinifex installer. Re-running it rewrites this file.
|
||||
[${NFV_SIG_REPO_ID}]
|
||||
name=CentOS NFV SIG Open vSwitch and OVN, for Spinifex
|
||||
baseurl=https://mirror.stream.centos.org/SIGs/10-stream/nfv/\$basearch/openvswitch-2/
|
||||
enabled=1
|
||||
gpgcheck=1
|
||||
gpgkey=file://${NFV_SIG_KEY_FILE}
|
||||
includepkgs=${NFV_SIG_INCLUDEPKGS}
|
||||
EOF
|
||||
|
||||
for repo in "$OL_UEK_REPO" "$OL_APPSTREAM_REPO" "$OL_KVM_UTILS_REPO" "$NFV_SIG_REPO_ID"; do
|
||||
$SUDO dnf -q makecache --repo "$repo" > /dev/null \
|
||||
|| fatal "Repository $repo is unreachable. Check this host's network access and proxy settings, then re-run this installer"
|
||||
done
|
||||
}
|
||||
|
||||
# Adds the QEMU excludes to AppStream, keeping any exclude already set there.
|
||||
exclude_from_appstream() {
|
||||
local current pattern merged="" patterns
|
||||
current=$(dnf config-manager --dump "$OL_APPSTREAM_REPO" | sed -n 's/^excludepkgs = //p' | tr ',' ' ')
|
||||
# read, not an unquoted expansion, so the patterns are never globbed.
|
||||
read -ra patterns <<< "$current $OL_APPSTREAM_EXCLUDES"
|
||||
for pattern in "${patterns[@]}"; do
|
||||
case " $merged " in
|
||||
*" $pattern "*) ;;
|
||||
*) merged="${merged:+$merged }$pattern" ;;
|
||||
esac
|
||||
done
|
||||
$SUDO dnf config-manager --save --setopt="${OL_APPSTREAM_REPO}.excludepkgs=${merged// /,}" > /dev/null
|
||||
}
|
||||
|
||||
# gpg_fingerprints <file> — the primary-key fingerprint of each key in file.
|
||||
gpg_fingerprints() {
|
||||
local home
|
||||
home=$(mktemp -d)
|
||||
GNUPGHOME="$home" gpg --batch --with-colons --show-keys "$1" 2>/dev/null \
|
||||
| awk -F: '$1 == "pub" { want = 1; next } want && $1 == "fpr" { print $10; want = 0 }'
|
||||
rm -rf "$home"
|
||||
}
|
||||
|
||||
install_nfv_sig_key() {
|
||||
if [ -f "$NFV_SIG_KEY_FILE" ] && [ "$(gpg_fingerprints "$NFV_SIG_KEY_FILE")" = "$NFV_SIG_KEY_FPR" ]; then
|
||||
$SUDO rpm --import "$NFV_SIG_KEY_FILE"
|
||||
return 0
|
||||
fi
|
||||
local tmp got
|
||||
tmp=$(mktemp)
|
||||
curl -fsSL "$NFV_SIG_KEY_URL" -o "$tmp" \
|
||||
|| { rm -f "$tmp"; fatal "Cannot download the CentOS NFV SIG key from $NFV_SIG_KEY_URL"; }
|
||||
got=$(gpg_fingerprints "$tmp")
|
||||
if [ "$got" != "$NFV_SIG_KEY_FPR" ]; then
|
||||
rm -f "$tmp"
|
||||
fatal "The key at $NFV_SIG_KEY_URL has fingerprint ${got:-<unreadable>}, not the CentOS NFV SIG's $NFV_SIG_KEY_FPR. Refusing to trust it"
|
||||
fi
|
||||
$SUDO install -D -m 0644 "$tmp" "$NFV_SIG_KEY_FILE"
|
||||
rm -f "$tmp"
|
||||
$SUDO rpm --import "$NFV_SIG_KEY_FILE"
|
||||
}
|
||||
|
||||
# Oracle ships kernel modules per kernel version. The unversioned package
|
||||
# follows the newest kernel and leaves the running one without openvswitch.ko.
|
||||
install_running_kernel_modules() {
|
||||
local pkg
|
||||
pkg="kernel-uek-modules-$(uname -r)"
|
||||
if rpm -q "$pkg" > /dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
if ! $SUDO dnf -q list --available "$pkg" > /dev/null 2>&1; then
|
||||
fatal "$pkg, the modules for the running kernel, is no longer in the repositories. Run 'sudo dnf update kernel-uek', reboot into the new kernel, then re-run this installer"
|
||||
fi
|
||||
info "Installing $pkg (it pulls in linux-firmware, about 800 MB)..."
|
||||
$SUDO dnf install -y -q "$pkg" > /dev/null
|
||||
}
|
||||
|
||||
load_kernel_modules() {
|
||||
local mod failed=""
|
||||
for mod in "$@"; do
|
||||
$SUDO modprobe "$mod" || failed="${failed:+$failed }$mod"
|
||||
done
|
||||
if [ -n "$failed" ]; then
|
||||
fatal "Cannot load kernel modules for the running kernel $(uname -r): $failed"
|
||||
fi
|
||||
}
|
||||
|
||||
# --- Install AWS CLI ---
|
||||
install_aws_cli() {
|
||||
stage "installing AWS CLI v2"
|
||||
@@ -891,6 +1142,13 @@ install_files() {
|
||||
info " /usr/local/share/spinifex/setup-ovn.sh"
|
||||
fi
|
||||
|
||||
# Sourced, not run, by setup-ovn.sh and uninstall-spx.sh wherever they are
|
||||
# installed, including the ISO's /usr/local/bin/setup-ovn.sh.
|
||||
if [ -f "$EXTRACT_DIR/lib/distro.sh" ]; then
|
||||
$SUDO install -D -m 0644 "$EXTRACT_DIR/lib/distro.sh" /usr/local/share/spinifex/lib/distro.sh
|
||||
info " /usr/local/share/spinifex/lib/distro.sh"
|
||||
fi
|
||||
|
||||
# Install setup.sh itself so firstboot and future re-runs can find it at a
|
||||
# stable path on both ISO and curl|bash installs.
|
||||
if [ -f "$EXTRACT_DIR/setup.sh" ]; then
|
||||
@@ -1451,7 +1709,7 @@ main() {
|
||||
download_spinifex
|
||||
fi
|
||||
|
||||
stage_enabled deps && install_apt_deps
|
||||
stage_enabled deps && install_deps
|
||||
stage_enabled aws && install_aws_cli
|
||||
stage_enabled users && create_service_users
|
||||
stage_enabled sudoers && install_sudoers
|
||||
|
||||
@@ -0,0 +1,172 @@
|
||||
#!/bin/sh
|
||||
# setup_test.sh — tests for setup.sh's supported-OS gate.
|
||||
#
|
||||
# setup.sh is sourced with INSTALL_SPINIFEX_LIB_ONLY=1 so main never runs, and
|
||||
# detect_os reads fixture os-release, cpuinfo and SELinux config files, and stub
|
||||
# uname and getenforce commands.
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
|
||||
SETUP="$SCRIPT_DIR/setup.sh"
|
||||
TMP=$(mktemp -d)
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
|
||||
fails=0
|
||||
pass() { echo " ok - $1"; }
|
||||
fail() { echo " FAIL - $1"; fails=$((fails + 1)); }
|
||||
|
||||
echo "-- setup.sh OS gate"
|
||||
|
||||
V3_FLAGS="fpu cx16 lahf_lm popcnt sse4_1 sse4_2 ssse3 avx avx2 bmi1 bmi2 f16c fma abm movbe xsave"
|
||||
V2_FLAGS="fpu cx16 lahf_lm popcnt sse4_1 sse4_2 ssse3"
|
||||
UEK_R8="6.12.0-207.111.5.4.el10uek.x86_64"
|
||||
|
||||
# gate <os-release body> [machine] [kernel] [cpu flags] — runs detect_os and
|
||||
# sets RC and OUT (stdout and stderr together). SELINUX_NOW is what getenforce
|
||||
# prints; SELINUX_BOOT is the config file's SELINUX=, or "none" for no file.
|
||||
gate() {
|
||||
printf '%s\n' "$1" > "$TMP/os-release"
|
||||
printf 'processor\t: 0\nflags\t\t: %s\n' "${4:-$V3_FLAGS}" > "$TMP/cpuinfo"
|
||||
rm -f "$TMP/selinux-config"
|
||||
if [ "${SELINUX_BOOT:-permissive}" != none ]; then
|
||||
printf '# fixture\nSELINUX=%s\nSELINUXTYPE=targeted\n' "${SELINUX_BOOT:-permissive}" > "$TMP/selinux-config"
|
||||
fi
|
||||
mkdir -p "$TMP/bin"
|
||||
printf '#!/bin/sh\necho %s\n' "${SELINUX_NOW:-Permissive}" > "$TMP/bin/getenforce"
|
||||
chmod +x "$TMP/bin/getenforce"
|
||||
cat > "$TMP/bin/uname" <<EOF
|
||||
#!/bin/sh
|
||||
case "\$1" in
|
||||
-m) echo "${2:-x86_64}" ;;
|
||||
-r) echo "${3:-$UEK_R8}" ;;
|
||||
*) exit 1 ;;
|
||||
esac
|
||||
EOF
|
||||
chmod +x "$TMP/bin/uname"
|
||||
RC=0
|
||||
OUT=$(PATH="$TMP/bin:$PATH" SPINIFEX_OS_RELEASE="$TMP/os-release" SPINIFEX_CPUINFO="$TMP/cpuinfo" \
|
||||
SPINIFEX_SELINUX_CONFIG="$TMP/selinux-config" INSTALL_SPINIFEX_LIB_ONLY=1 bash -c '. "$1"; detect_os' _ "$SETUP" 2>&1) || RC=$?
|
||||
# Some shells keep a prefix assignment to a function call after it returns.
|
||||
unset SELINUX_NOW SELINUX_BOOT
|
||||
}
|
||||
accepts() {
|
||||
if [ "$RC" = 0 ]; then pass "accepts $1"; else fail "rejects $1: $OUT"; fi
|
||||
}
|
||||
# rejects <desc> <word> — the refusal must say why, not just exit.
|
||||
rejects() {
|
||||
if [ "$RC" = 0 ]; then
|
||||
fail "accepts $1"
|
||||
else
|
||||
case "$OUT" in
|
||||
*"$2"*) pass "rejects $1" ;;
|
||||
*) fail "rejects $1 without naming '$2': $OUT" ;;
|
||||
esac
|
||||
fi
|
||||
}
|
||||
|
||||
OL='ID="ol"
|
||||
NAME="Oracle Linux Server"
|
||||
PRETTY_NAME="Oracle Linux Server 10.2"'
|
||||
|
||||
gate 'ID=debian
|
||||
VERSION_ID="13"
|
||||
PRETTY_NAME="Debian GNU/Linux 13 (trixie)"'
|
||||
accepts "Debian 13"
|
||||
|
||||
# Debian testing and sid carry no VERSION_ID. The gate has always let them
|
||||
# through, and Debian behaviour is not this gate's to change.
|
||||
gate 'ID=debian
|
||||
PRETTY_NAME="Debian GNU/Linux forky/sid"'
|
||||
accepts "Debian testing with no VERSION_ID"
|
||||
|
||||
gate 'ID=debian
|
||||
VERSION_ID="12"'
|
||||
rejects "Debian 12" "Minimum: Debian 13"
|
||||
|
||||
gate 'ID=ubuntu
|
||||
VERSION_ID="26.04"
|
||||
PRETTY_NAME="Ubuntu 26.04 LTS"'
|
||||
accepts "Ubuntu 26.04"
|
||||
|
||||
gate 'ID=ubuntu
|
||||
VERSION_ID="22.04"'
|
||||
rejects "Ubuntu 22.04" "Minimum: Ubuntu 24.04"
|
||||
|
||||
gate "$OL
|
||||
VERSION_ID=\"10.2\""
|
||||
accepts "Oracle Linux 10.2 on UEK R8, x86_64, x86-64-v3"
|
||||
|
||||
gate "$OL
|
||||
VERSION_ID=\"10.4\""
|
||||
accepts "Oracle Linux 10.4"
|
||||
|
||||
gate "$OL
|
||||
VERSION_ID=\"10.1\""
|
||||
rejects "Oracle Linux 10.1" "Oracle Linux 10.2 or a later 10.x"
|
||||
|
||||
gate "$OL
|
||||
VERSION_ID=\"10\""
|
||||
rejects "Oracle Linux with no minor version" "Oracle Linux 10.2 or a later 10.x"
|
||||
|
||||
gate "$OL
|
||||
VERSION_ID=\"11.0\""
|
||||
rejects "Oracle Linux 11" "Oracle Linux 10.2 or a later 10.x"
|
||||
|
||||
gate "$OL
|
||||
VERSION_ID=\"9.6\"" x86_64 "5.15.0-310.184.5.2.el9uek.x86_64"
|
||||
rejects "Oracle Linux 9" "Oracle Linux 10.2 or a later 10.x"
|
||||
|
||||
gate "$OL" x86_64
|
||||
rejects "Oracle Linux with no VERSION_ID" "Oracle Linux 10.2 or a later 10.x"
|
||||
|
||||
gate "$OL
|
||||
VERSION_ID=\"10.2\"" aarch64 "6.12.0-207.111.5.4.el10uek.aarch64"
|
||||
rejects "Oracle Linux on aarch64" "x86_64 only"
|
||||
|
||||
gate "$OL
|
||||
VERSION_ID=\"10.2\"" x86_64 "6.12.0-55.9.1.el10_0.x86_64"
|
||||
rejects "Oracle Linux on the Red Hat compatible kernel" "is not UEK R8"
|
||||
|
||||
gate "$OL
|
||||
VERSION_ID=\"10.2\"" x86_64 "5.15.0-310.184.5.2.el9uek.x86_64"
|
||||
rejects "Oracle Linux on a UEK older than R8" "is not UEK R8"
|
||||
|
||||
gate "$OL
|
||||
VERSION_ID=\"10.2\"" x86_64 "$UEK_R8" "$V2_FLAGS"
|
||||
rejects "Oracle Linux on an x86-64-v2 CPU" "missing: avx avx2 bmi1 bmi2 f16c fma abm movbe xsave"
|
||||
|
||||
OL102="$OL
|
||||
VERSION_ID=\"10.2\""
|
||||
|
||||
SELINUX_NOW=Enforcing gate "$OL102"
|
||||
rejects "Oracle Linux with SELinux enforcing" "SELinux is enforcing"
|
||||
|
||||
SELINUX_BOOT=enforcing gate "$OL102"
|
||||
rejects "Oracle Linux with SELinux permissive now but enforcing at next boot" "enforcing at next boot"
|
||||
|
||||
SELINUX_BOOT='"enforcing"' gate "$OL102"
|
||||
rejects "Oracle Linux with a quoted SELINUX=\"enforcing\"" "enforcing at next boot"
|
||||
|
||||
SELINUX_NOW=Disabled SELINUX_BOOT=disabled gate "$OL102"
|
||||
accepts "Oracle Linux with SELinux disabled"
|
||||
|
||||
SELINUX_NOW=Disabled SELINUX_BOOT=none gate "$OL102"
|
||||
accepts "Oracle Linux with no SELinux config file"
|
||||
|
||||
# Spinifex has no SELinux policy on any OS, but refusing enforcing on Debian
|
||||
# would change Debian behaviour, so only the Oracle Linux gate checks it.
|
||||
SELINUX_NOW=Enforcing SELINUX_BOOT=enforcing gate 'ID=debian
|
||||
VERSION_ID="13"'
|
||||
accepts "Debian 13 regardless of SELinux"
|
||||
|
||||
gate 'ID="rocky"
|
||||
VERSION_ID="10.0"'
|
||||
rejects "Rocky Linux 10" "Unsupported OS: rocky"
|
||||
|
||||
gate 'this is not an os-release file'
|
||||
rejects "a malformed os-release" "Unsupported OS"
|
||||
|
||||
if [ "$fails" -gt 0 ]; then
|
||||
echo "$fails failure(s)"
|
||||
exit 1
|
||||
fi
|
||||
@@ -73,6 +73,19 @@ while [[ $# -gt 0 ]]; do
|
||||
shift
|
||||
done
|
||||
|
||||
# Host-OS capability probes: beside this script in a checkout and in the
|
||||
# installed share dir, else where setup.sh installs them.
|
||||
DISTRO_LIB=""
|
||||
for candidate in "$(dirname "${BASH_SOURCE[0]}")/lib/distro.sh" /usr/local/share/spinifex/lib/distro.sh; do
|
||||
if [ -f "$candidate" ]; then DISTRO_LIB="$candidate"; break; fi
|
||||
done
|
||||
if [ -z "$DISTRO_LIB" ]; then
|
||||
echo "ERROR: distro.sh not found at $(dirname "${BASH_SOURCE[0]}")/lib/distro.sh or /usr/local/share/spinifex/lib/distro.sh; re-run setup.sh to install it" >&2
|
||||
exit 1
|
||||
fi
|
||||
# shellcheck source=lib/distro.sh
|
||||
. "$DISTRO_LIB"
|
||||
|
||||
log() { echo "[uninstall-spx] $*"; }
|
||||
run() {
|
||||
if $DRY_RUN; then
|
||||
@@ -208,7 +221,7 @@ fi
|
||||
#
|
||||
# It handles the guests, the OVS bridges, the OVN databases and the JetStream
|
||||
# store, and it carries correctness that is invisible in its code: QEMU matched
|
||||
# as ^qemu-system because comm truncates at 15 characters, a wait for QEMU to
|
||||
# by a comm prefix because comm truncates at 15 characters, a wait for QEMU to
|
||||
# exit rather than for the signal, and the firewall-mode stash that stops an
|
||||
# interrupted run leaving the node unarmed. A second copy of that would drift,
|
||||
# and the failure mode of the drift is data loss.
|
||||
@@ -294,12 +307,14 @@ rm_path /etc/systemd/system/spinifex-*.service \
|
||||
# exactly that way on the first dev-oci round trip.
|
||||
log "removing drop-ins written into other units"
|
||||
rm_path /etc/systemd/system/openvswitch-switch.service.d/spinifex-perms.conf \
|
||||
/etc/systemd/system/ovs-vswitchd.service.d/spinifex-perms.conf \
|
||||
/etc/systemd/system/ovn-controller.service.d/spinifex-perms.conf \
|
||||
/etc/systemd/system/ovn-controller.service.d/log-level.conf \
|
||||
/etc/systemd/system/openvswitch-ipsec.service.d/spinifex-perms.conf \
|
||||
/etc/systemd/system/ovn-northd.service.d/spinifex-perms.conf \
|
||||
/etc/systemd/system/ovn-northd.service.d/no-manage-ovsdb.conf
|
||||
# Only if we emptied them — another package may own the same override dir.
|
||||
for d in openvswitch-switch ovn-controller openvswitch-ipsec ovn-northd; do
|
||||
for d in openvswitch-switch ovs-vswitchd ovn-controller openvswitch-ipsec ovn-northd; do
|
||||
dir="/etc/systemd/system/${d}.service.d"
|
||||
if sudo test -d "$dir" && [ -z "$(sudo ls -A "$dir" 2>/dev/null)" ]; then
|
||||
rm_path "$dir"
|
||||
@@ -379,8 +394,12 @@ rm_path /etc/sysctl.d/*spinifex* \
|
||||
/etc/sudoers.d/spinifex-network \
|
||||
/etc/chrony/conf.d/spinifex.conf \
|
||||
/etc/logrotate.d/spinifex \
|
||||
/etc/udev/rules.d/*spinifex*
|
||||
/etc/udev/rules.d/*spinifex* \
|
||||
/etc/NetworkManager/conf.d/*spinifex*
|
||||
run sudo sysctl --system >/dev/null 2>&1 || true
|
||||
if systemctl is-active --quiet NetworkManager.service 2>/dev/null; then
|
||||
run sudo nmcli general reload conf 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 10. Spinifex directories.
|
||||
@@ -503,6 +522,7 @@ if ! $DRY_RUN; then
|
||||
for p in /etc/systemd/system/spinifex-*.service /etc/systemd/system/spinifex*.slice \
|
||||
/etc/systemd/system/spinifex.target /etc/sysctl.d/*spinifex* \
|
||||
/etc/systemd/network/1[5-8]-spinifex-* \
|
||||
/etc/NetworkManager/conf.d/*spinifex* \
|
||||
/etc/systemd/system/*.service.d/spinifex-perms.conf \
|
||||
/etc/systemd/system/ovn-northd.service.d/no-manage-ovsdb.conf \
|
||||
/etc/systemd/system/ovn-controller.service.d/log-level.conf; do
|
||||
|
||||
@@ -70,11 +70,6 @@ func NewClusterEngine(
|
||||
reapers = append(reapers, S3Reapers(buckets)...)
|
||||
reapers = append(reapers, ACMReapers(nc)...)
|
||||
reapers = append(reapers, IAMReapers(svc)...)
|
||||
|
||||
// Before the account counters and reservations: the seal must still be
|
||||
// able to find the account IAM holds as TERMINATING, which the account
|
||||
// record reaper does not remove.
|
||||
reapers = append(reapers, EC2AccountSettingsReapers(nc, nodeIDs)...)
|
||||
reapers = append(reapers, AccountReapers(svc, names, usage)...)
|
||||
|
||||
engine := NewEngine(NewIAMAccounts(svc), reapers...)
|
||||
|
||||
@@ -41,27 +41,6 @@ func TestNewClusterEngineCoversEveryStage(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The EC2 settings seal reaper must be registered in StageAccount on every
|
||||
// surface that builds a cluster engine, or teardown would finalise accounts
|
||||
// without ever sealing their settings.
|
||||
func TestNewClusterEngineRegistersTheEC2SettingsSealReaper(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
|
||||
masterKey, err := handlers_iam.GenerateMasterKey()
|
||||
require.NoError(t, err)
|
||||
svc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey)
|
||||
require.NoError(t, err)
|
||||
|
||||
engine, err := NewClusterEngine(t.Context(), nc, 1, []string{"node-1"}, svc, newFakeBuckets())
|
||||
require.NoError(t, err)
|
||||
|
||||
var kinds []string
|
||||
for _, reaper := range engine.reapersFor(StageAccount) {
|
||||
kinds = append(kinds, reaper.Kind())
|
||||
}
|
||||
assert.Contains(t, kinds, "ec2-account-settings")
|
||||
}
|
||||
|
||||
// A teardown must refuse the accounts no credential may delete before it reads
|
||||
// anything, and refuse a name confirmation that does not match the record.
|
||||
func TestClusterEnginePrechecks(t *testing.T) {
|
||||
|
||||
@@ -1,125 +0,0 @@
|
||||
package accountteardown
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
|
||||
natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// ec2AccountSettingsSealTimeout bounds one seal request; an earlier caller
|
||||
// deadline wins.
|
||||
const ec2AccountSettingsSealTimeout = 30 * time.Second
|
||||
|
||||
// ec2AccountSettingsSealer is EC2's seal of one account's settings record.
|
||||
type ec2AccountSettingsSealer interface {
|
||||
SealAccountSettings(ctx context.Context, accountID string) (revision uint64, err error)
|
||||
}
|
||||
|
||||
// ec2AccountSettingsSealClient reaches the EC2 settings-seal contract over
|
||||
// NATS. Only a reply for this account, disposition sealed and a non-zero
|
||||
// revision is a seal; anything else, including no responder, is an error.
|
||||
type ec2AccountSettingsSealClient struct {
|
||||
nc *nats.Conn
|
||||
timeout time.Duration
|
||||
}
|
||||
|
||||
var _ ec2AccountSettingsSealer = (*ec2AccountSettingsSealClient)(nil)
|
||||
|
||||
func newEC2AccountSettingsSealClient(nc *nats.Conn) *ec2AccountSettingsSealClient {
|
||||
return &ec2AccountSettingsSealClient{nc: nc, timeout: ec2AccountSettingsSealTimeout}
|
||||
}
|
||||
|
||||
func (c *ec2AccountSettingsSealClient) SealAccountSettings(ctx context.Context, accountID string) (uint64, error) {
|
||||
timeout := c.timeout
|
||||
if deadline, ok := ctx.Deadline(); ok {
|
||||
timeout = min(timeout, time.Until(deadline))
|
||||
}
|
||||
if ctx.Err() != nil || timeout <= 0 {
|
||||
return 0, fmt.Errorf("seal account settings: %w", context.DeadlineExceeded)
|
||||
}
|
||||
|
||||
reply, err := natsmsg.NATSRequest[ec2v1.SealAccountSettingsReply](ctx, c.nc, ec2v1.SealAccountSettingsSubject,
|
||||
&ec2v1.SealAccountSettingsRequest{AccountID: accountID}, timeout, accountID)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("seal account settings: %w", err)
|
||||
}
|
||||
if reply.AccountID != accountID {
|
||||
return 0, fmt.Errorf("seal account settings: reply named account %q, not %q", reply.AccountID, accountID)
|
||||
}
|
||||
if reply.Disposition != ec2v1.AccountSettingsDispositionSealed {
|
||||
return 0, fmt.Errorf("seal account settings: unexpected disposition %q", reply.Disposition)
|
||||
}
|
||||
if reply.Revision == 0 {
|
||||
return 0, errors.New("seal account settings: reply carried no revision")
|
||||
}
|
||||
return reply.Revision, nil
|
||||
}
|
||||
|
||||
// accountSettingsSealReaper seals EC2's settings record in StageAccount. List
|
||||
// does no I/O and stays pending until this instance has sealed, so DryRun never
|
||||
// seals; state is per engine, so every fresh teardown run seals again.
|
||||
type accountSettingsSealReaper struct {
|
||||
sealer ec2AccountSettingsSealer
|
||||
capability sealCapabilityChecker
|
||||
|
||||
mu sync.Mutex
|
||||
sealed map[string]uint64 // account ID -> leader revision, this run only
|
||||
}
|
||||
|
||||
func (r *accountSettingsSealReaper) Kind() string { return "ec2-account-settings" }
|
||||
func (r *accountSettingsSealReaper) Stage() Stage { return StageAccount }
|
||||
|
||||
func (r *accountSettingsSealReaper) alreadySealed(accountID string) bool {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
_, ok := r.sealed[accountID]
|
||||
return ok
|
||||
}
|
||||
|
||||
func (r *accountSettingsSealReaper) List(_ context.Context, accountID string) ([]Resource, error) {
|
||||
if r.alreadySealed(accountID) {
|
||||
return nil, nil
|
||||
}
|
||||
return []Resource{{Kind: r.Kind(), ID: accountID, Detail: "seal"}}, nil
|
||||
}
|
||||
|
||||
// Delete probes every configured daemon immediately before each seal attempt,
|
||||
// never reusing an earlier probe. Incomplete capability sends no seal and is
|
||||
// retryable; force is ignored, since nothing may force past the probe.
|
||||
func (r *accountSettingsSealReaper) Delete(ctx context.Context, accountID string, _ Resource, _ bool) error {
|
||||
report := r.capability.CheckSealCapability(ctx, accountID)
|
||||
if !report.Complete {
|
||||
return errSealCapabilityIncomplete(report)
|
||||
}
|
||||
|
||||
revision, err := r.sealer.SealAccountSettings(ctx, accountID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
r.mu.Lock()
|
||||
if r.sealed == nil {
|
||||
r.sealed = map[string]uint64{}
|
||||
}
|
||||
r.sealed[accountID] = revision
|
||||
r.mu.Unlock()
|
||||
slog.InfoContext(ctx, "Account teardown sealed EC2 account settings", "accountID", accountID, "revision", revision)
|
||||
return nil
|
||||
}
|
||||
|
||||
// EC2AccountSettingsReapers returns the StageAccount reaper that seals EC2's
|
||||
// settings record. nodeIDs is the configured node set the capability probe
|
||||
// treats as "every relevant writer"; an empty set is unknown, never complete.
|
||||
func EC2AccountSettingsReapers(nc *nats.Conn, nodeIDs []string) []Reaper {
|
||||
return []Reaper{&accountSettingsSealReaper{
|
||||
sealer: newEC2AccountSettingsSealClient(nc),
|
||||
capability: newSealCapabilityClient(nc, nodeIDs),
|
||||
}}
|
||||
}
|
||||
@@ -1,190 +0,0 @@
|
||||
package accountteardown
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
|
||||
awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// sealCapabilityWait bounds the whole probe; shortened by the caller's
|
||||
// deadline. Every configured node is probed in parallel against this one
|
||||
// wait, so N nodes cost one wait, not N.
|
||||
const sealCapabilityWait = 3 * time.Second
|
||||
|
||||
// sealCapabilityGrace is how long a node's probe keeps listening after its
|
||||
// one expected responder has answered, long enough for a genuine second
|
||||
// subscriber on the same node to be caught as a duplicate.
|
||||
const sealCapabilityGrace = 150 * time.Millisecond
|
||||
|
||||
// Seal capability reasons, one per failing node. Each names a category, never
|
||||
// wire error text.
|
||||
const (
|
||||
sealCapabilityReasonNoReply = "no-reply"
|
||||
sealCapabilityReasonDuplicate = "duplicate"
|
||||
sealCapabilityReasonWrongIdentity = "wrong-identity"
|
||||
sealCapabilityReasonErrorReply = "error-reply"
|
||||
sealCapabilityReasonMalformed = "malformed"
|
||||
sealCapabilityReasonOldVersion = "old-version"
|
||||
sealCapabilityReasonTransport = "transport"
|
||||
sealCapabilityReasonInvalidNodeID = "invalid-node-id"
|
||||
sealCapabilityReasonSetUnknown = "configured-node-set-unknown"
|
||||
)
|
||||
|
||||
// sealCapabilityChecker reports whether every configured daemon supports the
|
||||
// seal.
|
||||
type sealCapabilityChecker interface {
|
||||
CheckSealCapability(ctx context.Context, correlationRef string) sealCapabilityReport
|
||||
}
|
||||
|
||||
// sealCapabilityReport is one probe's result, one entry per configured node.
|
||||
type sealCapabilityReport struct {
|
||||
Complete bool
|
||||
Nodes []nodeSealCapability // one per configured node, sorted by node
|
||||
}
|
||||
|
||||
// nodeSealCapability is one configured node's compatibility. Reason is ""
|
||||
// when the node is compatible.
|
||||
type nodeSealCapability struct {
|
||||
Node string
|
||||
Version int // 0 unless a valid reply arrived
|
||||
Reason string // "" when compatible
|
||||
}
|
||||
|
||||
// sealCapabilityClient probes every configured node's seal-capability subject
|
||||
// in parallel. "Configured" is the same node set NewClusterEngine already
|
||||
// receives; there is no separate liveness or membership source.
|
||||
type sealCapabilityClient struct {
|
||||
nc *nats.Conn
|
||||
nodeIDs []string
|
||||
wait time.Duration // 3 s, shortened by the caller's deadline
|
||||
grace time.Duration // 150 ms ResponderGrace
|
||||
}
|
||||
|
||||
var _ sealCapabilityChecker = (*sealCapabilityClient)(nil)
|
||||
|
||||
func newSealCapabilityClient(nc *nats.Conn, nodeIDs []string) *sealCapabilityClient {
|
||||
return &sealCapabilityClient{nc: nc, nodeIDs: nodeIDs, wait: sealCapabilityWait, grace: sealCapabilityGrace}
|
||||
}
|
||||
|
||||
// CheckSealCapability probes every configured node in parallel, bounded by
|
||||
// one shared deadline, run immediately before each seal attempt. A cached or
|
||||
// earlier result is never reused.
|
||||
func (c *sealCapabilityClient) CheckSealCapability(ctx context.Context, correlationRef string) sealCapabilityReport {
|
||||
if len(c.nodeIDs) == 0 {
|
||||
return sealCapabilityReport{Complete: false}
|
||||
}
|
||||
|
||||
wait := c.wait
|
||||
if deadline, ok := ctx.Deadline(); ok {
|
||||
wait = min(wait, time.Until(deadline))
|
||||
}
|
||||
|
||||
nodes := append([]string(nil), c.nodeIDs...)
|
||||
sort.Strings(nodes)
|
||||
|
||||
results := make([]nodeSealCapability, len(nodes))
|
||||
if ctx.Err() != nil || wait <= 0 {
|
||||
for i, node := range nodes {
|
||||
results[i] = nodeSealCapability{Node: node, Reason: sealCapabilityReasonNoReply}
|
||||
}
|
||||
return sealCapabilityReport{Complete: false, Nodes: results}
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for i, node := range nodes {
|
||||
wg.Add(1)
|
||||
go func(i int, node string) {
|
||||
defer wg.Done()
|
||||
results[i] = c.checkNode(ctx, node, correlationRef, wait)
|
||||
}(i, node)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
complete := true
|
||||
for _, r := range results {
|
||||
if r.Reason != "" {
|
||||
complete = false
|
||||
break
|
||||
}
|
||||
}
|
||||
return sealCapabilityReport{Complete: complete, Nodes: results}
|
||||
}
|
||||
|
||||
// checkNode probes one node and classifies the result. A node is compatible
|
||||
// only when exactly one identified, decodable, matching, version >= 1 frame
|
||||
// arrived, with no duplicate, conflict or cap hit.
|
||||
func (c *sealCapabilityClient) checkNode(ctx context.Context, node, correlationRef string, wait time.Duration) nodeSealCapability {
|
||||
if !ec2v1.ValidSealCapabilityNode(node) {
|
||||
return nodeSealCapability{Node: node, Reason: sealCapabilityReasonInvalidNodeID}
|
||||
}
|
||||
|
||||
data, err := json.Marshal(ec2v1.AccountSettingsSealCapabilityRequest{Node: node, CorrelationRef: correlationRef})
|
||||
if err != nil {
|
||||
return nodeSealCapability{Node: node, Reason: sealCapabilityReasonMalformed}
|
||||
}
|
||||
|
||||
frames, sum, err := natsmsg.Gather(ctx, c.nc, ec2v1.AccountSettingsSealCapabilitySubject(node), data, natsmsg.GatherOpts{
|
||||
Timeout: wait,
|
||||
ExpectedResponders: 1,
|
||||
Mode: natsmsg.CollectUntilDeadline,
|
||||
ResponderGrace: c.grace,
|
||||
})
|
||||
if err != nil {
|
||||
return nodeSealCapability{Node: node, Reason: sealCapabilityReasonTransport}
|
||||
}
|
||||
|
||||
switch {
|
||||
case sum.CapHit, len(sum.ConflictNodes) > 0, sum.DuplicateFrames > 0:
|
||||
return nodeSealCapability{Node: node, Reason: sealCapabilityReasonDuplicate}
|
||||
case sum.Unidentified > 0:
|
||||
return nodeSealCapability{Node: node, Reason: sealCapabilityReasonWrongIdentity}
|
||||
case len(sum.ErrorResponders) > 0:
|
||||
return nodeSealCapability{Node: node, Reason: sealCapabilityReasonErrorReply}
|
||||
case sum.Received == 0, len(frames) == 0:
|
||||
return nodeSealCapability{Node: node, Reason: sealCapabilityReasonNoReply}
|
||||
}
|
||||
|
||||
frame := frames[0]
|
||||
if frame.NodeID != node {
|
||||
return nodeSealCapability{Node: node, Reason: sealCapabilityReasonWrongIdentity}
|
||||
}
|
||||
|
||||
var reply ec2v1.AccountSettingsSealCapabilityReply
|
||||
if json.Unmarshal(frame.Data, &reply) != nil {
|
||||
return nodeSealCapability{Node: node, Reason: sealCapabilityReasonMalformed}
|
||||
}
|
||||
if reply.Node != node {
|
||||
return nodeSealCapability{Node: node, Reason: sealCapabilityReasonWrongIdentity}
|
||||
}
|
||||
if reply.CapabilityVersion < 1 {
|
||||
return nodeSealCapability{Node: node, Version: reply.CapabilityVersion, Reason: sealCapabilityReasonOldVersion}
|
||||
}
|
||||
return nodeSealCapability{Node: node, Version: reply.CapabilityVersion}
|
||||
}
|
||||
|
||||
// errSealCapabilityIncomplete builds the retryable refusal from report. It
|
||||
// names node IDs and reason categories only, never wire error text.
|
||||
func errSealCapabilityIncomplete(report sealCapabilityReport) error {
|
||||
detail := sealCapabilityReasonSetUnknown
|
||||
var parts []string
|
||||
for _, n := range report.Nodes {
|
||||
if n.Reason != "" {
|
||||
parts = append(parts, n.Node+"="+n.Reason)
|
||||
}
|
||||
}
|
||||
if len(parts) > 0 {
|
||||
detail = strings.Join(parts, ", ")
|
||||
}
|
||||
return awserrors.WithRetryAfter(
|
||||
awserrors.Errorf(awserrors.ErrorServiceUnavailable, "EC2 account-settings seal capability incomplete: %s", detail),
|
||||
5*time.Second,
|
||||
)
|
||||
}
|
||||
@@ -1,311 +0,0 @@
|
||||
package accountteardown
|
||||
|
||||
//test:in-package — the probe client, its report types and reason constants
|
||||
// are unexported, and what they guarantee (incomplete never seals, complete
|
||||
// does) is only observable from inside the package.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
|
||||
"github.com/mulgadc/spinifex/internal/testkit"
|
||||
awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// testCapabilityClient builds a probe client with test-sized timeouts so the
|
||||
// incomplete cases below do not each pay the production 3 s wait.
|
||||
func testCapabilityClient(nc *nats.Conn, nodeIDs []string) *sealCapabilityClient {
|
||||
c := newSealCapabilityClient(nc, nodeIDs)
|
||||
c.wait = 200 * time.Millisecond
|
||||
c.grace = 30 * time.Millisecond
|
||||
return c
|
||||
}
|
||||
|
||||
// serveCapability subscribes reply on subject with no queue group, which is
|
||||
// how the real responder is addressed: per node, so two subscribers on the
|
||||
// same node's subject both answer.
|
||||
func serveCapability(t *testing.T, nc *nats.Conn, subject string, reply nats.MsgHandler) *nats.Subscription {
|
||||
t.Helper()
|
||||
sub, err := nc.Subscribe(subject, reply)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
require.NoError(t, nc.Flush())
|
||||
return sub
|
||||
}
|
||||
|
||||
// capabilityReply answers with a well-formed reply naming replyNode (which
|
||||
// may deliberately differ from the subject's node) and headerNode as the
|
||||
// X-Node-ID ("" omits the header).
|
||||
func capabilityReply(replyNode, headerNode string, version int) nats.MsgHandler {
|
||||
return func(msg *nats.Msg) {
|
||||
data, _ := json.Marshal(ec2v1.AccountSettingsSealCapabilityReply{Node: replyNode, CapabilityVersion: version})
|
||||
reply := nats.NewMsg(msg.Reply)
|
||||
reply.Data = data
|
||||
if headerNode != "" {
|
||||
reply.Header.Set(natsmsg.NodeIDHeader, headerNode)
|
||||
}
|
||||
_ = msg.RespondMsg(reply)
|
||||
}
|
||||
}
|
||||
|
||||
func nodeResult(t *testing.T, report sealCapabilityReport, node string) nodeSealCapability {
|
||||
t.Helper()
|
||||
for _, n := range report.Nodes {
|
||||
if n.Node == node {
|
||||
return n
|
||||
}
|
||||
}
|
||||
t.Fatalf("no result for node %q in %+v", node, report)
|
||||
return nodeSealCapability{}
|
||||
}
|
||||
|
||||
// Each sub-case names exactly one way a node fails to prove capability
|
||||
// version 1, and that failure alone makes the whole probe incomplete.
|
||||
func TestSealCapabilityIncomplete(t *testing.T) {
|
||||
t.Run("missing", func(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
report := testCapabilityClient(nc, []string{"node-1"}).CheckSealCapability(testCtx(t), "req")
|
||||
assert.False(t, report.Complete)
|
||||
assert.Equal(t, sealCapabilityReasonNoReply, nodeResult(t, report, "node-1").Reason)
|
||||
})
|
||||
|
||||
t.Run("duplicate", func(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
serveCapability(t, nc, ec2v1.AccountSettingsSealCapabilitySubject("node-1"), capabilityReply("node-1", "node-1", 1))
|
||||
serveCapability(t, nc, ec2v1.AccountSettingsSealCapabilitySubject("node-1"), capabilityReply("node-1", "node-1", 1))
|
||||
report := testCapabilityClient(nc, []string{"node-1"}).CheckSealCapability(testCtx(t), "req")
|
||||
assert.False(t, report.Complete)
|
||||
assert.Equal(t, sealCapabilityReasonDuplicate, nodeResult(t, report, "node-1").Reason)
|
||||
})
|
||||
|
||||
t.Run("unreachable timeout", func(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
serveCapability(t, nc, ec2v1.AccountSettingsSealCapabilitySubject("node-1"), func(msg *nats.Msg) {
|
||||
time.Sleep(time.Second)
|
||||
_ = msg.Respond(nil)
|
||||
})
|
||||
client := testCapabilityClient(nc, []string{"node-1"})
|
||||
report := client.CheckSealCapability(testCtx(t), "req")
|
||||
assert.False(t, report.Complete)
|
||||
assert.Equal(t, sealCapabilityReasonNoReply, nodeResult(t, report, "node-1").Reason)
|
||||
})
|
||||
|
||||
t.Run("transport closed connection", func(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
nc.Close()
|
||||
report := testCapabilityClient(nc, []string{"node-1"}).CheckSealCapability(testCtx(t), "req")
|
||||
assert.False(t, report.Complete)
|
||||
assert.Equal(t, sealCapabilityReasonTransport, nodeResult(t, report, "node-1").Reason)
|
||||
})
|
||||
|
||||
t.Run("malformed body", func(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
serveCapability(t, nc, ec2v1.AccountSettingsSealCapabilitySubject("node-1"), func(msg *nats.Msg) {
|
||||
reply := nats.NewMsg(msg.Reply)
|
||||
reply.Data = []byte("not json")
|
||||
reply.Header.Set(natsmsg.NodeIDHeader, "node-1")
|
||||
_ = msg.RespondMsg(reply)
|
||||
})
|
||||
report := testCapabilityClient(nc, []string{"node-1"}).CheckSealCapability(testCtx(t), "req")
|
||||
assert.False(t, report.Complete)
|
||||
assert.Equal(t, sealCapabilityReasonMalformed, nodeResult(t, report, "node-1").Reason)
|
||||
})
|
||||
|
||||
t.Run("error reply", func(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
serveCapability(t, nc, ec2v1.AccountSettingsSealCapabilitySubject("node-1"), func(msg *nats.Msg) {
|
||||
reply := nats.NewMsg(msg.Reply)
|
||||
reply.Data = awserrors.GenerateErrorPayload(awserrors.ErrorServerInternal)
|
||||
reply.Header.Set(natsmsg.NodeIDHeader, "node-1")
|
||||
_ = msg.RespondMsg(reply)
|
||||
})
|
||||
report := testCapabilityClient(nc, []string{"node-1"}).CheckSealCapability(testCtx(t), "req")
|
||||
assert.False(t, report.Complete)
|
||||
assert.Equal(t, sealCapabilityReasonErrorReply, nodeResult(t, report, "node-1").Reason)
|
||||
})
|
||||
|
||||
t.Run("wrong identity no header", func(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
serveCapability(t, nc, ec2v1.AccountSettingsSealCapabilitySubject("node-1"), capabilityReply("node-1", "", 1))
|
||||
report := testCapabilityClient(nc, []string{"node-1"}).CheckSealCapability(testCtx(t), "req")
|
||||
assert.False(t, report.Complete)
|
||||
assert.Equal(t, sealCapabilityReasonWrongIdentity, nodeResult(t, report, "node-1").Reason)
|
||||
})
|
||||
|
||||
t.Run("wrong identity another node header", func(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
serveCapability(t, nc, ec2v1.AccountSettingsSealCapabilitySubject("node-1"), capabilityReply("node-1", "node-9", 1))
|
||||
report := testCapabilityClient(nc, []string{"node-1"}).CheckSealCapability(testCtx(t), "req")
|
||||
assert.False(t, report.Complete)
|
||||
assert.Equal(t, sealCapabilityReasonWrongIdentity, nodeResult(t, report, "node-1").Reason)
|
||||
})
|
||||
|
||||
t.Run("wrong identity body node differs", func(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
serveCapability(t, nc, ec2v1.AccountSettingsSealCapabilitySubject("node-1"), capabilityReply("node-9", "node-1", 1))
|
||||
report := testCapabilityClient(nc, []string{"node-1"}).CheckSealCapability(testCtx(t), "req")
|
||||
assert.False(t, report.Complete)
|
||||
assert.Equal(t, sealCapabilityReasonWrongIdentity, nodeResult(t, report, "node-1").Reason)
|
||||
})
|
||||
|
||||
t.Run("lower version", func(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
serveCapability(t, nc, ec2v1.AccountSettingsSealCapabilitySubject("node-1"), capabilityReply("node-1", "node-1", 0))
|
||||
report := testCapabilityClient(nc, []string{"node-1"}).CheckSealCapability(testCtx(t), "req")
|
||||
assert.False(t, report.Complete)
|
||||
assert.Equal(t, sealCapabilityReasonOldVersion, nodeResult(t, report, "node-1").Reason)
|
||||
})
|
||||
|
||||
t.Run("configured set empty", func(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
report := testCapabilityClient(nc, nil).CheckSealCapability(testCtx(t), "req")
|
||||
assert.False(t, report.Complete)
|
||||
assert.Empty(t, report.Nodes)
|
||||
})
|
||||
|
||||
t.Run("invalid node id", func(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
report := testCapabilityClient(nc, []string{"bad.node"}).CheckSealCapability(testCtx(t), "req")
|
||||
assert.False(t, report.Complete)
|
||||
assert.Equal(t, sealCapabilityReasonInvalidNodeID, nodeResult(t, report, "bad.node").Reason)
|
||||
})
|
||||
}
|
||||
|
||||
// Every configured node answering version 1 or later makes the probe
|
||||
// complete, and the reaper sends exactly one seal request.
|
||||
func TestSealCapabilityCompleteAllowsSeal(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
serveCapability(t, nc, ec2v1.AccountSettingsSealCapabilitySubject("node-1"), capabilityReply("node-1", "node-1", 1))
|
||||
serveCapability(t, nc, ec2v1.AccountSettingsSealCapabilitySubject("node-2"), capabilityReply("node-2", "node-2", 2))
|
||||
|
||||
client := testCapabilityClient(nc, []string{"node-1", "node-2"})
|
||||
report := client.CheckSealCapability(testCtx(t), "req")
|
||||
require.True(t, report.Complete)
|
||||
assert.Equal(t, 1, nodeResult(t, report, "node-1").Version)
|
||||
assert.Equal(t, 2, nodeResult(t, report, "node-2").Version)
|
||||
|
||||
sealer := &fakeSealer{}
|
||||
reaper := &accountSettingsSealReaper{sealer: sealer, capability: client}
|
||||
require.NoError(t, reaper.Delete(testCtx(t), testAccountID, Resource{Kind: reaper.Kind(), ID: testAccountID}, false))
|
||||
assert.Equal(t, []string{testAccountID}, sealer.calls)
|
||||
}
|
||||
|
||||
// An incomplete probe keeps the account TERMINATING through the full engine,
|
||||
// names the silent node and its reason, sends no seal request, and leaves the
|
||||
// settings record untouched (the fake sealer records no call).
|
||||
func TestSealCapabilityIncompleteKeepsAccountTerminating(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
serveCapability(t, nc, ec2v1.AccountSettingsSealCapabilitySubject("node-1"), capabilityReply("node-1", "node-1", 1))
|
||||
// node-2 has no subscriber.
|
||||
|
||||
sealer := &fakeSealer{}
|
||||
reaper := &accountSettingsSealReaper{sealer: sealer, capability: testCapabilityClient(nc, []string{"node-1", "node-2"})}
|
||||
accounts := newFakeAccounts()
|
||||
engine := quietEngine(accounts, reaper)
|
||||
|
||||
result, err := engine.Teardown(testCtx(t), request())
|
||||
require.ErrorIs(t, err, ErrResourcesStuck)
|
||||
assert.False(t, result.AccountDeleted)
|
||||
assert.Equal(t, AccountStatusTerminating, accounts.account.Status)
|
||||
assert.Empty(t, sealer.calls)
|
||||
|
||||
var reason string
|
||||
for _, stage := range result.Stages {
|
||||
for _, stuck := range stage.Stuck {
|
||||
if stuck.Resource.Kind == "ec2-account-settings" {
|
||||
reason = stuck.Reason
|
||||
}
|
||||
}
|
||||
}
|
||||
assert.Contains(t, reason, "node-2="+sealCapabilityReasonNoReply)
|
||||
}
|
||||
|
||||
// The silent node starting to answer during the drain makes the very next
|
||||
// Delete re-probe and succeed, which proves the result is retryable and the
|
||||
// probe is never cached across attempts.
|
||||
func TestSealCapabilityRetriesThenSeals(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
serveCapability(t, nc, ec2v1.AccountSettingsSealCapabilitySubject("node-1"), capabilityReply("node-1", "node-1", 1))
|
||||
|
||||
var node2Up atomic.Bool
|
||||
sub, err := nc.Subscribe(ec2v1.AccountSettingsSealCapabilitySubject("node-2"), func(msg *nats.Msg) {
|
||||
if !node2Up.Load() {
|
||||
return
|
||||
}
|
||||
capabilityReply("node-2", "node-2", 1)(msg)
|
||||
})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
require.NoError(t, nc.Flush())
|
||||
|
||||
sealer := &fakeSealer{}
|
||||
reaper := &accountSettingsSealReaper{sealer: sealer, capability: testCapabilityClient(nc, []string{"node-1", "node-2"})}
|
||||
accounts := newFakeAccounts()
|
||||
engine := quietEngine(accounts, reaper)
|
||||
engine.Timeouts = Timeouts{StageDrain: time.Second, DrainPoll: 5 * time.Millisecond}
|
||||
|
||||
go func() {
|
||||
time.Sleep(80 * time.Millisecond)
|
||||
node2Up.Store(true)
|
||||
}()
|
||||
|
||||
result, err := engine.Teardown(testCtx(t), request())
|
||||
require.NoError(t, err)
|
||||
assert.True(t, result.AccountDeleted)
|
||||
assert.Equal(t, []string{testAccountID}, sealer.calls)
|
||||
}
|
||||
|
||||
// Two Delete calls on one reaper probe twice: the probe runs in Delete
|
||||
// itself, never cached from a previous attempt.
|
||||
func TestSealCapabilityProbedBeforeEverySeal(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
var probes int
|
||||
serveCapability(t, nc, ec2v1.AccountSettingsSealCapabilitySubject("node-1"), func(msg *nats.Msg) {
|
||||
probes++
|
||||
capabilityReply("node-1", "node-1", 1)(msg)
|
||||
})
|
||||
|
||||
sealer := &fakeSealer{}
|
||||
client := testCapabilityClient(nc, []string{"node-1"})
|
||||
reaper := &accountSettingsSealReaper{sealer: sealer, capability: client}
|
||||
|
||||
require.NoError(t, reaper.Delete(testCtx(t), testAccountID, Resource{Kind: reaper.Kind(), ID: testAccountID}, false))
|
||||
// alreadySealed now makes List empty, but Delete itself does not consult
|
||||
// it, so a second direct call still probes and seals again.
|
||||
require.NoError(t, reaper.Delete(testCtx(t), testAccountID, Resource{Kind: reaper.Kind(), ID: testAccountID}, false))
|
||||
assert.Equal(t, 2, probes)
|
||||
assert.Len(t, sealer.calls, 2)
|
||||
}
|
||||
|
||||
// Five silent nodes still return within one deadline, not five: every node
|
||||
// is probed in parallel against the same wait.
|
||||
func TestSealCapabilityBoundedByOneDeadline(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
client := testCapabilityClient(nc, []string{"node-1", "node-2", "node-3", "node-4", "node-5"})
|
||||
|
||||
started := time.Now()
|
||||
report := client.CheckSealCapability(testCtx(t), "req")
|
||||
elapsed := time.Since(started)
|
||||
|
||||
assert.False(t, report.Complete)
|
||||
assert.Less(t, elapsed, client.wait+time.Second, "five silent nodes must not cost five deadlines")
|
||||
}
|
||||
|
||||
// The stuck reason is logged and stored as one line naming each failing node;
|
||||
// the retry hint and the ServiceUnavailable code travel with it.
|
||||
func TestSealCapabilityIncompleteErrorIsOneLine(t *testing.T) {
|
||||
err := errSealCapabilityIncomplete(sealCapabilityReport{Nodes: []nodeSealCapability{{Node: "node-1", Reason: sealCapabilityReasonDuplicate}}})
|
||||
|
||||
assert.Equal(t, "EC2 account-settings seal capability incomplete: node-1=duplicate: ServiceUnavailable", err.Error())
|
||||
d, ok := awserrors.ResolveRetryAfter(err)
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, 5*time.Second, d)
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorServiceUnavailable))
|
||||
}
|
||||
@@ -1,273 +0,0 @@
|
||||
package accountteardown
|
||||
|
||||
//test:in-package — the reaper, sealer and capability checker are unexported;
|
||||
// no seal without a complete probe, completion only after a sealed reply and
|
||||
// idempotent fresh runs are only observable from inside the package.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
|
||||
"github.com/mulgadc/spinifex/internal/testkit"
|
||||
ec2account "github.com/mulgadc/spinifex/spinifex/domains/ec2/account"
|
||||
awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// fakeSealer is a recording ec2AccountSettingsSealer. Each call beyond
|
||||
// failFor succeeds with a fresh, strictly increasing revision, unless err is
|
||||
// set, in which case every call after failFor fails with it.
|
||||
type fakeSealer struct {
|
||||
mu sync.Mutex
|
||||
calls []string
|
||||
failFor int
|
||||
err error
|
||||
revision uint64
|
||||
}
|
||||
|
||||
func (f *fakeSealer) SealAccountSettings(_ context.Context, accountID string) (uint64, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.calls = append(f.calls, accountID)
|
||||
if len(f.calls) <= f.failFor {
|
||||
return 0, errors.New("account settings owner temporarily unavailable")
|
||||
}
|
||||
if f.err != nil {
|
||||
return 0, f.err
|
||||
}
|
||||
f.revision++
|
||||
return f.revision, nil
|
||||
}
|
||||
|
||||
// fakeSealCapability is a recording sealCapabilityChecker returning a fixed report.
|
||||
type fakeSealCapability struct {
|
||||
calls int
|
||||
report sealCapabilityReport
|
||||
}
|
||||
|
||||
func (f *fakeSealCapability) CheckSealCapability(context.Context, string) sealCapabilityReport {
|
||||
f.calls++
|
||||
return f.report
|
||||
}
|
||||
|
||||
// completeReport builds a sealCapabilityReport where every named node answers
|
||||
// compatible at version 1.
|
||||
func completeReport(nodeIDs ...string) sealCapabilityReport {
|
||||
var nodes []nodeSealCapability
|
||||
for _, node := range nodeIDs {
|
||||
nodes = append(nodes, nodeSealCapability{Node: node, Version: 1})
|
||||
}
|
||||
return sealCapabilityReport{Complete: true, Nodes: nodes}
|
||||
}
|
||||
|
||||
// alwaysTerminatingChecker stands in for IAM in the end-to-end test: the
|
||||
// account under test is always TERMINATING.
|
||||
type alwaysTerminatingChecker struct{}
|
||||
|
||||
func (alwaysTerminatingChecker) AccountTerminating(context.Context, string) (bool, error) {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// No subscriber on either subject makes the capability probe incomplete, so
|
||||
// no seal request is ever sent and the account stays TERMINATING with one
|
||||
// stuck ec2-account-settings item.
|
||||
func TestSealReaperNoResponderKeepsAccountTerminating(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
|
||||
capability := newSealCapabilityClient(nc, []string{"node-1"})
|
||||
capability.wait = 50 * time.Millisecond
|
||||
capability.grace = 10 * time.Millisecond
|
||||
sealer := newEC2AccountSettingsSealClient(nc)
|
||||
sealer.timeout = 50 * time.Millisecond
|
||||
|
||||
reaper := &accountSettingsSealReaper{sealer: sealer, capability: capability}
|
||||
accounts := newFakeAccounts()
|
||||
engine := quietEngine(accounts, reaper)
|
||||
|
||||
result, err := engine.Teardown(testCtx(t), request())
|
||||
require.ErrorIs(t, err, ErrResourcesStuck)
|
||||
assert.False(t, result.AccountDeleted)
|
||||
assert.Equal(t, AccountStatusTerminating, accounts.account.Status)
|
||||
|
||||
var stuck []Stuck
|
||||
for _, stage := range result.Stages {
|
||||
stuck = append(stuck, stage.Stuck...)
|
||||
}
|
||||
require.Len(t, stuck, 1)
|
||||
assert.Equal(t, "ec2-account-settings", stuck[0].Resource.Kind)
|
||||
}
|
||||
|
||||
// Whatever the owner refuses with, the account stays TERMINATING: an owner
|
||||
// refusal, a timeout, and each reply shape the client rejects (wrong account,
|
||||
// wrong disposition, no revision).
|
||||
func TestSealReaperOwnerFailureKeepsAccountTerminating(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
err error
|
||||
}{
|
||||
{"incorrect-state", errors.New(awserrors.ErrorIncorrectState)},
|
||||
{"internal-error", errors.New(awserrors.ErrorInternalError)},
|
||||
{"timeout", context.DeadlineExceeded},
|
||||
{"wrong-account", errors.New(`seal account settings: reply named account "999999999999", not "000000000042"`)},
|
||||
{"wrong-disposition", errors.New(`seal account settings: unexpected disposition "pending"`)},
|
||||
{"zero-revision", errors.New("seal account settings: reply carried no revision")},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
sealer := &fakeSealer{err: tc.err}
|
||||
reaper := &accountSettingsSealReaper{sealer: sealer, capability: &fakeSealCapability{report: completeReport("node-1")}}
|
||||
accounts := newFakeAccounts()
|
||||
engine := quietEngine(accounts, reaper)
|
||||
|
||||
result, err := engine.Teardown(testCtx(t), request())
|
||||
require.ErrorIs(t, err, ErrResourcesStuck)
|
||||
assert.False(t, result.AccountDeleted)
|
||||
assert.Equal(t, AccountStatusTerminating, accounts.account.Status)
|
||||
assert.NotEmpty(t, sealer.calls)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The reaper retries through the engine's own drain mechanics: it fails
|
||||
// twice, then succeeds, and only the success lets the account be deleted.
|
||||
// List is non-empty before the success and empty only after it.
|
||||
func TestSealReaperCompletesOnlyAfterSealed(t *testing.T) {
|
||||
sealer := &fakeSealer{failFor: 2}
|
||||
reaper := &accountSettingsSealReaper{sealer: sealer, capability: &fakeSealCapability{report: completeReport("node-1")}}
|
||||
accounts := newFakeAccounts()
|
||||
engine := quietEngine(accounts, reaper)
|
||||
|
||||
before, err := reaper.List(testCtx(t), testAccountID)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, before, 1)
|
||||
|
||||
result, err := engine.Teardown(testCtx(t), request())
|
||||
require.NoError(t, err)
|
||||
assert.True(t, result.AccountDeleted)
|
||||
assert.GreaterOrEqual(t, len(sealer.calls), 3)
|
||||
|
||||
after, err := reaper.List(testCtx(t), testAccountID)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, after)
|
||||
}
|
||||
|
||||
// Each teardown builds a fresh engine and a fresh reaper, so a retried or
|
||||
// taken-over run seals again: two engines against the same owner both
|
||||
// succeed, each with a higher revision than the last.
|
||||
func TestSealReaperFreshRunSealsAgain(t *testing.T) {
|
||||
sealer := &fakeSealer{}
|
||||
capability := &fakeSealCapability{report: completeReport("node-1")}
|
||||
|
||||
firstEngine := quietEngine(newFakeAccounts(), &accountSettingsSealReaper{sealer: sealer, capability: capability})
|
||||
firstResult, err := firstEngine.Teardown(testCtx(t), request())
|
||||
require.NoError(t, err)
|
||||
assert.True(t, firstResult.AccountDeleted)
|
||||
|
||||
secondEngine := quietEngine(newFakeAccounts(), &accountSettingsSealReaper{sealer: sealer, capability: capability})
|
||||
secondResult, err := secondEngine.Teardown(testCtx(t), request())
|
||||
require.NoError(t, err)
|
||||
assert.True(t, secondResult.AccountDeleted)
|
||||
|
||||
require.Len(t, sealer.calls, 2)
|
||||
assert.Equal(t, uint64(2), sealer.revision)
|
||||
}
|
||||
|
||||
// Inventory and the explicit dry run, and a Teardown request marked dry run,
|
||||
// all list the pending seal and send no seal request: only a real teardown
|
||||
// may seal.
|
||||
func TestSealReaperInventoryDoesNotSeal(t *testing.T) {
|
||||
sealer := &fakeSealer{}
|
||||
capability := &fakeSealCapability{report: completeReport("node-1")}
|
||||
reaper := &accountSettingsSealReaper{sealer: sealer, capability: capability}
|
||||
accounts := newFakeAccounts()
|
||||
engine := quietEngine(accounts, reaper)
|
||||
|
||||
inventory, err := engine.Inventory(testCtx(t), testAccountID)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, sealer.calls)
|
||||
assert.Zero(t, capability.calls)
|
||||
found := false
|
||||
for _, stage := range inventory.Stages {
|
||||
for _, resource := range stage.Deleted {
|
||||
if resource.Kind == "ec2-account-settings" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
}
|
||||
assert.True(t, found, "inventory must still list the pending seal")
|
||||
|
||||
dryRun := request()
|
||||
dryRun.DryRun = true
|
||||
_, err = engine.Teardown(testCtx(t), dryRun)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, sealer.calls)
|
||||
assert.Zero(t, capability.calls)
|
||||
}
|
||||
|
||||
// A real settings service subscribed on the real seal subject, behind a real
|
||||
// engine: teardown seals the record, replacing the characterisation test that
|
||||
// showed it used to survive.
|
||||
func TestTeardownSealsAccountSettings(t *testing.T) {
|
||||
_, nc, js := testutil.StartTestJetStream(t)
|
||||
ctx := testCtx(t)
|
||||
|
||||
settings, err := ec2account.NewAccountSettingsServiceImplWithNATS(ctx, nil, nc)
|
||||
require.NoError(t, err)
|
||||
settings.SetAccountTerminationChecker(alwaysTerminatingChecker{})
|
||||
_, err = settings.EnableEbsEncryptionByDefault(ctx, &ec2.EnableEbsEncryptionByDefaultInput{}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
sealSub, err := nc.QueueSubscribe(ec2v1.SealAccountSettingsSubject, "spinifex-workers", func(msg *nats.Msg) {
|
||||
natsmsg.ServeNATSRequestCtx(msg, func(ctx context.Context, req *ec2v1.SealAccountSettingsRequest) (*ec2v1.SealAccountSettingsReply, error) {
|
||||
seal, err := settings.SealAccount(ctx, req.AccountID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &ec2v1.SealAccountSettingsReply{
|
||||
AccountID: req.AccountID, Disposition: ec2v1.AccountSettingsDispositionSealed, Revision: seal.Revision,
|
||||
}, nil
|
||||
})
|
||||
})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = sealSub.Unsubscribe() })
|
||||
|
||||
capSub, err := nc.Subscribe(ec2v1.AccountSettingsSealCapabilitySubject("node-1"), func(msg *nats.Msg) {
|
||||
reply, err := json.Marshal(ec2v1.AccountSettingsSealCapabilityReply{Node: "node-1", CapabilityVersion: 1})
|
||||
require.NoError(t, err)
|
||||
out := nats.NewMsg("")
|
||||
out.Data = reply
|
||||
out.Header.Set(natsmsg.NodeIDHeader, "node-1")
|
||||
require.NoError(t, msg.RespondMsg(out))
|
||||
})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = capSub.Unsubscribe() })
|
||||
require.NoError(t, nc.Flush())
|
||||
|
||||
accounts := newFakeAccounts()
|
||||
engine := quietEngine(accounts, EC2AccountSettingsReapers(nc, []string{"node-1"})...)
|
||||
// The real capability probe holds CollectUntilDeadline's full ResponderGrace
|
||||
// (150 ms) even when the one expected node answers instantly, so the quiet
|
||||
// engine's default 200 ms StageDrain leaves no room for the sweep after it.
|
||||
engine.Timeouts = Timeouts{StageDrain: 2 * time.Second, DrainPoll: 10 * time.Millisecond}
|
||||
|
||||
result, err := engine.Teardown(ctx, request())
|
||||
require.NoError(t, err)
|
||||
assert.True(t, result.AccountDeleted)
|
||||
|
||||
bucket, err := js.KeyValue(ctx, ec2account.KVBucketAccountSettings)
|
||||
require.NoError(t, err)
|
||||
entry, err := bucket.Get(ctx, testAccountID)
|
||||
require.NoError(t, err)
|
||||
var rec ec2account.AccountSettingsRecord
|
||||
require.NoError(t, json.Unmarshal(entry.Value(), &rec))
|
||||
assert.Equal(t, ec2account.AccountSettingsStateSealed, rec.State)
|
||||
}
|
||||
@@ -1,61 +0,0 @@
|
||||
package accountteardown
|
||||
|
||||
//test:in-package — drives the real final-stage reapers against the real quota service to pin that teardown leaves the override bucket alone.
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/spinifex/internal/testkit"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/admission/quota"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
|
||||
handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// No reaper in this package references the quota override bucket or calls an
|
||||
// override delete. Running the real engine and the real StageAccount reapers
|
||||
// against an account with a raised vCPU override, with usage wired, shows the
|
||||
// consequence: the override outlives the account record it belongs to, while
|
||||
// the usage counter the reapers do own is removed.
|
||||
func TestCurrentBehaviour_AccountTermination_QuotaOverridesSurviveTeardown(t *testing.T) {
|
||||
_, _, js := testutil.StartTestJetStream(t)
|
||||
ctx := testCtx(t)
|
||||
|
||||
usage, err := kvutil.GetOrCreateBucket(ctx, js, quota.KVBucketAccountUsage, 1)
|
||||
require.NoError(t, err)
|
||||
_, err = usage.Put(ctx, testAccountID, []byte(`{"vcpus":4}`))
|
||||
require.NoError(t, err)
|
||||
|
||||
overridesKV, err := kvutil.GetOrCreateBucket(ctx, js, quota.KVBucketAccountQuota, 1)
|
||||
require.NoError(t, err)
|
||||
svc := quota.New(quota.Limits{Enabled: true, VCPUs: 16}, usage)
|
||||
svc.SetOverrides(overridesKV)
|
||||
unlimited := quota.Unlimited
|
||||
require.NoError(t, svc.PutAccountQuota(ctx, testAccountID, quota.Overrides{VCPUs: &unlimited}, "operator"))
|
||||
|
||||
accounts := newFakeAccounts()
|
||||
iam := &accountIAM{account: &handlers_iam.Account{AccountID: testAccountID, AccountName: testAccountName}}
|
||||
engine := quietEngine(accounts, AccountReapers(iam, nil, usage)...)
|
||||
|
||||
result, err := engine.Teardown(ctx, request())
|
||||
require.NoError(t, err)
|
||||
assert.True(t, result.AccountDeleted)
|
||||
assert.True(t, accounts.deleted)
|
||||
assert.Zero(t, result.StuckCount())
|
||||
|
||||
// The reaper this package does own removed the usage counter, which shows
|
||||
// which store teardown actually clears.
|
||||
_, err = usage.Get(ctx, testAccountID)
|
||||
assert.ErrorIs(t, err, jetstream.ErrKeyNotFound, "teardown's quota-usage reaper must remove the vCPU counter")
|
||||
|
||||
_, err = overridesKV.Get(ctx, testAccountID)
|
||||
require.NoError(t, err, "the override key must still be present after the account is torn down")
|
||||
|
||||
over, effective, err := svc.GetAccountQuota(ctx, testAccountID)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, over.VCPUs)
|
||||
assert.Equal(t, quota.Unlimited, *over.VCPUs)
|
||||
assert.Equal(t, quota.Unlimited, effective.VCPUs)
|
||||
}
|
||||
@@ -1,89 +0,0 @@
|
||||
package accountteardown
|
||||
|
||||
//test:in-package — pins ADR-0003:S4 and ADR-0001 Context item 2 for the EC2
|
||||
// settings seal: the legacy engine must reach the owner only through the
|
||||
// versioned contract and NATS, never by reading the settings bucket itself.
|
||||
|
||||
import (
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"io/fs"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// forbiddenSettingsImports are imports that would let this package reach the
|
||||
// EC2 settings owner or the configured node set by a path other than the
|
||||
// versioned contract and NewClusterEngine's parameters.
|
||||
var forbiddenSettingsImports = []string{
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/ec2/account",
|
||||
"github.com/mulgadc/spinifex/spinifex/bootstrap/config",
|
||||
}
|
||||
|
||||
// forbiddenSettingsIdentifiers would let this package name the settings
|
||||
// bucket or its key directly instead of reaching it only through
|
||||
// contracts/ec2/v1 and NATS.
|
||||
var forbiddenSettingsIdentifiers = []string{"spinifex-ec2-account-settings", "KVBucketAccountSettings"}
|
||||
|
||||
// Teardown reaches EC2 settings only over contracts/ec2/v1: no import of the
|
||||
// settings owner or bootstrap/config (nodes arrive via NewClusterEngine), and
|
||||
// no literal naming the settings bucket or its constant.
|
||||
func TestAccountTeardownHasNoSettingsKVAccess(t *testing.T) {
|
||||
_, file, _, ok := runtime.Caller(0)
|
||||
if !ok {
|
||||
t.Fatal("cannot locate test source")
|
||||
}
|
||||
root := filepath.Dir(file)
|
||||
|
||||
fset := token.NewFileSet()
|
||||
scanned := 0
|
||||
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil || d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {
|
||||
return err
|
||||
}
|
||||
f, perr := parser.ParseFile(fset, path, nil, parser.ParseComments)
|
||||
if perr != nil {
|
||||
return perr
|
||||
}
|
||||
scanned++
|
||||
|
||||
for _, imp := range f.Imports {
|
||||
p, _ := strconv.Unquote(imp.Path.Value)
|
||||
for _, bad := range forbiddenSettingsImports {
|
||||
if p == bad {
|
||||
t.Errorf("%s: imports %s; the legacy engine must reach the EC2 settings owner only through contracts/ec2/v1 and the configured node set NewClusterEngine already receives",
|
||||
fset.Position(imp.Pos()), p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ast.Inspect(f, func(n ast.Node) bool {
|
||||
lit, ok := n.(*ast.BasicLit)
|
||||
if !ok || lit.Kind != token.STRING {
|
||||
return true
|
||||
}
|
||||
value, uerr := strconv.Unquote(lit.Value)
|
||||
if uerr != nil {
|
||||
return true
|
||||
}
|
||||
for _, bad := range forbiddenSettingsIdentifiers {
|
||||
if value == bad {
|
||||
t.Errorf("%s: names %q; the settings bucket and its key are the owner's own, never this package's",
|
||||
fset.Position(lit.Pos()), value)
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if scanned == 0 {
|
||||
t.Fatalf("scanned no Go files under %s; the invariant would pass vacuously", root)
|
||||
}
|
||||
}
|
||||
@@ -18,29 +18,13 @@ import (
|
||||
// ClusterConfig is the root of spinifex.toml as returned by LoadConfig: the
|
||||
// cluster-wide settings plus a [nodes.<name>] Config for every node.
|
||||
type ClusterConfig struct {
|
||||
Epoch uint64 `mapstructure:"epoch"` // bump when leader commits changes
|
||||
Node string `mapstructure:"node"` // my node name
|
||||
Version string `mapstructure:"version"` // spinifex version
|
||||
Network NetworkConfig `mapstructure:"network"` // cluster-wide external network settings
|
||||
Bootstrap BootstrapConfig `mapstructure:"bootstrap"` // default VPC IDs for OVN reconciliation
|
||||
AWS AWSConfig `mapstructure:"aws"` // cluster-wide AWS-parity settings (region, endpoint suffix)
|
||||
Northstar NorthstarClusterConfig `mapstructure:"northstar"` // cluster-wide authoritative DNS settings
|
||||
Nodes map[string]Config `mapstructure:"nodes"` // full config for every node
|
||||
}
|
||||
|
||||
// MaxNameservers caps the nsN names a cluster publishes for its zones. Every
|
||||
// northstar node still serves every zone; the cap bounds only the delegation a
|
||||
// registrar has to carry.
|
||||
const MaxNameservers = 4
|
||||
|
||||
// NorthstarClusterConfig is the cluster-wide [northstar] block.
|
||||
type NorthstarClusterConfig struct {
|
||||
// NameserverCount is how many nsN names to publish, 1 to MaxNameservers.
|
||||
// Zero means one per northstar node, up to MaxNameservers.
|
||||
NameserverCount int `mapstructure:"nameserver_count"`
|
||||
// NameserverNodes pins ns1, ns2, ... to these nodes, in order. Unpinned
|
||||
// slots are filled from the remaining northstar nodes in name order.
|
||||
NameserverNodes []string `mapstructure:"nameserver_nodes"`
|
||||
Epoch uint64 `mapstructure:"epoch"` // bump when leader commits changes
|
||||
Node string `mapstructure:"node"` // my node name
|
||||
Version string `mapstructure:"version"` // spinifex version
|
||||
Network NetworkConfig `mapstructure:"network"` // cluster-wide external network settings
|
||||
Bootstrap BootstrapConfig `mapstructure:"bootstrap"` // default VPC IDs for OVN reconciliation
|
||||
AWS AWSConfig `mapstructure:"aws"` // cluster-wide AWS-parity settings (region, endpoint suffix)
|
||||
Nodes map[string]Config `mapstructure:"nodes"` // full config for every node
|
||||
}
|
||||
|
||||
// Defaults for cluster-wide AWS-parity settings.
|
||||
@@ -670,32 +654,6 @@ func LoadConfig(configPath string) (*ClusterConfig, error) {
|
||||
return &config, nil
|
||||
}
|
||||
|
||||
// validateNorthstarCluster checks the [northstar] nameserver settings against
|
||||
// the cap and the configured nodes.
|
||||
func validateNorthstarCluster(cc *ClusterConfig) error {
|
||||
ns := cc.Northstar
|
||||
if ns.NameserverCount < 0 || ns.NameserverCount > MaxNameservers {
|
||||
return fmt.Errorf("config: [northstar] nameserver_count=%d out of range; use 1-%d, or remove the key for one per northstar node up to %d", ns.NameserverCount, MaxNameservers, MaxNameservers)
|
||||
}
|
||||
if len(ns.NameserverNodes) > MaxNameservers {
|
||||
return fmt.Errorf("config: [northstar] nameserver_nodes lists %d nodes; at most %d are published", len(ns.NameserverNodes), MaxNameservers)
|
||||
}
|
||||
if ns.NameserverCount > 0 && len(ns.NameserverNodes) > ns.NameserverCount {
|
||||
return fmt.Errorf("config: [northstar] nameserver_nodes lists %d nodes but nameserver_count is %d", len(ns.NameserverNodes), ns.NameserverCount)
|
||||
}
|
||||
seen := make(map[string]bool, len(ns.NameserverNodes))
|
||||
for _, name := range ns.NameserverNodes {
|
||||
if _, ok := cc.Nodes[name]; !ok {
|
||||
return fmt.Errorf("config: [northstar] nameserver_nodes names %q, which is not a configured node", name)
|
||||
}
|
||||
if seen[name] {
|
||||
return fmt.Errorf("config: [northstar] nameserver_nodes names %q twice", name)
|
||||
}
|
||||
seen[name] = true
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateClusterConfig rejects legacy DHCP config keys and validates external pool ranges.
|
||||
func validateClusterConfig(cc *ClusterConfig) error {
|
||||
if viper.IsSet("network.external_dhcp") {
|
||||
@@ -726,10 +684,6 @@ func validateClusterConfig(cc *ClusterConfig) error {
|
||||
}
|
||||
}
|
||||
|
||||
if err := validateNorthstarCluster(cc); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(cc.Network.NATExemptCIDRs) > 0 && cc.Network.ExternalMode != "nat" {
|
||||
return fmt.Errorf("config: [network] nat_exempt_cidrs requires external_mode = \"nat\"")
|
||||
}
|
||||
|
||||
@@ -1426,43 +1426,3 @@ exoscale_instance_id = "0f1e2d3c-0000-4000-8000-000000000001"
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfig_NorthstarNameservers(t *testing.T) {
|
||||
load := func(t *testing.T, block string) (*ClusterConfig, error) {
|
||||
t.Helper()
|
||||
resetViper(t)
|
||||
path := filepath.Join(t.TempDir(), "spinifex.toml")
|
||||
body := "node = \"n1\"\n\n" + block + "\n[nodes.n1]\nregion = \"us-east-1\"\n\n[nodes.n2]\nregion = \"us-east-1\"\n"
|
||||
require.NoError(t, os.WriteFile(path, []byte(body), 0600))
|
||||
return LoadConfig(path)
|
||||
}
|
||||
|
||||
t.Run("absent block means automatic", func(t *testing.T) {
|
||||
cfg, err := load(t, "")
|
||||
require.NoError(t, err)
|
||||
assert.Zero(t, cfg.Northstar.NameserverCount)
|
||||
assert.Empty(t, cfg.Northstar.NameserverNodes)
|
||||
})
|
||||
|
||||
t.Run("count and pin load", func(t *testing.T) {
|
||||
cfg, err := load(t, "[northstar]\nnameserver_count = 2\nnameserver_nodes = [\"n2\", \"n1\"]\n")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 2, cfg.Northstar.NameserverCount)
|
||||
assert.Equal(t, []string{"n2", "n1"}, cfg.Northstar.NameserverNodes)
|
||||
})
|
||||
|
||||
for name, tc := range map[string]struct{ block, want string }{
|
||||
"count above the cap": {"[northstar]\nnameserver_count = 5\n", "nameserver_count=5"},
|
||||
"negative count": {"[northstar]\nnameserver_count = -1\n", "nameserver_count=-1"},
|
||||
"pin longer than count": {"[northstar]\nnameserver_count = 1\nnameserver_nodes = [\"n1\", \"n2\"]\n", "nameserver_count is 1"},
|
||||
"pin names an unknown node": {"[northstar]\nnameserver_nodes = [\"n9\"]\n", `"n9"`},
|
||||
"pin repeats a node": {"[northstar]\nnameserver_nodes = [\"n1\", \"n1\"]\n", "twice"},
|
||||
"pin above the cap": {"[northstar]\nnameserver_nodes = [\"n1\", \"n2\", \"n3\", \"n4\", \"n5\"]\n", "at most 4"},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
_, err := load(t, tc.block)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), tc.want)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
package preflight
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/host/hostnet"
|
||||
)
|
||||
|
||||
// hostnetUnit is the unit whose failure this check surfaces.
|
||||
const hostnetUnit = "spinifex-hostnet.service"
|
||||
|
||||
// hostnetOVNRemote is a variable so tests never query a real OVS.
|
||||
var hostnetOVNRemote = hostnet.DefaultEnv().OVNRemote
|
||||
|
||||
// checkHostnet runs the check the daemon and vpcd run before they start, so a
|
||||
// failed host network apply shows here, not only as a refused service start.
|
||||
func checkHostnet(root string) Result {
|
||||
env := hostnet.DefaultEnv()
|
||||
env.Paths = hostnet.PathsAt(root)
|
||||
env.OVNRemote = hostnetOVNRemote
|
||||
if err := hostnet.Check(context.Background(), env); err != nil {
|
||||
return Result{Path: hostnetUnit, Kind: "hostnet", Status: Failed, Detail: err.Error()}
|
||||
}
|
||||
return Result{Path: hostnetUnit, Kind: "hostnet", Status: OK}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package preflight
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCheckHostAt_Hostnet(t *testing.T) {
|
||||
for name, tc := range map[string]struct {
|
||||
backend string
|
||||
state bool
|
||||
want Status
|
||||
detail string
|
||||
}{
|
||||
"no record, nothing set up": {want: OK},
|
||||
"networkd backend": {backend: "networkd", want: OK},
|
||||
"hostnet backend, no state": {backend: "hostnet", want: Failed, detail: "hostnet.json"},
|
||||
"hostnet backend, no marker": {backend: "hostnet", state: true, want: Failed, detail: "host network not applied"},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
etc := filepath.Join(root, "etc/spinifex")
|
||||
if err := os.MkdirAll(etc, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if tc.backend != "" {
|
||||
if err := os.WriteFile(filepath.Join(etc, "hostnet-backend"), []byte(tc.backend+"\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if tc.state {
|
||||
state := `{"version": 1, "pending": false, "int_bridge": "br-int"}`
|
||||
if err := os.WriteFile(filepath.Join(etc, "hostnet.json"), []byte(state), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
r := findResult(t, CheckHostAt(root), hostnetUnit)
|
||||
if r.Status != tc.want || !strings.Contains(r.Detail, tc.detail) {
|
||||
t.Errorf("hostnet result = %s %q, want %s containing %q", r.Status, r.Detail, tc.want, tc.detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package preflight
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/host/hostos"
|
||||
)
|
||||
|
||||
// microvmMachine is the QEMU machine type load balancer system instances boot on.
|
||||
const microvmMachine = "microvm"
|
||||
|
||||
// microvmChecked reports whether this host runs system instances. The microvm
|
||||
// machine type exists only in x86 QEMU.
|
||||
var microvmChecked = runtime.GOARCH == "amd64"
|
||||
|
||||
// resolveQEMU and machineHelp are variables so tests never run a real QEMU.
|
||||
var (
|
||||
resolveQEMU = func() (string, error) { return hostos.QEMUBinary("x86_64") }
|
||||
machineHelp = func(binary string) ([]byte, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
return exec.CommandContext(ctx, binary, "-machine", "help").Output()
|
||||
}
|
||||
)
|
||||
|
||||
// checkMicroVM checks the QEMU binary the daemon will run, and the firmware it
|
||||
// passes, can boot a microvm. A QEMU build without microvm, such as Oracle
|
||||
// Linux's AppStream qemu-kvm, otherwise surfaces only at the first ALB launch.
|
||||
func checkMicroVM(root string) []Result {
|
||||
if !microvmChecked {
|
||||
return nil
|
||||
}
|
||||
return []Result{checkQEMUMicroVM(), checkMicroVMFirmware(root)}
|
||||
}
|
||||
|
||||
func checkQEMUMicroVM() Result {
|
||||
binary, err := resolveQEMU()
|
||||
if err != nil {
|
||||
return Result{Path: "qemu", Kind: "qemu", Status: Missing, Detail: err.Error()}
|
||||
}
|
||||
out, err := machineHelp(binary)
|
||||
if err != nil {
|
||||
return Result{Path: binary, Kind: "qemu", Status: Missing, Detail: fmt.Sprintf("%s -machine help: %v", binary, err)}
|
||||
}
|
||||
if !listsMachine(out, microvmMachine) {
|
||||
return Result{
|
||||
Path: binary,
|
||||
Kind: "qemu",
|
||||
Status: Unsupported,
|
||||
Detail: fmt.Sprintf("no %s machine type, so load balancers cannot boot; install a QEMU build that has it (on Oracle Linux, qemu-kvm from ol10_kvm_utils)", microvmMachine),
|
||||
}
|
||||
}
|
||||
return Result{Path: binary, Kind: "qemu", Status: OK}
|
||||
}
|
||||
|
||||
// listsMachine reports whether `-machine help` output has a row for name. Rows
|
||||
// are "<name> <description>", after a "Supported machines are:" header.
|
||||
func listsMachine(help []byte, name string) bool {
|
||||
sc := bufio.NewScanner(bytes.NewReader(help))
|
||||
for sc.Scan() {
|
||||
if fields := strings.Fields(sc.Text()); len(fields) > 0 && fields[0] == name {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func checkMicroVMFirmware(root string) Result {
|
||||
path := filepath.Join(vm.MicroVMDir, vm.MicroVMFirmware)
|
||||
if _, err := os.Stat(filepath.Join(root, path)); err != nil {
|
||||
return Result{Path: path, Kind: "firmware", Status: Missing, Detail: err.Error()}
|
||||
}
|
||||
return Result{Path: path, Kind: "firmware", Status: OK}
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
package preflight
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/host/hostos"
|
||||
)
|
||||
|
||||
// Debian's qemu-system-x86_64 -machine help, trimmed.
|
||||
const helpWithMicroVM = `Supported machines are:
|
||||
microvm microvm (i386)
|
||||
pc Standard PC (i440FX + PIIX, 1996) (alias of pc-i440fx-10.0)
|
||||
q35 Standard PC (Q35 + ICH9, 2009) (alias of pc-q35-10.0)
|
||||
`
|
||||
|
||||
// Oracle Linux 10's AppStream qemu-kvm 10.1.0 -machine help, trimmed. It has
|
||||
// no microvm row.
|
||||
const helpWithoutMicroVM = `Supported machines are:
|
||||
pc RHEL 10.0.0 PC (i440FX + PIIX, 1996) (alias of pc-i440fx-rhel10.0.0)
|
||||
pc-i440fx-rhel10.0.0 RHEL 10.0.0 PC (i440FX + PIIX, 1996) (default) (deprecated)
|
||||
pc-q35-rhel9.8.0 RHEL-9.8.0 PC (Q35 + ICH9, 2009) (deprecated)
|
||||
`
|
||||
|
||||
var microvmFirmwarePath = filepath.Join(vm.MicroVMDir, vm.MicroVMFirmware)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
microvmChecked = true
|
||||
resolveQEMU = func() (string, error) { return hostos.QEMUKVMPath, nil }
|
||||
machineHelp = func(string) ([]byte, error) { return []byte(helpWithMicroVM), nil }
|
||||
hostnetOVNRemote = func(context.Context) (string, error) { return "", nil }
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
func stubQEMU(t *testing.T, binary string, resolveErr error, help string, helpErr error) {
|
||||
t.Helper()
|
||||
origResolve, origHelp := resolveQEMU, machineHelp
|
||||
t.Cleanup(func() { resolveQEMU, machineHelp = origResolve, origHelp })
|
||||
var asked string
|
||||
resolveQEMU = func() (string, error) { return binary, resolveErr }
|
||||
machineHelp = func(b string) ([]byte, error) {
|
||||
asked = b
|
||||
return []byte(help), helpErr
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if resolveErr == nil && asked != binary {
|
||||
t.Errorf("machine help ran %q, want the resolved binary %q", asked, binary)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func qemuResult(t *testing.T, results []Result) Result {
|
||||
t.Helper()
|
||||
for _, r := range results {
|
||||
if r.Kind == "qemu" {
|
||||
return r
|
||||
}
|
||||
}
|
||||
t.Fatalf("no qemu result in %+v", results)
|
||||
return Result{}
|
||||
}
|
||||
|
||||
func TestCheckHostAt_QEMUMicroVM(t *testing.T) {
|
||||
t.Run("resolved binary lists microvm", func(t *testing.T) {
|
||||
stubQEMU(t, hostos.QEMUKVMPath, nil, helpWithMicroVM, nil)
|
||||
r := qemuResult(t, CheckHostAt(t.TempDir()))
|
||||
if r.Status != OK || r.Path != hostos.QEMUKVMPath {
|
||||
t.Errorf("got %+v, want OK for %s", r, hostos.QEMUKVMPath)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("resolved binary has no microvm", func(t *testing.T) {
|
||||
stubQEMU(t, hostos.QEMUKVMPath, nil, helpWithoutMicroVM, nil)
|
||||
r := qemuResult(t, CheckHostAt(t.TempDir()))
|
||||
if r.Status != Unsupported || !strings.Contains(r.Detail, "ol10_kvm_utils") {
|
||||
t.Errorf("got %+v, want Unsupported naming ol10_kvm_utils", r)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("no QEMU resolves", func(t *testing.T) {
|
||||
noQEMU := errors.New("no QEMU for x86_64 guests: found neither /usr/libexec/qemu-kvm (package qemu-kvm) nor qemu-system-x86_64 (package qemu-system-x86) on PATH")
|
||||
stubQEMU(t, "", noQEMU, "", nil)
|
||||
r := qemuResult(t, CheckHostAt(t.TempDir()))
|
||||
if r.Status != Missing || r.Detail != noQEMU.Error() {
|
||||
t.Errorf("got %+v, want Missing carrying the resolver's error", r)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("machine help fails", func(t *testing.T) {
|
||||
stubQEMU(t, "/usr/bin/qemu-system-x86_64", nil, "", errors.New("exit status 1"))
|
||||
r := qemuResult(t, CheckHostAt(t.TempDir()))
|
||||
if r.Status != Missing || !strings.Contains(r.Detail, "exit status 1") {
|
||||
t.Errorf("got %+v, want Missing with the exec error", r)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestCheckHostAt_MicroVMFirmware(t *testing.T) {
|
||||
t.Run("absent", func(t *testing.T) {
|
||||
r := findResult(t, CheckHostAt(t.TempDir()), microvmFirmwarePath)
|
||||
if r.Status != Missing {
|
||||
t.Errorf("status = %v, want Missing", r.Status)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("present", func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
writeHelper(t, root, microvmFirmwarePath, []byte("qboot"))
|
||||
r := findResult(t, CheckHostAt(root), microvmFirmwarePath)
|
||||
if r.Status != OK {
|
||||
t.Errorf("status = %v, want OK (detail: %s)", r.Status, r.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestCheckHostAt_NoMicroVMChecksOffX86(t *testing.T) {
|
||||
orig := microvmChecked
|
||||
microvmChecked = false
|
||||
t.Cleanup(func() { microvmChecked = orig })
|
||||
|
||||
for _, r := range CheckHostAt(t.TempDir()) {
|
||||
if r.Kind == "qemu" || r.Kind == "firmware" {
|
||||
t.Errorf("unexpected %s result on a host without system instances: %+v", r.Kind, r)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -29,6 +29,10 @@ const (
|
||||
Stale
|
||||
// Ungranted means the sudoers grant for a helper is absent or missing its NOPASSWD line.
|
||||
Ungranted
|
||||
// Unsupported means the asset is present but lacks a capability this build needs.
|
||||
Unsupported
|
||||
// Failed means a host check that gates the services did not pass.
|
||||
Failed
|
||||
)
|
||||
|
||||
// String returns the human-readable name of the status, used in table output and log lines.
|
||||
@@ -42,6 +46,10 @@ func (s Status) String() string {
|
||||
return "Stale"
|
||||
case Ungranted:
|
||||
return "Ungranted"
|
||||
case Unsupported:
|
||||
return "Unsupported"
|
||||
case Failed:
|
||||
return "Failed"
|
||||
default:
|
||||
return "Unknown"
|
||||
}
|
||||
@@ -50,7 +58,7 @@ func (s Status) String() string {
|
||||
// Result is the outcome of checking one managed asset against this build's manifest.
|
||||
type Result struct {
|
||||
Path string
|
||||
Kind string // "helper" | "sudoers-grant" | "service"
|
||||
Kind string // "helper" | "sudoers-grant" | "service" | "qemu" | "firmware" | "hostnet"
|
||||
Status Status
|
||||
Detail string
|
||||
}
|
||||
@@ -82,6 +90,8 @@ func CheckHostAt(root string) []Result {
|
||||
}
|
||||
results = append(results, checkSudoersGrant(root))
|
||||
results = append(results, checkRunningBinaries(root)...)
|
||||
results = append(results, checkMicroVM(root)...)
|
||||
results = append(results, checkHostnet(root))
|
||||
return results
|
||||
}
|
||||
|
||||
|
||||
@@ -105,7 +105,7 @@ func TestHasProblem(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestStatusString(t *testing.T) {
|
||||
cases := map[Status]string{OK: "OK", Missing: "Missing", Stale: "Stale", Ungranted: "Ungranted"}
|
||||
cases := map[Status]string{OK: "OK", Missing: "Missing", Stale: "Stale", Ungranted: "Ungranted", Unsupported: "Unsupported", Failed: "Failed"}
|
||||
for status, want := range cases {
|
||||
if got := status.String(); got != want {
|
||||
t.Errorf("Status(%d).String() = %q, want %q", status, got, want)
|
||||
|
||||
@@ -71,7 +71,6 @@ import (
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/ochre"
|
||||
ochrevector "github.com/mulgadc/spinifex/spinifex/domains/ochre/vector"
|
||||
rdsengine "github.com/mulgadc/spinifex/spinifex/domains/rds/engine"
|
||||
route53domain "github.com/mulgadc/spinifex/spinifex/domains/route53"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/clustersize"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
|
||||
@@ -176,8 +175,6 @@ type Daemon struct {
|
||||
bedrockReaper *ochre.Reaper
|
||||
acmService *acmdomain.ACMServiceImpl
|
||||
acmRenewalWorker *acmdomain.Worker
|
||||
route53Service *route53domain.Service
|
||||
route53Verifier *route53domain.Verifier
|
||||
ochreVectorService ochrevector.VectorService
|
||||
ochreAppliance *ochrevector.Appliance
|
||||
ochreBackupService *ochrevector.BackupService
|
||||
@@ -1029,7 +1026,6 @@ func (d *Daemon) subscribeAll() error {
|
||||
{"ec2.GetSerialConsoleAccessStatus", handleNATSRequest(d.node, d.accountService.GetSerialConsoleAccessStatus), "spinifex-workers"},
|
||||
{"ec2.EnableSerialConsoleAccess", handleNATSRequest(d.node, d.accountService.EnableSerialConsoleAccess), "spinifex-workers"},
|
||||
{"ec2.DisableSerialConsoleAccess", handleNATSRequest(d.node, d.accountService.DisableSerialConsoleAccess), "spinifex-workers"},
|
||||
{ec2v1.SealAccountSettingsSubject, handleNATSRequest(d.node, d.sealAccountSettings), "spinifex-workers"},
|
||||
{clusterv1.NodeHealthSubject(d.node), d.handleHealthCheck, ""},
|
||||
{clusterv1.NodesDiscoverSubject, d.handleNodeDiscover, ""},
|
||||
{clusterv1.NodeStatusSubject, d.handleNodeStatus, ""},
|
||||
@@ -1235,31 +1231,6 @@ func (d *Daemon) subscribeAll() error {
|
||||
)
|
||||
}
|
||||
|
||||
// Route 53 is global: any daemon serves any account's zones from shared KV.
|
||||
if d.route53Service != nil {
|
||||
subs = append(subs,
|
||||
natsSub{"route53.CreateHostedZone", handleNATSRequest(d.node, d.route53Service.CreateHostedZone), "spinifex-workers"},
|
||||
natsSub{"route53.GetHostedZone", handleNATSRequest(d.node, d.route53Service.GetHostedZone), "spinifex-workers"},
|
||||
natsSub{"route53.GetHostedZoneCount", handleNATSRequest(d.node, d.route53Service.GetHostedZoneCount), "spinifex-workers"},
|
||||
natsSub{"route53.ListHostedZones", handleNATSRequest(d.node, d.route53Service.ListHostedZones), "spinifex-workers"},
|
||||
natsSub{"route53.ListHostedZonesByName", handleNATSRequest(d.node, d.route53Service.ListHostedZonesByName), "spinifex-workers"},
|
||||
natsSub{"route53.UpdateHostedZoneComment", handleNATSRequest(d.node, d.route53Service.UpdateHostedZoneComment), "spinifex-workers"},
|
||||
natsSub{"route53.DeleteHostedZone", handleNATSRequest(d.node, d.route53Service.DeleteHostedZone), "spinifex-workers"},
|
||||
natsSub{"route53.ChangeResourceRecordSets", handleNATSRequest(d.node, d.route53Service.ChangeResourceRecordSets), "spinifex-workers"},
|
||||
natsSub{"route53.ListResourceRecordSets", handleNATSRequest(d.node, d.route53Service.ListResourceRecordSets), "spinifex-workers"},
|
||||
natsSub{"route53.GetChange", handleNATSRequest(d.node, d.route53Service.GetChange), "spinifex-workers"},
|
||||
natsSub{"route53.ChangeTagsForResource", handleNATSRequest(d.node, d.route53Service.ChangeTagsForResource), "spinifex-workers"},
|
||||
natsSub{"route53.ListTagsForResource", handleNATSRequest(d.node, d.route53Service.ListTagsForResource), "spinifex-workers"},
|
||||
natsSub{"route53.ListTagsForResources", handleNATSRequest(d.node, d.route53Service.ListTagsForResources), "spinifex-workers"},
|
||||
natsSub{"route53.GetHostedZoneActivation", handleNATSRequest(d.node, d.route53Service.GetHostedZoneActivation), "spinifex-workers"},
|
||||
natsSub{"route53.VerifyHostedZoneDelegation", handleNATSRequest(d.node, d.route53Service.VerifyHostedZoneDelegation), "spinifex-workers"},
|
||||
natsSub{"route53.admin.FindZones", handleNATSRequest(d.node, d.route53Service.AdminFindZones), "spinifex-workers"},
|
||||
natsSub{"route53.admin.Verify", handleNATSRequest(d.node, d.route53Service.AdminVerify), "spinifex-workers"},
|
||||
natsSub{"route53.admin.Override", handleNATSRequest(d.node, d.route53Service.AdminOverride), "spinifex-workers"},
|
||||
natsSub{"route53.admin.Delegate", handleNATSRequest(d.node, d.route53Delegate), "spinifex-workers"},
|
||||
)
|
||||
}
|
||||
|
||||
// Ochre vector store tenant surface is deliberately NOT registered here:
|
||||
// its subjects only exist once the platform appliance has finished
|
||||
// launching (startOchreVector, run in the background after this method
|
||||
@@ -1317,15 +1288,6 @@ func (d *Daemon) subscribeAll() error {
|
||||
natsSub{viperblocklegacyv1.VolumeFencedSubject(d.node), d.handleVolumeFenced, ""},
|
||||
)
|
||||
|
||||
// Per-node, no queue group: a second daemon claiming this node ID also
|
||||
// answers, so the teardown probe sees a duplicate and refuses to seal. An
|
||||
// invalid node ID serves nothing, so this daemon reads as incompatible.
|
||||
if ec2v1.ValidSealCapabilityNode(d.node) {
|
||||
subs = append(subs, natsSub{ec2v1.AccountSettingsSealCapabilitySubject(d.node), handleNATSRequest(d.node, d.accountSettingsSealCapability), ""})
|
||||
} else {
|
||||
slog.Error("Not serving account-settings seal capability: node ID is not a single NATS subject token", "node", d.node)
|
||||
}
|
||||
|
||||
return d.registerNatsSubs(subs)
|
||||
}
|
||||
|
||||
@@ -2030,12 +1992,6 @@ func (d *Daemon) startCluster() error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to initialize account settings service: %w", err)
|
||||
}
|
||||
accountStatusJS, err := jetstream.New(d.natsConn)
|
||||
if err != nil {
|
||||
return fmt.Errorf("jetstream handle for account status reader: %w", err)
|
||||
}
|
||||
var _ ec2account.AccountTerminationChecker = (*handlers_iam.AccountStatusReader)(nil)
|
||||
d.accountService.SetAccountTerminationChecker(handlers_iam.NewAccountStatusReader(accountStatusJS))
|
||||
|
||||
// ELBv2 resolves listener certificate private keys through its own ACM
|
||||
// Store over the same JetStream bucket the ACM service writes, so it needs
|
||||
@@ -2201,14 +2157,6 @@ func (d *Daemon) startCluster() error {
|
||||
return dns.HostsZone(d.config, domain)
|
||||
}
|
||||
|
||||
route53JS, err := jetstream.New(d.natsConn)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to initialize Route 53 service: %w", err)
|
||||
}
|
||||
d.route53Service = d.newRoute53Service(route53JS)
|
||||
d.route53Verifier = d.newRoute53Verifier()
|
||||
d.route53Service.Verifier = d.route53Verifier
|
||||
|
||||
// The tenant private CA is optional, unlike the master key above: a
|
||||
// deployment issuing only public (PROVIDER_API/MANUAL_TXT) certificates has
|
||||
// no reason to have one, so its absence must not block daemon startup.
|
||||
@@ -2351,10 +2299,6 @@ func (d *Daemon) startCluster() error {
|
||||
go d.dnsReconciler.Run(d.ctx)
|
||||
slog.Info("Started DNS reconcile backstop", "interval_ms", otelsetup.Millis(dns.DefaultReconcileInterval))
|
||||
}
|
||||
if d.route53Service != nil {
|
||||
d.startRoute53Realizer()
|
||||
go d.route53Verifier.Run(d.ctx)
|
||||
}
|
||||
|
||||
// Initialize per-instance-type NATS subscriptions for capacity-aware routing.
|
||||
d.resourceMgr.initSubscriptions(d.natsConn, d.handleEC2RunInstances, d.handleSystemLaunchInstance, d.node)
|
||||
|
||||
@@ -1,23 +0,0 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
|
||||
)
|
||||
|
||||
// sealAccountSettings serves the irreversible EC2 settings seal. The account
|
||||
// comes from the request body, not X-Account-ID: the sealed account is part
|
||||
// of the versioned contract, so the header is ignored.
|
||||
func (d *Daemon) sealAccountSettings(ctx context.Context, req *ec2v1.SealAccountSettingsRequest, _ string) (*ec2v1.SealAccountSettingsReply, error) {
|
||||
seal, err := d.accountService.SealAccount(ctx, req.AccountID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &ec2v1.SealAccountSettingsReply{
|
||||
AccountID: req.AccountID,
|
||||
CorrelationRef: req.CorrelationRef,
|
||||
Disposition: ec2v1.AccountSettingsDispositionSealed,
|
||||
Revision: seal.Revision,
|
||||
}, nil
|
||||
}
|
||||
@@ -1,23 +0,0 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
|
||||
awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
)
|
||||
|
||||
// accountSettingsSealCapability serves this daemon's read-only seal capability.
|
||||
// It reads no KV and takes no account; a request naming another node is a
|
||||
// caller mistake, since the subject is already addressed to this node.
|
||||
func (d *Daemon) accountSettingsSealCapability(_ context.Context, req *ec2v1.AccountSettingsSealCapabilityRequest, _ string) (*ec2v1.AccountSettingsSealCapabilityReply, error) {
|
||||
if req.Node != d.node {
|
||||
return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue,
|
||||
"seal capability request for node %q answered by node %q", req.Node, d.node)
|
||||
}
|
||||
return &ec2v1.AccountSettingsSealCapabilityReply{
|
||||
Node: d.node,
|
||||
CapabilityVersion: ec2v1.AccountSettingsSealCapabilityVersion,
|
||||
CorrelationRef: req.CorrelationRef,
|
||||
}, nil
|
||||
}
|
||||
@@ -1,115 +0,0 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
|
||||
"github.com/mulgadc/spinifex/internal/testkit"
|
||||
awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// newCapabilityTestDaemon builds a daemon serving its real subscription table
|
||||
// with node as its configured node ID.
|
||||
func newCapabilityTestDaemon(t *testing.T, node string) (*nats.Conn, *nats.Conn) {
|
||||
t.Helper()
|
||||
ns, nc, _ := testutil.StartTestJetStream(t)
|
||||
|
||||
d := newSubscribeTestDaemon(t, nc, "", "")
|
||||
d.node = node
|
||||
require.NoError(t, d.subscribeAll())
|
||||
require.NoError(t, nc.Flush())
|
||||
|
||||
caller, err := nats.Connect(ns.ClientURL())
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(caller.Close)
|
||||
return nc, caller
|
||||
}
|
||||
|
||||
func capabilityRequest(t *testing.T, nc *nats.Conn, node string) *nats.Msg {
|
||||
t.Helper()
|
||||
data, err := json.Marshal(ec2v1.AccountSettingsSealCapabilityRequest{Node: node, CorrelationRef: "req-1"})
|
||||
require.NoError(t, err)
|
||||
reply, err := nc.Request(ec2v1.AccountSettingsSealCapabilitySubject(node), data, 5*time.Second)
|
||||
require.NoError(t, err)
|
||||
return reply
|
||||
}
|
||||
|
||||
// The responder answers its own node with version 1, the configured node and
|
||||
// no KV access (no settings service is wired into the test daemon at all).
|
||||
func TestAccountSettingsSealCapabilityAnswersOwnNode(t *testing.T) {
|
||||
_, caller := newCapabilityTestDaemon(t, "node-1")
|
||||
|
||||
raw := capabilityRequest(t, caller, "node-1")
|
||||
assert.Equal(t, "node-1", raw.Header.Get("X-Node-ID"))
|
||||
|
||||
var reply ec2v1.AccountSettingsSealCapabilityReply
|
||||
require.NoError(t, json.Unmarshal(raw.Data, &reply))
|
||||
assert.Equal(t, "node-1", reply.Node)
|
||||
assert.Equal(t, ec2v1.AccountSettingsSealCapabilityVersion, reply.CapabilityVersion)
|
||||
assert.Equal(t, "req-1", reply.CorrelationRef)
|
||||
}
|
||||
|
||||
// A request naming a node other than this daemon's own is refused, not
|
||||
// answered as if it were this node.
|
||||
func TestAccountSettingsSealCapabilityRefusesOtherNode(t *testing.T) {
|
||||
_, caller := newCapabilityTestDaemon(t, "node-1")
|
||||
|
||||
data, err := json.Marshal(ec2v1.AccountSettingsSealCapabilityRequest{Node: "node-2"})
|
||||
require.NoError(t, err)
|
||||
raw, err := caller.Request(ec2v1.AccountSettingsSealCapabilitySubject("node-1"), data, 5*time.Second)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, awserrors.ErrorInvalidParameterValue, replyErrCode(t, raw.Data))
|
||||
}
|
||||
|
||||
// The subject is per node with no queue group, so two daemons configured
|
||||
// with the same node ID both answer a request addressed to that node.
|
||||
func TestAccountSettingsSealCapabilityHasNoQueueGroup(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
|
||||
first := newSubscribeTestDaemon(t, nc, "", "")
|
||||
first.node = "node-1"
|
||||
require.NoError(t, first.subscribeAll())
|
||||
|
||||
second := newSubscribeTestDaemon(t, nc, "", "")
|
||||
second.node = "node-1"
|
||||
require.NoError(t, second.subscribeAll())
|
||||
require.NoError(t, nc.Flush())
|
||||
|
||||
sub, err := nc.SubscribeSync(nats.NewInbox())
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
|
||||
data, err := json.Marshal(ec2v1.AccountSettingsSealCapabilityRequest{Node: "node-1"})
|
||||
require.NoError(t, err)
|
||||
msg := nats.NewMsg(ec2v1.AccountSettingsSealCapabilitySubject("node-1"))
|
||||
msg.Data = data
|
||||
msg.Reply = sub.Subject
|
||||
require.NoError(t, nc.PublishMsg(msg))
|
||||
require.NoError(t, nc.Flush())
|
||||
|
||||
seen := 0
|
||||
for {
|
||||
_, err := sub.NextMsg(300 * time.Millisecond)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
seen++
|
||||
}
|
||||
assert.Equal(t, 2, seen)
|
||||
}
|
||||
|
||||
// A wildcard node ID must not become a wildcard subscription: built unchecked,
|
||||
// "node-*" would answer probes addressed to node-1 on node-1's behalf.
|
||||
func TestAccountSettingsSealCapabilityNotServedForInvalidNodeID(t *testing.T) {
|
||||
_, caller := newCapabilityTestDaemon(t, "node-*")
|
||||
|
||||
data, err := json.Marshal(ec2v1.AccountSettingsSealCapabilityRequest{Node: "node-1"})
|
||||
require.NoError(t, err)
|
||||
_, err = caller.Request(ec2v1.AccountSettingsSealCapabilitySubject("node-1"), data, time.Second)
|
||||
require.ErrorIs(t, err, nats.ErrNoResponders)
|
||||
}
|
||||
@@ -1,101 +0,0 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
|
||||
"github.com/mulgadc/spinifex/internal/testkit"
|
||||
ec2account "github.com/mulgadc/spinifex/spinifex/domains/ec2/account"
|
||||
awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// sealTestChecker answers AccountTerminating from a fixed set, standing in
|
||||
// for IAM in these daemon-level tests.
|
||||
type sealTestChecker struct {
|
||||
terminating map[string]bool
|
||||
}
|
||||
|
||||
func (c *sealTestChecker) AccountTerminating(_ context.Context, accountID string) (bool, error) {
|
||||
return c.terminating[accountID], nil
|
||||
}
|
||||
|
||||
// newSealTestDaemon builds a daemon serving its real subscription table over
|
||||
// a real account settings service, wired with checker as its termination
|
||||
// precondition.
|
||||
func newSealTestDaemon(t *testing.T, checker ec2account.AccountTerminationChecker) (*Daemon, *nats.Conn) {
|
||||
t.Helper()
|
||||
ns, nc, _ := testutil.StartTestJetStream(t)
|
||||
|
||||
acctSvc, err := ec2account.NewAccountSettingsServiceImplWithNATS(t.Context(), nil, nc)
|
||||
require.NoError(t, err)
|
||||
acctSvc.SetAccountTerminationChecker(checker)
|
||||
|
||||
d := newSubscribeTestDaemon(t, nc, "", "")
|
||||
d.accountService = acctSvc
|
||||
require.NoError(t, d.subscribeAll())
|
||||
require.NoError(t, nc.Flush())
|
||||
|
||||
caller, err := nats.Connect(ns.ClientURL())
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(caller.Close)
|
||||
return d, caller
|
||||
}
|
||||
|
||||
const (
|
||||
sealTestTerminating = "111122223333"
|
||||
sealTestActive = "444455556666"
|
||||
)
|
||||
|
||||
func sealRequest(t *testing.T, nc *nats.Conn, body any, header string) []byte {
|
||||
t.Helper()
|
||||
data, err := json.Marshal(body)
|
||||
require.NoError(t, err)
|
||||
reply, err := natsRequestAs(nc, ec2v1.SealAccountSettingsSubject, data, header, 5*time.Second)
|
||||
require.NoError(t, err)
|
||||
return reply.Data
|
||||
}
|
||||
|
||||
// The header is ignored: the body account is what gets sealed, even when
|
||||
// X-Account-ID names a different one.
|
||||
func TestSealAccountSettingsSubjectSealsTheBodyAccount(t *testing.T) {
|
||||
_, nc := newSealTestDaemon(t, &sealTestChecker{terminating: map[string]bool{sealTestTerminating: true}})
|
||||
|
||||
raw := sealRequest(t, nc, ec2v1.SealAccountSettingsRequest{AccountID: sealTestTerminating, CorrelationRef: "req-1"}, sealTestActive)
|
||||
require.Empty(t, replyErrCode(t, raw), string(raw))
|
||||
|
||||
var reply ec2v1.SealAccountSettingsReply
|
||||
require.NoError(t, json.Unmarshal(raw, &reply))
|
||||
assert.Equal(t, sealTestTerminating, reply.AccountID)
|
||||
assert.Equal(t, "req-1", reply.CorrelationRef)
|
||||
assert.Equal(t, ec2v1.AccountSettingsDispositionSealed, reply.Disposition)
|
||||
assert.NotZero(t, reply.Revision)
|
||||
}
|
||||
|
||||
// Only an account IAM holds as TERMINATING may be sealed; ACTIVE is refused
|
||||
// before any write.
|
||||
func TestSealAccountSettingsSubjectRefusesAnActiveAccount(t *testing.T) {
|
||||
_, nc := newSealTestDaemon(t, &sealTestChecker{terminating: map[string]bool{sealTestTerminating: true}})
|
||||
|
||||
raw := sealRequest(t, nc, ec2v1.SealAccountSettingsRequest{AccountID: sealTestActive}, "")
|
||||
assert.Equal(t, awserrors.ErrorIncorrectState, replyErrCode(t, raw))
|
||||
}
|
||||
|
||||
// A request without a valid account is refused, not sealed.
|
||||
func TestSealAccountSettingsSubjectRefusesAMissingAccount(t *testing.T) {
|
||||
_, nc := newSealTestDaemon(t, &sealTestChecker{terminating: map[string]bool{sealTestTerminating: true}})
|
||||
|
||||
assert.Equal(t, awserrors.ErrorMissingParameter, replyErrCode(t, sealRequest(t, nc, ec2v1.SealAccountSettingsRequest{}, "")))
|
||||
}
|
||||
|
||||
// A daemon that does not serve the subject leaves the request with no responder.
|
||||
func TestSealAccountSettingsSubjectHasNoResponderWithoutTheDaemon(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
_, err := natsRequestAs(nc, ec2v1.SealAccountSettingsSubject, []byte(`{"account_id":"111122223333"}`), "", time.Second)
|
||||
require.ErrorIs(t, err, nats.ErrNoResponders)
|
||||
}
|
||||
@@ -774,10 +774,14 @@ func (d *Daemon) WaitForSystemInstance(instanceID string, timeout time.Duration)
|
||||
return fmt.Errorf("instance %s did not reach running state within %s", instanceID, timeout)
|
||||
}
|
||||
|
||||
// microvmDir holds the microvm kernel, initramfs and firmware. A variable so
|
||||
// tests can supply their own.
|
||||
var microvmDir = vm.MicroVMDir
|
||||
|
||||
// buildDirectBootConfig constructs a vm.Config for a direct-boot microVM
|
||||
// launch. It does not call buildBaseVMConfig — the microvm machine type
|
||||
// requires no PCIe root ports, no UEFI/BIOS, and no drives. Runtime paths
|
||||
// (PID file, console log, serial socket) are filled in later by startQEMU.
|
||||
// requires no PCIe root ports, no UEFI, and no drives; it boots qboot. Runtime
|
||||
// paths (PID file, console log, serial socket) are filled in later by startQEMU.
|
||||
//
|
||||
// Network topology from input.NICs is serialised to three fw_cfg tmpfiles
|
||||
// (netcfg, lb-agent-env, ca-cert) under hostprocess.RuntimeDir().
|
||||
@@ -794,7 +798,10 @@ func (d *Daemon) buildDirectBootConfig(instanceID string, input *handlers_elbv2.
|
||||
memMiB = int(instanceTypeMemoryMiB(it))
|
||||
}
|
||||
|
||||
const imagePath = "/usr/share/spinifex/microvm"
|
||||
firmware := filepath.Join(microvmDir, vm.MicroVMFirmware)
|
||||
if _, err := os.Stat(firmware); err != nil {
|
||||
return vm.Config{}, fmt.Errorf("microvm firmware %s is missing, so load balancer instances cannot boot; reinstall Spinifex to restore it: %w", firmware, err)
|
||||
}
|
||||
|
||||
fwCfg, err := d.writeFwCfgBlobs(instanceID, input)
|
||||
if err != nil {
|
||||
@@ -846,8 +853,9 @@ func (d *Daemon) buildDirectBootConfig(instanceID string, input *handlers_elbv2.
|
||||
CPUCount: vcpus,
|
||||
Memory: memMiB,
|
||||
MachineType: machineType,
|
||||
KernelImage: filepath.Join(imagePath, "vmlinuz"),
|
||||
Initrd: filepath.Join(imagePath, "initramfs.cpio.gz"),
|
||||
BIOS: firmware,
|
||||
KernelImage: filepath.Join(microvmDir, "vmlinuz"),
|
||||
Initrd: filepath.Join(microvmDir, "initramfs.cpio.gz"),
|
||||
KernelCmdline: cmdline,
|
||||
FwCfg: fwCfg,
|
||||
}
|
||||
|
||||
@@ -804,6 +804,37 @@ func TestBuildDirectBootConfig_InstanceTypeShape(t *testing.T) {
|
||||
assert.Len(t, cfg.FwCfg, 3)
|
||||
}
|
||||
|
||||
func TestBuildDirectBootConfig_BootsTheShippedQboot(t *testing.T) {
|
||||
t.Setenv("XDG_RUNTIME_DIR", t.TempDir())
|
||||
d := &Daemon{resourceMgr: &ResourceManager{}}
|
||||
|
||||
cfg, err := d.buildDirectBootConfig("i-qboot", &handlers_elbv2.SystemInstanceInput{
|
||||
NICs: []handlers_elbv2.NICConfig{{MAC: "02:00:00:00:00:05", IsDefault: true}},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, filepath.Join(microvmDir, vm.MicroVMFirmware), cfg.BIOS)
|
||||
for _, e := range cfg.FwCfg {
|
||||
assert.NoError(t, os.Remove(e.File))
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildDirectBootConfig_MissingFirmwareFailsByName(t *testing.T) {
|
||||
runtimeDir := t.TempDir()
|
||||
t.Setenv("XDG_RUNTIME_DIR", runtimeDir)
|
||||
orig := microvmDir
|
||||
microvmDir = t.TempDir()
|
||||
t.Cleanup(func() { microvmDir = orig })
|
||||
|
||||
_, err := (&Daemon{resourceMgr: &ResourceManager{}}).buildDirectBootConfig("i-nofw", &handlers_elbv2.SystemInstanceInput{
|
||||
NICs: []handlers_elbv2.NICConfig{{MAC: "02:00:00:00:00:06", IsDefault: true}},
|
||||
})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), filepath.Join(microvmDir, vm.MicroVMFirmware))
|
||||
entries, readErr := os.ReadDir(runtimeDir)
|
||||
require.NoError(t, readErr)
|
||||
assert.Empty(t, entries, "a launch refused for missing firmware must leave no fw_cfg blobs behind")
|
||||
}
|
||||
|
||||
func TestBuildDirectBootConfig_FwCfgWriteError(t *testing.T) {
|
||||
notADir := filepath.Join(t.TempDir(), "file")
|
||||
require.NoError(t, os.WriteFile(notADir, nil, 0o600))
|
||||
|
||||
@@ -3,10 +3,12 @@ package daemon
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/clustersize"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
)
|
||||
|
||||
@@ -31,6 +33,19 @@ func TestMain(m *testing.M) {
|
||||
// has its own tests that inject a reader.
|
||||
os.Setenv("SPINIFEX_ADMISSION_LIVE_MEM", "0")
|
||||
|
||||
// System-instance launches refuse to build a config without the microvm
|
||||
// firmware, which a test host does not have installed.
|
||||
fwDir, err := os.MkdirTemp("", "microvm-*")
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Failed to create microvm fixture dir: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(fwDir, vm.MicroVMFirmware), []byte("qboot"), 0o644); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Failed to write microvm firmware fixture: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
microvmDir = fwDir
|
||||
|
||||
// Start a plain NATS server (no JetStream)
|
||||
ns, err := server.NewServer(&server.Options{
|
||||
Host: "127.0.0.1",
|
||||
@@ -88,6 +103,7 @@ func TestMain(m *testing.M) {
|
||||
ns.Shutdown()
|
||||
jsNS.Shutdown()
|
||||
os.RemoveAll(jsTmpDir)
|
||||
os.RemoveAll(fwDir)
|
||||
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
@@ -1,92 +0,0 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/dns"
|
||||
reconcilelock "github.com/mulgadc/spinifex/spinifex/domains/network/reconcile"
|
||||
route53domain "github.com/mulgadc/spinifex/spinifex/domains/route53"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
var _ route53domain.Publisher = (*dns.ZonePublisher)(nil)
|
||||
|
||||
// newRoute53Service builds the Route 53 control plane. The nameservers and the
|
||||
// reserved names are read from the cluster's DNS settings on every call, so a
|
||||
// node joining or a domain change needs no restart.
|
||||
func (d *Daemon) newRoute53Service(js jetstream.JetStream) *route53domain.Service {
|
||||
svc := route53domain.NewService(route53domain.NewStore(js))
|
||||
svc.Nameservers = func() []string {
|
||||
base := dns.ResolveBaseDomain(d.config)
|
||||
if base == "" || d.clusterConfig == nil {
|
||||
return nil
|
||||
}
|
||||
published := dns.PublishedNameservers(d.clusterConfig)
|
||||
hosts := make([]string, 0, len(published))
|
||||
for _, ns := range published {
|
||||
hosts = append(hosts, ns.Host+"."+base)
|
||||
}
|
||||
return hosts
|
||||
}
|
||||
svc.Reserved = func(name string) bool {
|
||||
return dns.ReservedZoneName(name, dns.ResolveBaseDomain(d.config), dns.ResolveInternalDomain(d.config))
|
||||
}
|
||||
return svc
|
||||
}
|
||||
|
||||
// startRoute53Realizer publishes ACTIVE hosted zones to northstar. Every node
|
||||
// runs it, and a per-pass election picks the one that writes. Without a zone
|
||||
// store the zones stay unpublished and their changes stay PENDING.
|
||||
func (d *Daemon) startRoute53Realizer() {
|
||||
pub, ok := dns.NewZonePublisher(d.config, func() *config.ClusterConfig { return d.clusterConfig }, d.natsConn)
|
||||
if !ok {
|
||||
slog.Info("Route 53 realizer not started: no northstar zone store is configured")
|
||||
return
|
||||
}
|
||||
r := route53domain.NewRealizer(d.route53Service.Store(), pub)
|
||||
r.Elect = func(ctx context.Context) (func(), bool) {
|
||||
return reconcilelock.AcquireLeader(ctx, d.natsConn, route53domain.KVBucketRoute53Realize, d.node)
|
||||
}
|
||||
r.Reserved = d.route53Service.Reserved
|
||||
go r.Run(d.ctx)
|
||||
slog.Info("Started Route 53 realizer")
|
||||
}
|
||||
|
||||
// newRoute53Verifier checks each zone's delegation: from this cluster's own
|
||||
// records when the parent is a zone it serves, and from the parent's public
|
||||
// servers otherwise.
|
||||
func (d *Daemon) newRoute53Verifier() *route53domain.Verifier {
|
||||
v := route53domain.NewVerifier(d.route53Service.Store())
|
||||
v.Nameservers = d.route53Service.Nameservers
|
||||
if d.dnsWriter.Enabled() {
|
||||
v.Platform = func(_ context.Context, name string) (route53domain.Delegation, bool, error) {
|
||||
servers, zone, ok, err := d.dnsWriter.PlatformDelegation(name)
|
||||
return route53domain.Delegation{Source: "platform zone " + zone, Servers: servers}, ok, err
|
||||
}
|
||||
}
|
||||
lookup := &dns.ParentLookup{}
|
||||
v.Public = func(ctx context.Context, name string) (route53domain.Delegation, error) {
|
||||
servers, parent, err := lookup.Delegation(ctx, name)
|
||||
return route53domain.Delegation{Source: "the " + parent + " name servers", Servers: servers}, err
|
||||
}
|
||||
v.Elect = func(ctx context.Context) (func(), bool) {
|
||||
return reconcilelock.AcquireLeader(ctx, d.natsConn, route53domain.KVBucketRoute53Verify, d.node)
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// route53Delegate writes or withdraws platform-zone delegations for the
|
||||
// operator's spx admin dns delegate.
|
||||
func (d *Daemon) route53Delegate(ctx context.Context, in *route53domain.AdminDelegateInput, _ string) (*route53domain.AdminDelegateOutput, error) {
|
||||
if len(in.Names) == 0 {
|
||||
return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "no names to delegate")
|
||||
}
|
||||
res, err := d.dnsWriter.Delegate(ctx, d.natsConn, d.node, in.Names, in.Remove)
|
||||
if err != nil {
|
||||
return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "%v", err)
|
||||
}
|
||||
return &route53domain.AdminDelegateOutput{Applied: res.Applied}, nil
|
||||
}
|
||||
@@ -1,50 +0,0 @@
|
||||
//test:in-package — newRoute53Service is an unexported Daemon method and reads
|
||||
//the unexported config and clusterConfig fields.
|
||||
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/route53"
|
||||
testutil "github.com/mulgadc/spinifex/internal/testkit"
|
||||
"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestRoute53ServiceDelegatesToThePublishedNameservers(t *testing.T) {
|
||||
_, _, js := testutil.StartTestJetStream(t)
|
||||
ns := config.NorthstarConfig{ConfigPath: "/etc/spinifex/northstar.toml", DefaultDomain: "spx3.net"}
|
||||
nodes := map[string]config.Config{
|
||||
"node1": {AdvertiseIP: "10.0.0.1", Northstar: ns},
|
||||
"node2": {AdvertiseIP: "10.0.0.2", Northstar: ns},
|
||||
}
|
||||
local := nodes["node1"]
|
||||
d := &Daemon{config: &local, clusterConfig: &config.ClusterConfig{Node: "node1", Nodes: nodes}}
|
||||
svc := d.newRoute53Service(js)
|
||||
|
||||
out, err := svc.CreateHostedZone(context.Background(), &route53.CreateHostedZoneInput{
|
||||
Name: aws.String("example.com"), CallerReference: aws.String("ref-1"),
|
||||
}, "000000000001")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"ns1.spx3.net", "ns2.spx3.net"}, aws.StringValueSlice(out.DelegationSet.NameServers))
|
||||
|
||||
_, err = svc.CreateHostedZone(context.Background(), &route53.CreateHostedZoneInput{
|
||||
Name: aws.String("lb.elb.spx3.net"), CallerReference: aws.String("ref-2"),
|
||||
}, "000000000001")
|
||||
assert.ErrorContains(t, err, "InvalidDomainName")
|
||||
}
|
||||
|
||||
func TestRoute53ServiceRefusesZonesWithoutABaseDomain(t *testing.T) {
|
||||
_, _, js := testutil.StartTestJetStream(t)
|
||||
local := config.Config{AdvertiseIP: "10.0.0.1"}
|
||||
d := &Daemon{config: &local, clusterConfig: &config.ClusterConfig{Node: "node1", Nodes: map[string]config.Config{"node1": local}}}
|
||||
|
||||
_, err := d.newRoute53Service(js).CreateHostedZone(context.Background(), &route53.CreateHostedZoneInput{
|
||||
Name: aws.String("example.com"), CallerReference: aws.String("ref-1"),
|
||||
}, "000000000001")
|
||||
assert.ErrorContains(t, err, "no nameservers")
|
||||
}
|
||||
@@ -39,11 +39,7 @@ func (s *Service) EnforceVPCs(ctx context.Context, natsConn *nats.Conn, accountI
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
limits, err := s.limitsFor(ctx, accountID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return exceeds(len(out.Vpcs), want, limits.VPCs)
|
||||
return exceeds(len(out.Vpcs), want, s.limitsFor(ctx, accountID).VPCs)
|
||||
}
|
||||
|
||||
// EnforceSubnets gates CreateSubnet on the account's live DescribeSubnets count.
|
||||
@@ -56,11 +52,7 @@ func (s *Service) EnforceSubnets(ctx context.Context, natsConn *nats.Conn, accou
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
limits, err := s.limitsFor(ctx, accountID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return exceeds(len(out.Subnets), want, limits.Subnets)
|
||||
return exceeds(len(out.Subnets), want, s.limitsFor(ctx, accountID).Subnets)
|
||||
}
|
||||
|
||||
// EnforceEIPs gates AllocateAddress on the account's live DescribeAddresses count.
|
||||
@@ -72,11 +64,7 @@ func (s *Service) EnforceEIPs(ctx context.Context, natsConn *nats.Conn, accountI
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
limits, err := s.limitsFor(ctx, accountID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return exceeds(len(out.Addresses), want, limits.EIPs)
|
||||
return exceeds(len(out.Addresses), want, s.limitsFor(ctx, accountID).EIPs)
|
||||
}
|
||||
|
||||
// EnforceLoadBalancers gates CreateLoadBalancer on the account's live count.
|
||||
@@ -89,11 +77,7 @@ func (s *Service) EnforceLoadBalancers(ctx context.Context, natsConn *nats.Conn,
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
limits, err := s.limitsFor(ctx, accountID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return exceeds(len(out.LoadBalancers), want, limits.LoadBalancers)
|
||||
return exceeds(len(out.LoadBalancers), want, s.limitsFor(ctx, accountID).LoadBalancers)
|
||||
}
|
||||
|
||||
// EnforceRDSInstances gates CreateDBInstance on the account's live count. A DB
|
||||
@@ -107,9 +91,5 @@ func (s *Service) EnforceRDSInstances(ctx context.Context, natsConn *nats.Conn,
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
limits, err := s.limitsFor(ctx, accountID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return exceeds(len(out.DBInstances), want, limits.RDSInstances)
|
||||
return exceeds(len(out.DBInstances), want, s.limitsFor(ctx, accountID).RDSInstances)
|
||||
}
|
||||
|
||||
@@ -8,16 +8,12 @@ import (
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// Overrides raises or lowers individual dimensions for one account. Every field
|
||||
// is a pointer so an explicit 0 (a limit that denies everything) is distinct
|
||||
// from an absent field, which inherits the configured value. This is also the
|
||||
// admin wire type: an admin request body can carry only these fields, so it can
|
||||
// never set or clear a stored record's state.
|
||||
// from an absent field, which inherits the configured value.
|
||||
type Overrides struct {
|
||||
VCPUs *int `json:"vcpus,omitempty"`
|
||||
VPCs *int `json:"vpcs,omitempty"`
|
||||
@@ -33,62 +29,11 @@ type Overrides struct {
|
||||
UpdatedBy string `json:"updated_by,omitempty"`
|
||||
}
|
||||
|
||||
// OverrideState is a stored override record's lifecycle state. "" is active;
|
||||
// a record written before the state field existed has none and stays active.
|
||||
type OverrideState string
|
||||
|
||||
// OverrideStateSealed marks a record no writer may change or recreate.
|
||||
const OverrideStateSealed OverrideState = "sealed"
|
||||
|
||||
// overrideRecord is the stored form: the admin-settable fields inline, plus a
|
||||
// state no admin request can read or write.
|
||||
type overrideRecord struct {
|
||||
Overrides
|
||||
|
||||
State OverrideState `json:"state,omitempty"`
|
||||
}
|
||||
|
||||
// errOverridesSealed marks a read or write that found a sealed record, so a
|
||||
// caller maps it to Blocked without logging it as a fault.
|
||||
var errOverridesSealed = errors.New(awserrors.ErrorBlocked)
|
||||
|
||||
// errOverridesContended marks a write whose CAS retries were exhausted under
|
||||
// concurrent writers, bounding the retry loop instead of spinning.
|
||||
var errOverridesContended = errors.New("quota override: CAS retries exhausted")
|
||||
|
||||
// overrideMaxAttempts bounds a write's retries against a revision that keeps
|
||||
// moving under concurrent writers, so it fails closed instead of spinning.
|
||||
const overrideMaxAttempts = 5
|
||||
|
||||
// checkActive rejects a sealed or unknown-state record; "" is active and
|
||||
// passes through. A caller must never read or mutate a record that failed
|
||||
// this as if it were active.
|
||||
func checkActive(rec overrideRecord) error {
|
||||
switch rec.State {
|
||||
case "":
|
||||
return nil
|
||||
case OverrideStateSealed:
|
||||
return errOverridesSealed
|
||||
default:
|
||||
return fmt.Errorf("quota override: unknown state %q", rec.State)
|
||||
}
|
||||
}
|
||||
|
||||
// mapOverrideError turns a recordFor, PutAccountQuota or clear failure into
|
||||
// the AWS error a caller returns: sealed is an expected, unlogged Blocked;
|
||||
// everything else is InternalError, logged with the account ID.
|
||||
func mapOverrideError(ctx context.Context, accountID string, err error) error {
|
||||
switch {
|
||||
case err == nil:
|
||||
return nil
|
||||
case errors.Is(err, errOverridesSealed):
|
||||
return errors.New(awserrors.ErrorBlocked)
|
||||
case errors.Is(err, errOverridesContended):
|
||||
slog.ErrorContext(ctx, "Quota override CAS retries exhausted under contention", "accountID", accountID)
|
||||
default:
|
||||
slog.ErrorContext(ctx, "Quota override operation failed", "accountID", accountID, "err", err)
|
||||
}
|
||||
return errors.New(awserrors.ErrorInternalError)
|
||||
// Empty reports whether no dimension is overridden, in which case the record
|
||||
// carries nothing but provenance and the account resolves to the config.
|
||||
func (o Overrides) Empty() bool {
|
||||
return o.VCPUs == nil && o.VPCs == nil && o.Subnets == nil && o.EIPs == nil &&
|
||||
o.VolumesGiB == nil && o.Volumes == nil && o.RDSInstances == nil && o.LoadBalancers == nil
|
||||
}
|
||||
|
||||
// apply overlays the set fields onto base, leaving Enabled and the Bedrock
|
||||
@@ -115,128 +60,72 @@ func (o Overrides) apply(base Limits) Limits {
|
||||
return out
|
||||
}
|
||||
|
||||
// Empty reports whether no dimension is overridden, in which case the record
|
||||
// carries nothing but provenance and the account resolves to the config.
|
||||
func (o Overrides) Empty() bool {
|
||||
return o.VCPUs == nil && o.VPCs == nil && o.Subnets == nil && o.EIPs == nil &&
|
||||
o.VolumesGiB == nil && o.Volumes == nil && o.RDSInstances == nil && o.LoadBalancers == nil
|
||||
}
|
||||
|
||||
// recordFor reads and decodes accountID's stored override record. An absent
|
||||
// key is the active zero value, not an error; only a genuine read or decode
|
||||
// failure is returned.
|
||||
func (s *Service) recordFor(ctx context.Context, accountID string) (overrideRecord, error) {
|
||||
if s == nil || s.overrides == nil {
|
||||
return overrideRecord{}, nil
|
||||
}
|
||||
entry, err := s.overrides.Get(ctx, accountID)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return overrideRecord{}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return overrideRecord{}, fmt.Errorf("read quota override for %s: %w", accountID, err)
|
||||
}
|
||||
var rec overrideRecord
|
||||
if err := json.Unmarshal(entry.Value(), &rec); err != nil {
|
||||
return overrideRecord{}, fmt.Errorf("decode quota override for %s: %w", accountID, err)
|
||||
}
|
||||
return rec, nil
|
||||
}
|
||||
|
||||
// limitsFor resolves accountID's effective limits: overrides where set, the
|
||||
// config elsewhere. A read or decode failure falls back to the config so an
|
||||
// outage cannot lift a cap; a sealed or unknown-state record refuses instead.
|
||||
func (s *Service) limitsFor(ctx context.Context, accountID string) (Limits, error) {
|
||||
rec, err := s.recordFor(ctx, accountID)
|
||||
// limitsFor resolves the effective limits for accountID: its overrides where
|
||||
// set, the configured limits everywhere else. A read failure falls back to the
|
||||
// configured limits rather than to no limit, so a KV outage cannot lift a cap.
|
||||
func (s *Service) limitsFor(ctx context.Context, accountID string) Limits {
|
||||
over, err := s.overridesFor(ctx, accountID)
|
||||
if err != nil {
|
||||
slog.WarnContext(ctx, "quota: override read failed, using configured limits",
|
||||
"account", accountID, "err", err)
|
||||
return s.limits, nil
|
||||
return s.limits
|
||||
}
|
||||
if err := checkActive(rec); err != nil {
|
||||
return Limits{}, mapOverrideError(ctx, accountID, err)
|
||||
}
|
||||
return rec.apply(s.limits), nil
|
||||
return over.apply(s.limits)
|
||||
}
|
||||
|
||||
// GetAccountQuota returns accountID's stored overrides and resolved limits.
|
||||
// Read failures surface rather than show the fallback as real; a sealed or
|
||||
// unknown-state record returns Blocked or InternalError.
|
||||
// overridesFor reads accountID's override record. An absent key is the normal
|
||||
// case for nearly every account and yields the zero value, not an error; only a
|
||||
// genuine read or decode failure is returned.
|
||||
func (s *Service) overridesFor(ctx context.Context, accountID string) (Overrides, error) {
|
||||
if s == nil || s.overrides == nil {
|
||||
return Overrides{}, nil
|
||||
}
|
||||
entry, err := s.overrides.Get(ctx, accountID)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return Overrides{}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return Overrides{}, fmt.Errorf("read quota override for %s: %w", accountID, err)
|
||||
}
|
||||
var over Overrides
|
||||
if err := json.Unmarshal(entry.Value(), &over); err != nil {
|
||||
return Overrides{}, fmt.Errorf("decode quota override for %s: %w", accountID, err)
|
||||
}
|
||||
return over, nil
|
||||
}
|
||||
|
||||
// GetAccountQuota returns accountID's stored overrides and the limits they
|
||||
// resolve to. A read failure is surfaced here rather than masked, because an
|
||||
// operator inspecting a quota must not be shown the fallback as if it were real.
|
||||
func (s *Service) GetAccountQuota(ctx context.Context, accountID string) (Overrides, Limits, error) {
|
||||
rec, err := s.recordFor(ctx, accountID)
|
||||
over, err := s.overridesFor(ctx, accountID)
|
||||
if err != nil {
|
||||
return Overrides{}, Limits{}, err
|
||||
}
|
||||
if err := checkActive(rec); err != nil {
|
||||
return Overrides{}, Limits{}, mapOverrideError(ctx, accountID, err)
|
||||
}
|
||||
return rec.Overrides, rec.apply(s.limits), nil
|
||||
return over, over.apply(s.limits), nil
|
||||
}
|
||||
|
||||
// PutAccountQuota stores or clears accountID's overrides, stamping provenance
|
||||
// on a store. Both branches are revision-guarded and refuse a sealed record,
|
||||
// so a seal landing between a read and a write fails the CAS.
|
||||
// PutAccountQuota stores accountID's overrides, stamping provenance. An empty
|
||||
// set deletes the record so the account resolves to the config, which keeps
|
||||
// "has this account been touched?" a single existence check.
|
||||
func (s *Service) PutAccountQuota(ctx context.Context, accountID string, over Overrides, updatedBy string) error {
|
||||
if s == nil || s.overrides == nil {
|
||||
return errors.New("quota: override bucket not configured")
|
||||
}
|
||||
if over.Empty() {
|
||||
return s.clearAccountQuota(ctx, accountID)
|
||||
if err := s.overrides.Delete(ctx, accountID); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return fmt.Errorf("clear quota override for %s: %w", accountID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
over.UpdatedAt = time.Now().UTC().Format(time.RFC3339)
|
||||
over.UpdatedBy = updatedBy
|
||||
|
||||
_, err := kvutil.Update(ctx, s.overrides, accountID, kvutil.CASConfig{
|
||||
CreateIfAbsent: true,
|
||||
Attempts: overrideMaxAttempts,
|
||||
Exhausted: func(string, int) error { return errOverridesContended },
|
||||
}, func(rec *overrideRecord) (bool, error) {
|
||||
if err := checkActive(*rec); err != nil {
|
||||
return false, err
|
||||
}
|
||||
rec.Overrides = over
|
||||
// Every success is a leader-accepted write over an active value: a
|
||||
// seal landing between the read and the write fails this CAS.
|
||||
return true, nil
|
||||
})
|
||||
return mapOverrideError(ctx, accountID, err)
|
||||
}
|
||||
|
||||
// clearAccountQuota deletes an active override at the revision it read. An
|
||||
// observed-absent key writes nothing, so a stale read cannot unseal; a sealed
|
||||
// record refuses rather than being erased and later recreated active.
|
||||
func (s *Service) clearAccountQuota(ctx context.Context, accountID string) error {
|
||||
for range overrideMaxAttempts {
|
||||
entry, err := s.overrides.Get(ctx, accountID)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return mapOverrideError(ctx, accountID, fmt.Errorf("read quota override for clear: %w", err))
|
||||
}
|
||||
|
||||
var rec overrideRecord
|
||||
if err := json.Unmarshal(entry.Value(), &rec); err != nil {
|
||||
return mapOverrideError(ctx, accountID, fmt.Errorf("decode quota override for clear: %w", err))
|
||||
}
|
||||
if err := checkActive(rec); err != nil {
|
||||
return mapOverrideError(ctx, accountID, err)
|
||||
}
|
||||
|
||||
err = s.overrides.Delete(ctx, accountID, jetstream.LastRevision(entry.Revision()))
|
||||
switch {
|
||||
case err == nil:
|
||||
return nil
|
||||
case errors.Is(err, jetstream.ErrKeyNotFound):
|
||||
return nil
|
||||
case isRevisionConflict(err):
|
||||
// A concurrent write landed between the read and the guarded
|
||||
// delete; retry against whatever revision now holds.
|
||||
continue
|
||||
default:
|
||||
return mapOverrideError(ctx, accountID, fmt.Errorf("clear quota override: %w", err))
|
||||
}
|
||||
data, err := json.Marshal(over)
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode quota override for %s: %w", accountID, err)
|
||||
}
|
||||
return mapOverrideError(ctx, accountID, errOverridesContended)
|
||||
if _, err := s.overrides.Put(ctx, accountID, data); err != nil {
|
||||
return fmt.Errorf("store quota override for %s: %w", accountID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -31,8 +31,7 @@ func newOverrideService(t *testing.T) (*Service, jetstream.KeyValue) {
|
||||
func TestLimitsForAbsentKeyInherits(t *testing.T) {
|
||||
svc, _ := newOverrideService(t)
|
||||
|
||||
got, err := svc.limitsFor(t.Context(), "123456789012")
|
||||
require.NoError(t, err)
|
||||
got := svc.limitsFor(t.Context(), "123456789012")
|
||||
require.Equal(t, baseLimits(), got)
|
||||
}
|
||||
|
||||
@@ -45,14 +44,10 @@ func TestLimitsForIsPerAccount(t *testing.T) {
|
||||
require.NoError(t, svc.PutAccountQuota(t.Context(), upgraded,
|
||||
Overrides{VCPUs: ptr(32), RDSInstances: ptr(8)}, "operator"))
|
||||
|
||||
upgradedLimits, err := svc.limitsFor(t.Context(), upgraded)
|
||||
require.NoError(t, err)
|
||||
sandboxLimits, err := svc.limitsFor(t.Context(), sandbox)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 32, upgradedLimits.VCPUs)
|
||||
require.Equal(t, 8, upgradedLimits.RDSInstances)
|
||||
require.Equal(t, baseLimits().VCPUs, sandboxLimits.VCPUs)
|
||||
require.Equal(t, baseLimits().RDSInstances, sandboxLimits.RDSInstances)
|
||||
require.Equal(t, 32, svc.limitsFor(t.Context(), upgraded).VCPUs)
|
||||
require.Equal(t, 8, svc.limitsFor(t.Context(), upgraded).RDSInstances)
|
||||
require.Equal(t, baseLimits().VCPUs, svc.limitsFor(t.Context(), sandbox).VCPUs)
|
||||
require.Equal(t, baseLimits().RDSInstances, svc.limitsFor(t.Context(), sandbox).RDSInstances)
|
||||
}
|
||||
|
||||
// A stored override must round-trip through KV with its provenance, and an
|
||||
@@ -84,9 +79,7 @@ func TestPutAccountQuotaEmptyClears(t *testing.T) {
|
||||
|
||||
_, err := kv.Get(t.Context(), account)
|
||||
require.ErrorIs(t, err, jetstream.ErrKeyNotFound)
|
||||
limits, err := svc.limitsFor(t.Context(), account)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, baseLimits(), limits)
|
||||
require.Equal(t, baseLimits(), svc.limitsFor(t.Context(), account))
|
||||
}
|
||||
|
||||
// Clearing an account that never had a record must succeed rather than fail on
|
||||
@@ -104,10 +97,8 @@ func TestPutAccountQuotaExplicitZero(t *testing.T) {
|
||||
|
||||
require.NoError(t, svc.PutAccountQuota(t.Context(), account, Overrides{VPCs: ptr(0)}, "operator"))
|
||||
|
||||
limits, err := svc.limitsFor(t.Context(), account)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 0, limits.VPCs)
|
||||
require.Error(t, exceeds(0, 1, limits.VPCs))
|
||||
require.Equal(t, 0, svc.limitsFor(t.Context(), account).VPCs)
|
||||
require.Error(t, exceeds(0, 1, svc.limitsFor(t.Context(), account).VPCs))
|
||||
}
|
||||
|
||||
// A corrupt record is a read failure, not an absent one: it must fall back to
|
||||
@@ -119,9 +110,7 @@ func TestLimitsForCorruptRecordFailsClosed(t *testing.T) {
|
||||
_, err := kv.Put(t.Context(), account, []byte("{not json"))
|
||||
require.NoError(t, err)
|
||||
|
||||
limits, err := svc.limitsFor(t.Context(), account)
|
||||
require.NoError(t, err, "a decode failure must fall back, not error, from limitsFor")
|
||||
require.Equal(t, baseLimits(), limits,
|
||||
require.Equal(t, baseLimits(), svc.limitsFor(t.Context(), account),
|
||||
"a corrupt override must fall back to the configured limits")
|
||||
|
||||
_, _, err = svc.GetAccountQuota(t.Context(), account)
|
||||
|
||||
@@ -126,11 +126,7 @@ func TestOverridesEmpty(t *testing.T) {
|
||||
// fall through to the configured limits rather than panicking.
|
||||
func TestLimitsForWithoutBucket(t *testing.T) {
|
||||
svc := New(baseLimits(), nil)
|
||||
got, err := svc.limitsFor(t.Context(), "123456789012")
|
||||
if err != nil {
|
||||
t.Fatalf("limitsFor error = %v, want nil", err)
|
||||
}
|
||||
if got != baseLimits() {
|
||||
if got := svc.limitsFor(t.Context(), "123456789012"); got != baseLimits() {
|
||||
t.Fatalf("limitsFor = %+v, want the configured limits", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,75 +0,0 @@
|
||||
package quota
|
||||
|
||||
//test:in-package — pins what AccountQuotaOverrideSealCapabilityVersion 1 will
|
||||
// assert: nothing in this package writes the override bucket except through
|
||||
// PutAccountQuota's CAS loop, clearAccountQuota's guarded delete or
|
||||
// SealAccountOverrides' CAS loop.
|
||||
|
||||
import (
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// overrideWriteFiles are the only files in this package that touch the
|
||||
// override bucket. The package also holds the vCPU usage and Bedrock usage
|
||||
// buckets, which write through their own stores and are out of scope here;
|
||||
// scanning the whole package directory, as the EC2 settings equivalent does,
|
||||
// would false-flag those unrelated writes.
|
||||
var overrideWriteFiles = []string{"override.go", "seal.go"}
|
||||
|
||||
func quotaPackageDir(t *testing.T) string {
|
||||
t.Helper()
|
||||
_, file, _, ok := runtime.Caller(0)
|
||||
if !ok {
|
||||
t.Fatal("cannot locate test source")
|
||||
}
|
||||
return filepath.Dir(file)
|
||||
}
|
||||
|
||||
// No file owning the override bucket may call Put, PutString, or Delete
|
||||
// without a revision option: every write must go through PutAccountQuota's
|
||||
// CAS loop, clearAccountQuota's guarded delete, or SealAccountOverrides' CAS
|
||||
// loop.
|
||||
func TestOverrideWritesAreRevisionGuarded(t *testing.T) {
|
||||
root := quotaPackageDir(t)
|
||||
fset := token.NewFileSet()
|
||||
scanned := 0
|
||||
|
||||
for _, name := range overrideWriteFiles {
|
||||
path := filepath.Join(root, name)
|
||||
f, err := parser.ParseFile(fset, path, nil, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("parse %s: %v", path, err)
|
||||
}
|
||||
scanned++
|
||||
|
||||
ast.Inspect(f, func(n ast.Node) bool {
|
||||
call, ok := n.(*ast.CallExpr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
sel, ok := call.Fun.(*ast.SelectorExpr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
switch sel.Sel.Name {
|
||||
case "Put", "PutString":
|
||||
t.Errorf("%s: unconditional %s call bypasses the revision-guarded override write; use PutAccountQuota or SealAccountOverrides instead",
|
||||
fset.Position(call.Pos()), sel.Sel.Name)
|
||||
case "Delete":
|
||||
if len(call.Args) < 3 {
|
||||
t.Errorf("%s: Delete call with no revision option bypasses the guarded clear; pin it to the read revision",
|
||||
fset.Position(call.Pos()))
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
}
|
||||
if scanned == 0 {
|
||||
t.Fatalf("scanned no override-write files under %s; the invariant would pass vacuously", root)
|
||||
}
|
||||
}
|
||||
@@ -90,14 +90,6 @@ type Service struct {
|
||||
// overrides holds the per-account limit overrides. Nil leaves every account
|
||||
// on the configured limits, which is also what an absent key means.
|
||||
overrides jetstream.KeyValue
|
||||
|
||||
// js opens the override bucket for SealAccountOverrides when quotas were
|
||||
// disabled at startup; nil makes such a seal refuse rather than create it.
|
||||
js jetstream.KeyValueManager
|
||||
|
||||
// terminationChecker is the seal precondition; SealAccountOverrides
|
||||
// refuses until it is wired.
|
||||
terminationChecker AccountTerminationChecker
|
||||
}
|
||||
|
||||
// New constructs a quota Service from the configured limits and the gateway-owned
|
||||
@@ -133,16 +125,6 @@ func (s *Service) SetOverrides(kv jetstream.KeyValue) {
|
||||
s.overrides = kv
|
||||
}
|
||||
|
||||
// SetJetStream wires the client SealAccountOverrides uses to get-or-create
|
||||
// the override bucket when this gateway disabled quotas. Called once at
|
||||
// wiring time, regardless of whether quotas are enabled here.
|
||||
func (s *Service) SetJetStream(js jetstream.KeyValueManager) {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
s.js = js
|
||||
}
|
||||
|
||||
// Exempt returns true for the global/system account and whenever quotas are
|
||||
// disabled, so those callers bypass every quota check. A nil Service (quotas
|
||||
// never built) is treated as disabled so callers need no nil guard.
|
||||
|
||||
@@ -1,163 +0,0 @@
|
||||
package quota
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log/slog"
|
||||
|
||||
awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// sealMaxAttempts bounds SealAccountOverrides' retry loop against a revision
|
||||
// that keeps moving under concurrent writes, so it fails closed instead of
|
||||
// spinning.
|
||||
const sealMaxAttempts = 5
|
||||
|
||||
// sealedRecord is the fixed value a seal writes. No dimension or provenance
|
||||
// survives, so no tenant commercial setting outlives the seal.
|
||||
var sealedRecord = &overrideRecord{State: OverrideStateSealed}
|
||||
|
||||
// OverrideSeal is SealAccountOverrides' result: the revision at which the
|
||||
// stream leader accepted the sealed value.
|
||||
type OverrideSeal struct {
|
||||
Revision uint64
|
||||
}
|
||||
|
||||
// AccountTerminationChecker reports whether IAM holds accountID as
|
||||
// TERMINATING. A missing record is (false, nil); only a failed read is an
|
||||
// error.
|
||||
type AccountTerminationChecker interface {
|
||||
AccountTerminating(ctx context.Context, accountID string) (bool, error)
|
||||
}
|
||||
|
||||
// SetAccountTerminationChecker wires the seal precondition; until it is set,
|
||||
// SealAccountOverrides refuses.
|
||||
func (s *Service) SetAccountTerminationChecker(c AccountTerminationChecker) {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
s.terminationChecker = c
|
||||
}
|
||||
|
||||
// checkTerminating runs before any override-KV call, so a refusal never
|
||||
// writes or opens the bucket. Not TERMINATING (incl. missing) is
|
||||
// IncorrectState; a read failure or unwired checker is InternalError.
|
||||
func (s *Service) checkTerminating(ctx context.Context, accountID string) error {
|
||||
if s.terminationChecker == nil {
|
||||
slog.ErrorContext(ctx, "Quota override seal has no termination checker wired", "accountID", accountID)
|
||||
return errors.New(awserrors.ErrorInternalError)
|
||||
}
|
||||
terminating, err := s.terminationChecker.AccountTerminating(ctx, accountID)
|
||||
if err != nil {
|
||||
slog.ErrorContext(ctx, "Failed to read account status for seal precondition", "accountID", accountID, "err", err)
|
||||
return errors.New(awserrors.ErrorInternalError)
|
||||
}
|
||||
if !terminating {
|
||||
return awserrors.Errorf(awserrors.ErrorIncorrectState, "account %s is not TERMINATING", accountID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// isRevisionConflict reports a lost CAS race to retry, including the bare
|
||||
// wrong-last-sequence APIError from Create's retry against a delete marker.
|
||||
// A private copy of domains/ec2/account's helper, which this package may not import.
|
||||
func isRevisionConflict(err error) bool {
|
||||
if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
return true
|
||||
}
|
||||
if apiErr, ok := errors.AsType[*jetstream.APIError](err); ok {
|
||||
return apiErr.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequence ||
|
||||
apiErr.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequenceConstant
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// sealBucket returns this gateway's override bucket, or get-or-creates it
|
||||
// when quotas are disabled here. Called only after checkTerminating accepts,
|
||||
// so a refused seal never creates the bucket.
|
||||
func (s *Service) sealBucket(ctx context.Context) (jetstream.KeyValue, error) {
|
||||
if s.overrides != nil {
|
||||
return s.overrides, nil
|
||||
}
|
||||
if s.js == nil {
|
||||
slog.ErrorContext(ctx, "Quota override seal has no JetStream client wired to open the override bucket")
|
||||
return nil, errors.New(awserrors.ErrorInternalError)
|
||||
}
|
||||
kv, err := kvutil.GetOrCreateBucket(ctx, s.js, KVBucketAccountQuota, 1)
|
||||
if err != nil {
|
||||
slog.ErrorContext(ctx, "Failed to open quota override bucket for seal", "err", err)
|
||||
return nil, errors.New(awserrors.ErrorInternalError)
|
||||
}
|
||||
return kv, nil
|
||||
}
|
||||
|
||||
// SealAccountOverrides seals accountID's override record so no later writer
|
||||
// can recreate it. Success is only a leader-accepted write, evidenced by the
|
||||
// revision; a missing key is created sealed, never reported from absence.
|
||||
func (s *Service) SealAccountOverrides(ctx context.Context, accountID string) (*OverrideSeal, error) {
|
||||
if accountID == "" {
|
||||
return nil, awserrors.Errorf(awserrors.ErrorMissingParameter, "accountID is required to seal quota overrides")
|
||||
}
|
||||
if !awsidentifiers.IsAccountID(accountID) {
|
||||
return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "accountID %q is not a valid AWS account ID", accountID)
|
||||
}
|
||||
if accountID == awsidentifiers.GlobalAccountID {
|
||||
return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "quota override seal must not target the global account")
|
||||
}
|
||||
|
||||
if err := s.checkTerminating(ctx, accountID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
overrides, err := s.sealBucket(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
data, err := json.Marshal(sealedRecord)
|
||||
if err != nil {
|
||||
slog.ErrorContext(ctx, "Failed to encode sealed quota override record", "accountID", accountID, "err", err)
|
||||
return nil, errors.New(awserrors.ErrorInternalError)
|
||||
}
|
||||
|
||||
// accountID is never resolved through any fallback here: a seal must act
|
||||
// on exactly accountID and never fall back to the global key.
|
||||
for range sealMaxAttempts {
|
||||
var rev uint64
|
||||
entry, getErr := overrides.Get(ctx, accountID)
|
||||
switch {
|
||||
case getErr == nil:
|
||||
rev = entry.Revision()
|
||||
case errors.Is(getErr, jetstream.ErrKeyNotFound):
|
||||
rev = 0
|
||||
default:
|
||||
slog.ErrorContext(ctx, "Failed to read quota override for seal", "accountID", accountID, "err", getErr)
|
||||
return nil, errors.New(awserrors.ErrorInternalError)
|
||||
}
|
||||
|
||||
var newRev uint64
|
||||
var writeErr error
|
||||
if rev == 0 {
|
||||
newRev, writeErr = overrides.Create(ctx, accountID, data)
|
||||
} else {
|
||||
newRev, writeErr = overrides.Update(ctx, accountID, data, rev)
|
||||
}
|
||||
|
||||
switch {
|
||||
case writeErr == nil:
|
||||
return &OverrideSeal{Revision: newRev}, nil
|
||||
case isRevisionConflict(writeErr):
|
||||
continue
|
||||
default:
|
||||
slog.ErrorContext(ctx, "Failed to write sealed quota override record", "accountID", accountID, "err", writeErr)
|
||||
return nil, errors.New(awserrors.ErrorInternalError)
|
||||
}
|
||||
}
|
||||
|
||||
slog.ErrorContext(ctx, "Quota override seal exhausted attempts against concurrent writes", "accountID", accountID, "attempts", sealMaxAttempts)
|
||||
return nil, errors.New(awserrors.ErrorInternalError)
|
||||
}
|
||||
@@ -1,806 +0,0 @@
|
||||
package quota
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/spinifex/internal/testkit"
|
||||
awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// stubTerminationChecker answers AccountTerminating from a fixed set, for
|
||||
// tests that exercise the KV paths rather than IAM itself.
|
||||
type stubTerminationChecker struct {
|
||||
terminating map[string]bool
|
||||
err error
|
||||
}
|
||||
|
||||
func (c *stubTerminationChecker) AccountTerminating(_ context.Context, accountID string) (bool, error) {
|
||||
if c.err != nil {
|
||||
return false, c.err
|
||||
}
|
||||
return c.terminating[accountID], nil
|
||||
}
|
||||
|
||||
// terminatingChecker answers true for exactly accountID.
|
||||
func terminatingChecker(accountID string) *stubTerminationChecker {
|
||||
return &stubTerminationChecker{terminating: map[string]bool{accountID: true}}
|
||||
}
|
||||
|
||||
// newSealService builds a Service over a real JetStream KV bucket with the
|
||||
// override bucket already attached, mirroring a quota-enabled gateway.
|
||||
func newSealService(t *testing.T) (*Service, jetstream.JetStream) {
|
||||
t.Helper()
|
||||
_, _, js := testutil.StartTestJetStream(t)
|
||||
kv, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{
|
||||
Bucket: KVBucketAccountQuota,
|
||||
History: 1,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
usage, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{
|
||||
Bucket: KVBucketAccountUsage,
|
||||
History: 1,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
svc := New(baseLimits(), usage)
|
||||
svc.SetOverrides(kv)
|
||||
svc.SetJetStream(js)
|
||||
return svc, js
|
||||
}
|
||||
|
||||
// newDisabledSealService builds a Service with only the JetStream client
|
||||
// wired, mirroring a quota-disabled gateway: the override bucket does not
|
||||
// exist until a seal creates it.
|
||||
func newDisabledSealService(t *testing.T) (*Service, jetstream.JetStream) {
|
||||
t.Helper()
|
||||
_, _, js := testutil.StartTestJetStream(t)
|
||||
svc := New(baseLimits(), nil)
|
||||
svc.SetJetStream(js)
|
||||
return svc, js
|
||||
}
|
||||
|
||||
// lastOverrideSeq reports the override stream's last sequence, so a test can
|
||||
// assert a failed attempt wrote nothing.
|
||||
func lastOverrideSeq(t *testing.T, js jetstream.JetStream) uint64 {
|
||||
t.Helper()
|
||||
stream, err := js.Stream(context.Background(), "KV_"+KVBucketAccountQuota)
|
||||
require.NoError(t, err)
|
||||
info, err := stream.Info(context.Background())
|
||||
require.NoError(t, err)
|
||||
return info.State.LastSeq
|
||||
}
|
||||
|
||||
// mutationCase is one of PutAccountQuota's two mutations, called uniformly so
|
||||
// a test can table over them.
|
||||
type mutationCase struct {
|
||||
name string
|
||||
call func(svc *Service, ctx context.Context) error
|
||||
}
|
||||
|
||||
func mutationTable() []mutationCase {
|
||||
return []mutationCase{
|
||||
{"Set", func(svc *Service, ctx context.Context) error {
|
||||
return svc.PutAccountQuota(ctx, testAccount, Overrides{VCPUs: ptr(32)}, "operator")
|
||||
}},
|
||||
{"Clear", func(svc *Service, ctx context.Context) error {
|
||||
return svc.PutAccountQuota(ctx, testAccount, Overrides{}, "operator")
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
// seedOverrideState prepares testAccount's record for a race test: present
|
||||
// (an active record), absent (never written) or tombstone (written then
|
||||
// deleted, leaving a delete marker Get hides).
|
||||
func seedOverrideState(t *testing.T, svc *Service, state string) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
switch state {
|
||||
case "present":
|
||||
require.NoError(t, svc.PutAccountQuota(ctx, testAccount, Overrides{VCPUs: ptr(8)}, "operator"))
|
||||
case "tombstone":
|
||||
require.NoError(t, svc.PutAccountQuota(ctx, testAccount, Overrides{VCPUs: ptr(8)}, "operator"))
|
||||
require.NoError(t, svc.overrides.Delete(ctx, testAccount))
|
||||
case "absent":
|
||||
default:
|
||||
t.Fatalf("unknown seed state %q", state)
|
||||
}
|
||||
}
|
||||
|
||||
// sealRaceKV wraps a real KeyValue so a test can inject a deterministic race
|
||||
// between a caller's read and its guarded write (Create, Update or Delete),
|
||||
// instead of relying on goroutine timing.
|
||||
type sealRaceKV struct {
|
||||
jetstream.KeyValue
|
||||
|
||||
writeCalls int
|
||||
beforeWrite func(attempt int) error
|
||||
}
|
||||
|
||||
func (f *sealRaceKV) Create(ctx context.Context, key string, value []byte, opts ...jetstream.KVCreateOpt) (uint64, error) {
|
||||
f.writeCalls++
|
||||
if f.beforeWrite != nil {
|
||||
if err := f.beforeWrite(f.writeCalls); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
return f.KeyValue.Create(ctx, key, value, opts...)
|
||||
}
|
||||
|
||||
func (f *sealRaceKV) Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error) {
|
||||
f.writeCalls++
|
||||
if f.beforeWrite != nil {
|
||||
if err := f.beforeWrite(f.writeCalls); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
return f.KeyValue.Update(ctx, key, value, revision)
|
||||
}
|
||||
|
||||
func (f *sealRaceKV) Delete(ctx context.Context, key string, opts ...jetstream.KVDeleteOpt) error {
|
||||
f.writeCalls++
|
||||
if f.beforeWrite != nil {
|
||||
if err := f.beforeWrite(f.writeCalls); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return f.KeyValue.Delete(ctx, key, opts...)
|
||||
}
|
||||
|
||||
// staleGetKV always answers Get with a fixed snapshot taken at construction,
|
||||
// so a caller's CAS keeps retrying against a revision that can never win.
|
||||
type staleGetKV struct {
|
||||
jetstream.KeyValue
|
||||
|
||||
entry jetstream.KeyValueEntry
|
||||
}
|
||||
|
||||
func (f *staleGetKV) Get(context.Context, string) (jetstream.KeyValueEntry, error) {
|
||||
return f.entry, nil
|
||||
}
|
||||
|
||||
// staleNotFoundKV always answers Get as absent, regardless of what the real
|
||||
// bucket holds.
|
||||
type staleNotFoundKV struct {
|
||||
jetstream.KeyValue
|
||||
}
|
||||
|
||||
func (f *staleNotFoundKV) Get(context.Context, string) (jetstream.KeyValueEntry, error) {
|
||||
return nil, jetstream.ErrKeyNotFound
|
||||
}
|
||||
|
||||
// errGetKV always fails Get with a fixed store error.
|
||||
type errGetKV struct {
|
||||
jetstream.KeyValue
|
||||
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *errGetKV) Get(context.Context, string) (jetstream.KeyValueEntry, error) {
|
||||
return nil, f.err
|
||||
}
|
||||
|
||||
func decodeOverride(t *testing.T, entry jetstream.KeyValueEntry) overrideRecord {
|
||||
t.Helper()
|
||||
var rec overrideRecord
|
||||
require.NoError(t, json.Unmarshal(entry.Value(), &rec))
|
||||
return rec
|
||||
}
|
||||
|
||||
// A seal landing between a mutation's read and its guarded write is not
|
||||
// outlived: the mutation's retry sees the seal and is rejected, and nothing
|
||||
// it wrote survives. Clear on an absent or tombstoned key never reaches a
|
||||
// write at all (see TestClearOnMissingKeyWritesNothing), so only the four
|
||||
// cells with a real write are exercised here.
|
||||
func TestSealRaceMutation_SealLandsBetweenReadAndWrite(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
state string
|
||||
call func(svc *Service, ctx context.Context) error
|
||||
}{
|
||||
{"Set_present", "present", mutationTable()[0].call},
|
||||
{"Set_absent", "absent", mutationTable()[0].call},
|
||||
{"Set_tombstone", "tombstone", mutationTable()[0].call},
|
||||
{"Clear_present", "present", mutationTable()[1].call},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
svc, _ := newSealService(t)
|
||||
ctx := context.Background()
|
||||
seedOverrideState(t, svc, tc.state)
|
||||
|
||||
realKV := svc.overrides
|
||||
sealer := New(baseLimits(), nil)
|
||||
sealer.SetOverrides(realKV)
|
||||
sealer.SetAccountTerminationChecker(terminatingChecker(testAccount))
|
||||
|
||||
var sealed *OverrideSeal
|
||||
race := &sealRaceKV{KeyValue: realKV}
|
||||
race.beforeWrite = func(attempt int) error {
|
||||
if attempt != 1 {
|
||||
return nil
|
||||
}
|
||||
var sealErr error
|
||||
sealed, sealErr = sealer.SealAccountOverrides(ctx, testAccount)
|
||||
return sealErr
|
||||
}
|
||||
svc.overrides = race
|
||||
|
||||
err := tc.call(svc, ctx)
|
||||
require.Error(t, err)
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorBlocked), "got %v", err)
|
||||
require.NotNil(t, sealed)
|
||||
|
||||
entry, err := realKV.Get(ctx, testAccount)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, OverrideStateSealed, decodeOverride(t, entry).State)
|
||||
assert.Equal(t, sealed.Revision, entry.Revision(), "no write landed after the seal")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A mutation landing inside a seal's read-write window does not survive it:
|
||||
// the seal retries against the mutation's write and wins, leaving the sealed
|
||||
// value with no overridden dimension.
|
||||
func TestSealRaceMutation_MutationLandsBetweenSealReadAndWrite(t *testing.T) {
|
||||
for _, m := range mutationTable() {
|
||||
for _, state := range []string{"present", "absent", "tombstone"} {
|
||||
t.Run(m.name+"_"+state, func(t *testing.T) {
|
||||
svc, _ := newSealService(t)
|
||||
ctx := context.Background()
|
||||
seedOverrideState(t, svc, state)
|
||||
|
||||
realKV := svc.overrides
|
||||
race := &sealRaceKV{KeyValue: realKV}
|
||||
race.beforeWrite = func(attempt int) error {
|
||||
if attempt != 1 {
|
||||
return nil
|
||||
}
|
||||
_ = m.call(svc, ctx)
|
||||
return nil
|
||||
}
|
||||
|
||||
sealer := New(baseLimits(), nil)
|
||||
sealer.SetOverrides(race)
|
||||
sealer.SetAccountTerminationChecker(terminatingChecker(testAccount))
|
||||
|
||||
seal, err := sealer.SealAccountOverrides(ctx, testAccount)
|
||||
require.NoError(t, err)
|
||||
|
||||
entry, err := realKV.Get(ctx, testAccount)
|
||||
require.NoError(t, err)
|
||||
rec := decodeOverride(t, entry)
|
||||
assert.Equal(t, OverrideStateSealed, rec.State)
|
||||
assert.Nil(t, rec.VCPUs)
|
||||
assert.Equal(t, seal.Revision, entry.Revision())
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Neither mutation can act on an already-sealed record.
|
||||
func TestMutationOnSealedRecordIsRejected(t *testing.T) {
|
||||
for _, m := range mutationTable() {
|
||||
t.Run(m.name, func(t *testing.T) {
|
||||
svc, _ := newSealService(t)
|
||||
ctx := context.Background()
|
||||
svc.SetAccountTerminationChecker(terminatingChecker(testAccount))
|
||||
_, err := svc.SealAccountOverrides(ctx, testAccount)
|
||||
require.NoError(t, err)
|
||||
|
||||
before, err := svc.overrides.Get(ctx, testAccount)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = m.call(svc, ctx)
|
||||
require.Error(t, err)
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorBlocked), "got %v", err)
|
||||
|
||||
after, err := svc.overrides.Get(ctx, testAccount)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, before.Revision(), after.Revision())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Every override mutation races one seal on a real bucket; afterwards the
|
||||
// record is sealed and stays sealed, so no success among them could have
|
||||
// landed after the seal.
|
||||
func TestSealConcurrentMutations(t *testing.T) {
|
||||
svc, _ := newSealService(t)
|
||||
ctx := context.Background()
|
||||
svc.SetAccountTerminationChecker(terminatingChecker(testAccount))
|
||||
require.NoError(t, svc.PutAccountQuota(ctx, testAccount, Overrides{VCPUs: ptr(8)}, "operator"))
|
||||
|
||||
mutations := mutationTable()
|
||||
errs := make([]error, len(mutations))
|
||||
var wg sync.WaitGroup
|
||||
for i, m := range mutations {
|
||||
wg.Add(1)
|
||||
go func(i int, m mutationCase) {
|
||||
defer wg.Done()
|
||||
errs[i] = m.call(svc, ctx)
|
||||
}(i, m)
|
||||
}
|
||||
|
||||
var sealResult *OverrideSeal
|
||||
var sealErr error
|
||||
wg.Go(func() {
|
||||
sealResult, sealErr = svc.SealAccountOverrides(ctx, testAccount)
|
||||
})
|
||||
wg.Wait()
|
||||
|
||||
require.NoError(t, sealErr)
|
||||
require.NotNil(t, sealResult)
|
||||
for i, err := range errs {
|
||||
if err != nil {
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorBlocked), "mutation %s: got %v", mutations[i].name, err)
|
||||
}
|
||||
}
|
||||
|
||||
entry, err := svc.overrides.Get(ctx, testAccount)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, OverrideStateSealed, decodeOverride(t, entry).State)
|
||||
assert.Equal(t, sealResult.Revision, entry.Revision())
|
||||
|
||||
_, _, err = svc.GetAccountQuota(ctx, testAccount)
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorBlocked))
|
||||
}
|
||||
|
||||
// A Get pinned to a revision the record has already moved past exhausts the
|
||||
// mutation's retries without ever writing.
|
||||
func TestMutationStaleReadExhaustsWithoutWriting(t *testing.T) {
|
||||
svc, js := newSealService(t)
|
||||
ctx := context.Background()
|
||||
|
||||
require.NoError(t, svc.PutAccountQuota(ctx, testAccount, Overrides{VCPUs: ptr(8)}, "operator"))
|
||||
stale, err := svc.overrides.Get(ctx, testAccount)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, svc.PutAccountQuota(ctx, testAccount, Overrides{VCPUs: ptr(16)}, "operator"))
|
||||
|
||||
before := lastOverrideSeq(t, js)
|
||||
svc.overrides = &staleGetKV{KeyValue: svc.overrides, entry: stale}
|
||||
|
||||
err = svc.PutAccountQuota(ctx, testAccount, Overrides{VCPUs: ptr(32)}, "operator")
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInternalError), "got %v", err)
|
||||
assert.Equal(t, before, lastOverrideSeq(t, js))
|
||||
}
|
||||
|
||||
// A Get that always reports absent, while a sealed record actually exists,
|
||||
// never gets to recreate it: every Create attempt conflicts.
|
||||
func TestMutationStaleAbsentDoesNotRecreate(t *testing.T) {
|
||||
svc, js := newSealService(t)
|
||||
ctx := context.Background()
|
||||
svc.SetAccountTerminationChecker(terminatingChecker(testAccount))
|
||||
_, err := svc.SealAccountOverrides(ctx, testAccount)
|
||||
require.NoError(t, err)
|
||||
|
||||
before := lastOverrideSeq(t, js)
|
||||
svc.overrides = &staleNotFoundKV{KeyValue: svc.overrides}
|
||||
|
||||
err = svc.PutAccountQuota(ctx, testAccount, Overrides{VCPUs: ptr(8)}, "operator")
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInternalError), "got %v", err)
|
||||
assert.Equal(t, before, lastOverrideSeq(t, js))
|
||||
}
|
||||
|
||||
// A revision that keeps moving under sustained concurrent writes must not let
|
||||
// the seal spin forever: it gives up, bounded by its attempt budget, and its
|
||||
// own write never lands.
|
||||
func TestSealSustainedConflictFailsWithoutWriting(t *testing.T) {
|
||||
svc, _ := newSealService(t)
|
||||
ctx := context.Background()
|
||||
svc.SetAccountTerminationChecker(terminatingChecker(testAccount))
|
||||
require.NoError(t, svc.PutAccountQuota(ctx, testAccount, Overrides{VCPUs: ptr(8)}, "operator"))
|
||||
|
||||
realKV := svc.overrides
|
||||
race := &sealRaceKV{KeyValue: realKV}
|
||||
race.beforeWrite = func(int) error {
|
||||
data, merr := json.Marshal(&overrideRecord{Overrides: Overrides{VCPUs: ptr(16)}})
|
||||
require.NoError(t, merr)
|
||||
_, perr := realKV.Put(ctx, testAccount, data)
|
||||
return perr
|
||||
}
|
||||
svc.overrides = race
|
||||
|
||||
_, err := svc.SealAccountOverrides(ctx, testAccount)
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInternalError), "got %v", err)
|
||||
assert.Equal(t, sealMaxAttempts, race.writeCalls)
|
||||
|
||||
entry, err := realKV.Get(ctx, testAccount)
|
||||
require.NoError(t, err)
|
||||
assert.NotEqual(t, OverrideStateSealed, decodeOverride(t, entry).State, "the seal's own write never landed")
|
||||
}
|
||||
|
||||
// A store error on the seal's read reports failed, never written.
|
||||
func TestSealStoreErrorWritesNothing(t *testing.T) {
|
||||
svc, js := newSealService(t)
|
||||
ctx := context.Background()
|
||||
svc.SetAccountTerminationChecker(terminatingChecker(testAccount))
|
||||
|
||||
before := lastOverrideSeq(t, js)
|
||||
svc.overrides = &errGetKV{KeyValue: svc.overrides, err: errors.New("kv unavailable")}
|
||||
|
||||
_, err := svc.SealAccountOverrides(ctx, testAccount)
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInternalError), "got %v", err)
|
||||
assert.Equal(t, before, lastOverrideSeq(t, js))
|
||||
}
|
||||
|
||||
func TestIsRevisionConflictRecognisesTombstoneRetry(t *testing.T) {
|
||||
assert.True(t, isRevisionConflict(jetstream.ErrKeyExists))
|
||||
assert.True(t, isRevisionConflict(jetstream.ErrKeyRevisionMismatch))
|
||||
assert.True(t, isRevisionConflict(&jetstream.APIError{ErrorCode: jetstream.JSErrCodeStreamWrongLastSequence}))
|
||||
assert.True(t, isRevisionConflict(&jetstream.APIError{ErrorCode: jetstream.JSErrCodeStreamWrongLastSequenceConstant}))
|
||||
assert.True(t, isRevisionConflict(fmt.Errorf("wrapped: %w", &jetstream.APIError{ErrorCode: jetstream.JSErrCodeStreamWrongLastSequence})))
|
||||
assert.False(t, isRevisionConflict(errors.New("unrelated failure")))
|
||||
assert.False(t, isRevisionConflict(nil))
|
||||
}
|
||||
|
||||
// Clearing never writes when there is nothing active to clear: an absent key,
|
||||
// a tombstoned key, and a sealed key hidden behind a stale not-found Get all
|
||||
// leave the stream exactly where they found it.
|
||||
func TestClearOnMissingKeyWritesNothing(t *testing.T) {
|
||||
t.Run("absent", func(t *testing.T) {
|
||||
svc, js := newSealService(t)
|
||||
ctx := context.Background()
|
||||
before := lastOverrideSeq(t, js)
|
||||
|
||||
require.NoError(t, svc.PutAccountQuota(ctx, testAccount, Overrides{}, "operator"))
|
||||
assert.Equal(t, before, lastOverrideSeq(t, js))
|
||||
})
|
||||
|
||||
t.Run("tombstone", func(t *testing.T) {
|
||||
svc, js := newSealService(t)
|
||||
ctx := context.Background()
|
||||
require.NoError(t, svc.PutAccountQuota(ctx, testAccount, Overrides{VCPUs: ptr(8)}, "operator"))
|
||||
require.NoError(t, svc.overrides.Delete(ctx, testAccount))
|
||||
before := lastOverrideSeq(t, js)
|
||||
|
||||
require.NoError(t, svc.PutAccountQuota(ctx, testAccount, Overrides{}, "operator"))
|
||||
assert.Equal(t, before, lastOverrideSeq(t, js))
|
||||
})
|
||||
|
||||
t.Run("sealed_hidden_by_stale_not_found", func(t *testing.T) {
|
||||
svc, js := newSealService(t)
|
||||
ctx := context.Background()
|
||||
svc.SetAccountTerminationChecker(terminatingChecker(testAccount))
|
||||
sealed, err := svc.SealAccountOverrides(ctx, testAccount)
|
||||
require.NoError(t, err)
|
||||
before := lastOverrideSeq(t, js)
|
||||
|
||||
realKV := svc.overrides
|
||||
svc.overrides = &staleNotFoundKV{KeyValue: realKV}
|
||||
|
||||
require.NoError(t, svc.PutAccountQuota(ctx, testAccount, Overrides{}, "operator"))
|
||||
assert.Equal(t, before, lastOverrideSeq(t, js))
|
||||
|
||||
entry, err := realKV.Get(ctx, testAccount)
|
||||
require.NoError(t, err)
|
||||
rec := decodeOverride(t, entry)
|
||||
assert.Equal(t, OverrideStateSealed, rec.State)
|
||||
assert.Equal(t, sealed.Revision, entry.Revision())
|
||||
})
|
||||
}
|
||||
|
||||
// A seal against an absent or a tombstoned key creates the sealed record.
|
||||
func TestSealOnMissingKeyCreatesSealedRecord(t *testing.T) {
|
||||
for _, state := range []string{"absent", "tombstone"} {
|
||||
t.Run(state, func(t *testing.T) {
|
||||
svc, _ := newSealService(t)
|
||||
ctx := context.Background()
|
||||
svc.SetAccountTerminationChecker(terminatingChecker(testAccount))
|
||||
if state == "tombstone" {
|
||||
require.NoError(t, svc.PutAccountQuota(ctx, testAccount, Overrides{VCPUs: ptr(8)}, "operator"))
|
||||
require.NoError(t, svc.overrides.Delete(ctx, testAccount))
|
||||
}
|
||||
_, err := svc.overrides.Get(ctx, testAccount)
|
||||
require.ErrorIs(t, err, jetstream.ErrKeyNotFound)
|
||||
|
||||
seal, err := svc.SealAccountOverrides(ctx, testAccount)
|
||||
require.NoError(t, err)
|
||||
|
||||
entry, err := svc.overrides.Get(ctx, testAccount)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, seal.Revision, entry.Revision())
|
||||
assert.Equal(t, OverrideStateSealed, decodeOverride(t, entry).State)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// alwaysConflictThenErrorKV answers Get as absent and every Create attempt
|
||||
// with a conflict, then a store error on the final attempt: a recording
|
||||
// wrapper proving the seal tried a real write on every attempt before giving
|
||||
// up, rather than ever reporting success from an observed absence.
|
||||
type alwaysConflictThenErrorKV struct {
|
||||
jetstream.KeyValue
|
||||
|
||||
calls int
|
||||
}
|
||||
|
||||
func (f *alwaysConflictThenErrorKV) Get(context.Context, string) (jetstream.KeyValueEntry, error) {
|
||||
return nil, jetstream.ErrKeyNotFound
|
||||
}
|
||||
|
||||
func (f *alwaysConflictThenErrorKV) Create(context.Context, string, []byte, ...jetstream.KVCreateOpt) (uint64, error) {
|
||||
f.calls++
|
||||
if f.calls < sealMaxAttempts {
|
||||
return 0, jetstream.ErrKeyExists
|
||||
}
|
||||
return 0, errors.New("store unavailable")
|
||||
}
|
||||
|
||||
func TestSealNeverSucceedsWithoutAWrite(t *testing.T) {
|
||||
svc, _ := newSealService(t)
|
||||
ctx := context.Background()
|
||||
svc.SetAccountTerminationChecker(terminatingChecker(testAccount))
|
||||
fake := &alwaysConflictThenErrorKV{}
|
||||
svc.overrides = fake
|
||||
|
||||
seal, err := svc.SealAccountOverrides(ctx, testAccount)
|
||||
assert.Error(t, err)
|
||||
assert.Nil(t, seal)
|
||||
assert.Equal(t, sealMaxAttempts, fake.calls, "every attempt must have tried a real write before giving up")
|
||||
}
|
||||
|
||||
// An account IAM does not hold as TERMINATING refuses the seal before any
|
||||
// override-KV write, whether the key is present or absent.
|
||||
func TestSealRefusedForActiveAccount(t *testing.T) {
|
||||
for _, state := range []string{"present", "absent"} {
|
||||
t.Run(state, func(t *testing.T) {
|
||||
svc, js := newSealService(t)
|
||||
ctx := context.Background()
|
||||
if state == "present" {
|
||||
require.NoError(t, svc.PutAccountQuota(ctx, testAccount, Overrides{VCPUs: ptr(8)}, "operator"))
|
||||
}
|
||||
svc.SetAccountTerminationChecker(&stubTerminationChecker{terminating: map[string]bool{}})
|
||||
|
||||
before := lastOverrideSeq(t, js)
|
||||
_, err := svc.SealAccountOverrides(ctx, testAccount)
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorIncorrectState), "got %v", err)
|
||||
assert.Equal(t, before, lastOverrideSeq(t, js))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSealRefusedForMissingIAMRecord(t *testing.T) {
|
||||
svc, js := newSealService(t)
|
||||
ctx := context.Background()
|
||||
svc.SetAccountTerminationChecker(&stubTerminationChecker{terminating: map[string]bool{}})
|
||||
|
||||
before := lastOverrideSeq(t, js)
|
||||
_, err := svc.SealAccountOverrides(ctx, testAccount)
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorIncorrectState), "got %v", err)
|
||||
assert.Equal(t, before, lastOverrideSeq(t, js))
|
||||
}
|
||||
|
||||
func TestSealRefusedWhenIAMReadFails(t *testing.T) {
|
||||
svc, js := newSealService(t)
|
||||
ctx := context.Background()
|
||||
svc.SetAccountTerminationChecker(&stubTerminationChecker{err: errors.New("iam unavailable")})
|
||||
|
||||
before := lastOverrideSeq(t, js)
|
||||
_, err := svc.SealAccountOverrides(ctx, testAccount)
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInternalError), "got %v", err)
|
||||
assert.Equal(t, before, lastOverrideSeq(t, js))
|
||||
}
|
||||
|
||||
func TestSealRefusedWithNoCheckerWired(t *testing.T) {
|
||||
svc, js := newSealService(t)
|
||||
ctx := context.Background()
|
||||
|
||||
before := lastOverrideSeq(t, js)
|
||||
_, err := svc.SealAccountOverrides(ctx, testAccount)
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInternalError), "got %v", err)
|
||||
assert.Equal(t, before, lastOverrideSeq(t, js))
|
||||
}
|
||||
|
||||
func TestSealAcceptedForTerminatingAccount(t *testing.T) {
|
||||
svc, _ := newSealService(t)
|
||||
ctx := context.Background()
|
||||
svc.SetAccountTerminationChecker(terminatingChecker(testAccount))
|
||||
|
||||
seal, err := svc.SealAccountOverrides(ctx, testAccount)
|
||||
require.NoError(t, err)
|
||||
assert.NotZero(t, seal.Revision)
|
||||
|
||||
entry, err := svc.overrides.Get(ctx, testAccount)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, seal.Revision, entry.Revision())
|
||||
}
|
||||
|
||||
// A refused seal, on a gateway that disabled quotas and so holds no override
|
||||
// bucket, never creates one.
|
||||
func TestSealRefusalDoesNotCreateBucket(t *testing.T) {
|
||||
t.Run("not_terminating", func(t *testing.T) {
|
||||
svc, js := newDisabledSealService(t)
|
||||
ctx := context.Background()
|
||||
svc.SetAccountTerminationChecker(&stubTerminationChecker{terminating: map[string]bool{}})
|
||||
|
||||
_, err := svc.SealAccountOverrides(ctx, testAccount)
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorIncorrectState), "got %v", err)
|
||||
_, err = js.KeyValue(context.Background(), KVBucketAccountQuota)
|
||||
assert.ErrorIs(t, err, jetstream.ErrBucketNotFound)
|
||||
})
|
||||
|
||||
t.Run("iam_read_fails", func(t *testing.T) {
|
||||
svc, js := newDisabledSealService(t)
|
||||
ctx := context.Background()
|
||||
svc.SetAccountTerminationChecker(&stubTerminationChecker{err: errors.New("iam unavailable")})
|
||||
|
||||
_, err := svc.SealAccountOverrides(ctx, testAccount)
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInternalError), "got %v", err)
|
||||
_, err = js.KeyValue(context.Background(), KVBucketAccountQuota)
|
||||
assert.ErrorIs(t, err, jetstream.ErrBucketNotFound)
|
||||
})
|
||||
}
|
||||
|
||||
// A seal accepted on a gateway whose quotas were disabled at startup creates
|
||||
// the override bucket at seal time, never at startup.
|
||||
func TestSealOnQuotaDisabledGatewayCreatesBucket(t *testing.T) {
|
||||
svc, js := newDisabledSealService(t)
|
||||
ctx := context.Background()
|
||||
svc.SetAccountTerminationChecker(terminatingChecker(testAccount))
|
||||
|
||||
seal, err := svc.SealAccountOverrides(ctx, testAccount)
|
||||
require.NoError(t, err)
|
||||
|
||||
kv, err := js.KeyValue(context.Background(), KVBucketAccountQuota)
|
||||
require.NoError(t, err)
|
||||
entry, err := kv.Get(context.Background(), testAccount)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, seal.Revision, entry.Revision())
|
||||
assert.Equal(t, OverrideStateSealed, decodeOverride(t, entry).State)
|
||||
}
|
||||
|
||||
// A sealed record stays sealed after the service is rebuilt over the same
|
||||
// JetStream, every read and write refuses it, and a re-seal gets a higher
|
||||
// revision.
|
||||
func TestSealSurvivesServiceReconstruction(t *testing.T) {
|
||||
_, _, js := testutil.StartTestJetStream(t)
|
||||
ctx := context.Background()
|
||||
|
||||
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: KVBucketAccountQuota, History: 1})
|
||||
require.NoError(t, err)
|
||||
|
||||
svc := New(baseLimits(), nil)
|
||||
svc.SetOverrides(kv)
|
||||
svc.SetJetStream(js)
|
||||
svc.SetAccountTerminationChecker(terminatingChecker(testAccount))
|
||||
first, err := svc.SealAccountOverrides(ctx, testAccount)
|
||||
require.NoError(t, err)
|
||||
|
||||
rebuiltKV, err := js.KeyValue(ctx, KVBucketAccountQuota)
|
||||
require.NoError(t, err)
|
||||
rebuilt := New(baseLimits(), nil)
|
||||
rebuilt.SetOverrides(rebuiltKV)
|
||||
rebuilt.SetJetStream(js)
|
||||
|
||||
entry, err := rebuilt.overrides.Get(ctx, testAccount)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, OverrideStateSealed, decodeOverride(t, entry).State)
|
||||
|
||||
_, _, err = rebuilt.GetAccountQuota(ctx, testAccount)
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorBlocked))
|
||||
assert.True(t, awserrors.IsErrorCode(rebuilt.PutAccountQuota(ctx, testAccount, Overrides{VCPUs: ptr(8)}, "operator"), awserrors.ErrorBlocked))
|
||||
assert.True(t, awserrors.IsErrorCode(rebuilt.PutAccountQuota(ctx, testAccount, Overrides{}, "operator"), awserrors.ErrorBlocked))
|
||||
|
||||
limits, err := rebuilt.limitsFor(ctx, testAccount)
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorBlocked))
|
||||
assert.Zero(t, limits)
|
||||
|
||||
rebuilt.SetAccountTerminationChecker(terminatingChecker(testAccount))
|
||||
second, err := rebuilt.SealAccountOverrides(ctx, testAccount)
|
||||
require.NoError(t, err)
|
||||
assert.Greater(t, second.Revision, first.Revision)
|
||||
}
|
||||
|
||||
func TestSealRefusesEmptyInvalidAndGlobalAccount(t *testing.T) {
|
||||
svc, _ := newSealService(t)
|
||||
ctx := context.Background()
|
||||
|
||||
_, err := svc.SealAccountOverrides(ctx, "")
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorMissingParameter), "got %v", err)
|
||||
|
||||
_, err = svc.SealAccountOverrides(ctx, "not-an-account")
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInvalidParameterValue), "got %v", err)
|
||||
|
||||
_, err = svc.SealAccountOverrides(ctx, awsidentifiers.GlobalAccountID)
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInvalidParameterValue), "got %v", err)
|
||||
}
|
||||
|
||||
func TestSealLeavesOtherAccountsUntouched(t *testing.T) {
|
||||
svc, _ := newSealService(t)
|
||||
ctx := context.Background()
|
||||
accountB := "222222222222"
|
||||
require.NoError(t, svc.PutAccountQuota(ctx, accountB, Overrides{VCPUs: ptr(8)}, "operator"))
|
||||
|
||||
svc.SetAccountTerminationChecker(terminatingChecker(testAccount))
|
||||
_, err := svc.SealAccountOverrides(ctx, testAccount)
|
||||
require.NoError(t, err)
|
||||
|
||||
over, _, err := svc.GetAccountQuota(ctx, accountB)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, over.VCPUs)
|
||||
assert.Equal(t, 8, *over.VCPUs)
|
||||
}
|
||||
|
||||
// A record written before the state field existed has none and is treated as
|
||||
// active by both reads and writes; a write on it preserves the old shape.
|
||||
func TestRecordWithoutStateIsActive(t *testing.T) {
|
||||
svc, _ := newSealService(t)
|
||||
ctx := context.Background()
|
||||
|
||||
data, err := json.Marshal(Overrides{VCPUs: ptr(8)})
|
||||
require.NoError(t, err)
|
||||
_, err = svc.overrides.Put(ctx, testAccount, data)
|
||||
require.NoError(t, err)
|
||||
|
||||
over, limits, err := svc.GetAccountQuota(ctx, testAccount)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, over.VCPUs)
|
||||
assert.Equal(t, 8, limits.VCPUs)
|
||||
|
||||
require.NoError(t, svc.PutAccountQuota(ctx, testAccount, Overrides{VCPUs: ptr(16)}, "operator"))
|
||||
entry, err := svc.overrides.Get(ctx, testAccount)
|
||||
require.NoError(t, err)
|
||||
assert.NotContains(t, string(entry.Value()), `"state"`)
|
||||
}
|
||||
|
||||
// A record whose state is neither active nor sealed fails every read and
|
||||
// write closed, rather than being treated as active.
|
||||
func TestUnknownStateFailsClosed(t *testing.T) {
|
||||
svc, _ := newSealService(t)
|
||||
ctx := context.Background()
|
||||
|
||||
_, err := svc.overrides.Put(ctx, testAccount, []byte(`{"state":"frozen"}`))
|
||||
require.NoError(t, err)
|
||||
|
||||
_, _, err = svc.GetAccountQuota(ctx, testAccount)
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInternalError), "got %v", err)
|
||||
|
||||
limits, err := svc.limitsFor(ctx, testAccount)
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInternalError), "got %v", err)
|
||||
assert.Zero(t, limits)
|
||||
|
||||
assert.True(t, awserrors.IsErrorCode(svc.PutAccountQuota(ctx, testAccount, Overrides{VCPUs: ptr(8)}, "operator"), awserrors.ErrorInternalError))
|
||||
assert.True(t, awserrors.IsErrorCode(svc.PutAccountQuota(ctx, testAccount, Overrides{}, "operator"), awserrors.ErrorInternalError))
|
||||
}
|
||||
|
||||
// A sealed record is Blocked for admission's own resolver, which every
|
||||
// caller (admin read, ELBv2 DescribeAccountLimits, every Enforce* method)
|
||||
// shares.
|
||||
func TestAdmissionOnSealedRecordIsBlocked(t *testing.T) {
|
||||
svc, _ := newSealService(t)
|
||||
ctx := context.Background()
|
||||
svc.SetAccountTerminationChecker(terminatingChecker(testAccount))
|
||||
_, err := svc.SealAccountOverrides(ctx, testAccount)
|
||||
require.NoError(t, err)
|
||||
|
||||
limits, err := svc.limitsFor(ctx, testAccount)
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorBlocked), "got %v", err)
|
||||
assert.Zero(t, limits)
|
||||
|
||||
assert.Error(t, svc.CheckVCPU(ctx, testAccount, 1))
|
||||
assert.True(t, awserrors.IsErrorCode(svc.CheckVCPU(ctx, testAccount, 1), awserrors.ErrorBlocked))
|
||||
}
|
||||
|
||||
// The existing read-failure fallback in limitsFor (a decode or store error
|
||||
// falls back to the configured limits) is unchanged by the seal work: it is
|
||||
// distinct from a successfully read sealed record, which refuses instead.
|
||||
func TestAdmissionReadFailureFallbackUnchanged(t *testing.T) {
|
||||
svc, kv := newOverrideService(t)
|
||||
|
||||
_, err := kv.Put(t.Context(), testAccount, []byte("{not json"))
|
||||
require.NoError(t, err)
|
||||
|
||||
limits, err := svc.limitsFor(t.Context(), testAccount)
|
||||
require.NoError(t, err, "a decode failure must still fall back, not error, from limitsFor")
|
||||
require.Equal(t, baseLimits(), limits)
|
||||
}
|
||||
@@ -23,10 +23,7 @@ func (s *Service) EnforceVolumeCreate(ctx context.Context, natsConn *nats.Conn,
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
limits, err := s.limitsFor(ctx, accountID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
limits := s.limitsFor(ctx, accountID)
|
||||
if err := exceeds(len(out.Volumes), 1, limits.Volumes); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -46,11 +43,7 @@ func (s *Service) EnforceVolumeModify(ctx context.Context, natsConn *nats.Conn,
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
limits, err := s.limitsFor(ctx, accountID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return exceeds(total-oldGiB, newGiB, limits.VolumesGiB)
|
||||
return exceeds(total-oldGiB, newGiB, s.limitsFor(ctx, accountID).VolumesGiB)
|
||||
}
|
||||
|
||||
// volumeUsage sums the account's volume sizes in GiB via a single DescribeVolumes
|
||||
|
||||
@@ -37,11 +37,7 @@ func (s *Service) CheckVCPU(ctx context.Context, accountID string, want int) err
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
limits, err := s.limitsFor(ctx, accountID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return exceeds(current, want, limits.VCPUs)
|
||||
return exceeds(current, want, s.limitsFor(ctx, accountID).VCPUs)
|
||||
}
|
||||
|
||||
// casVCPU applies next to accountID's counter under bounded JetStream CAS so
|
||||
|
||||
@@ -1,104 +0,0 @@
|
||||
package dns
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
nsconfig "github.com/mulgadc/northstar/pkg/config"
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
reconcilelock "github.com/mulgadc/spinifex/spinifex/domains/network/reconcile"
|
||||
)
|
||||
|
||||
// delegateLockPoll is how often Delegate retries the zone-writer election
|
||||
// while another node holds it.
|
||||
var delegateLockPoll = time.Second
|
||||
|
||||
// PlatformDelegation returns the NS set the base zone publishes for name and
|
||||
// the zone it read. ok is false when name is not strictly below the base
|
||||
// domain, so the zone's parent is somewhere else.
|
||||
func (w *Writer) PlatformDelegation(name string) (servers []string, zone string, ok bool, err error) {
|
||||
if !w.Enabled() {
|
||||
return nil, "", false, nil
|
||||
}
|
||||
base := normalizeZone(w.baseDomain)
|
||||
n := normalizeZone(name)
|
||||
if base == "" || !strings.HasSuffix(n, "."+base) {
|
||||
return nil, "", false, nil
|
||||
}
|
||||
cfg, exists, err := nsconfig.ReadZoneRaw(w.s3cfg, base)
|
||||
if err != nil {
|
||||
return nil, base, true, fmt.Errorf("read zone %s: %w", base, err)
|
||||
}
|
||||
if !exists {
|
||||
return nil, base, true, nil
|
||||
}
|
||||
label := relativeLabel(n, base)
|
||||
for _, r := range cfg.Records {
|
||||
if r.Type == nsconfig.TypeNS && strings.EqualFold(r.Domain, label) {
|
||||
servers = append(servers, r.Address)
|
||||
}
|
||||
}
|
||||
return servers, base, true, nil
|
||||
}
|
||||
|
||||
// Delegate writes, or with remove withdraws, an NS set in the base zone that
|
||||
// delegates each name to this cluster's nameservers. It takes the zone-writer
|
||||
// election first, so it never races the reconcile's own writes to the zone.
|
||||
func (w *Writer) Delegate(ctx context.Context, nc *nats.Conn, holder string, names []string, remove bool) (*ChangeResult, error) {
|
||||
if !w.Enabled() {
|
||||
return nil, errors.New("northstar is not configured on this node")
|
||||
}
|
||||
base := normalizeZone(w.baseDomain)
|
||||
values := make([]string, 0, len(w.nameservers))
|
||||
for _, ns := range w.nameservers {
|
||||
values = append(values, ns.Host+"."+base+".")
|
||||
}
|
||||
batch := &ChangeBatch{}
|
||||
for _, name := range names {
|
||||
n := normalizeZone(name)
|
||||
if !strings.HasSuffix(n, "."+base) {
|
||||
return nil, fmt.Errorf("%s is not below the base domain %s", name, base)
|
||||
}
|
||||
if ReservedZoneName(n, base, w.internalDomain) {
|
||||
return nil, fmt.Errorf("%s is a platform name and cannot be delegated", name)
|
||||
}
|
||||
if remove {
|
||||
for _, v := range values {
|
||||
batch.Changes = append(batch.Changes, Change{Action: ActionDelete, Zone: base, Name: n + ".", Type: "NS", Value: v})
|
||||
}
|
||||
continue
|
||||
}
|
||||
batch.Changes = append(batch.Changes, Change{Action: ActionUpsertSet, Zone: base, Name: n + ".", Type: "NS", Values: values})
|
||||
}
|
||||
if len(batch.Changes) == 0 {
|
||||
return &ChangeResult{}, nil
|
||||
}
|
||||
release, err := w.awaitWriterElection(ctx, nc, holder)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer release()
|
||||
return w.ApplyBatch(batch)
|
||||
}
|
||||
|
||||
func (w *Writer) awaitWriterElection(ctx context.Context, nc *nats.Conn, holder string) (func(), error) {
|
||||
if nc == nil {
|
||||
return func() {}, nil
|
||||
}
|
||||
tick := time.NewTicker(delegateLockPoll)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
if release, ok := reconcilelock.AcquireLeader(ctx, nc, KVBucketDNSReconcile, holder); ok {
|
||||
return release, nil
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, fmt.Errorf("another node is writing the zone: %w", ctx.Err())
|
||||
case <-tick.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,82 +0,0 @@
|
||||
package dns
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
testutil "github.com/mulgadc/spinifex/internal/testkit"
|
||||
reconcilelock "github.com/mulgadc/spinifex/spinifex/domains/network/reconcile"
|
||||
)
|
||||
|
||||
func TestDelegateWritesAndWithdrawsPlatformDelegations(t *testing.T) {
|
||||
w, _ := newMultiNodeTestWriter(t)
|
||||
_, err := w.Delegate(t.Context(), nil, "node1", []string{"a.r53e2e.spx3.net", "B.r53e2e.spx3.net."}, false)
|
||||
require.NoError(t, err)
|
||||
|
||||
for _, name := range []string{"a.r53e2e.spx3.net.", "b.r53e2e.spx3.net"} {
|
||||
servers, zone, ok, err := w.PlatformDelegation(name)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, "spx3.net", zone)
|
||||
assert.ElementsMatch(t, []string{"ns1.spx3.net.", "ns2.spx3.net."}, servers, name)
|
||||
}
|
||||
servers, _, ok, err := w.PlatformDelegation("c.r53e2e.spx3.net")
|
||||
require.NoError(t, err)
|
||||
assert.True(t, ok, "a name below the base domain is decided by the base zone")
|
||||
assert.Empty(t, servers)
|
||||
|
||||
_, err = w.Delegate(t.Context(), nil, "node1", []string{"a.r53e2e.spx3.net"}, true)
|
||||
require.NoError(t, err)
|
||||
servers, _, _, err = w.PlatformDelegation("a.r53e2e.spx3.net")
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, servers)
|
||||
servers, _, _, err = w.PlatformDelegation("b.r53e2e.spx3.net")
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, servers, 2, "withdrawing one delegation leaves the others")
|
||||
}
|
||||
|
||||
func TestPlatformDelegationIgnoresNamesOutsideTheBaseDomain(t *testing.T) {
|
||||
w, _ := newMultiNodeTestWriter(t)
|
||||
for _, name := range []string{"example.com", "spx3.net", "notspx3.net"} {
|
||||
_, _, ok, err := w.PlatformDelegation(name)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, ok, name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelegateRefusesPlatformAndForeignNames(t *testing.T) {
|
||||
w, _ := newMultiNodeTestWriter(t)
|
||||
for _, name := range []string{"ns1.spx3.net", "x.elb.spx3.net", "compute.spx3.net", "example.com", "spx3.net"} {
|
||||
_, err := w.Delegate(t.Context(), nil, "node1", []string{"ok.spx3.net", name}, false)
|
||||
require.Error(t, err, name)
|
||||
}
|
||||
servers, _, _, err := w.PlatformDelegation("ok.spx3.net")
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, servers, "a refused batch writes nothing")
|
||||
}
|
||||
|
||||
func TestDelegateWaitsForTheZoneWriterElection(t *testing.T) {
|
||||
w, _ := newMultiNodeTestWriter(t)
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
prev := delegateLockPoll
|
||||
delegateLockPoll = 10 * time.Millisecond
|
||||
t.Cleanup(func() { delegateLockPoll = prev })
|
||||
|
||||
release, ok := reconcilelock.AcquireLeader(t.Context(), nc, KVBucketDNSReconcile, "node2")
|
||||
require.True(t, ok)
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
|
||||
defer cancel()
|
||||
_, err := w.Delegate(ctx, nc, "node1", []string{"a.spx3.net"}, false)
|
||||
require.ErrorContains(t, err, "another node is writing the zone")
|
||||
|
||||
release()
|
||||
_, err = w.Delegate(t.Context(), nc, "node1", []string{"a.spx3.net"}, false)
|
||||
require.NoError(t, err)
|
||||
servers, _, _, err := w.PlatformDelegation("a.spx3.net")
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, servers, 2)
|
||||
}
|
||||
@@ -10,18 +10,17 @@ import (
|
||||
"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
|
||||
)
|
||||
|
||||
// NameserverSeeds derives the published nameservers (nsN → node IP) from the
|
||||
// cluster nodes that run northstar and are reachable by their peers: the
|
||||
// [northstar] pins first, then the rest in name order, capped at
|
||||
// nameserver_count or MaxNameservers. Every node computes the same set. Falls
|
||||
// back to the local node when none qualifies (single-node / dev).
|
||||
// NameserverSeeds derives one nameserver (nsN → node IP) per cluster node that
|
||||
// runs northstar and is reachable by its peers, ordered deterministically so
|
||||
// every node computes the same set. Falls back to the local node when none
|
||||
// qualifies (single-node / dev).
|
||||
//
|
||||
// Both zone creators build from this: the northstar bootstrap seed and the
|
||||
// writer's on-demand materialisation. Both are create-if-absent and neither
|
||||
// overwrites, so they must agree — a set derived independently would be baked
|
||||
// into the zone permanently by whichever creator ran first.
|
||||
func NameserverSeeds(cluster *config.ClusterConfig) []nsconfig.NameserverSeed {
|
||||
names := publishedNameserverNodes(usableNorthstarNodeNames(cluster), cluster.Northstar)
|
||||
names := usableNorthstarNodeNames(cluster)
|
||||
// A zone needs an SOA and at least one NS whatever the topology, so unlike the
|
||||
// resolver list this falls back to the local node. On single-node dev that node
|
||||
// is loopback, which is correct there: it is the zone's only reader.
|
||||
@@ -30,37 +29,17 @@ func NameserverSeeds(cluster *config.ClusterConfig) []nsconfig.NameserverSeed {
|
||||
}
|
||||
|
||||
seeds := make([]nsconfig.NameserverSeed, 0, len(names))
|
||||
for _, ns := range nameserverAssignments(cluster, names) {
|
||||
seeds = append(seeds, nsconfig.NameserverSeed{Host: ns.Host, IP: ns.IP})
|
||||
for i, name := range names {
|
||||
seeds = append(seeds, nsconfig.NameserverSeed{
|
||||
Host: fmt.Sprintf("ns%d", i+1),
|
||||
IP: nameserverIP(cluster.Nodes[name]),
|
||||
})
|
||||
}
|
||||
return seeds
|
||||
}
|
||||
|
||||
// Nameserver is one published nameserver: its label under the base domain,
|
||||
// the node it is assigned to and that node's address.
|
||||
type Nameserver struct {
|
||||
Host string
|
||||
Node string
|
||||
IP string
|
||||
}
|
||||
|
||||
// PublishedNameservers returns the nsN assignment NameserverSeeds publishes,
|
||||
// with the node behind each name. Unlike NameserverSeeds it has no local-node
|
||||
// fallback, so an empty result means no node qualifies.
|
||||
func PublishedNameservers(cluster *config.ClusterConfig) []Nameserver {
|
||||
return nameserverAssignments(cluster, publishedNameserverNodes(usableNorthstarNodeNames(cluster), cluster.Northstar))
|
||||
}
|
||||
|
||||
func nameserverAssignments(cluster *config.ClusterConfig, names []string) []Nameserver {
|
||||
out := make([]Nameserver, 0, len(names))
|
||||
for i, name := range names {
|
||||
out = append(out, Nameserver{Host: fmt.Sprintf("ns%d", i+1), Node: name, IP: nameserverIP(cluster.Nodes[name])})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ResolverNameserverIPs returns the WAN IPs of every cluster node running
|
||||
// northstar in name order, not only the published nameservers. vpcd's per-tap DNS
|
||||
// ResolverNameserverIPs returns the WAN IPs of cluster nodes running northstar,
|
||||
// in the same deterministic order as the seeded nameservers. vpcd's per-tap DNS
|
||||
// shim uses these as forward targets (northstar's :5300 listener), so internal
|
||||
// names resolve authoritatively and external names via upstream forwarders.
|
||||
//
|
||||
@@ -76,36 +55,6 @@ func ResolverNameserverIPs(cluster *config.ClusterConfig) []string {
|
||||
return ips
|
||||
}
|
||||
|
||||
// publishedNameserverNodes orders the usable northstar nodes for publication
|
||||
// and truncates them to the configured count. A pinned node that is not usable
|
||||
// is skipped rather than published with an address its peers cannot reach.
|
||||
func publishedNameserverNodes(usable []string, ns config.NorthstarClusterConfig) []string {
|
||||
limit := ns.NameserverCount
|
||||
if limit <= 0 || limit > config.MaxNameservers {
|
||||
limit = config.MaxNameservers
|
||||
}
|
||||
ok := make(map[string]bool, len(usable))
|
||||
for _, name := range usable {
|
||||
ok[name] = true
|
||||
}
|
||||
ordered := make([]string, 0, len(usable))
|
||||
for _, name := range ns.NameserverNodes {
|
||||
if ok[name] {
|
||||
ordered = append(ordered, name)
|
||||
delete(ok, name)
|
||||
}
|
||||
}
|
||||
for _, name := range usable {
|
||||
if ok[name] {
|
||||
ordered = append(ordered, name)
|
||||
}
|
||||
}
|
||||
if len(ordered) > limit {
|
||||
ordered = ordered[:limit]
|
||||
}
|
||||
return ordered
|
||||
}
|
||||
|
||||
// usableNorthstarNodeNames returns the deterministically ordered nodes that run
|
||||
// northstar and can be reached by their peers.
|
||||
//
|
||||
|
||||
@@ -1,10 +1,8 @@
|
||||
package dns
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
nsconfig "github.com/mulgadc/northstar/pkg/config"
|
||||
"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -134,62 +132,3 @@ func TestResolverNameserverIPs(t *testing.T) {
|
||||
}}
|
||||
assert.Empty(t, ResolverNameserverIPs(dev))
|
||||
}
|
||||
|
||||
func northstarNodes(n int) map[string]config.Config {
|
||||
nodes := make(map[string]config.Config, n)
|
||||
for i := 1; i <= n; i++ {
|
||||
nodes[fmt.Sprintf("node%02d", i)] = config.Config{
|
||||
AdvertiseIP: fmt.Sprintf("10.0.0.%d", i),
|
||||
Northstar: config.NorthstarConfig{ConfigPath: "/etc/n.toml"},
|
||||
}
|
||||
}
|
||||
return nodes
|
||||
}
|
||||
|
||||
func seedIPs(seeds []nsconfig.NameserverSeed) []string {
|
||||
ips := make([]string, 0, len(seeds))
|
||||
for i, s := range seeds {
|
||||
if s.Host != fmt.Sprintf("ns%d", i+1) {
|
||||
return []string{"misnumbered " + s.Host}
|
||||
}
|
||||
ips = append(ips, s.IP)
|
||||
}
|
||||
return ips
|
||||
}
|
||||
|
||||
// A large cluster publishes at most four nameservers, while every northstar
|
||||
// node stays a resolver backend for the guest shim.
|
||||
func TestNameserverSeedsCappedAtFour(t *testing.T) {
|
||||
cluster := &config.ClusterConfig{Node: "node01", Nodes: northstarNodes(12)}
|
||||
assert.Equal(t, []string{"10.0.0.1", "10.0.0.2", "10.0.0.3", "10.0.0.4"}, seedIPs(NameserverSeeds(cluster)))
|
||||
assert.Len(t, ResolverNameserverIPs(cluster), 12)
|
||||
}
|
||||
|
||||
func TestNameserverSeedsHonourCountAndPins(t *testing.T) {
|
||||
nodes := northstarNodes(6)
|
||||
nodes["node06"] = config.Config{AdvertiseIP: "127.0.0.1", Northstar: config.NorthstarConfig{ConfigPath: "/etc/n.toml"}}
|
||||
|
||||
for name, tc := range map[string]struct {
|
||||
ns config.NorthstarClusterConfig
|
||||
want []string
|
||||
}{
|
||||
"count below the node count": {config.NorthstarClusterConfig{NameserverCount: 2}, []string{"10.0.0.1", "10.0.0.2"}},
|
||||
"pins first, rest in name order": {
|
||||
config.NorthstarClusterConfig{NameserverNodes: []string{"node05", "node03"}},
|
||||
[]string{"10.0.0.5", "10.0.0.3", "10.0.0.1", "10.0.0.2"},
|
||||
},
|
||||
"pins fill the count": {
|
||||
config.NorthstarClusterConfig{NameserverCount: 2, NameserverNodes: []string{"node04", "node02"}},
|
||||
[]string{"10.0.0.4", "10.0.0.2"},
|
||||
},
|
||||
"unusable pin is skipped": {
|
||||
config.NorthstarClusterConfig{NameserverCount: 2, NameserverNodes: []string{"node06", "node05"}},
|
||||
[]string{"10.0.0.5", "10.0.0.1"},
|
||||
},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
cluster := &config.ClusterConfig{Node: "node01", Nodes: nodes, Northstar: tc.ns}
|
||||
assert.Equal(t, tc.want, seedIPs(NameserverSeeds(cluster)))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,179 +0,0 @@
|
||||
package dns
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
mdns "github.com/miekg/dns"
|
||||
)
|
||||
|
||||
// RootServers are the IANA root servers, by IPv4 address.
|
||||
var RootServers = []string{
|
||||
"198.41.0.4", "170.247.170.2", "192.33.4.12", "199.7.91.13", "192.203.230.10",
|
||||
"192.5.5.241", "192.112.36.4", "198.97.190.53", "192.36.148.17", "192.58.128.30",
|
||||
"193.0.14.129", "199.7.83.42", "202.12.27.33",
|
||||
}
|
||||
|
||||
const (
|
||||
parentQueryTimeout = 3 * time.Second
|
||||
maxReferrals = 16
|
||||
)
|
||||
|
||||
// ParentLookup reads a domain's delegation from its parent zone's own servers,
|
||||
// found by following referrals down from the roots. It never asks a recursive
|
||||
// resolver, whose cache can hold a delegation the parent has since withdrawn.
|
||||
type ParentLookup struct {
|
||||
// Roots are the servers the walk starts from; nil means RootServers.
|
||||
Roots []string
|
||||
// Exchange sends one query; nil means UDP to port 53, retried over TCP
|
||||
// when truncated.
|
||||
Exchange func(ctx context.Context, m *mdns.Msg, server string) (*mdns.Msg, error)
|
||||
// Resolve finds a name server's addresses when a referral has no glue.
|
||||
Resolve func(ctx context.Context, host string) ([]string, error)
|
||||
}
|
||||
|
||||
// Delegation returns the NS names the parent of name publishes for it and the
|
||||
// parent zone that answered. An empty set with a nil error means the parent
|
||||
// answered and does not delegate name.
|
||||
func (p *ParentLookup) Delegation(ctx context.Context, name string) (servers []string, parent string, err error) {
|
||||
target := mdns.Fqdn(strings.ToLower(name))
|
||||
addrs := p.Roots
|
||||
if addrs == nil {
|
||||
addrs = RootServers
|
||||
}
|
||||
zone := "."
|
||||
for range maxReferrals {
|
||||
resp, err := p.ask(ctx, target, addrs)
|
||||
if err != nil {
|
||||
return nil, zone, fmt.Errorf("ask the %s servers: %w", zone, err)
|
||||
}
|
||||
owner, ns := referral(resp, target)
|
||||
if owner == "" || resp.Authoritative {
|
||||
// The servers hold target's own zone or its parent: what they list
|
||||
// for target is the answer, and nothing listed is no delegation.
|
||||
return answerNS(resp, target), strings.TrimSuffix(zone, "."), nil
|
||||
}
|
||||
if owner == target {
|
||||
return ns, strings.TrimSuffix(zone, "."), nil
|
||||
}
|
||||
if !mdns.IsSubDomain(zone, owner) || owner == zone {
|
||||
return nil, zone, fmt.Errorf("the %s servers referred %s back to %s", zone, target, owner)
|
||||
}
|
||||
zone = owner
|
||||
if addrs, err = p.addresses(ctx, resp, ns); err != nil {
|
||||
return nil, zone, err
|
||||
}
|
||||
}
|
||||
return nil, zone, fmt.Errorf("more than %d referrals for %s", maxReferrals, target)
|
||||
}
|
||||
|
||||
// ask sends the query to each server in turn and returns the first answer
|
||||
// that is NOERROR or NXDOMAIN.
|
||||
func (p *ParentLookup) ask(ctx context.Context, target string, servers []string) (*mdns.Msg, error) {
|
||||
m := new(mdns.Msg)
|
||||
m.SetQuestion(target, mdns.TypeNS)
|
||||
m.RecursionDesired = false
|
||||
exchange := p.Exchange
|
||||
if exchange == nil {
|
||||
exchange = exchangeUDP
|
||||
}
|
||||
lastErr := errors.New("no servers")
|
||||
for _, s := range servers {
|
||||
resp, err := exchange(ctx, m, s)
|
||||
switch {
|
||||
case err != nil:
|
||||
lastErr = err
|
||||
case resp.Rcode != mdns.RcodeSuccess && resp.Rcode != mdns.RcodeNameError:
|
||||
lastErr = fmt.Errorf("%s answered %s", s, mdns.RcodeToString[resp.Rcode])
|
||||
default:
|
||||
return resp, nil
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
return nil, lastErr
|
||||
}
|
||||
|
||||
// referral returns the deepest zone at or above target that the response's
|
||||
// authority section delegates, and its NS names.
|
||||
func referral(resp *mdns.Msg, target string) (string, []string) {
|
||||
var owner string
|
||||
var ns []string
|
||||
for _, rr := range resp.Ns {
|
||||
r, ok := rr.(*mdns.NS)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
o := strings.ToLower(r.Hdr.Name)
|
||||
if !mdns.IsSubDomain(o, target) {
|
||||
continue
|
||||
}
|
||||
if o != owner && (owner == "" || mdns.CountLabel(o) > mdns.CountLabel(owner)) {
|
||||
owner, ns = o, nil
|
||||
}
|
||||
if o == owner {
|
||||
ns = append(ns, strings.ToLower(r.Ns))
|
||||
}
|
||||
}
|
||||
return owner, ns
|
||||
}
|
||||
|
||||
func answerNS(resp *mdns.Msg, target string) []string {
|
||||
var ns []string
|
||||
for _, rr := range append(resp.Answer, resp.Ns...) {
|
||||
if r, ok := rr.(*mdns.NS); ok && strings.EqualFold(r.Hdr.Name, target) {
|
||||
ns = append(ns, strings.ToLower(r.Ns))
|
||||
}
|
||||
}
|
||||
return ns
|
||||
}
|
||||
|
||||
// addresses returns where to send the next query: the referral's glue, or a
|
||||
// lookup of the server names when the parent gave none.
|
||||
func (p *ParentLookup) addresses(ctx context.Context, resp *mdns.Msg, ns []string) ([]string, error) {
|
||||
var addrs []string
|
||||
for _, rr := range resp.Extra {
|
||||
if a, ok := rr.(*mdns.A); ok && slices.ContainsFunc(ns, func(n string) bool { return strings.EqualFold(n, a.Hdr.Name) }) {
|
||||
addrs = append(addrs, a.A.String())
|
||||
}
|
||||
}
|
||||
if len(addrs) > 0 {
|
||||
return addrs, nil
|
||||
}
|
||||
resolve := p.Resolve
|
||||
if resolve == nil {
|
||||
resolve = net.DefaultResolver.LookupHost
|
||||
}
|
||||
var lastErr error
|
||||
for _, host := range ns {
|
||||
found, err := resolve(ctx, strings.TrimSuffix(host, "."))
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
for _, a := range found {
|
||||
if ip := net.ParseIP(a); ip != nil && ip.To4() != nil {
|
||||
addrs = append(addrs, a)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(addrs) == 0 {
|
||||
return nil, fmt.Errorf("no address for any of %s: %w", strings.Join(ns, ", "), lastErr)
|
||||
}
|
||||
return addrs, nil
|
||||
}
|
||||
|
||||
func exchangeUDP(ctx context.Context, m *mdns.Msg, server string) (*mdns.Msg, error) {
|
||||
addr := net.JoinHostPort(server, "53")
|
||||
resp, _, err := (&mdns.Client{Timeout: parentQueryTimeout}).ExchangeContext(ctx, m, addr)
|
||||
if err == nil && resp.Truncated {
|
||||
resp, _, err = (&mdns.Client{Net: "tcp", Timeout: parentQueryTimeout}).ExchangeContext(ctx, m, addr)
|
||||
}
|
||||
return resp, err
|
||||
}
|
||||
@@ -1,139 +0,0 @@
|
||||
package dns_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
mdns "github.com/miekg/dns"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/dns"
|
||||
)
|
||||
|
||||
// fakeTree is the DNS hierarchy as each server's address answers it.
|
||||
type fakeTree struct {
|
||||
servers map[string]func(q mdns.Question) *mdns.Msg
|
||||
asked []string
|
||||
}
|
||||
|
||||
func (f *fakeTree) exchange(_ context.Context, m *mdns.Msg, server string) (*mdns.Msg, error) {
|
||||
f.asked = append(f.asked, server)
|
||||
if m.RecursionDesired {
|
||||
return nil, errors.New("recursion requested")
|
||||
}
|
||||
h, ok := f.servers[server]
|
||||
if !ok {
|
||||
return nil, errors.New("i/o timeout")
|
||||
}
|
||||
resp := h(m.Question[0])
|
||||
rcode, aa := resp.Rcode, resp.Authoritative
|
||||
resp.SetReply(m)
|
||||
resp.Rcode, resp.Authoritative = rcode, aa
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func ns(owner string, hosts ...string) []mdns.RR {
|
||||
var out []mdns.RR
|
||||
for _, h := range hosts {
|
||||
out = append(out, &mdns.NS{Hdr: mdns.RR_Header{Name: owner, Rrtype: mdns.TypeNS, Class: mdns.ClassINET, Ttl: 172800}, Ns: h})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func glue(host, ip string) mdns.RR {
|
||||
rr, _ := mdns.NewRR(host + " 172800 IN A " + ip)
|
||||
return rr
|
||||
}
|
||||
|
||||
func referralTo(owner string, extra []mdns.RR, hosts ...string) *mdns.Msg {
|
||||
return &mdns.Msg{Ns: ns(owner, hosts...), Extra: extra}
|
||||
}
|
||||
|
||||
func authoritative(rcode int) *mdns.Msg {
|
||||
m := &mdns.Msg{}
|
||||
m.Authoritative = true
|
||||
m.Rcode = rcode
|
||||
return m
|
||||
}
|
||||
|
||||
func newTree() *fakeTree {
|
||||
return &fakeTree{servers: map[string]func(mdns.Question) *mdns.Msg{
|
||||
"10.0.0.1": func(q mdns.Question) *mdns.Msg {
|
||||
if mdns.IsSubDomain("org.", q.Name) {
|
||||
return referralTo("org.", nil, "a0.org-servers.test.") // no glue
|
||||
}
|
||||
return referralTo("com.", []mdns.RR{glue("a.gtld.test.", "10.0.1.1")}, "a.gtld.test.")
|
||||
},
|
||||
"10.0.1.1": func(q mdns.Question) *mdns.Msg {
|
||||
switch {
|
||||
case mdns.IsSubDomain("example.com.", q.Name):
|
||||
return referralTo("example.com.", nil, "NS1.spx3.net.", "ns2.spx3.net.")
|
||||
case mdns.IsSubDomain("deep.com.", q.Name):
|
||||
return referralTo("deep.com.", []mdns.RR{glue("ns.deep.com.", "10.0.2.1")}, "ns.deep.com.")
|
||||
default:
|
||||
return authoritative(mdns.RcodeNameError)
|
||||
}
|
||||
},
|
||||
"10.0.2.1": func(q mdns.Question) *mdns.Msg {
|
||||
if q.Name == "shop.deep.com." {
|
||||
return referralTo("shop.deep.com.", nil, "ns1.spx3.net.")
|
||||
}
|
||||
return authoritative(mdns.RcodeSuccess) // NODATA: not delegated
|
||||
},
|
||||
"10.0.3.1": func(q mdns.Question) *mdns.Msg {
|
||||
return referralTo(q.Name, nil, "ns3.spx3.net.")
|
||||
},
|
||||
}}
|
||||
}
|
||||
|
||||
func TestParentLookupFollowsReferralsToTheParent(t *testing.T) {
|
||||
tree := newTree()
|
||||
p := &dns.ParentLookup{
|
||||
Roots: []string{"10.0.0.9", "10.0.0.1"}, // the first root is down
|
||||
Exchange: tree.exchange,
|
||||
Resolve: func(_ context.Context, host string) ([]string, error) {
|
||||
if host == "a0.org-servers.test" {
|
||||
return []string{"2001:db8::1", "10.0.3.1"}, nil
|
||||
}
|
||||
return nil, errors.New("no such host")
|
||||
},
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
servers []string
|
||||
parent string
|
||||
}{
|
||||
{"Example.COM", []string{"ns1.spx3.net.", "ns2.spx3.net."}, "com"},
|
||||
{"shop.deep.com.", []string{"ns1.spx3.net."}, "deep.com"},
|
||||
{"other.deep.com", nil, "deep.com"},
|
||||
{"unregistered.com", nil, "com"},
|
||||
{"example.org", []string{"ns3.spx3.net."}, "org"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
servers, parent, err := p.Delegation(context.Background(), tc.name)
|
||||
require.NoError(t, err, tc.name)
|
||||
assert.Equal(t, tc.servers, servers, tc.name)
|
||||
assert.Equal(t, tc.parent, parent, tc.name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParentLookupFailsWhenNoServerAnswers(t *testing.T) {
|
||||
tree := newTree()
|
||||
tree.servers["10.0.1.1"] = func(mdns.Question) *mdns.Msg { return &mdns.Msg{MsgHdr: mdns.MsgHdr{Rcode: mdns.RcodeServerFailure}} }
|
||||
p := &dns.ParentLookup{Roots: []string{"10.0.0.1"}, Exchange: tree.exchange}
|
||||
_, parent, err := p.Delegation(context.Background(), "example.com")
|
||||
require.ErrorContains(t, err, "SERVFAIL")
|
||||
assert.Equal(t, "com.", parent)
|
||||
}
|
||||
|
||||
func TestParentLookupRefusesAReferralUpwards(t *testing.T) {
|
||||
tree := newTree()
|
||||
tree.servers["10.0.1.1"] = func(mdns.Question) *mdns.Msg {
|
||||
return referralTo(".", []mdns.RR{glue("a.root.test.", "10.0.0.1")}, "a.root.test.")
|
||||
}
|
||||
p := &dns.ParentLookup{Roots: []string{"10.0.0.1"}, Exchange: tree.exchange}
|
||||
_, _, err := p.Delegation(context.Background(), "example.com")
|
||||
require.Error(t, err)
|
||||
}
|
||||
@@ -1,78 +0,0 @@
|
||||
package dns
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
nsconfig "github.com/mulgadc/northstar/pkg/config"
|
||||
"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// zoneReloadTimeout bounds one wait for every northstar server to load a zone.
|
||||
const zoneReloadTimeout = 5 * time.Second
|
||||
|
||||
// ZonePublisher writes whole zone documents to the northstar zone store and
|
||||
// asks every northstar server to load them, for callers that own a zone
|
||||
// outright rather than editing records in one.
|
||||
type ZonePublisher struct {
|
||||
s3 *nsconfig.S3Config
|
||||
nc *nats.Conn
|
||||
cluster func() *config.ClusterConfig
|
||||
}
|
||||
|
||||
// NewZonePublisher returns a publisher, or false when northstar's S3 zone store
|
||||
// is not configured on this node. cluster is read on every reload, so a node
|
||||
// joining changes how many acknowledgements are awaited.
|
||||
func NewZonePublisher(cfg *config.Config, cluster func() *config.ClusterConfig, nc *nats.Conn) (*ZonePublisher, bool) {
|
||||
zoneCfg, ok := zoneS3Config(cfg)
|
||||
if !ok || nc == nil {
|
||||
return nil, false
|
||||
}
|
||||
return &ZonePublisher{s3: zoneCfg.s3, nc: nc, cluster: cluster}, true
|
||||
}
|
||||
|
||||
// PutZone writes apex's zone document.
|
||||
func (p *ZonePublisher) PutZone(_ context.Context, apex string, body []byte) error {
|
||||
return nsconfig.WriteZoneFile(p.s3, apex, body)
|
||||
}
|
||||
|
||||
// DeleteZone removes apex's zone document.
|
||||
func (p *ZonePublisher) DeleteZone(_ context.Context, apex string) error {
|
||||
return nsconfig.DeleteZoneFile(p.s3, apex)
|
||||
}
|
||||
|
||||
// Reload asks every northstar server to load apex and returns the version each
|
||||
// reports, "" where it serves nothing. A server that failed to load reports a
|
||||
// value no digest can equal. complete is whether every server answered,
|
||||
// counted by node so a repeated acknowledgement cannot stand in for another.
|
||||
func (p *ZonePublisher) Reload(ctx context.Context, apex string) ([]string, bool, error) {
|
||||
want := NorthstarServerCount(p.cluster())
|
||||
ctx, cancel := context.WithTimeout(ctx, zoneReloadTimeout)
|
||||
defer cancel()
|
||||
acks, err := RequestZoneReload(ctx, p.nc, apex, want)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
served := make([]string, 0, len(acks))
|
||||
nodes := map[string]bool{}
|
||||
for _, ack := range acks {
|
||||
nodes[ack.Node] = true
|
||||
if ack.Error != "" {
|
||||
served = append(served, "error: "+ack.Error)
|
||||
continue
|
||||
}
|
||||
served = append(served, ack.ETag)
|
||||
}
|
||||
return served, len(nodes) >= want, nil
|
||||
}
|
||||
|
||||
// NorthstarServerCount is how many northstar servers the cluster runs, and so
|
||||
// how many must acknowledge a reload. A cluster with none configured still runs
|
||||
// the local one.
|
||||
func NorthstarServerCount(cluster *config.ClusterConfig) int {
|
||||
if cluster == nil {
|
||||
return 1
|
||||
}
|
||||
return max(1, len(configuredNorthstarNodeNames(cluster)))
|
||||
}
|
||||
@@ -1,135 +0,0 @@
|
||||
package dns_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
testutil "github.com/mulgadc/spinifex/internal/testkit"
|
||||
"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/dns"
|
||||
)
|
||||
|
||||
// zoneBucket is a path-style S3 endpoint holding whole objects.
|
||||
func zoneBucket(t *testing.T) (endpoint string, get func(key string) (string, bool)) {
|
||||
t.Helper()
|
||||
var mu sync.Mutex
|
||||
objects := map[string]string{}
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
key := strings.TrimPrefix(r.URL.Path, "/northstar/")
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
switch r.Method {
|
||||
case http.MethodPut:
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
objects[key] = string(body)
|
||||
case http.MethodDelete:
|
||||
delete(objects, key)
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
default:
|
||||
http.Error(w, "unsupported", http.StatusMethodNotAllowed)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
return srv.URL, func(key string) (string, bool) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
v, ok := objects[key]
|
||||
return v, ok
|
||||
}
|
||||
}
|
||||
|
||||
func publisherConfig(t *testing.T, endpoint string) *config.Config {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "northstar.toml")
|
||||
body := fmt.Sprintf("listen = \"0.0.0.0:5300\"\ndefault_domain = \"spx3.net\"\n[s3]\nendpoint = %q\nbucket = \"northstar\"\nregion = \"us-east-1\"\naccess_key = \"RO\"\nsecret_key = \"RO\"\n", endpoint)
|
||||
require.NoError(t, os.WriteFile(path, []byte(body), 0o600))
|
||||
return &config.Config{
|
||||
Predastore: config.PredastoreConfig{AccessKey: "SYSTEM", SecretKey: "SYSTEMSECRET"},
|
||||
Northstar: config.NorthstarConfig{ConfigPath: path},
|
||||
}
|
||||
}
|
||||
|
||||
func cluster(nodes ...string) *config.ClusterConfig {
|
||||
c := &config.ClusterConfig{Node: nodes[0], Nodes: map[string]config.Config{}}
|
||||
for _, n := range nodes {
|
||||
c.Nodes[n] = config.Config{Northstar: config.NorthstarConfig{ConfigPath: "/etc/spinifex/northstar.toml"}}
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func TestZonePublisherWritesAndDeletesZoneObjects(t *testing.T) {
|
||||
endpoint, get := zoneBucket(t)
|
||||
_, nc := testutil.StartTestNATS(t)
|
||||
pub, ok := dns.NewZonePublisher(publisherConfig(t, endpoint), func() *config.ClusterConfig { return cluster("node1") }, nc)
|
||||
require.True(t, ok)
|
||||
|
||||
require.NoError(t, pub.PutZone(context.Background(), "Example.com", []byte("body")))
|
||||
body, ok := get("example.com.toml")
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, "body", body)
|
||||
|
||||
require.NoError(t, pub.DeleteZone(context.Background(), "example.com"))
|
||||
_, ok = get("example.com.toml")
|
||||
assert.False(t, ok)
|
||||
}
|
||||
|
||||
func TestZonePublisherNeedsTheZoneStore(t *testing.T) {
|
||||
_, nc := testutil.StartTestNATS(t)
|
||||
_, ok := dns.NewZonePublisher(&config.Config{}, func() *config.ClusterConfig { return nil }, nc)
|
||||
assert.False(t, ok)
|
||||
}
|
||||
|
||||
// A reload is complete only when every northstar node has answered; a failed
|
||||
// node reports a version no digest matches, and a node answering twice does not
|
||||
// stand in for one that never answered.
|
||||
func TestZonePublisherReloadCountsEveryNorthstarNode(t *testing.T) {
|
||||
endpoint, _ := zoneBucket(t)
|
||||
nodes := cluster("node1", "node2", "node3")
|
||||
cases := []struct {
|
||||
name string
|
||||
acks []dns.ZoneReloadAck
|
||||
served []string
|
||||
complete bool
|
||||
}{
|
||||
{"one failed, one silent", []dns.ZoneReloadAck{{Node: "node1", ETag: "v1"}, {Node: "node2", Error: "s3 down"}}, []string{"v1", "error: s3 down"}, false},
|
||||
{"one node twice", []dns.ZoneReloadAck{{Node: "node1", ETag: "v1"}, {Node: "node1", ETag: "v1"}}, []string{"v1", "v1"}, false},
|
||||
{"every node", []dns.ZoneReloadAck{{Node: "node1", ETag: "v1"}, {Node: "node2", ETag: "v1"}, {Node: "node3", ETag: "v1"}}, []string{"v1", "v1", "v1"}, true},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, nc := testutil.StartTestNATS(t)
|
||||
pub, ok := dns.NewZonePublisher(publisherConfig(t, endpoint), func() *config.ClusterConfig { return nodes }, nc)
|
||||
require.True(t, ok)
|
||||
ackAs(t, nc, func(zone string) []dns.ZoneReloadAck {
|
||||
acks := slices.Clone(tc.acks)
|
||||
for i := range acks {
|
||||
acks[i].Zone = zone
|
||||
}
|
||||
return acks
|
||||
})
|
||||
|
||||
served, complete, err := pub.Reload(context.Background(), "example.com")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.complete, complete)
|
||||
assert.ElementsMatch(t, tc.served, served)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNorthstarServerCount(t *testing.T) {
|
||||
assert.Equal(t, 1, dns.NorthstarServerCount(nil))
|
||||
assert.Equal(t, 1, dns.NorthstarServerCount(&config.ClusterConfig{Nodes: map[string]config.Config{"node1": {}}}))
|
||||
assert.Equal(t, 3, dns.NorthstarServerCount(cluster("node1", "node2", "node3")))
|
||||
}
|
||||
@@ -1,46 +0,0 @@
|
||||
package dns
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// RequestZoneReload asks every northstar server to reload zone and collects
|
||||
// their acknowledgements until want have arrived or ctx ends. The acks gathered
|
||||
// so far are returned either way; the error reports only a transport failure.
|
||||
func RequestZoneReload(ctx context.Context, nc *nats.Conn, zone string, want int) ([]ZoneReloadAck, error) {
|
||||
payload, err := json.Marshal(ZoneReload{Zone: zone})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal zone reload: %w", err)
|
||||
}
|
||||
inbox := nc.NewInbox()
|
||||
sub, err := nc.SubscribeSync(inbox)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("subscribe reload acks: %w", err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
if err := nc.PublishRequest(SubjectZoneReload, inbox, payload); err != nil {
|
||||
return nil, fmt.Errorf("publish zone reload: %w", err)
|
||||
}
|
||||
|
||||
var acks []ZoneReloadAck
|
||||
for len(acks) < want {
|
||||
msg, err := sub.NextMsgWithContext(ctx)
|
||||
if err != nil {
|
||||
if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
|
||||
return acks, nil
|
||||
}
|
||||
return acks, fmt.Errorf("receive reload ack: %w", err)
|
||||
}
|
||||
var ack ZoneReloadAck
|
||||
if json.Unmarshal(msg.Data, &ack) != nil || ack.Zone != zone {
|
||||
continue
|
||||
}
|
||||
acks = append(acks, ack)
|
||||
}
|
||||
return acks, nil
|
||||
}
|
||||
@@ -1,87 +0,0 @@
|
||||
package dns_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
testutil "github.com/mulgadc/spinifex/internal/testkit"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/dns"
|
||||
)
|
||||
|
||||
// ackAs subscribes a fake northstar that answers every reload with the acks
|
||||
// reply returns for the requested zone.
|
||||
func ackAs(t *testing.T, nc *nats.Conn, reply func(zone string) []dns.ZoneReloadAck) {
|
||||
t.Helper()
|
||||
sub, err := nc.Subscribe(dns.SubjectZoneReload, func(msg *nats.Msg) {
|
||||
var evt dns.ZoneReload
|
||||
require.NoError(t, json.Unmarshal(msg.Data, &evt))
|
||||
for _, ack := range reply(evt.Zone) {
|
||||
body, err := json.Marshal(ack)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, msg.Respond(body))
|
||||
}
|
||||
})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
}
|
||||
|
||||
func TestRequestZoneReloadCollectsOneAckPerServer(t *testing.T) {
|
||||
_, nc := testutil.StartTestNATS(t)
|
||||
for _, node := range []string{"node1", "node2", "node3"} {
|
||||
ackAs(t, nc, func(zone string) []dns.ZoneReloadAck {
|
||||
return []dns.ZoneReloadAck{{Zone: zone, Node: node, ETag: "v2"}}
|
||||
})
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
acks, err := dns.RequestZoneReload(ctx, nc, "example.com", 3)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, acks, 3)
|
||||
nodes := []string{acks[0].Node, acks[1].Node, acks[2].Node}
|
||||
assert.ElementsMatch(t, []string{"node1", "node2", "node3"}, nodes)
|
||||
for _, ack := range acks {
|
||||
assert.Equal(t, "v2", ack.ETag)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestZoneReloadIgnoresAcksForOtherZones(t *testing.T) {
|
||||
_, nc := testutil.StartTestNATS(t)
|
||||
ackAs(t, nc, func(zone string) []dns.ZoneReloadAck {
|
||||
return []dns.ZoneReloadAck{{Zone: "other.com", Node: "stray"}, {Zone: zone, Node: "node1"}}
|
||||
})
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
acks, err := dns.RequestZoneReload(ctx, nc, "example.com", 1)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, acks, 1)
|
||||
assert.Equal(t, "node1", acks[0].Node)
|
||||
}
|
||||
|
||||
func TestRequestZoneReloadReturnsPartialAcksAtDeadline(t *testing.T) {
|
||||
_, nc := testutil.StartTestNATS(t)
|
||||
ackAs(t, nc, func(zone string) []dns.ZoneReloadAck {
|
||||
return []dns.ZoneReloadAck{{Zone: zone, Node: "node1", Error: "s3 unavailable"}}
|
||||
})
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond)
|
||||
defer cancel()
|
||||
acks, err := dns.RequestZoneReload(ctx, nc, "example.com", 2)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, acks, 1)
|
||||
assert.Equal(t, "s3 unavailable", acks[0].Error)
|
||||
}
|
||||
|
||||
func TestRequestZoneReloadFailsOnClosedConnection(t *testing.T) {
|
||||
_, nc := testutil.StartTestNATS(t)
|
||||
nc.Close()
|
||||
_, err := dns.RequestZoneReload(context.Background(), nc, "example.com", 1)
|
||||
require.Error(t, err)
|
||||
}
|
||||
@@ -1,44 +0,0 @@
|
||||
package dns
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// managedLabels are the labels directly below the base domain under which the
|
||||
// platform writes its own records.
|
||||
var managedLabels = map[string]bool{"compute": true, "elb": true, "eks": true, "rds": true}
|
||||
|
||||
var nameserverLabel = regexp.MustCompile(`^ns[0-9]+$`)
|
||||
|
||||
// ReservedZoneName reports whether a tenant may not create a hosted zone named
|
||||
// name. It refuses the base and internal domains, any ancestor of them, every
|
||||
// name below the internal domain, and any name below the base domain that falls
|
||||
// under a platform-managed label or a published nameserver.
|
||||
func ReservedZoneName(name, baseDomain, internalDomain string) bool {
|
||||
name = normalizeZone(name)
|
||||
if name == "" {
|
||||
return true
|
||||
}
|
||||
for _, zone := range []string{normalizeZone(baseDomain), normalizeZone(privateZoneOrDefault(internalDomain))} {
|
||||
if zone == "" {
|
||||
continue
|
||||
}
|
||||
if name == zone || strings.HasSuffix(zone, "."+name) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
if internal := normalizeZone(privateZoneOrDefault(internalDomain)); strings.HasSuffix(name, "."+internal) {
|
||||
return true
|
||||
}
|
||||
if base := normalizeZone(baseDomain); base != "" && strings.HasSuffix(name, "."+base) {
|
||||
labels := strings.Split(strings.TrimSuffix(name, "."+base), ".")
|
||||
top := labels[len(labels)-1]
|
||||
return managedLabels[top] || nameserverLabel.MatchString(top)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func normalizeZone(name string) string {
|
||||
return strings.TrimSuffix(strings.ToLower(strings.TrimSpace(name)), ".")
|
||||
}
|
||||
@@ -1,46 +0,0 @@
|
||||
package dns_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/dns"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestReservedZoneName(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
reserved bool
|
||||
}{
|
||||
{"spx3.net.", true},
|
||||
{"SPX3.NET", true},
|
||||
{"net", true},
|
||||
{"compute.internal", true},
|
||||
{"ip-10-0-0-1.us-west-1.compute.internal.", true},
|
||||
{"internal", true},
|
||||
{"compute.spx3.net", true},
|
||||
{"ec2-1-2-3-4.us-west-1.compute.spx3.net", true},
|
||||
{"lb.us-west-1.elb.spx3.net", true},
|
||||
{"x.eks.spx3.net", true},
|
||||
{"db.rds.spx3.net", true},
|
||||
{"ns1.spx3.net", true},
|
||||
{"a.ns4.spx3.net", true},
|
||||
{"app.spx3.net", false},
|
||||
{"nsx.spx3.net", false},
|
||||
{"compute.app.spx3.net", false},
|
||||
{"example.com", false},
|
||||
{"compute.example.com", false},
|
||||
{"", true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
assert.Equal(t, c.reserved, dns.ReservedZoneName(c.name, "spx3.net", ""), c.name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReservedZoneNameHonoursTheConfiguredDomains(t *testing.T) {
|
||||
assert.True(t, dns.ReservedZoneName("cloud.example.com", "cloud.example.com", "corp.internal"))
|
||||
assert.True(t, dns.ReservedZoneName("example.com", "cloud.example.com", "corp.internal"))
|
||||
assert.True(t, dns.ReservedZoneName("host.corp.internal", "cloud.example.com", "corp.internal"))
|
||||
assert.False(t, dns.ReservedZoneName("spx3.net", "cloud.example.com", "corp.internal"))
|
||||
assert.False(t, dns.ReservedZoneName("app.spx3.net", "", ""))
|
||||
}
|
||||
@@ -67,13 +67,3 @@ type ChangeResult struct {
|
||||
type ZoneReload struct {
|
||||
Zone string `json:"zone"`
|
||||
}
|
||||
|
||||
// ZoneReloadAck is one northstar server's reply to a ZoneReload sent with a
|
||||
// reply subject. ETag is the version of the zone it now serves, "" when the
|
||||
// zone is absent; Error is set when the reload failed.
|
||||
type ZoneReloadAck struct {
|
||||
Zone string `json:"zone"`
|
||||
Node string `json:"node"`
|
||||
ETag string `json:"etag,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
@@ -19,16 +19,14 @@ import (
|
||||
// of zone TOML files in s3://northstar/ using the system predastore credentials.
|
||||
// Only the reconcile's elected node calls it, so a zone has one writer at a time.
|
||||
type Writer struct {
|
||||
enabled bool
|
||||
s3cfg *nsconfig.S3Config
|
||||
baseDomain string
|
||||
// internalDomain is the private zone, for refusing reserved delegations.
|
||||
internalDomain string
|
||||
nameservers []nsconfig.NameserverSeed
|
||||
ttl uint32
|
||||
nc *nats.Conn
|
||||
quotaEnabled bool
|
||||
quotas Quotas
|
||||
enabled bool
|
||||
s3cfg *nsconfig.S3Config
|
||||
baseDomain string
|
||||
nameservers []nsconfig.NameserverSeed
|
||||
ttl uint32
|
||||
nc *nats.Conn
|
||||
quotaEnabled bool
|
||||
quotas Quotas
|
||||
}
|
||||
|
||||
// NewWriter resolves the northstar S3 endpoint/bucket from the node's
|
||||
@@ -47,7 +45,6 @@ func NewWriter(cfg *config.Config, cluster *config.ClusterConfig, nc *nats.Conn)
|
||||
w.enabled = true
|
||||
w.s3cfg = zoneCfg.s3
|
||||
w.baseDomain = strings.TrimSpace(zoneCfg.server.DefaultDomain)
|
||||
w.internalDomain = ResolveInternalDomain(cfg)
|
||||
// Share the bootstrap's derivation so a zone this writer materialises carries
|
||||
// the same NS topology as one the bootstrap seeds; whichever wins the race,
|
||||
// the zone is correct.
|
||||
|
||||
@@ -1,135 +0,0 @@
|
||||
package account
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log/slog"
|
||||
|
||||
awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// sealMaxAttempts bounds SealAccount's retry loop against a revision that
|
||||
// keeps moving under concurrent writes, so it fails closed instead of
|
||||
// spinning.
|
||||
const sealMaxAttempts = 5
|
||||
|
||||
// sealedRecord is the fixed value a seal writes. Clearing both booleans
|
||||
// retains no tenant setting; an older reader that ignores state sees the
|
||||
// AWS defaults.
|
||||
var sealedRecord = &AccountSettingsRecord{State: AccountSettingsStateSealed}
|
||||
|
||||
// AccountSettingsSeal is SealAccount's result: the revision at which the
|
||||
// stream leader accepted the sealed value.
|
||||
type AccountSettingsSeal struct {
|
||||
Revision uint64
|
||||
}
|
||||
|
||||
// AccountTerminationChecker reports whether IAM holds accountID as TERMINATING.
|
||||
// A missing record is (false, nil); only a failed read is an error.
|
||||
type AccountTerminationChecker interface {
|
||||
AccountTerminating(ctx context.Context, accountID string) (bool, error)
|
||||
}
|
||||
|
||||
// SetAccountTerminationChecker wires the precondition; until it is set, SealAccount refuses.
|
||||
func (s *AccountSettingsServiceImpl) SetAccountTerminationChecker(c AccountTerminationChecker) {
|
||||
s.terminationChecker = c
|
||||
}
|
||||
|
||||
// checkTerminating runs before any settings-KV call, so a refusal never writes.
|
||||
// Not TERMINATING, including a missing IAM record, is IncorrectState; a read
|
||||
// failure or an unwired checker is InternalError.
|
||||
func (s *AccountSettingsServiceImpl) checkTerminating(ctx context.Context, accountID string) error {
|
||||
if s.terminationChecker == nil {
|
||||
slog.ErrorContext(ctx, "Account settings seal has no termination checker wired", "accountID", accountID)
|
||||
return errors.New(awserrors.ErrorInternalError)
|
||||
}
|
||||
|
||||
terminating, err := s.terminationChecker.AccountTerminating(ctx, accountID)
|
||||
if err != nil {
|
||||
slog.ErrorContext(ctx, "Failed to read account status for seal precondition", "accountID", accountID, "err", err)
|
||||
return errors.New(awserrors.ErrorInternalError)
|
||||
}
|
||||
if !terminating {
|
||||
return awserrors.Errorf(awserrors.ErrorIncorrectState, "account %s is not TERMINATING", accountID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// isRevisionConflict reports a lost CAS race SealAccount should retry. Besides
|
||||
// the two sentinels, Create's retry against a delete marker returns a bare
|
||||
// *jetstream.APIError carrying a wrong-last-sequence code.
|
||||
func isRevisionConflict(err error) bool {
|
||||
if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
return true
|
||||
}
|
||||
if apiErr, ok := errors.AsType[*jetstream.APIError](err); ok {
|
||||
return apiErr.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequence ||
|
||||
apiErr.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequenceConstant
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// SealAccount seals accountID's settings record so no later mutation can
|
||||
// recreate it. It never relies on a read for its claim: the only success is a
|
||||
// leader-accepted write, evidenced by the returned revision.
|
||||
func (s *AccountSettingsServiceImpl) SealAccount(ctx context.Context, accountID string) (*AccountSettingsSeal, error) {
|
||||
if accountID == "" {
|
||||
return nil, awserrors.Errorf(awserrors.ErrorMissingParameter, "accountID is required to seal settings")
|
||||
}
|
||||
if !awsidentifiers.IsAccountID(accountID) {
|
||||
return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "accountID %q is not a valid AWS account ID", accountID)
|
||||
}
|
||||
if accountID == awsidentifiers.GlobalAccountID {
|
||||
return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "settings seal must not target the global account")
|
||||
}
|
||||
|
||||
if err := s.checkTerminating(ctx, accountID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
data, err := json.Marshal(sealedRecord)
|
||||
if err != nil {
|
||||
slog.ErrorContext(ctx, "Failed to encode sealed account settings record", "accountID", accountID, "err", err)
|
||||
return nil, errors.New(awserrors.ErrorInternalError)
|
||||
}
|
||||
|
||||
// settingsKey is never used here: a seal must act on exactly accountID
|
||||
// and never fall back to the global key.
|
||||
for range sealMaxAttempts {
|
||||
var rev uint64
|
||||
entry, getErr := s.settingsKV.Get(ctx, accountID)
|
||||
switch {
|
||||
case getErr == nil:
|
||||
rev = entry.Revision()
|
||||
case errors.Is(getErr, jetstream.ErrKeyNotFound):
|
||||
rev = 0
|
||||
default:
|
||||
slog.ErrorContext(ctx, "Failed to read account settings for seal", "accountID", accountID, "err", getErr)
|
||||
return nil, errors.New(awserrors.ErrorInternalError)
|
||||
}
|
||||
|
||||
var newRev uint64
|
||||
var writeErr error
|
||||
if rev == 0 {
|
||||
newRev, writeErr = s.settingsKV.Create(ctx, accountID, data)
|
||||
} else {
|
||||
newRev, writeErr = s.settingsKV.Update(ctx, accountID, data, rev)
|
||||
}
|
||||
|
||||
switch {
|
||||
case writeErr == nil:
|
||||
return &AccountSettingsSeal{Revision: newRev}, nil
|
||||
case isRevisionConflict(writeErr):
|
||||
continue
|
||||
default:
|
||||
slog.ErrorContext(ctx, "Failed to write sealed account settings record", "accountID", accountID, "err", writeErr)
|
||||
return nil, errors.New(awserrors.ErrorInternalError)
|
||||
}
|
||||
}
|
||||
|
||||
slog.ErrorContext(ctx, "Account settings seal exhausted attempts against concurrent writes", "accountID", accountID, "attempts", sealMaxAttempts)
|
||||
return nil, errors.New(awserrors.ErrorInternalError)
|
||||
}
|
||||
@@ -1,559 +0,0 @@
|
||||
package account
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/mulgadc/spinifex/internal/testkit"
|
||||
awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// stubTerminationChecker answers AccountTerminating from a fixed set, for
|
||||
// tests that exercise the KV paths rather than IAM itself.
|
||||
type stubTerminationChecker struct {
|
||||
terminating map[string]bool
|
||||
err error
|
||||
}
|
||||
|
||||
func (c *stubTerminationChecker) AccountTerminating(_ context.Context, accountID string) (bool, error) {
|
||||
if c.err != nil {
|
||||
return false, c.err
|
||||
}
|
||||
return c.terminating[accountID], nil
|
||||
}
|
||||
|
||||
// terminatingChecker answers true for exactly accountID.
|
||||
func terminatingChecker(accountID string) *stubTerminationChecker {
|
||||
return &stubTerminationChecker{terminating: map[string]bool{accountID: true}}
|
||||
}
|
||||
|
||||
// lastSettingsSeq reports the account-settings stream's last sequence, so a
|
||||
// test can assert a failed attempt wrote nothing.
|
||||
func lastSettingsSeq(t *testing.T, svc *AccountSettingsServiceImpl) uint64 {
|
||||
t.Helper()
|
||||
stream, err := svc.js.Stream(context.Background(), "KV_"+KVBucketAccountSettings)
|
||||
require.NoError(t, err)
|
||||
info, err := stream.Info(context.Background())
|
||||
require.NoError(t, err)
|
||||
return info.State.LastSeq
|
||||
}
|
||||
|
||||
// mutationCase is one of the four settings mutations, called uniformly so a
|
||||
// test can table over them.
|
||||
type mutationCase struct {
|
||||
name string
|
||||
call func(svc *AccountSettingsServiceImpl, ctx context.Context) error
|
||||
}
|
||||
|
||||
func mutationTable() []mutationCase {
|
||||
return []mutationCase{
|
||||
{"EnableEbs", func(svc *AccountSettingsServiceImpl, ctx context.Context) error {
|
||||
_, err := svc.EnableEbsEncryptionByDefault(ctx, &ec2.EnableEbsEncryptionByDefaultInput{}, testAccountID)
|
||||
return err
|
||||
}},
|
||||
{"DisableEbs", func(svc *AccountSettingsServiceImpl, ctx context.Context) error {
|
||||
_, err := svc.DisableEbsEncryptionByDefault(ctx, &ec2.DisableEbsEncryptionByDefaultInput{}, testAccountID)
|
||||
return err
|
||||
}},
|
||||
{"EnableSerial", func(svc *AccountSettingsServiceImpl, ctx context.Context) error {
|
||||
_, err := svc.EnableSerialConsoleAccess(ctx, &ec2.EnableSerialConsoleAccessInput{}, testAccountID)
|
||||
return err
|
||||
}},
|
||||
{"DisableSerial", func(svc *AccountSettingsServiceImpl, ctx context.Context) error {
|
||||
_, err := svc.DisableSerialConsoleAccess(ctx, &ec2.DisableSerialConsoleAccessInput{}, testAccountID)
|
||||
return err
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
// seedSettingsState prepares testAccountID's record for a race test: present
|
||||
// (an active record), absent (never written) or tombstone (written then
|
||||
// deleted, leaving a delete marker Get hides).
|
||||
func seedSettingsState(t *testing.T, svc *AccountSettingsServiceImpl, state string) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
switch state {
|
||||
case "present":
|
||||
_, err := svc.EnableEbsEncryptionByDefault(ctx, &ec2.EnableEbsEncryptionByDefaultInput{}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
case "tombstone":
|
||||
_, err := svc.EnableEbsEncryptionByDefault(ctx, &ec2.EnableEbsEncryptionByDefaultInput{}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, svc.settingsKV.Delete(ctx, testAccountID))
|
||||
case "absent":
|
||||
default:
|
||||
t.Fatalf("unknown seed state %q", state)
|
||||
}
|
||||
}
|
||||
|
||||
// sealRaceKV wraps a real KeyValue so a test can inject a deterministic race
|
||||
// between a caller's read and its guarded write (Create or Update), instead
|
||||
// of relying on goroutine timing.
|
||||
type sealRaceKV struct {
|
||||
jetstream.KeyValue
|
||||
|
||||
writeCalls int
|
||||
beforeWrite func(attempt int) error
|
||||
}
|
||||
|
||||
func (f *sealRaceKV) Create(ctx context.Context, key string, value []byte, opts ...jetstream.KVCreateOpt) (uint64, error) {
|
||||
f.writeCalls++
|
||||
if f.beforeWrite != nil {
|
||||
if err := f.beforeWrite(f.writeCalls); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
return f.KeyValue.Create(ctx, key, value, opts...)
|
||||
}
|
||||
|
||||
func (f *sealRaceKV) Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error) {
|
||||
f.writeCalls++
|
||||
if f.beforeWrite != nil {
|
||||
if err := f.beforeWrite(f.writeCalls); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
return f.KeyValue.Update(ctx, key, value, revision)
|
||||
}
|
||||
|
||||
// staleGetKV always answers Get with a fixed snapshot taken at construction,
|
||||
// so a caller's CAS keeps retrying against a revision that can never win.
|
||||
type staleGetKV struct {
|
||||
jetstream.KeyValue
|
||||
|
||||
entry jetstream.KeyValueEntry
|
||||
}
|
||||
|
||||
func (f *staleGetKV) Get(context.Context, string) (jetstream.KeyValueEntry, error) {
|
||||
return f.entry, nil
|
||||
}
|
||||
|
||||
// staleNotFoundKV always answers Get as absent, regardless of what the real
|
||||
// bucket holds.
|
||||
type staleNotFoundKV struct {
|
||||
jetstream.KeyValue
|
||||
}
|
||||
|
||||
func (f *staleNotFoundKV) Get(context.Context, string) (jetstream.KeyValueEntry, error) {
|
||||
return nil, jetstream.ErrKeyNotFound
|
||||
}
|
||||
|
||||
// errGetKV always fails Get with a fixed store error.
|
||||
type errGetKV struct {
|
||||
jetstream.KeyValue
|
||||
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *errGetKV) Get(context.Context, string) (jetstream.KeyValueEntry, error) {
|
||||
return nil, f.err
|
||||
}
|
||||
|
||||
func decodeSettings(t *testing.T, entry jetstream.KeyValueEntry) AccountSettingsRecord {
|
||||
t.Helper()
|
||||
var rec AccountSettingsRecord
|
||||
require.NoError(t, json.Unmarshal(entry.Value(), &rec))
|
||||
return rec
|
||||
}
|
||||
|
||||
// A seal landing between a mutation's read and its guarded write is not
|
||||
// outlived: the mutation's retry sees the seal and is rejected, and nothing
|
||||
// it wrote survives.
|
||||
func TestSealRaceMutation_SealLandsBetweenReadAndWrite(t *testing.T) {
|
||||
for _, m := range mutationTable() {
|
||||
for _, state := range []string{"present", "absent", "tombstone"} {
|
||||
t.Run(m.name+"_"+state, func(t *testing.T) {
|
||||
svc := setupTestAccountService(t)
|
||||
ctx := context.Background()
|
||||
seedSettingsState(t, svc, state)
|
||||
|
||||
realKV := svc.settingsKV
|
||||
sealer := &AccountSettingsServiceImpl{js: svc.js, settingsKV: realKV}
|
||||
sealer.SetAccountTerminationChecker(terminatingChecker(testAccountID))
|
||||
|
||||
var sealed *AccountSettingsSeal
|
||||
race := &sealRaceKV{KeyValue: realKV}
|
||||
race.beforeWrite = func(attempt int) error {
|
||||
if attempt != 1 {
|
||||
return nil
|
||||
}
|
||||
var sealErr error
|
||||
sealed, sealErr = sealer.SealAccount(ctx, testAccountID)
|
||||
return sealErr
|
||||
}
|
||||
svc.settingsKV = race
|
||||
|
||||
err := m.call(svc, ctx)
|
||||
require.Error(t, err)
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorBlocked), "got %v", err)
|
||||
require.NotNil(t, sealed)
|
||||
|
||||
entry, err := realKV.Get(ctx, testAccountID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, AccountSettingsStateSealed, decodeSettings(t, entry).State)
|
||||
assert.Equal(t, sealed.Revision, entry.Revision(), "no write landed after the seal")
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A mutation landing inside a seal's read-write window does not survive it:
|
||||
// the seal retries against the mutation's write and wins, leaving the
|
||||
// sealed value with both booleans false.
|
||||
func TestSealRaceMutation_MutationLandsBetweenSealReadAndWrite(t *testing.T) {
|
||||
for _, m := range mutationTable() {
|
||||
for _, state := range []string{"present", "absent", "tombstone"} {
|
||||
t.Run(m.name+"_"+state, func(t *testing.T) {
|
||||
svc := setupTestAccountService(t)
|
||||
ctx := context.Background()
|
||||
seedSettingsState(t, svc, state)
|
||||
|
||||
realKV := svc.settingsKV
|
||||
race := &sealRaceKV{KeyValue: realKV}
|
||||
race.beforeWrite = func(attempt int) error {
|
||||
if attempt != 1 {
|
||||
return nil
|
||||
}
|
||||
_ = m.call(svc, ctx)
|
||||
return nil
|
||||
}
|
||||
|
||||
sealer := &AccountSettingsServiceImpl{js: svc.js, settingsKV: race}
|
||||
sealer.SetAccountTerminationChecker(terminatingChecker(testAccountID))
|
||||
|
||||
seal, err := sealer.SealAccount(ctx, testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
entry, err := realKV.Get(ctx, testAccountID)
|
||||
require.NoError(t, err)
|
||||
rec := decodeSettings(t, entry)
|
||||
assert.Equal(t, AccountSettingsStateSealed, rec.State)
|
||||
assert.False(t, rec.EbsEncryptionByDefault)
|
||||
assert.False(t, rec.SerialConsoleAccess)
|
||||
assert.Equal(t, seal.Revision, entry.Revision())
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// None of the four mutations can act on an already-sealed record.
|
||||
func TestMutationOnSealedRecordIsRejected(t *testing.T) {
|
||||
for _, m := range mutationTable() {
|
||||
t.Run(m.name, func(t *testing.T) {
|
||||
svc := setupTestAccountService(t)
|
||||
ctx := context.Background()
|
||||
svc.SetAccountTerminationChecker(terminatingChecker(testAccountID))
|
||||
_, err := svc.SealAccount(ctx, testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
before, err := svc.settingsKV.Get(ctx, testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = m.call(svc, ctx)
|
||||
require.Error(t, err)
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorBlocked), "got %v", err)
|
||||
|
||||
after, err := svc.settingsKV.Get(ctx, testAccountID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, before.Revision(), after.Revision())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Every settings mutation races one seal on a real bucket; afterwards the
|
||||
// record is sealed and stays sealed, so no success among them could have
|
||||
// landed after the seal.
|
||||
func TestSealConcurrentMutations(t *testing.T) {
|
||||
svc := setupTestAccountService(t)
|
||||
ctx := context.Background()
|
||||
svc.SetAccountTerminationChecker(terminatingChecker(testAccountID))
|
||||
_, err := svc.EnableEbsEncryptionByDefault(ctx, &ec2.EnableEbsEncryptionByDefaultInput{}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
mutations := mutationTable()
|
||||
errs := make([]error, len(mutations))
|
||||
var wg sync.WaitGroup
|
||||
for i, m := range mutations {
|
||||
wg.Add(1)
|
||||
go func(i int, m mutationCase) {
|
||||
defer wg.Done()
|
||||
errs[i] = m.call(svc, ctx)
|
||||
}(i, m)
|
||||
}
|
||||
|
||||
var sealResult *AccountSettingsSeal
|
||||
var sealErr error
|
||||
wg.Go(func() {
|
||||
sealResult, sealErr = svc.SealAccount(ctx, testAccountID)
|
||||
})
|
||||
wg.Wait()
|
||||
|
||||
require.NoError(t, sealErr)
|
||||
require.NotNil(t, sealResult)
|
||||
for i, err := range errs {
|
||||
if err != nil {
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorBlocked), "mutation %s: got %v", mutations[i].name, err)
|
||||
}
|
||||
}
|
||||
|
||||
entry, err := svc.settingsKV.Get(ctx, testAccountID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, AccountSettingsStateSealed, decodeSettings(t, entry).State)
|
||||
assert.Equal(t, sealResult.Revision, entry.Revision())
|
||||
|
||||
_, err = svc.GetEbsEncryptionByDefault(ctx, &ec2.GetEbsEncryptionByDefaultInput{}, testAccountID)
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorBlocked))
|
||||
}
|
||||
|
||||
// A Get pinned to a revision the record has already moved past exhausts the
|
||||
// mutation's retries without ever writing.
|
||||
func TestMutationStaleReadExhaustsWithoutWriting(t *testing.T) {
|
||||
svc := setupTestAccountService(t)
|
||||
ctx := context.Background()
|
||||
|
||||
_, err := svc.EnableEbsEncryptionByDefault(ctx, &ec2.EnableEbsEncryptionByDefaultInput{}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
stale, err := svc.settingsKV.Get(ctx, testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = svc.DisableEbsEncryptionByDefault(ctx, &ec2.DisableEbsEncryptionByDefaultInput{}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
before := lastSettingsSeq(t, svc)
|
||||
svc.settingsKV = &staleGetKV{KeyValue: svc.settingsKV, entry: stale}
|
||||
|
||||
_, err = svc.EnableSerialConsoleAccess(ctx, &ec2.EnableSerialConsoleAccessInput{}, testAccountID)
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInternalError), "got %v", err)
|
||||
assert.Equal(t, before, lastSettingsSeq(t, svc))
|
||||
}
|
||||
|
||||
// A Get that always reports absent, while a sealed record actually exists,
|
||||
// never gets to recreate it: every Create attempt conflicts.
|
||||
func TestMutationStaleAbsentDoesNotRecreate(t *testing.T) {
|
||||
svc := setupTestAccountService(t)
|
||||
ctx := context.Background()
|
||||
svc.SetAccountTerminationChecker(terminatingChecker(testAccountID))
|
||||
_, err := svc.SealAccount(ctx, testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
before := lastSettingsSeq(t, svc)
|
||||
svc.settingsKV = &staleNotFoundKV{KeyValue: svc.settingsKV}
|
||||
|
||||
_, err = svc.EnableEbsEncryptionByDefault(ctx, &ec2.EnableEbsEncryptionByDefaultInput{}, testAccountID)
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInternalError), "got %v", err)
|
||||
assert.Equal(t, before, lastSettingsSeq(t, svc))
|
||||
}
|
||||
|
||||
// A revision that keeps moving under sustained concurrent writes must not
|
||||
// let the seal spin forever: it gives up, bounded by its attempt budget,
|
||||
// and its own write never lands.
|
||||
func TestSealSustainedConflictFailsWithoutWriting(t *testing.T) {
|
||||
svc := setupTestAccountService(t)
|
||||
ctx := context.Background()
|
||||
svc.SetAccountTerminationChecker(terminatingChecker(testAccountID))
|
||||
_, err := svc.EnableEbsEncryptionByDefault(ctx, &ec2.EnableEbsEncryptionByDefaultInput{}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
realKV := svc.settingsKV
|
||||
race := &sealRaceKV{KeyValue: realKV}
|
||||
race.beforeWrite = func(int) error {
|
||||
data, merr := json.Marshal(&AccountSettingsRecord{SerialConsoleAccess: true})
|
||||
require.NoError(t, merr)
|
||||
_, perr := realKV.Put(ctx, testAccountID, data)
|
||||
return perr
|
||||
}
|
||||
svc.settingsKV = race
|
||||
|
||||
_, err = svc.SealAccount(ctx, testAccountID)
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInternalError), "got %v", err)
|
||||
assert.Equal(t, sealMaxAttempts, race.writeCalls)
|
||||
|
||||
entry, err := realKV.Get(ctx, testAccountID)
|
||||
require.NoError(t, err)
|
||||
assert.NotEqual(t, AccountSettingsStateSealed, decodeSettings(t, entry).State, "the seal's own write never landed")
|
||||
}
|
||||
|
||||
// A store error on the seal's read reports failed, never written.
|
||||
func TestSealStoreErrorWritesNothing(t *testing.T) {
|
||||
svc := setupTestAccountService(t)
|
||||
ctx := context.Background()
|
||||
svc.SetAccountTerminationChecker(terminatingChecker(testAccountID))
|
||||
|
||||
before := lastSettingsSeq(t, svc)
|
||||
svc.settingsKV = &errGetKV{KeyValue: svc.settingsKV, err: errors.New("kv unavailable")}
|
||||
|
||||
_, err := svc.SealAccount(ctx, testAccountID)
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInternalError), "got %v", err)
|
||||
assert.Equal(t, before, lastSettingsSeq(t, svc))
|
||||
}
|
||||
|
||||
func TestIsRevisionConflictRecognisesTombstoneRetry(t *testing.T) {
|
||||
assert.True(t, isRevisionConflict(jetstream.ErrKeyExists))
|
||||
assert.True(t, isRevisionConflict(jetstream.ErrKeyRevisionMismatch))
|
||||
assert.True(t, isRevisionConflict(&jetstream.APIError{ErrorCode: jetstream.JSErrCodeStreamWrongLastSequence}))
|
||||
assert.True(t, isRevisionConflict(&jetstream.APIError{ErrorCode: jetstream.JSErrCodeStreamWrongLastSequenceConstant}))
|
||||
assert.True(t, isRevisionConflict(fmt.Errorf("wrapped: %w", &jetstream.APIError{ErrorCode: jetstream.JSErrCodeStreamWrongLastSequence})))
|
||||
assert.False(t, isRevisionConflict(errors.New("unrelated failure")))
|
||||
assert.False(t, isRevisionConflict(nil))
|
||||
}
|
||||
|
||||
// An account IAM does not hold as TERMINATING refuses the seal before any
|
||||
// settings-KV write, whether the key is present or absent.
|
||||
func TestSealRefusedForActiveAccount(t *testing.T) {
|
||||
for _, state := range []string{"present", "absent"} {
|
||||
t.Run(state, func(t *testing.T) {
|
||||
svc := setupTestAccountService(t)
|
||||
ctx := context.Background()
|
||||
if state == "present" {
|
||||
_, err := svc.EnableEbsEncryptionByDefault(ctx, &ec2.EnableEbsEncryptionByDefaultInput{}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
svc.SetAccountTerminationChecker(&stubTerminationChecker{terminating: map[string]bool{}})
|
||||
|
||||
before := lastSettingsSeq(t, svc)
|
||||
_, err := svc.SealAccount(ctx, testAccountID)
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorIncorrectState), "got %v", err)
|
||||
assert.Equal(t, before, lastSettingsSeq(t, svc))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSealRefusedForMissingIAMRecord(t *testing.T) {
|
||||
svc := setupTestAccountService(t)
|
||||
ctx := context.Background()
|
||||
svc.SetAccountTerminationChecker(&stubTerminationChecker{terminating: map[string]bool{}})
|
||||
|
||||
before := lastSettingsSeq(t, svc)
|
||||
_, err := svc.SealAccount(ctx, testAccountID)
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorIncorrectState), "got %v", err)
|
||||
assert.Equal(t, before, lastSettingsSeq(t, svc))
|
||||
}
|
||||
|
||||
func TestSealRefusedWhenIAMReadFails(t *testing.T) {
|
||||
svc := setupTestAccountService(t)
|
||||
ctx := context.Background()
|
||||
svc.SetAccountTerminationChecker(&stubTerminationChecker{err: errors.New("iam unavailable")})
|
||||
|
||||
before := lastSettingsSeq(t, svc)
|
||||
_, err := svc.SealAccount(ctx, testAccountID)
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInternalError), "got %v", err)
|
||||
assert.Equal(t, before, lastSettingsSeq(t, svc))
|
||||
}
|
||||
|
||||
func TestSealRefusedWithNoCheckerWired(t *testing.T) {
|
||||
svc := setupTestAccountService(t)
|
||||
ctx := context.Background()
|
||||
|
||||
before := lastSettingsSeq(t, svc)
|
||||
_, err := svc.SealAccount(ctx, testAccountID)
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInternalError), "got %v", err)
|
||||
assert.Equal(t, before, lastSettingsSeq(t, svc))
|
||||
}
|
||||
|
||||
func TestSealAcceptedForTerminatingAccount(t *testing.T) {
|
||||
svc := setupTestAccountService(t)
|
||||
ctx := context.Background()
|
||||
svc.SetAccountTerminationChecker(terminatingChecker(testAccountID))
|
||||
|
||||
seal, err := svc.SealAccount(ctx, testAccountID)
|
||||
require.NoError(t, err)
|
||||
assert.NotZero(t, seal.Revision)
|
||||
|
||||
entry, err := svc.settingsKV.Get(ctx, testAccountID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, seal.Revision, entry.Revision())
|
||||
}
|
||||
|
||||
// A sealed record stays sealed after the service is rebuilt over the same
|
||||
// JetStream, and every one of the six settings actions still refuses it.
|
||||
func TestSealSurvivesServiceReconstruction(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
ctx := context.Background()
|
||||
|
||||
svc, err := NewAccountSettingsServiceImplWithNATS(ctx, nil, nc)
|
||||
require.NoError(t, err)
|
||||
svc.SetAccountTerminationChecker(terminatingChecker(testAccountID))
|
||||
first, err := svc.SealAccount(ctx, testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
rebuilt, err := NewAccountSettingsServiceImplWithNATS(ctx, nil, nc)
|
||||
require.NoError(t, err)
|
||||
|
||||
entry, err := rebuilt.settingsKV.Get(ctx, testAccountID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, AccountSettingsStateSealed, decodeSettings(t, entry).State)
|
||||
|
||||
_, err = rebuilt.GetEbsEncryptionByDefault(ctx, &ec2.GetEbsEncryptionByDefaultInput{}, testAccountID)
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorBlocked))
|
||||
_, err = rebuilt.GetSerialConsoleAccessStatus(ctx, &ec2.GetSerialConsoleAccessStatusInput{}, testAccountID)
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorBlocked))
|
||||
_, err = rebuilt.EnableEbsEncryptionByDefault(ctx, &ec2.EnableEbsEncryptionByDefaultInput{}, testAccountID)
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorBlocked))
|
||||
_, err = rebuilt.DisableEbsEncryptionByDefault(ctx, &ec2.DisableEbsEncryptionByDefaultInput{}, testAccountID)
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorBlocked))
|
||||
_, err = rebuilt.EnableSerialConsoleAccess(ctx, &ec2.EnableSerialConsoleAccessInput{}, testAccountID)
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorBlocked))
|
||||
_, err = rebuilt.DisableSerialConsoleAccess(ctx, &ec2.DisableSerialConsoleAccessInput{}, testAccountID)
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorBlocked))
|
||||
|
||||
rebuilt.SetAccountTerminationChecker(terminatingChecker(testAccountID))
|
||||
second, err := rebuilt.SealAccount(ctx, testAccountID)
|
||||
require.NoError(t, err)
|
||||
assert.Greater(t, second.Revision, first.Revision)
|
||||
}
|
||||
|
||||
func TestSealAbsentKeyCreatesSealedRecord(t *testing.T) {
|
||||
svc := setupTestAccountService(t)
|
||||
ctx := context.Background()
|
||||
svc.SetAccountTerminationChecker(terminatingChecker(testAccountID))
|
||||
|
||||
_, err := svc.settingsKV.Get(ctx, testAccountID)
|
||||
require.ErrorIs(t, err, jetstream.ErrKeyNotFound)
|
||||
|
||||
seal, err := svc.SealAccount(ctx, testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
entry, err := svc.settingsKV.Get(ctx, testAccountID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, seal.Revision, entry.Revision())
|
||||
assert.Equal(t, AccountSettingsStateSealed, decodeSettings(t, entry).State)
|
||||
}
|
||||
|
||||
func TestSealRefusesEmptyInvalidAndGlobalAccount(t *testing.T) {
|
||||
svc := setupTestAccountService(t)
|
||||
ctx := context.Background()
|
||||
|
||||
_, err := svc.SealAccount(ctx, "")
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorMissingParameter), "got %v", err)
|
||||
|
||||
_, err = svc.SealAccount(ctx, "not-an-account")
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInvalidParameterValue), "got %v", err)
|
||||
|
||||
_, err = svc.SealAccount(ctx, awsidentifiers.GlobalAccountID)
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInvalidParameterValue), "got %v", err)
|
||||
}
|
||||
|
||||
func TestSealLeavesOtherAccountsUntouched(t *testing.T) {
|
||||
svc := setupTestAccountService(t)
|
||||
ctx := context.Background()
|
||||
accountB := "222222222222"
|
||||
_, err := svc.EnableEbsEncryptionByDefault(ctx, &ec2.EnableEbsEncryptionByDefaultInput{}, accountB)
|
||||
require.NoError(t, err)
|
||||
|
||||
svc.SetAccountTerminationChecker(terminatingChecker(testAccountID))
|
||||
_, err = svc.SealAccount(ctx, testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
out, err := svc.GetEbsEncryptionByDefault(ctx, &ec2.GetEbsEncryptionByDefaultInput{}, accountB)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, *out.EbsEncryptionByDefault)
|
||||
}
|
||||
@@ -5,13 +5,12 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
|
||||
"log/slog"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
|
||||
awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
|
||||
"github.com/nats-io/nats.go"
|
||||
@@ -21,19 +20,14 @@ import (
|
||||
const (
|
||||
KVBucketAccountSettings = "spinifex-ec2-account-settings"
|
||||
KVBucketAccountSettingsVersion = 1
|
||||
KeyEbsEncryptionDefault = "ebs-encryption-default"
|
||||
KeySerialConsoleAccess = "serial-console-access"
|
||||
)
|
||||
|
||||
// AccountSettingsState is a settings record's lifecycle state. "" is active;
|
||||
// a record written before the state field existed has none and stays active.
|
||||
type AccountSettingsState string
|
||||
|
||||
const AccountSettingsStateSealed AccountSettingsState = "sealed"
|
||||
|
||||
// AccountSettingsRecord represents stored account settings.
|
||||
type AccountSettingsRecord struct {
|
||||
EbsEncryptionByDefault bool `json:"ebs_encryption_by_default"`
|
||||
SerialConsoleAccess bool `json:"serial_console_access"`
|
||||
State AccountSettingsState `json:"state,omitempty"`
|
||||
EbsEncryptionByDefault bool `json:"ebs_encryption_by_default"`
|
||||
SerialConsoleAccess bool `json:"serial_console_access"`
|
||||
}
|
||||
|
||||
// AccountSettingsServiceImpl implements account settings operations with NATS JetStream persistence.
|
||||
@@ -41,10 +35,6 @@ type AccountSettingsServiceImpl struct {
|
||||
config *config.Config
|
||||
js jetstream.JetStream
|
||||
settingsKV jetstream.KeyValue
|
||||
|
||||
// terminationChecker is the seal precondition; SealAccount refuses until
|
||||
// it is wired.
|
||||
terminationChecker AccountTerminationChecker
|
||||
}
|
||||
|
||||
var _ AccountSettingsService = (*AccountSettingsServiceImpl)(nil)
|
||||
@@ -82,99 +72,50 @@ func settingsKey(accountID string) string {
|
||||
return accountID
|
||||
}
|
||||
|
||||
// errSettingsSealed marks a mutation or read that found a sealed record, so
|
||||
// callers map it to Blocked without logging it as a fault.
|
||||
var errSettingsSealed = errors.New(awserrors.ErrorBlocked)
|
||||
|
||||
// errSettingsContended marks a mutation whose CAS retries were exhausted
|
||||
// under concurrent writers, bounding the retry loop instead of spinning.
|
||||
var errSettingsContended = errors.New("account settings: CAS retries exhausted")
|
||||
|
||||
// checkActive rejects a sealed or unknown-state record; "" is active and
|
||||
// passes through. A mutation must never act on a record that failed this.
|
||||
func checkActive(rec *AccountSettingsRecord) error {
|
||||
switch rec.State {
|
||||
case "":
|
||||
return nil
|
||||
case AccountSettingsStateSealed:
|
||||
return errSettingsSealed
|
||||
default:
|
||||
return fmt.Errorf("account settings: unknown state %q", rec.State)
|
||||
}
|
||||
}
|
||||
|
||||
// mapSettingsError turns a mutateSettings or getSettings failure into the AWS
|
||||
// error a caller returns: sealed is an expected, unlogged Blocked; everything
|
||||
// else is InternalError, logged with the account ID.
|
||||
func mapSettingsError(ctx context.Context, accountID string, err error) error {
|
||||
switch {
|
||||
case err == nil:
|
||||
return nil
|
||||
case errors.Is(err, errSettingsSealed):
|
||||
return errors.New(awserrors.ErrorBlocked)
|
||||
case errors.Is(err, errSettingsContended):
|
||||
slog.ErrorContext(ctx, "Account settings CAS retries exhausted under contention", "accountID", accountID)
|
||||
case errors.Is(err, kvutil.ErrRead):
|
||||
slog.ErrorContext(ctx, "Failed to read account settings from KV", "accountID", accountID, "err", err)
|
||||
case errors.Is(err, kvutil.ErrDecode):
|
||||
slog.ErrorContext(ctx, "Corrupt account settings record in KV", "accountID", accountID, "err", err)
|
||||
case errors.Is(err, kvutil.ErrEncode):
|
||||
slog.ErrorContext(ctx, "Failed to marshal account settings record", "accountID", accountID, "err", err)
|
||||
case errors.Is(err, kvutil.ErrWrite):
|
||||
slog.ErrorContext(ctx, "Failed to write account settings to KV", "accountID", accountID, "err", err)
|
||||
default:
|
||||
slog.ErrorContext(ctx, "Account settings operation failed", "accountID", accountID, "err", err)
|
||||
}
|
||||
return errors.New(awserrors.ErrorInternalError)
|
||||
}
|
||||
|
||||
// mutateSettings applies set to accountID's record at the revision it read.
|
||||
// changed is always true, so every success is a leader-accepted write over an
|
||||
// active value; a seal landing between the read and the write fails the CAS.
|
||||
func (s *AccountSettingsServiceImpl) mutateSettings(ctx context.Context, accountID string, set func(*AccountSettingsRecord)) error {
|
||||
_, err := kvutil.Update(ctx, s.settingsKV, settingsKey(accountID), kvutil.CASConfig{
|
||||
CreateIfAbsent: true,
|
||||
Exhausted: func(string, int) error { return errSettingsContended },
|
||||
}, func(rec *AccountSettingsRecord) (bool, error) {
|
||||
if err := checkActive(rec); err != nil {
|
||||
return false, err
|
||||
}
|
||||
set(rec)
|
||||
return true, nil
|
||||
})
|
||||
return mapSettingsError(ctx, accountID, err)
|
||||
}
|
||||
|
||||
// getSettings retrieves current account settings, rejecting a sealed or
|
||||
// unknown-state record so a read never answers from behind a seal. Absent is
|
||||
// the AWS defaults; a read is a direct get and can be stale.
|
||||
// getSettings retrieves current account settings.
|
||||
func (s *AccountSettingsServiceImpl) getSettings(ctx context.Context, accountID string) (*AccountSettingsRecord, error) {
|
||||
entry, err := s.settingsKV.Get(ctx, settingsKey(accountID))
|
||||
if err != nil {
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return &AccountSettingsRecord{}, nil
|
||||
return &AccountSettingsRecord{
|
||||
EbsEncryptionByDefault: false,
|
||||
SerialConsoleAccess: false,
|
||||
}, nil
|
||||
}
|
||||
slog.ErrorContext(ctx, "Failed to read account settings from KV", "accountID", accountID, "err", err)
|
||||
return nil, errors.New(awserrors.ErrorInternalError)
|
||||
return nil, fmt.Errorf("failed to get account settings: %w", err)
|
||||
}
|
||||
|
||||
var settings AccountSettingsRecord
|
||||
if err := json.Unmarshal(entry.Value(), &settings); err != nil {
|
||||
slog.ErrorContext(ctx, "Corrupt account settings record in KV", "accountID", accountID, "err", err)
|
||||
return nil, errors.New(awserrors.ErrorInternalError)
|
||||
}
|
||||
if err := checkActive(&settings); err != nil {
|
||||
return nil, mapSettingsError(ctx, accountID, err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &settings, nil
|
||||
}
|
||||
|
||||
// saveSettings saves current account settings.
|
||||
func (s *AccountSettingsServiceImpl) saveSettings(ctx context.Context, settings *AccountSettingsRecord, accountID string) error {
|
||||
data, err := json.Marshal(settings)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = s.settingsKV.Put(ctx, settingsKey(accountID), data)
|
||||
return err
|
||||
}
|
||||
|
||||
// EnableEbsEncryptionByDefault enables EBS encryption by default for the account.
|
||||
func (s *AccountSettingsServiceImpl) EnableEbsEncryptionByDefault(ctx context.Context, input *ec2.EnableEbsEncryptionByDefaultInput, accountID string) (*ec2.EnableEbsEncryptionByDefaultOutput, error) {
|
||||
slog.InfoContext(ctx, "EnableEbsEncryptionByDefault called", "accountID", accountID)
|
||||
|
||||
if err := s.mutateSettings(ctx, accountID, func(rec *AccountSettingsRecord) { rec.EbsEncryptionByDefault = true }); err != nil {
|
||||
settings, err := s.getSettings(ctx, accountID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
settings.EbsEncryptionByDefault = true
|
||||
|
||||
if err := s.saveSettings(ctx, settings, accountID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -187,7 +128,14 @@ func (s *AccountSettingsServiceImpl) EnableEbsEncryptionByDefault(ctx context.Co
|
||||
func (s *AccountSettingsServiceImpl) DisableEbsEncryptionByDefault(ctx context.Context, input *ec2.DisableEbsEncryptionByDefaultInput, accountID string) (*ec2.DisableEbsEncryptionByDefaultOutput, error) {
|
||||
slog.InfoContext(ctx, "DisableEbsEncryptionByDefault called", "accountID", accountID)
|
||||
|
||||
if err := s.mutateSettings(ctx, accountID, func(rec *AccountSettingsRecord) { rec.EbsEncryptionByDefault = false }); err != nil {
|
||||
settings, err := s.getSettings(ctx, accountID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
settings.EbsEncryptionByDefault = false
|
||||
|
||||
if err := s.saveSettings(ctx, settings, accountID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -228,7 +176,14 @@ func (s *AccountSettingsServiceImpl) GetSerialConsoleAccessStatus(ctx context.Co
|
||||
func (s *AccountSettingsServiceImpl) EnableSerialConsoleAccess(ctx context.Context, input *ec2.EnableSerialConsoleAccessInput, accountID string) (*ec2.EnableSerialConsoleAccessOutput, error) {
|
||||
slog.InfoContext(ctx, "EnableSerialConsoleAccess called", "accountID", accountID)
|
||||
|
||||
if err := s.mutateSettings(ctx, accountID, func(rec *AccountSettingsRecord) { rec.SerialConsoleAccess = true }); err != nil {
|
||||
settings, err := s.getSettings(ctx, accountID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
settings.SerialConsoleAccess = true
|
||||
|
||||
if err := s.saveSettings(ctx, settings, accountID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -241,7 +196,14 @@ func (s *AccountSettingsServiceImpl) EnableSerialConsoleAccess(ctx context.Conte
|
||||
func (s *AccountSettingsServiceImpl) DisableSerialConsoleAccess(ctx context.Context, input *ec2.DisableSerialConsoleAccessInput, accountID string) (*ec2.DisableSerialConsoleAccessOutput, error) {
|
||||
slog.InfoContext(ctx, "DisableSerialConsoleAccess called", "accountID", accountID)
|
||||
|
||||
if err := s.mutateSettings(ctx, accountID, func(rec *AccountSettingsRecord) { rec.SerialConsoleAccess = false }); err != nil {
|
||||
settings, err := s.getSettings(ctx, accountID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
settings.SerialConsoleAccess = false
|
||||
|
||||
if err := s.saveSettings(ctx, settings, accountID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
|
||||
@@ -2,13 +2,11 @@ package account
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/mulgadc/spinifex/internal/testkit"
|
||||
awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -145,44 +143,3 @@ func TestSettingsKey_EmptyAccountIDFallback(t *testing.T) {
|
||||
assert.Equal(t, awsidentifiers.GlobalAccountID, settingsKey(""))
|
||||
assert.Equal(t, "123456789012", settingsKey("123456789012"))
|
||||
}
|
||||
|
||||
// A raw pre-seal record, with no state field at all, is readable and
|
||||
// mutable, and a mutation writes it back without adding one.
|
||||
func TestRecordWithoutStateIsActive(t *testing.T) {
|
||||
svc := setupTestAccountService(t)
|
||||
ctx := context.Background()
|
||||
_, err := svc.settingsKV.Put(ctx, testAccountID, []byte(`{"ebs_encryption_by_default":true,"serial_console_access":false}`))
|
||||
require.NoError(t, err)
|
||||
|
||||
out, err := svc.GetEbsEncryptionByDefault(ctx, &ec2.GetEbsEncryptionByDefaultInput{}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, *out.EbsEncryptionByDefault)
|
||||
|
||||
_, err = svc.EnableSerialConsoleAccess(ctx, &ec2.EnableSerialConsoleAccessInput{}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
entry, err := svc.settingsKV.Get(ctx, testAccountID)
|
||||
require.NoError(t, err)
|
||||
assert.NotContains(t, string(entry.Value()), "state")
|
||||
}
|
||||
|
||||
// A record with an unrecognised state fails closed rather than being treated
|
||||
// as active or sealed, for both a read and a mutation.
|
||||
func TestUnknownStateFailsClosed(t *testing.T) {
|
||||
svc := setupTestAccountService(t)
|
||||
ctx := context.Background()
|
||||
_, err := svc.settingsKV.Put(ctx, testAccountID, []byte(`{"ebs_encryption_by_default":false,"serial_console_access":false,"state":"quarantined"}`))
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = svc.GetEbsEncryptionByDefault(ctx, &ec2.GetEbsEncryptionByDefaultInput{}, testAccountID)
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInternalError), "got %v", err)
|
||||
|
||||
_, err = svc.EnableEbsEncryptionByDefault(ctx, &ec2.EnableEbsEncryptionByDefaultInput{}, testAccountID)
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInternalError), "got %v", err)
|
||||
|
||||
entry, err := svc.settingsKV.Get(ctx, testAccountID)
|
||||
require.NoError(t, err)
|
||||
var rec AccountSettingsRecord
|
||||
require.NoError(t, json.Unmarshal(entry.Value(), &rec))
|
||||
assert.EqualValues(t, "quarantined", rec.State, "the unknown-state record was never overwritten")
|
||||
}
|
||||
|
||||
@@ -1,68 +0,0 @@
|
||||
package account
|
||||
|
||||
//test:in-package — pins what AccountSettingsSealCapabilityVersion 1 asserts:
|
||||
// nothing in this package writes the settings KV except through the
|
||||
// revision-guarded helpers.
|
||||
|
||||
import (
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"io/fs"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func accountPackageDir(t *testing.T) string {
|
||||
t.Helper()
|
||||
_, file, _, ok := runtime.Caller(0)
|
||||
if !ok {
|
||||
t.Fatal("cannot locate test source")
|
||||
}
|
||||
return filepath.Dir(file)
|
||||
}
|
||||
|
||||
// The capability daemons report as version 1 is a claim about every writer in
|
||||
// the binary, not just the responder: no non-test file in this package may
|
||||
// call Put or PutString, which bypass the revision-guarded mutate/seal paths.
|
||||
func TestAccountSettingsWritesAreRevisionGuarded(t *testing.T) {
|
||||
root := accountPackageDir(t)
|
||||
fset := token.NewFileSet()
|
||||
scanned := 0
|
||||
|
||||
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil || d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {
|
||||
return err
|
||||
}
|
||||
f, perr := parser.ParseFile(fset, path, nil, 0)
|
||||
if perr != nil {
|
||||
return perr
|
||||
}
|
||||
scanned++
|
||||
|
||||
ast.Inspect(f, func(n ast.Node) bool {
|
||||
call, ok := n.(*ast.CallExpr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
sel, ok := call.Fun.(*ast.SelectorExpr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
if sel.Sel.Name == "Put" || sel.Sel.Name == "PutString" {
|
||||
t.Errorf("%s: unconditional %s call bypasses the revision-guarded settings write; use mutateSettings or SealAccount's CAS loop instead",
|
||||
fset.Position(call.Pos()), sel.Sel.Name)
|
||||
}
|
||||
return true
|
||||
})
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if scanned == 0 {
|
||||
t.Fatalf("scanned no Go files under %s; the invariant would pass vacuously", root)
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package instance
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
|
||||
"log/slog"
|
||||
@@ -100,10 +101,10 @@ func DescribeInstanceTypes(ctx context.Context, input *ec2.DescribeInstanceTypes
|
||||
|
||||
if missing := missingInstanceTypes(finalInstanceTypes, input.InstanceTypes); len(missing) > 0 {
|
||||
if !proveAbsence || !everyNodeAnswered(nodeIDs, answered, sum) {
|
||||
return nil, awserrors.WithRetryAfter(
|
||||
return nil, errors.Join(
|
||||
awserrors.Errorf(awserrors.ErrorServiceUnavailable,
|
||||
"cannot confirm instance type(s) %s: describe fan-out did not complete", strings.Join(missing, ", ")),
|
||||
describeIncompleteRetryAfter,
|
||||
awserrors.RetryAfter(awserrors.ErrorServiceUnavailable, describeIncompleteRetryAfter),
|
||||
)
|
||||
}
|
||||
return nil, awserrors.Errorf(awserrors.ErrorInvalidInstanceType,
|
||||
|
||||
@@ -126,10 +126,10 @@ func DescribeInstancesChecked(ctx context.Context, input *ec2.DescribeInstancesI
|
||||
// An incomplete sweep cannot prove absence, so a missing id is
|
||||
// unresolved, not gone. NotFound would be a false negative and an
|
||||
// empty success would look identical to a real one, so return 503.
|
||||
return nil, awserrors.WithRetryAfter(
|
||||
return nil, errors.Join(
|
||||
awserrors.Errorf(awserrors.ErrorServiceUnavailable,
|
||||
"cannot confirm status of instance(s) %s: describe fan-out did not complete", strings.Join(missing, ", ")),
|
||||
describeIncompleteRetryAfter,
|
||||
awserrors.RetryAfter(awserrors.ErrorServiceUnavailable, describeIncompleteRetryAfter),
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -198,10 +198,10 @@ func DescribeAccountInstancesComplete(ctx context.Context, natsConn *nats.Conn,
|
||||
}
|
||||
|
||||
func incompleteListingError() error {
|
||||
return awserrors.WithRetryAfter(
|
||||
return errors.Join(
|
||||
awserrors.Errorf(awserrors.ErrorServiceUnavailable,
|
||||
"cannot list instances: not every node and instance store answered"),
|
||||
describeIncompleteRetryAfter,
|
||||
awserrors.RetryAfter(awserrors.ErrorServiceUnavailable, describeIncompleteRetryAfter),
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -1,187 +0,0 @@
|
||||
package route53
|
||||
|
||||
import (
|
||||
"context"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/route53/ext"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
|
||||
)
|
||||
|
||||
// GetHostedZoneActivation reports whether a zone is served and why not.
|
||||
func (s *Service) GetHostedZoneActivation(ctx context.Context, in *ext.GetHostedZoneActivationInput, account string) (*ext.GetHostedZoneActivationOutput, error) {
|
||||
id := bareZoneID(aws.StringValue(in.Id))
|
||||
zone, err := s.store.GetZone(ctx, account, id)
|
||||
if err != nil {
|
||||
return nil, s.zoneErr(id, err)
|
||||
}
|
||||
return &ext.GetHostedZoneActivationOutput{HostedZoneId: aws.String(hostedZonePrefix + zone.ID), Activation: activation(zone)}, nil
|
||||
}
|
||||
|
||||
// VerifyHostedZoneDelegation checks a zone's delegation now, the console's
|
||||
// "Check now", rather than at its next scheduled check.
|
||||
func (s *Service) VerifyHostedZoneDelegation(ctx context.Context, in *ext.VerifyHostedZoneDelegationInput, account string) (*ext.VerifyHostedZoneDelegationOutput, error) {
|
||||
id := bareZoneID(aws.StringValue(in.Id))
|
||||
if s.Verifier == nil {
|
||||
return nil, awserrors.Errorf(awserrors.ErrorServiceUnavailable, "delegation checks are not available on this cluster")
|
||||
}
|
||||
zone, err := s.Verifier.Verify(ctx, account, id)
|
||||
if err != nil {
|
||||
return nil, s.zoneErr(id, err)
|
||||
}
|
||||
return &ext.VerifyHostedZoneDelegationOutput{HostedZoneId: aws.String(hostedZonePrefix + zone.ID), Activation: activation(zone)}, nil
|
||||
}
|
||||
|
||||
func activation(z *ZoneRecord) *ext.Activation {
|
||||
a := &ext.Activation{
|
||||
State: aws.String(z.State),
|
||||
Reason: nonEmpty(z.StateReason),
|
||||
NameServers: aws.StringSlice(z.Nameservers),
|
||||
DelegatedTo: aws.StringSlice(z.Delegation),
|
||||
DelegationSource: nonEmpty(z.DelegationSource),
|
||||
Override: nonEmpty(z.Override),
|
||||
}
|
||||
if !z.CheckedAt.IsZero() {
|
||||
a.CheckedAt = aws.Time(z.CheckedAt)
|
||||
}
|
||||
if z.Override != "" {
|
||||
a.Reason = nonEmpty(z.OverrideReason)
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
// AdminZoneQuery selects zones across every account, by ID or by name.
|
||||
type AdminZoneQuery struct {
|
||||
ZoneID string `json:"zone_id,omitempty"`
|
||||
Name string `json:"name,omitempty"`
|
||||
}
|
||||
|
||||
// AdminZones is the zones a query matched.
|
||||
type AdminZones struct {
|
||||
Zones []ZoneRecord `json:"zones"`
|
||||
}
|
||||
|
||||
// AdminOverrideInput sets or clears an operator override on a zone.
|
||||
type AdminOverrideInput struct {
|
||||
ZoneID string `json:"zone_id"`
|
||||
// Override is StateActive, StateSuspended, or "" to hand the zone back to
|
||||
// the verifier.
|
||||
Override string `json:"override"`
|
||||
Reason string `json:"reason"`
|
||||
}
|
||||
|
||||
// AdminDelegateInput names the platform-zone delegations to write or, with
|
||||
// Remove, withdraw.
|
||||
type AdminDelegateInput struct {
|
||||
Names []string `json:"names"`
|
||||
Remove bool `json:"remove,omitempty"`
|
||||
}
|
||||
|
||||
// AdminDelegateOutput reports how many record changes were applied.
|
||||
type AdminDelegateOutput struct {
|
||||
Applied int `json:"applied"`
|
||||
}
|
||||
|
||||
// AdminFindZones lists the zones matching q in every account. It is for the
|
||||
// operator and is never reachable through the gateway.
|
||||
func (s *Service) AdminFindZones(ctx context.Context, q *AdminZoneQuery, _ string) (*AdminZones, error) {
|
||||
zones, err := s.store.ListAllZones(ctx)
|
||||
if err != nil {
|
||||
return nil, internal("list hosted zones", err)
|
||||
}
|
||||
id, name := bareZoneID(q.ZoneID), normalizeName(q.Name)
|
||||
out := &AdminZones{Zones: []ZoneRecord{}}
|
||||
for _, z := range zones {
|
||||
if (id == "" || z.ID == id) && (name == "" || z.Name == name) {
|
||||
out.Zones = append(out.Zones, z)
|
||||
}
|
||||
}
|
||||
slices.SortFunc(out.Zones, func(a, b ZoneRecord) int { return olderFirst(&a, &b) })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// AdminVerify checks one zone now, in whichever account holds it.
|
||||
func (s *Service) AdminVerify(ctx context.Context, q *AdminZoneQuery, _ string) (*AdminZones, error) {
|
||||
z, err := s.adminZone(ctx, q.ZoneID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if s.Verifier == nil {
|
||||
return nil, awserrors.Errorf(awserrors.ErrorServiceUnavailable, "delegation checks are not available on this cluster")
|
||||
}
|
||||
checked, err := s.Verifier.Verify(ctx, z.AccountID, z.ID)
|
||||
if err != nil {
|
||||
return nil, s.zoneErr(z.ID, err)
|
||||
}
|
||||
return &AdminZones{Zones: []ZoneRecord{*checked}}, nil
|
||||
}
|
||||
|
||||
// AdminOverride activates or suspends a zone whatever its delegation, or
|
||||
// clears an override. Activation refuses a name active in another zone.
|
||||
func (s *Service) AdminOverride(ctx context.Context, in *AdminOverrideInput, _ string) (*AdminZones, error) {
|
||||
if in.Override != "" && strings.TrimSpace(in.Reason) == "" {
|
||||
return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "an override needs a reason")
|
||||
}
|
||||
z, err := s.adminZone(ctx, in.ZoneID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch in.Override {
|
||||
case StateActive:
|
||||
zones, err := s.store.ListAllZones(ctx)
|
||||
if err != nil {
|
||||
return nil, internal("list hosted zones", err)
|
||||
}
|
||||
for _, other := range zones {
|
||||
if other.Name == z.Name && other.ID != z.ID && other.State == StateActive {
|
||||
return nil, awserrors.Errorf(awserrors.ErrorConflictingDomainExists, "%s is already active in account %s as %s", strings.TrimSuffix(z.Name, "."), other.AccountID, other.ID)
|
||||
}
|
||||
}
|
||||
case StateSuspended, "":
|
||||
default:
|
||||
return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "override must be %s, %s or empty", StateActive, StateSuspended)
|
||||
}
|
||||
err = s.store.MutateZone(ctx, z.AccountID, z.ID, func(r *ZoneRecord) (bool, error) {
|
||||
switch in.Override {
|
||||
case "":
|
||||
// The verifier decides again from here, starting with a check.
|
||||
if r.State == StateSuspended || r.Override == StateActive {
|
||||
r.State = StatePendingNameservers
|
||||
}
|
||||
r.CheckedAt = time.Time{}
|
||||
default:
|
||||
r.State = in.Override
|
||||
}
|
||||
r.Override, r.OverrideReason = in.Override, strings.TrimSpace(in.Reason)
|
||||
r.FailingSince = time.Time{}
|
||||
return true, nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, s.zoneErr(z.ID, err)
|
||||
}
|
||||
updated, err := s.store.GetZone(ctx, z.AccountID, z.ID)
|
||||
if err != nil {
|
||||
return nil, s.zoneErr(z.ID, err)
|
||||
}
|
||||
return &AdminZones{Zones: []ZoneRecord{*updated}}, nil
|
||||
}
|
||||
|
||||
func (s *Service) adminZone(ctx context.Context, rawID string) (*ZoneRecord, error) {
|
||||
id := bareZoneID(rawID)
|
||||
if id == "" {
|
||||
return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "a hosted zone ID is required")
|
||||
}
|
||||
found, err := s.AdminFindZones(ctx, &AdminZoneQuery{ZoneID: id}, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(found.Zones) == 0 {
|
||||
return nil, s.zoneErr(id, kvstore.ErrNotFound)
|
||||
}
|
||||
return &found.Zones[0], nil
|
||||
}
|
||||
@@ -1,111 +0,0 @@
|
||||
package route53_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/route53"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/route53/ext"
|
||||
)
|
||||
|
||||
func TestActivationIsReportedAndCheckedOnDemand(t *testing.T) {
|
||||
svc, v, dns, _ := newVerifier(t)
|
||||
svc.Verifier = v
|
||||
id := createZone(t, svc, acctA, "demand.example")
|
||||
|
||||
got, err := svc.GetHostedZoneActivation(ctx, &ext.GetHostedZoneActivationInput{Id: aws.String(id)}, acctA)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, route53.StatePendingNameservers, aws.StringValue(got.Activation.State))
|
||||
assert.Nil(t, got.Activation.CheckedAt, "never checked")
|
||||
|
||||
dns.parents["demand.example"] = ours()
|
||||
verified, err := svc.VerifyHostedZoneDelegation(ctx, &ext.VerifyHostedZoneDelegationInput{Id: aws.String(id)}, acctA)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, route53.StateActive, aws.StringValue(verified.Activation.State))
|
||||
assert.Equal(t, ours(), aws.StringValueSlice(verified.Activation.DelegatedTo))
|
||||
assert.NotNil(t, verified.Activation.CheckedAt)
|
||||
|
||||
_, err = svc.GetHostedZoneActivation(ctx, &ext.GetHostedZoneActivationInput{Id: aws.String(id)}, acctB)
|
||||
requireCode(t, err, "NoSuchHostedZone")
|
||||
_, err = svc.VerifyHostedZoneDelegation(ctx, &ext.VerifyHostedZoneDelegationInput{Id: aws.String(id)}, acctB)
|
||||
requireCode(t, err, "NoSuchHostedZone")
|
||||
}
|
||||
|
||||
func TestAdminFindsZonesAcrossAccounts(t *testing.T) {
|
||||
svc, _, _, c := newVerifier(t)
|
||||
a := createZone(t, svc, acctA, "shared.example")
|
||||
c.advance(time.Second)
|
||||
createZone(t, svc, acctB, "shared.example")
|
||||
createZone(t, svc, acctB, "other.example")
|
||||
|
||||
byName, err := svc.AdminFindZones(ctx, &route53.AdminZoneQuery{Name: "Shared.Example"}, "")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, byName.Zones, 2)
|
||||
assert.Equal(t, acctA, byName.Zones[0].AccountID, "oldest first")
|
||||
|
||||
byID, err := svc.AdminFindZones(ctx, &route53.AdminZoneQuery{ZoneID: a}, "")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, byID.Zones, 1)
|
||||
assert.Equal(t, zoneID(a), byID.Zones[0].ID)
|
||||
}
|
||||
|
||||
func TestAdminOverrideActivatesSuspendsAndReleases(t *testing.T) {
|
||||
svc, v, dns, c := newVerifier(t)
|
||||
svc.Verifier = v
|
||||
held := createZone(t, svc, acctA, "contested.example")
|
||||
c.advance(time.Second)
|
||||
squat := createZone(t, svc, acctB, "contested.example")
|
||||
|
||||
_, err := svc.AdminOverride(ctx, &route53.AdminOverrideInput{ZoneID: held, Override: route53.StateActive}, "")
|
||||
requireCode(t, err, "InvalidInput")
|
||||
|
||||
out, err := svc.AdminOverride(ctx, &route53.AdminOverrideInput{ZoneID: held, Override: route53.StateActive, Reason: "air-gapped root"}, "")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, route53.StateActive, out.Zones[0].State)
|
||||
|
||||
_, err = svc.AdminOverride(ctx, &route53.AdminOverrideInput{ZoneID: squat, Override: route53.StateActive, Reason: "mine"}, "")
|
||||
requireCode(t, err, "ConflictingDomainExists")
|
||||
|
||||
// The delegation check never demotes an operator's activation.
|
||||
pass(t, v)
|
||||
assert.Equal(t, route53.StateActive, zone(t, svc, acctA, held).State)
|
||||
|
||||
_, err = svc.AdminOverride(ctx, &route53.AdminOverrideInput{ZoneID: held, Override: route53.StateSuspended, Reason: "dispute"}, "")
|
||||
require.NoError(t, err)
|
||||
dns.parents["contested.example"] = ours()
|
||||
c.advance(time.Minute)
|
||||
pass(t, v)
|
||||
assert.Equal(t, route53.StateSuspended, zone(t, svc, acctA, held).State)
|
||||
assert.Equal(t, route53.StateActive, zone(t, svc, acctB, squat).State, "suspending a zone frees its name")
|
||||
|
||||
_, err = svc.AdminOverride(ctx, &route53.AdminOverrideInput{ZoneID: held}, "")
|
||||
require.NoError(t, err)
|
||||
z := zone(t, svc, acctA, held)
|
||||
assert.Equal(t, route53.StatePendingNameservers, z.State)
|
||||
assert.Empty(t, z.Override)
|
||||
pass(t, v)
|
||||
assert.Equal(t, "the domain is active in another account", zone(t, svc, acctA, held).StateReason)
|
||||
|
||||
_, err = svc.AdminOverride(ctx, &route53.AdminOverrideInput{ZoneID: "ZNOPE"}, "")
|
||||
requireCode(t, err, "NoSuchHostedZone")
|
||||
_, err = svc.AdminOverride(ctx, &route53.AdminOverrideInput{ZoneID: held, Override: "BOGUS", Reason: "x"}, "")
|
||||
requireCode(t, err, "InvalidInput")
|
||||
}
|
||||
|
||||
func TestAdminVerifyChecksInTheOwningAccount(t *testing.T) {
|
||||
svc, v, dns, _ := newVerifier(t)
|
||||
id := createZone(t, svc, acctB, "admin.example")
|
||||
_, err := svc.AdminVerify(ctx, &route53.AdminZoneQuery{ZoneID: id}, "")
|
||||
requireCode(t, err, "ServiceUnavailable")
|
||||
|
||||
svc.Verifier = v
|
||||
dns.parents["admin.example"] = ours()
|
||||
out, err := svc.AdminVerify(ctx, &route53.AdminZoneQuery{ZoneID: id}, "")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, route53.StateActive, out.Zones[0].State)
|
||||
assert.Equal(t, acctB, out.Zones[0].AccountID)
|
||||
}
|
||||
@@ -1,60 +0,0 @@
|
||||
package awsapi
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/route53"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/route53/ext"
|
||||
)
|
||||
|
||||
// Route 53 ARNs carry no region or account, so the policy gate cannot scope
|
||||
// them to the caller; the daemon looks every ID up within the caller's account.
|
||||
const arnPrefix = "arn:aws:route53:::"
|
||||
|
||||
// ResourceARNs returns the resources an action acts on, for the policy gate.
|
||||
// Actions that name no zone or change are authorized against "*".
|
||||
func ResourceARNs(input any) []string {
|
||||
switch in := input.(type) {
|
||||
case *route53.GetHostedZoneInput:
|
||||
return zoneARN(in.Id)
|
||||
case *route53.UpdateHostedZoneCommentInput:
|
||||
return zoneARN(in.Id)
|
||||
case *route53.DeleteHostedZoneInput:
|
||||
return zoneARN(in.Id)
|
||||
case *route53.ChangeResourceRecordSetsInput:
|
||||
return zoneARN(in.HostedZoneId)
|
||||
case *route53.ListResourceRecordSetsInput:
|
||||
return zoneARN(in.HostedZoneId)
|
||||
case *ext.GetHostedZoneActivationInput:
|
||||
return zoneARN(in.Id)
|
||||
case *ext.VerifyHostedZoneDelegationInput:
|
||||
return zoneARN(in.Id)
|
||||
case *route53.GetChangeInput:
|
||||
return []string{arnPrefix + "change/" + strings.TrimPrefix(aws.StringValue(in.Id), "/change/")}
|
||||
case *route53.ChangeTagsForResourceInput:
|
||||
return tagARNs(in.ResourceType, in.ResourceId)
|
||||
case *route53.ListTagsForResourceInput:
|
||||
return tagARNs(in.ResourceType, in.ResourceId)
|
||||
case *route53.ListTagsForResourcesInput:
|
||||
var out []string
|
||||
for _, id := range in.ResourceIds {
|
||||
out = append(out, tagARNs(in.ResourceType, id)...)
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return []string{"*"}
|
||||
}
|
||||
return out
|
||||
default:
|
||||
return []string{"*"}
|
||||
}
|
||||
}
|
||||
|
||||
func zoneARN(id *string) []string {
|
||||
return []string{arnPrefix + "hostedzone/" + strings.TrimPrefix(aws.StringValue(id), "/hostedzone/")}
|
||||
}
|
||||
|
||||
func tagARNs(resourceType, id *string) []string {
|
||||
return []string{arnPrefix + aws.StringValue(resourceType) + "/" + aws.StringValue(id)}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user