mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-26 05:59:41 +00:00
Compare commits
22 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| bcf28b7af4 | |||
| c628ae97c4 | |||
| 48b029bfae | |||
| 58248c9d28 | |||
| 3187764fd1 | |||
| 6a2438e7b6 | |||
| 12b09ac296 | |||
| 66313bd342 | |||
| c3b5bb1873 | |||
| 464be8972c | |||
| 83cf04f4d8 | |||
| e739cf3a27 | |||
| abcbaeb2c2 | |||
| 4a065a7546 | |||
| d548535fcf | |||
| a54ea1beb5 | |||
| 44df3096ac | |||
| 98b864249c | |||
| aebc2f0efb | |||
| 0c488d9470 | |||
| 9a3ab0468d | |||
| 520478dc37 |
@@ -190,5 +190,38 @@
|
||||
"extra": "",
|
||||
"suite": "kvquorum",
|
||||
"timeout_minutes": 55
|
||||
},
|
||||
{
|
||||
"cell": 34,
|
||||
"install": "binary",
|
||||
"topo": "single",
|
||||
"net": "dhcp",
|
||||
"host": "oracle-10",
|
||||
"arch": "x86_64",
|
||||
"runner": "ci-single",
|
||||
"extra": "",
|
||||
"suite": ""
|
||||
},
|
||||
{
|
||||
"cell": 35,
|
||||
"install": "binary",
|
||||
"topo": "single",
|
||||
"net": "static",
|
||||
"host": "oracle-10",
|
||||
"arch": "x86_64",
|
||||
"runner": "ci-single",
|
||||
"extra": "reboot-single",
|
||||
"suite": ""
|
||||
},
|
||||
{
|
||||
"cell": 36,
|
||||
"install": "binary",
|
||||
"topo": "single",
|
||||
"net": "nat",
|
||||
"host": "oracle-10",
|
||||
"arch": "x86_64",
|
||||
"runner": "ci-single",
|
||||
"extra": "nat-single",
|
||||
"suite": ""
|
||||
}
|
||||
]
|
||||
|
||||
@@ -2,7 +2,8 @@ name: Apt Packages
|
||||
|
||||
# Path-filtered like image-scripts.yml: this resolves setup.sh's runtime package
|
||||
# list against each supported suite's real indices, so a renamed, dropped, or
|
||||
# ambiguously virtual package fails here rather than on a user's machine.
|
||||
# ambiguously virtual package fails here rather than on a user's machine. The
|
||||
# dnf job does the same for Oracle Linux, through setup.sh's own repo setup.
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
@@ -11,6 +12,7 @@ on:
|
||||
paths:
|
||||
- scripts/setup.sh
|
||||
- scripts/lint-apt-packages.sh
|
||||
- scripts/lint-dnf-packages.sh
|
||||
- .github/workflows/apt-packages.yml
|
||||
pull_request:
|
||||
branches:
|
||||
@@ -19,6 +21,7 @@ on:
|
||||
paths:
|
||||
- scripts/setup.sh
|
||||
- scripts/lint-apt-packages.sh
|
||||
- scripts/lint-dnf-packages.sh
|
||||
- .github/workflows/apt-packages.yml
|
||||
workflow_dispatch:
|
||||
|
||||
@@ -44,3 +47,14 @@ jobs:
|
||||
|
||||
- name: Resolve runtime package list
|
||||
run: ./scripts/lint-apt-packages.sh ${{ matrix.image }}
|
||||
|
||||
dnf_packages:
|
||||
name: Resolve dnf list (oraclelinux:10)
|
||||
runs-on: ubuntu-26.04
|
||||
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v7.0.1
|
||||
|
||||
- name: Resolve runtime package list
|
||||
run: ./scripts/lint-dnf-packages.sh oraclelinux:10
|
||||
|
||||
@@ -102,7 +102,7 @@ jobs:
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
|
||||
- name: Build microVM artifacts (lb-agent + kernel + initramfs)
|
||||
- name: Build microVM artifacts (lb-agent + kernel + initramfs + qboot)
|
||||
if: steps.version.outputs.exists != 'true'
|
||||
run: cd spinifex && make build-microvm-image
|
||||
|
||||
@@ -144,6 +144,7 @@ jobs:
|
||||
--notes-file "$RUNNER_TEMP/notes.md" \
|
||||
"dist/spinifex-$TAG-linux-amd64.tar.gz" \
|
||||
"dist/spinifex-$TAG-linux-amd64.tar.gz.sha256" \
|
||||
dist/qboot-*.tar.gz \
|
||||
"scripts/setup.sh"
|
||||
|
||||
# The assertion that matters. A prerelease must never become the latest
|
||||
|
||||
@@ -10,7 +10,7 @@ on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
cells:
|
||||
description: "Comma-separated cell numbers to run, no spaces (blank = all). 1-3, 8-10, 17-19 tofu permutation matrix, 20 baremetal, 21-22 ISO, 23 predastore throughput, 24 predastore fault injection, 25-27 service suites (eks, ecs, rds), 28 guest disk performance, 29 storage fault injection, 30 routed NAT on three nodes, 31 instance recovery on three nodes, 32 load balancer recovery on three nodes, 33 KV bucket quorum under node loss on three nodes"
|
||||
description: "Comma-separated cell numbers to run, no spaces (blank = all). 1-3, 8-10, 17-19 tofu permutation matrix, 34-36 Oracle Linux single node (basic, reboot, routed NAT), 20 baremetal, 21-22 ISO, 23 predastore throughput, 24 predastore fault injection, 25-27 service suites (eks, ecs, rds), 28 guest disk performance, 29 storage fault injection, 30 routed NAT on three nodes, 31 instance recovery on three nodes, 32 load balancer recovery on three nodes, 33 KV bucket quorum under node loss on three nodes"
|
||||
required: false
|
||||
|
||||
permissions:
|
||||
@@ -33,6 +33,7 @@ concurrency:
|
||||
env:
|
||||
GIT_BRANCH: ${{ github.ref_name }}
|
||||
UBUNTU_GOLD_IMAGE: /mnt/disk1/gold-images/spinifex-image-builder-ubuntu-26.04.qcow2
|
||||
ORACLE_GOLD_IMAGE: /mnt/disk1/gold-images/spinifex-image-builder-oracle-10.qcow2
|
||||
SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
|
||||
SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}
|
||||
|
||||
@@ -195,7 +196,8 @@ jobs:
|
||||
# back onto the debian gold.
|
||||
TOFU_VAR_ARGS=""
|
||||
case "$MATRIX_HOST" in
|
||||
ubuntu-*) TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}" ;;
|
||||
ubuntu-*) TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE} -var=host_os=ubuntu" ;;
|
||||
oracle-*) TOFU_VAR_ARGS="-var=host_image_override=${ORACLE_GOLD_IMAGE} -var=host_os=oracle" ;;
|
||||
esac
|
||||
|
||||
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
|
||||
@@ -687,7 +689,7 @@ jobs:
|
||||
'echo "host: $(hostname)"; echo "loadavg: $(cat /proc/loadavg)";
|
||||
grep -E "^(MemTotal|MemAvailable|SwapFree):" /proc/meminfo;
|
||||
echo "cpu(steal is field 8): $(grep "^cpu " /proc/stat)";
|
||||
echo "running qemu: $(pgrep -c qemu-system-x86 || echo 0)"' 2>&1
|
||||
echo "running qemu: $(pgrep -c "qemu-system-x86|qemu-kvm" || echo 0)"' 2>&1
|
||||
} >> "${ARTIFACT_DIR}/node-load.txt"
|
||||
exit 0
|
||||
|
||||
@@ -858,7 +860,8 @@ jobs:
|
||||
# different image path than apply did.
|
||||
TOFU_VAR_ARGS=""
|
||||
case "$MATRIX_HOST" in
|
||||
ubuntu-*) TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}" ;;
|
||||
ubuntu-*) TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE} -var=host_os=ubuntu" ;;
|
||||
oracle-*) TOFU_VAR_ARGS="-var=host_image_override=${ORACLE_GOLD_IMAGE} -var=host_os=oracle" ;;
|
||||
esac
|
||||
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
|
||||
tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
|
||||
|
||||
@@ -57,7 +57,7 @@ jobs:
|
||||
go-version-file: spinifex/go.mod
|
||||
cache: false
|
||||
|
||||
- name: Build microVM artifacts (lb-agent + kernel + initramfs)
|
||||
- name: Build microVM artifacts (lb-agent + kernel + initramfs + qboot)
|
||||
run: cd spinifex && make build-microvm-image
|
||||
|
||||
- name: Build amd64 Tarball
|
||||
@@ -72,6 +72,7 @@ jobs:
|
||||
path: |
|
||||
spinifex/dist/spinifex-${{ github.ref_name }}-linux-amd64.tar.gz
|
||||
spinifex/dist/spinifex-${{ github.ref_name }}-linux-amd64.tar.gz.sha256
|
||||
spinifex/dist/qboot-*.tar.gz
|
||||
spinifex/scripts/setup.sh
|
||||
|
||||
distro-arm64:
|
||||
|
||||
@@ -19,6 +19,9 @@ bin/
|
||||
dist/
|
||||
build/microvm/vmlinuz
|
||||
build/microvm/initramfs.cpio.gz
|
||||
build/microvm/qboot.rom
|
||||
build/microvm/qboot.LICENSE
|
||||
build/qboot/
|
||||
!spinifex/runtime/roles/spinifexui/frontend/dist/
|
||||
tmp/
|
||||
*.test
|
||||
|
||||
@@ -140,12 +140,23 @@ $(MICROVM_ARTIFACTS) &: $(MICROVM_INPUTS)
|
||||
bin/lb-agent:
|
||||
$(MAKE) build-lb-agent
|
||||
|
||||
build-microvm-image: $(MICROVM_ARTIFACTS) ## Build microVM kernel + initramfs (incremental — skips when up to date)
|
||||
# qboot is the microvm firmware. The source tarball ships with each release
|
||||
# because qboot is GPL-2.0.
|
||||
QBOOT_COMMIT := $(shell sed -n 's/^QBOOT_COMMIT="\(.*\)"$$/\1/p' scripts/build-qboot.sh)
|
||||
QBOOT_SOURCE := build/qboot/qboot-$(QBOOT_COMMIT).tar.gz
|
||||
QBOOT_ARTIFACTS := $(MICROVM_OUT_DIR)/qboot.rom $(MICROVM_OUT_DIR)/qboot.LICENSE $(QBOOT_SOURCE)
|
||||
|
||||
install-microvm: $(MICROVM_ARTIFACTS) ## Install microVM artifacts to /usr/share/spinifex/microvm/
|
||||
$(QBOOT_ARTIFACTS) &: scripts/build-qboot.sh
|
||||
./scripts/build-qboot.sh
|
||||
|
||||
build-microvm-image: $(MICROVM_ARTIFACTS) $(QBOOT_ARTIFACTS) ## Build microVM kernel, initramfs and qboot firmware (incremental — skips when up to date)
|
||||
|
||||
install-microvm: $(MICROVM_ARTIFACTS) $(QBOOT_ARTIFACTS) ## Install microVM artifacts to /usr/share/spinifex/microvm/
|
||||
sudo install -d /usr/share/spinifex/microvm
|
||||
sudo install -m 0644 $(MICROVM_OUT_DIR)/vmlinuz /usr/share/spinifex/microvm/vmlinuz
|
||||
sudo install -m 0644 $(MICROVM_OUT_DIR)/initramfs.cpio.gz /usr/share/spinifex/microvm/initramfs.cpio.gz
|
||||
sudo install -m 0644 $(MICROVM_OUT_DIR)/qboot.rom /usr/share/spinifex/microvm/qboot.rom
|
||||
sudo install -m 0644 $(MICROVM_OUT_DIR)/qboot.LICENSE /usr/share/spinifex/microvm/qboot.LICENSE
|
||||
|
||||
# the pre-commit gate: manifest checks, lint, vuln, and the unit and e2e-harness tiers.
|
||||
# integration and race tests skipped to keep quick, they run in CI.
|
||||
@@ -167,7 +178,7 @@ preflight:
|
||||
# reason: between them they install the software, wire OVN, erase the state and
|
||||
# remove the install, so a shellcheck warning in one is an outage or a wrong
|
||||
# deletion, not a style note.
|
||||
NODE_SCRIPTS := scripts/setup.sh scripts/setup-ovn.sh scripts/install-node.sh scripts/smoke-test.sh scripts/node-reset.sh scripts/uninstall-spx.sh
|
||||
NODE_SCRIPTS := scripts/setup.sh scripts/setup-ovn.sh scripts/install-node.sh scripts/smoke-test.sh scripts/node-reset.sh scripts/uninstall-spx.sh scripts/lib/distro.sh
|
||||
|
||||
test-build-scripts:
|
||||
@echo -e "\n....Running build/scripts/**/*_test.sh...."
|
||||
@@ -175,8 +186,8 @@ test-build-scripts:
|
||||
echo "-- $$t"; \
|
||||
sh "$$t" || exit 1; \
|
||||
done
|
||||
@echo -e "\n....Running scripts/*_test.sh...."
|
||||
@for t in $$(find scripts -maxdepth 1 -name '*_test.sh' | sort); do \
|
||||
@echo -e "\n....Running scripts/*_test.sh and scripts/lib/*_test.sh...."
|
||||
@for t in $$( (find scripts -maxdepth 1 -name '*_test.sh'; find scripts/lib -name '*_test.sh') | sort); do \
|
||||
echo "-- $$t"; \
|
||||
sh "$$t" || exit 1; \
|
||||
done
|
||||
@@ -333,14 +344,20 @@ fuzz:
|
||||
# Microvm artifacts are reinstalled when they already exist on disk — the rule's
|
||||
# input timestamps drive a rebuild only if anything actually changed. On a fresh
|
||||
# checkout (no build/microvm/vmlinuz yet) the install step is skipped; run
|
||||
# `make install-microvm` explicitly the first time.
|
||||
# `make install-microvm` explicitly the first time. They go in before the
|
||||
# preflight, which checks the microvm firmware.
|
||||
deploy: build
|
||||
sudo install -m 755 bin/spx /usr/local/bin/spx
|
||||
@if [ -f $(MICROVM_OUT_DIR)/vmlinuz ]; then \
|
||||
$(MAKE) install-microvm; \
|
||||
else \
|
||||
echo "[deploy] microvm artifacts absent — run 'make install-microvm' for first-time setup"; \
|
||||
fi
|
||||
@if [ "$${SKIP_ASSET_PREFLIGHT:-}" != "1" ]; then \
|
||||
if ! sudo /usr/local/bin/spx admin preflight; then \
|
||||
echo ""; \
|
||||
echo "[deploy] host-asset preflight failed: this node's helper scripts or sudoers"; \
|
||||
echo " grants don't match the binary just installed. Remediate with:"; \
|
||||
echo "[deploy] host-asset preflight failed: this node's helper scripts, sudoers"; \
|
||||
echo " grants, QEMU or microvm firmware don't match the binary just installed. Remediate with:"; \
|
||||
echo " scripts/update-nodes.sh (from the mulga root)"; \
|
||||
echo " or:"; \
|
||||
echo " make reinstall"; \
|
||||
@@ -350,11 +367,6 @@ deploy: build
|
||||
else \
|
||||
echo "[deploy] SKIP_ASSET_PREFLIGHT=1 — skipping host-asset preflight"; \
|
||||
fi
|
||||
@if [ -f $(MICROVM_OUT_DIR)/vmlinuz ]; then \
|
||||
$(MAKE) install-microvm; \
|
||||
else \
|
||||
echo "[deploy] microvm artifacts absent — run 'make install-microvm' for first-time setup"; \
|
||||
fi
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl restart spinifex.target
|
||||
|
||||
@@ -367,16 +379,25 @@ clean:
|
||||
rm -f ./bin/$(GO_PROJECT_NAME)
|
||||
rm -rf spinifex/runtime/roles/spinifexui/frontend/dist
|
||||
|
||||
# On dnf hosts the runtime packages and repositories come from setup.sh, so the
|
||||
# two cannot drift; nbdkit-devel, for building the plugin, is only in CodeReady.
|
||||
install-system:
|
||||
@echo -e "\n....Installing system dependencies for $(ARCH)...."
|
||||
sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \
|
||||
-o Dpkg::Options::="--force-confdef" \
|
||||
-o Dpkg::Options::="--force-confold" \
|
||||
nbdkit nbdkit-plugin-dev pkg-config qemu-system-x86 qemu-system-arm qemu-utils \
|
||||
ovmf qemu-efi-aarch64 \
|
||||
libvirt-daemon-system libvirt-clients libvirt-dev make gcc jq curl \
|
||||
iproute2 netcat-openbsd openssh-client wget git unzip sudo xz-utils file \
|
||||
ovn-central ovn-host openvswitch-switch dhcpcd-base
|
||||
@pm=$$(INSTALL_SPINIFEX_LIB_ONLY=1 bash -c '. scripts/setup.sh && distro_pkg_manager') || exit 1; \
|
||||
if [ "$$pm" = dnf ]; then \
|
||||
INSTALL_SPINIFEX_LIB_ONLY=1 bash -c '. scripts/setup.sh && setup_sudo && detect_os && install_dnf_deps' && \
|
||||
sudo dnf config-manager --set-enabled ol10_codeready_builder && \
|
||||
sudo dnf install -y nbdkit-devel pkgconf-pkg-config make gcc openssh-clients git sudo; \
|
||||
else \
|
||||
sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \
|
||||
-o Dpkg::Options::="--force-confdef" \
|
||||
-o Dpkg::Options::="--force-confold" \
|
||||
nbdkit nbdkit-plugin-dev pkg-config qemu-system-x86 qemu-system-arm qemu-utils \
|
||||
ovmf qemu-efi-aarch64 \
|
||||
libvirt-daemon-system libvirt-clients libvirt-dev make gcc jq curl \
|
||||
iproute2 netcat-openbsd openssh-client wget git unzip sudo xz-utils file \
|
||||
ovn-central ovn-host openvswitch-switch dhcpcd-base; \
|
||||
fi
|
||||
|
||||
install-go:
|
||||
@echo -e "\n....Installing Go 1.27.3 for $(ARCH) ($(GO_ARCH))...."
|
||||
@@ -438,10 +459,11 @@ distro-amd64:
|
||||
-f build/Dockerfile.distro \
|
||||
--output type=local,dest=dist/amd64/ \
|
||||
../
|
||||
@if [ -f $(MICROVM_OUT_DIR)/vmlinuz ] && [ -f $(MICROVM_OUT_DIR)/initramfs.cpio.gz ]; then \
|
||||
@if [ -f $(MICROVM_OUT_DIR)/vmlinuz ] && [ -f $(MICROVM_OUT_DIR)/initramfs.cpio.gz ] && [ -f $(MICROVM_OUT_DIR)/qboot.rom ] && [ -f $(QBOOT_SOURCE) ]; then \
|
||||
echo "[distro-amd64] staging microvm artifacts into tarball"; \
|
||||
mkdir -p dist/amd64/microvm; \
|
||||
cp $(MICROVM_OUT_DIR)/vmlinuz $(MICROVM_OUT_DIR)/initramfs.cpio.gz dist/amd64/microvm/; \
|
||||
cp $(MICROVM_OUT_DIR)/vmlinuz $(MICROVM_OUT_DIR)/initramfs.cpio.gz $(MICROVM_OUT_DIR)/qboot.rom $(MICROVM_OUT_DIR)/qboot.LICENSE dist/amd64/microvm/; \
|
||||
cp $(QBOOT_SOURCE) dist/; \
|
||||
else \
|
||||
echo "[distro-amd64] WARNING: microvm artifacts missing — tarball will not include them"; \
|
||||
echo "[distro-amd64] run 'make build-microvm-image' before 'make distro-amd64'"; \
|
||||
|
||||
@@ -10,28 +10,28 @@ type allowance struct {
|
||||
// docs/PACKAGE_BOUNDARY_MIGRATION.md. It only shrinks: delete an entry as soon
|
||||
// as its use is gone, and never add one to make a new consumer pass.
|
||||
var allowlist = []allowance{
|
||||
{File: "spinifex/runtime/daemon/daemon.go", Symbol: "daemon.LocalStatePath", Inv: "INV-01"},
|
||||
{File: "spinifex/runtime/daemon/daemon.go", Symbol: "daemon.ReadLocalState", Inv: "INV-01"},
|
||||
{File: "spinifex/runtime/daemon/daemon.go", Symbol: "daemon.MarshalLocalState", Inv: "INV-01"},
|
||||
{File: "spinifex/runtime/daemon/daemon.go", Symbol: "daemon.WriteLocalStateBytes", Inv: "INV-01"},
|
||||
{File: "spinifex/runtime/daemon/daemon.go", Symbol: "JetStreamManager.InitKVBucket", Inv: "INV-02"},
|
||||
{File: "spinifex/runtime/daemon/daemon.go", Symbol: "JetStreamManager.InitTerminatedInstanceBucket", Inv: "INV-02"},
|
||||
{File: "spinifex/runtime/daemon/daemon.go", Symbol: "JetStreamManager.WriteNodeMarkerBestEffort", Inv: "INV-03"},
|
||||
{File: "spinifex/runtime/daemon/daemon.go", Symbol: "JetStreamManager.WriteRunningSet", Inv: "INV-03"},
|
||||
{File: "spinifex/runtime/daemon/instance_record_repair.go", Symbol: "JetStreamManager.WriteRunningSet", Inv: "INV-03"},
|
||||
{File: "spinifex/runtime/daemon/daemon.go", Symbol: "JetStreamManager.ListTerminatedInstances", Inv: "INV-04"},
|
||||
{File: "spinifex/daemon/daemon.go", Symbol: "daemon.LocalStatePath", Inv: "INV-01"},
|
||||
{File: "spinifex/daemon/daemon.go", Symbol: "daemon.ReadLocalState", Inv: "INV-01"},
|
||||
{File: "spinifex/daemon/daemon.go", Symbol: "daemon.MarshalLocalState", Inv: "INV-01"},
|
||||
{File: "spinifex/daemon/daemon.go", Symbol: "daemon.WriteLocalStateBytes", Inv: "INV-01"},
|
||||
{File: "spinifex/daemon/daemon.go", Symbol: "JetStreamManager.InitKVBucket", Inv: "INV-02"},
|
||||
{File: "spinifex/daemon/daemon.go", Symbol: "JetStreamManager.InitTerminatedInstanceBucket", Inv: "INV-02"},
|
||||
{File: "spinifex/daemon/daemon.go", Symbol: "JetStreamManager.WriteNodeMarkerBestEffort", Inv: "INV-03"},
|
||||
{File: "spinifex/daemon/daemon.go", Symbol: "JetStreamManager.WriteRunningSet", Inv: "INV-03"},
|
||||
{File: "spinifex/daemon/instance_record_repair.go", Symbol: "JetStreamManager.WriteRunningSet", Inv: "INV-03"},
|
||||
{File: "spinifex/daemon/daemon.go", Symbol: "JetStreamManager.ListTerminatedInstances", Inv: "INV-04"},
|
||||
|
||||
{File: "spinifex/runtime/daemon/vm_adapters.go", Symbol: "JetStreamManager.LoadState", Inv: "INV-05"},
|
||||
{File: "spinifex/runtime/daemon/vm_adapters.go", Symbol: "JetStreamManager.WriteStoppedInstance", Inv: "INV-05"},
|
||||
{File: "spinifex/runtime/daemon/vm_adapters.go", Symbol: "JetStreamManager.LoadStoppedInstance", Inv: "INV-05"},
|
||||
{File: "spinifex/runtime/daemon/vm_adapters.go", Symbol: "JetStreamManager.DeleteStoppedInstance", Inv: "INV-05"},
|
||||
{File: "spinifex/runtime/daemon/vm_adapters.go", Symbol: "JetStreamManager.UpdateStoppedInstance", Inv: "INV-05"},
|
||||
{File: "spinifex/runtime/daemon/vm_adapters.go", Symbol: "JetStreamManager.ClaimStoppedInstance", Inv: "INV-05"},
|
||||
{File: "spinifex/runtime/daemon/vm_adapters.go", Symbol: "JetStreamManager.ListStoppedInstances", Inv: "INV-05"},
|
||||
{File: "spinifex/runtime/daemon/vm_adapters.go", Symbol: "JetStreamManager.WriteTerminatedInstance", Inv: "INV-05"},
|
||||
{File: "spinifex/runtime/daemon/vm_adapters.go", Symbol: "JetStreamManager.UpdateTerminatedInstance", Inv: "INV-05"},
|
||||
{File: "spinifex/runtime/daemon/vm_adapters.go", Symbol: "JetStreamManager.ListTerminatedInstances", Inv: "INV-05"},
|
||||
{File: "spinifex/runtime/daemon/vm_adapters.go", Symbol: "JetStreamManager.DeleteTerminatedInstance", Inv: "INV-05"},
|
||||
{File: "spinifex/daemon/vm_adapters.go", Symbol: "JetStreamManager.LoadState", Inv: "INV-05"},
|
||||
{File: "spinifex/daemon/vm_adapters.go", Symbol: "JetStreamManager.WriteStoppedInstance", Inv: "INV-05"},
|
||||
{File: "spinifex/daemon/vm_adapters.go", Symbol: "JetStreamManager.LoadStoppedInstance", Inv: "INV-05"},
|
||||
{File: "spinifex/daemon/vm_adapters.go", Symbol: "JetStreamManager.DeleteStoppedInstance", Inv: "INV-05"},
|
||||
{File: "spinifex/daemon/vm_adapters.go", Symbol: "JetStreamManager.UpdateStoppedInstance", Inv: "INV-05"},
|
||||
{File: "spinifex/daemon/vm_adapters.go", Symbol: "JetStreamManager.ClaimStoppedInstance", Inv: "INV-05"},
|
||||
{File: "spinifex/daemon/vm_adapters.go", Symbol: "JetStreamManager.ListStoppedInstances", Inv: "INV-05"},
|
||||
{File: "spinifex/daemon/vm_adapters.go", Symbol: "JetStreamManager.WriteTerminatedInstance", Inv: "INV-05"},
|
||||
{File: "spinifex/daemon/vm_adapters.go", Symbol: "JetStreamManager.UpdateTerminatedInstance", Inv: "INV-05"},
|
||||
{File: "spinifex/daemon/vm_adapters.go", Symbol: "JetStreamManager.ListTerminatedInstances", Inv: "INV-05"},
|
||||
{File: "spinifex/daemon/vm_adapters.go", Symbol: "JetStreamManager.DeleteTerminatedInstance", Inv: "INV-05"},
|
||||
|
||||
{File: "spinifex/runtime/compute/vm/migrate.go", Symbol: "JetStreamManager.WriteStoppedInstance", Inv: "INV-06"},
|
||||
{File: "spinifex/runtime/compute/vm/migrate.go", Symbol: "JetStreamManager.WriteTerminatedInstance", Inv: "INV-06"},
|
||||
@@ -50,23 +50,23 @@ var allowlist = []allowance{
|
||||
{File: "spinifex/domains/ec2/instance/service_impl.go", Symbol: "JetStreamManager.WriteTerminatedInstance", Inv: "INV-07"},
|
||||
{File: "spinifex/domains/ec2/instance/service_impl.go", Symbol: "JetStreamManager.DeleteStoppedInstance", Inv: "INV-07"},
|
||||
|
||||
{File: "spinifex/runtime/daemon/daemon_handlers_image.go", Symbol: "JetStreamManager.LoadStoppedInstance", Inv: "INV-08"},
|
||||
{File: "spinifex/runtime/daemon/daemon_handlers_image.go", Symbol: "JetStreamManager.UpdateStoppedInstance", Inv: "INV-08"},
|
||||
{File: "spinifex/daemon/daemon_handlers_image.go", Symbol: "JetStreamManager.LoadStoppedInstance", Inv: "INV-08"},
|
||||
{File: "spinifex/daemon/daemon_handlers_image.go", Symbol: "JetStreamManager.UpdateStoppedInstance", Inv: "INV-08"},
|
||||
|
||||
{File: "spinifex/runtime/daemon/instance_recovery.go", Symbol: "vm.InstanceRecord", Inv: "INV-10"},
|
||||
{File: "spinifex/runtime/daemon/instance_recovery.go", Symbol: "JetStreamManager.ListInstanceRecords", Inv: "INV-10"},
|
||||
{File: "spinifex/runtime/daemon/instance_recovery.go", Symbol: "JetStreamManager.ClaimRecoverableInstance", Inv: "INV-10"},
|
||||
{File: "spinifex/runtime/daemon/instance_recovery.go", Symbol: "JetStreamManager.ReleaseRecoveredInstance", Inv: "INV-10"},
|
||||
{File: "spinifex/runtime/daemon/instance_recovery.go", Symbol: "JetStreamManager.AbandonRecovery", Inv: "INV-10"},
|
||||
{File: "spinifex/runtime/daemon/instance_recovery.go", Symbol: "JetStreamManager.LoadInstanceRecord", Inv: "INV-10"},
|
||||
{File: "spinifex/daemon/instance_recovery.go", Symbol: "vm.InstanceRecord", Inv: "INV-10"},
|
||||
{File: "spinifex/daemon/instance_recovery.go", Symbol: "JetStreamManager.ListInstanceRecords", Inv: "INV-10"},
|
||||
{File: "spinifex/daemon/instance_recovery.go", Symbol: "JetStreamManager.ClaimRecoverableInstance", Inv: "INV-10"},
|
||||
{File: "spinifex/daemon/instance_recovery.go", Symbol: "JetStreamManager.ReleaseRecoveredInstance", Inv: "INV-10"},
|
||||
{File: "spinifex/daemon/instance_recovery.go", Symbol: "JetStreamManager.AbandonRecovery", Inv: "INV-10"},
|
||||
{File: "spinifex/daemon/instance_recovery.go", Symbol: "JetStreamManager.LoadInstanceRecord", Inv: "INV-10"},
|
||||
|
||||
{File: "spinifex/runtime/daemon/dns_reconcile.go", Symbol: "daemon.InstanceRecordPrefix", Inv: "INV-11"},
|
||||
{File: "spinifex/runtime/daemon/dns_reconcile.go", Symbol: "daemon.InstanceStateBucket", Inv: "INV-11"},
|
||||
{File: "spinifex/runtime/daemon/dns_reconcile.go", Symbol: "JetStreamManager.ListInstanceRecords", Inv: "INV-11"},
|
||||
{File: "spinifex/daemon/dns_reconcile.go", Symbol: "daemon.InstanceRecordPrefix", Inv: "INV-11"},
|
||||
{File: "spinifex/daemon/dns_reconcile.go", Symbol: "daemon.InstanceStateBucket", Inv: "INV-11"},
|
||||
{File: "spinifex/daemon/dns_reconcile.go", Symbol: "JetStreamManager.ListInstanceRecords", Inv: "INV-11"},
|
||||
|
||||
{File: "spinifex/runtime/daemon/eni_orphan_reaper.go", Symbol: "vm.InstanceRecord", Inv: "INV-12"},
|
||||
{File: "spinifex/runtime/daemon/eni_orphan_reaper.go", Symbol: "JetStreamManager.ListInstanceRecords", Inv: "INV-12"},
|
||||
{File: "spinifex/runtime/daemon/eni_orphan_reaper.go", Symbol: "JetStreamManager.ListTerminatedInstanceRecords", Inv: "INV-12"},
|
||||
{File: "spinifex/daemon/eni_orphan_reaper.go", Symbol: "vm.InstanceRecord", Inv: "INV-12"},
|
||||
{File: "spinifex/daemon/eni_orphan_reaper.go", Symbol: "JetStreamManager.ListInstanceRecords", Inv: "INV-12"},
|
||||
{File: "spinifex/daemon/eni_orphan_reaper.go", Symbol: "JetStreamManager.ListTerminatedInstanceRecords", Inv: "INV-12"},
|
||||
|
||||
{File: "spinifex/domains/ec2/instance/state/imds_readers.go", Symbol: "daemon.LocalState", Inv: "INV-13"},
|
||||
{File: "spinifex/domains/ec2/instance/state/imds_readers.go", Symbol: "daemon.LocalStatePath", Inv: "INV-13"},
|
||||
|
||||
@@ -95,7 +95,7 @@ const faultSource = `package %s
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
spxd "github.com/mulgadc/spinifex/spinifex/runtime/daemon"
|
||||
spxd "github.com/mulgadc/spinifex/spinifex/daemon"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
|
||||
)
|
||||
|
||||
@@ -176,11 +176,11 @@ func TestADR0007_S6_RatchetDetectsFaults(t *testing.T) {
|
||||
// must see those too.
|
||||
func TestADR0007_S6_RatchetSeesBareIdentifiersInOwnerPackages(t *testing.T) {
|
||||
src := map[string]string{
|
||||
"spinifex/runtime/daemon/new_consumer.go": "package daemon\n\nfunc f() string { return InstanceStateBucket + LocalStatePath(\"\") }\n",
|
||||
"spinifex/daemon/new_consumer.go": "package daemon\n\nfunc f() string { return InstanceStateBucket + LocalStatePath(\"\") }\n",
|
||||
"spinifex/runtime/compute/vm/new_reader.go": "package vm\n\nfunc g(r *InstanceRecord) *VM { return VMFromRecord(r) }\n",
|
||||
}
|
||||
want := map[string][]string{
|
||||
"spinifex/runtime/daemon/new_consumer.go": {"daemon.InstanceStateBucket", "daemon.LocalStatePath"},
|
||||
"spinifex/daemon/new_consumer.go": {"daemon.InstanceStateBucket", "daemon.LocalStatePath"},
|
||||
"spinifex/runtime/compute/vm/new_reader.go": {"vm.InstanceRecord", "vm.VMFromRecord"},
|
||||
}
|
||||
for rel, code := range src {
|
||||
|
||||
@@ -20,9 +20,9 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
daemonPath = "github.com/mulgadc/spinifex/spinifex/runtime/daemon"
|
||||
daemonPath = "github.com/mulgadc/spinifex/spinifex/daemon"
|
||||
vmPath = "github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
|
||||
daemonDir = "spinifex/runtime/daemon"
|
||||
daemonDir = "spinifex/daemon"
|
||||
vmDir = "spinifex/runtime/compute/vm"
|
||||
)
|
||||
|
||||
@@ -69,12 +69,12 @@ var literals = set(`"spinifex-instance-state"`, `"spinifex-terminated-instances"
|
||||
// ownerFiles define the symbols above. They are the implementation being
|
||||
// migrated, not consumers of it, and are listed so a deleted one is noticed.
|
||||
var ownerFiles = []string{
|
||||
"spinifex/runtime/daemon/instance_membership.go",
|
||||
"spinifex/runtime/daemon/instance_records.go",
|
||||
"spinifex/runtime/daemon/instance_records_migrate.go",
|
||||
"spinifex/runtime/daemon/instance_running_set.go",
|
||||
"spinifex/runtime/daemon/jetstream.go",
|
||||
"spinifex/runtime/daemon/local_state.go",
|
||||
"spinifex/daemon/instance_membership.go",
|
||||
"spinifex/daemon/instance_records.go",
|
||||
"spinifex/daemon/instance_records_migrate.go",
|
||||
"spinifex/daemon/instance_running_set.go",
|
||||
"spinifex/daemon/jetstream.go",
|
||||
"spinifex/daemon/local_state.go",
|
||||
"spinifex/runtime/compute/vm/record.go",
|
||||
}
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ var allowlist = []allowance{
|
||||
{From: "accountteardown", To: "domains/ec2/awsapi/volume", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
|
||||
{From: "accountteardown", To: "domains/ec2/awsapi/vpc", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
|
||||
|
||||
{From: "agents/ecs/agent", To: "domains/ecs", Debt: "agents/ecs/agent", Reason: "assignment/state wire types from the ECS implementation; needs a guest/controller contract"},
|
||||
{From: "agents/ecs/agent", To: "handlers/ecs", Debt: "agents/ecs/agent", Reason: "assignment/state wire types from the ECS implementation; needs a guest/controller contract"},
|
||||
|
||||
{From: "agents/eks/tokenwebhook", To: "domains/eks/cluster", Debt: "agents/eks/tokenwebhook", Reason: "WebhookTokenReviewResult from the EKS implementation; needs an EKS contract or projection"},
|
||||
|
||||
@@ -32,7 +32,7 @@ var allowlist = []allowance{
|
||||
{From: "cmd/aws-model-coverage", To: "domains/acm/awsapi", Debt: "cmd/aws-model-coverage", Reason: "reads declared ECR/ACM inventories; named exception, must not construct a registration"},
|
||||
{From: "cmd/aws-model-coverage", To: "domains/ecr/awsapi", Debt: "cmd/aws-model-coverage", Reason: "reads declared ECR/ACM inventories; named exception, must not construct a registration"},
|
||||
|
||||
{From: "runtime/daemon", To: "domains/ec2/awsapi/instance", Debt: "Instance adapter consumers outside EC2", Reason: "runtime/daemon/eks_cp_control.go and rds_deps.go call the instance adapter; needs an EC2 capability"},
|
||||
{From: "daemon", To: "domains/ec2/awsapi/instance", Debt: "Instance adapter consumers outside EC2", Reason: "daemon/eks_cp_control.go and rds_deps.go call the instance adapter; needs an EC2 capability"},
|
||||
|
||||
{From: "domains/acm", To: "domains/iam", Debt: "ACM: the IAM AES-GCM helper import (ACM relocation residual)", Reason: "IAM AES-GCM helpers; needs a shared crypto boundary"},
|
||||
|
||||
@@ -66,7 +66,7 @@ var allowlist = []allowance{
|
||||
{From: "domains/ec2/instance", To: "domains/network/topology", Debt: "EC2 → network implementation", Reason: "EC2 reads network implementation; clears with the EC2-owned projection"},
|
||||
{From: "domains/ec2/instance", To: "runtime/compute/gpu", Debt: "domains/ec2/instance", Reason: "VM/GPU runtime state read directly; needs a domain-owned projection"},
|
||||
{From: "domains/ec2/instance", To: "runtime/compute/vm", Debt: "domains/ec2/instance", Reason: "VM/GPU runtime state read directly; needs a domain-owned projection"},
|
||||
{From: "domains/ec2/instance/state", To: "runtime/daemon", Debt: "domains/ec2/instance/state (dependency, IMDS readers)", Reason: "wraps the daemon's local-file reader and JetStreamManager bucket open unchanged for vpcd's IMDS lookup"},
|
||||
{From: "domains/ec2/instance/state", To: "daemon", Debt: "domains/ec2/instance/state (dependency, IMDS readers)", Reason: "wraps the daemon's local-file reader and JetStreamManager bucket open unchanged for vpcd's IMDS lookup"},
|
||||
{From: "domains/ec2/instance/state", To: "runtime/compute/vm", Debt: "domains/ec2/instance/state (dependency)", Reason: "the vpcd IMDS local reader returns vm.VM"},
|
||||
|
||||
{From: "domains/ec2/volume", To: "runtime/compute/vm", Debt: "Domain → runtime", Reason: "VM runtime state read directly; needs a domain-owned projection"},
|
||||
@@ -89,9 +89,8 @@ var allowlist = []allowance{
|
||||
{From: "domains/network/reconcile", To: "runtime/compute/vm", Debt: "domains/network", Reason: "domain-to-runtime import"},
|
||||
|
||||
{From: "domains/ochre", To: "domains/ec2/systeminstance", Debt: "Cross-domain", Reason: "consumes another domain's implementation rather than a capability"},
|
||||
{From: "domains/ochre", To: "domains/iam", Debt: "IAM implementation consumers in STS, Ochre and ELBv2", Reason: "secret encrypt/decrypt helpers; needs a shared crypto boundary"},
|
||||
{From: "domains/ochre", To: "domains/network/systemvpc", Debt: "Cross-domain", Reason: "consumes another domain's implementation rather than a capability"},
|
||||
{From: "domains/ochre", To: "ingress/aws/bodyscope", Debt: "domains/ochre (action adapters)", Reason: "Bedrock action adapters still live in the Ochre package; they move to domains/ochre/awsapi"},
|
||||
{From: "domains/ochre", To: "gateway/bedrock", Debt: "gateway/bedrock", Reason: "Ochre domain core still lives in gateway/bedrock; needs the recorded split"},
|
||||
{From: "domains/ochre", To: "runtime/compute/gpu", Debt: "Domain → runtime", Reason: "GPU runtime state read directly; needs a domain-owned projection"},
|
||||
|
||||
{From: "domains/ochre/vector", To: "domains/iam", Debt: "Cross-domain", Reason: "consumes another domain's implementation rather than a capability"},
|
||||
@@ -117,6 +116,8 @@ var allowlist = []allowance{
|
||||
{From: "gateway", To: "domains/ec2/awsapi/vpc", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
|
||||
{From: "gateway", To: "domains/ec2/awsapi/zone", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
|
||||
|
||||
{From: "gateway/bedrock", To: "domains/iam", Debt: "IAM implementation consumers in STS, Ochre and ELBv2", Reason: "secret encrypt/decrypt helpers; needs a shared crypto boundary"},
|
||||
|
||||
{From: "domains/rds/awsapi", To: "domains/ec2/awsapi/instance", Debt: "Instance adapter consumers outside EC2", Reason: "enginecatalog.go calls the instance adapter; needs an EC2 capability"},
|
||||
|
||||
{From: "domains/sts/awsapi", To: "domains/iam", Debt: "IAM implementation consumers in STS, Ochre and ELBv2", Reason: "GetCallerIdentity takes IAMService; needs an IAM capability"},
|
||||
@@ -125,11 +126,11 @@ var allowlist = []allowance{
|
||||
{From: "gateway", To: "domains/iam/awsapi", Debt: "gateway (service registration)", Reason: "IAM, STS, EKS and RDS dispatch stays in gateway until the ingress registration seam"},
|
||||
{From: "gateway", To: "domains/rds/awsapi", Debt: "gateway (service registration)", Reason: "IAM, STS, EKS and RDS dispatch stays in gateway until the ingress registration seam"},
|
||||
{From: "gateway", To: "domains/sts/awsapi", Debt: "gateway (service registration)", Reason: "IAM, STS, EKS and RDS dispatch stays in gateway until the ingress registration seam"},
|
||||
{From: "domains/ecs", To: "domains/ec2/ebs/metadata", Debt: "close-out: handlers/ecs (ADR-0004 blocked-dependency list; inventory-ecs boundary imports)", Reason: "legacy ECS calls another domain's implementation as an internal API"},
|
||||
{From: "domains/ecs", To: "domains/ec2/eip", Debt: "close-out: handlers/ecs (ADR-0004 blocked-dependency list; inventory-ecs boundary imports)", Reason: "legacy ECS calls another domain's implementation as an internal API"},
|
||||
{From: "domains/ecs", To: "domains/ec2/instancetypes", Debt: "close-out: handlers/ecs (ADR-0004 blocked-dependency list; inventory-ecs boundary imports)", Reason: "legacy ECS calls another domain's implementation as an internal API"},
|
||||
{From: "domains/ecs", To: "domains/elbv2", Debt: "close-out: handlers/ecs (ADR-0004 blocked-dependency list; inventory-ecs boundary imports)", Reason: "legacy ECS calls another domain's implementation as an internal API"},
|
||||
{From: "domains/ecs", To: "domains/iam", Debt: "close-out: handlers/ecs (ADR-0004 blocked-dependency list; inventory-ecs boundary imports)", Reason: "legacy ECS calls another domain's implementation as an internal API"},
|
||||
{From: "handlers/ecs", To: "domains/ec2/ebs/metadata", Debt: "close-out: handlers/ecs (ADR-0004 blocked-dependency list; inventory-ecs boundary imports)", Reason: "legacy ECS calls another domain's implementation as an internal API"},
|
||||
{From: "handlers/ecs", To: "domains/ec2/eip", Debt: "close-out: handlers/ecs (ADR-0004 blocked-dependency list; inventory-ecs boundary imports)", Reason: "legacy ECS calls another domain's implementation as an internal API"},
|
||||
{From: "handlers/ecs", To: "domains/ec2/instancetypes", Debt: "close-out: handlers/ecs (ADR-0004 blocked-dependency list; inventory-ecs boundary imports)", Reason: "legacy ECS calls another domain's implementation as an internal API"},
|
||||
{From: "handlers/ecs", To: "handlers/elbv2", Debt: "close-out: handlers/ecs (ADR-0004 blocked-dependency list; inventory-ecs boundary imports)", Reason: "legacy ECS calls another domain's implementation as an internal API"},
|
||||
{From: "handlers/ecs", To: "domains/iam", Debt: "close-out: handlers/ecs (ADR-0004 blocked-dependency list; inventory-ecs boundary imports)", Reason: "legacy ECS calls another domain's implementation as an internal API"},
|
||||
|
||||
{From: "domains/eks/cluster", To: "domains/dns", Debt: "close-out: handlers/eks (ADR-0004 blocked-dependency list; inventory-eks boundary imports)", Reason: "legacy EKS calls another domain's or runtime implementation directly"},
|
||||
{From: "domains/eks/cluster", To: "domains/ec2/ebs/metadata", Debt: "close-out: handlers/eks (ADR-0004 blocked-dependency list; inventory-eks boundary imports)", Reason: "legacy EKS calls another domain's or runtime implementation directly"},
|
||||
@@ -140,13 +141,13 @@ var allowlist = []allowance{
|
||||
{From: "domains/eks/cluster", To: "domains/iam", Debt: "close-out: handlers/eks (ADR-0004 blocked-dependency list; inventory-eks boundary imports)", Reason: "legacy EKS calls another domain's or runtime implementation directly"},
|
||||
{From: "domains/eks/cluster", To: "runtime/compute/vm", Debt: "close-out: handlers/eks (ADR-0004 blocked-dependency list; inventory-eks boundary imports)", Reason: "legacy EKS calls another domain's or runtime implementation directly"},
|
||||
|
||||
{From: "domains/elbv2", To: "domains/acm", Debt: "domains/elbv2 (cross-domain)", Reason: "ELBv2 cross-domain import; clears when ELBv2 is extracted behind consumer-owned capabilities"},
|
||||
{From: "domains/elbv2", To: "domains/dns", Debt: "domains/elbv2 (cross-domain)", Reason: "ELBv2 cross-domain import; clears when ELBv2 is extracted behind consumer-owned capabilities"},
|
||||
{From: "domains/elbv2", To: "domains/ec2/systeminstance", Debt: "domains/elbv2 (cross-domain)", Reason: "ELBv2 cross-domain import; clears when ELBv2 is extracted behind consumer-owned capabilities"},
|
||||
{From: "domains/elbv2", To: "domains/ec2/vpc", Debt: "domains/elbv2 (cross-domain)", Reason: "ELBv2 cross-domain import; clears when ELBv2 is extracted behind consumer-owned capabilities"},
|
||||
{From: "domains/elbv2", To: "domains/network/topology", Debt: "domains/elbv2 (cross-domain)", Reason: "ELBv2 cross-domain import; clears when ELBv2 is extracted behind consumer-owned capabilities"},
|
||||
{From: "domains/elbv2", To: "domains/iam", Debt: "IAM implementation consumers in STS, Ochre and ELBv2", Reason: "system-instance role ensurer; needs an IAM capability"},
|
||||
{From: "domains/elbv2", To: "agents/elbv2", Debt: "domains/elbv2/{agent_types,haproxy,health_checker,nginx}.go import lbagent", Reason: "health-report and cert/PID-path vocabulary; needs a guest/controller contract"},
|
||||
{From: "handlers/elbv2", To: "domains/acm", Debt: "handlers/elbv2 (cross-domain)", Reason: "legacy ELBv2 cross-domain import; clears when ELBv2 is extracted behind consumer-owned capabilities"},
|
||||
{From: "handlers/elbv2", To: "domains/dns", Debt: "handlers/elbv2 (cross-domain)", Reason: "legacy ELBv2 cross-domain import; clears when ELBv2 is extracted behind consumer-owned capabilities"},
|
||||
{From: "handlers/elbv2", To: "domains/ec2/systeminstance", Debt: "handlers/elbv2 (cross-domain)", Reason: "legacy ELBv2 cross-domain import; clears when ELBv2 is extracted behind consumer-owned capabilities"},
|
||||
{From: "handlers/elbv2", To: "domains/ec2/vpc", Debt: "handlers/elbv2 (cross-domain)", Reason: "legacy ELBv2 cross-domain import; clears when ELBv2 is extracted behind consumer-owned capabilities"},
|
||||
{From: "handlers/elbv2", To: "domains/network/topology", Debt: "handlers/elbv2 (cross-domain)", Reason: "legacy ELBv2 cross-domain import; clears when ELBv2 is extracted behind consumer-owned capabilities"},
|
||||
{From: "handlers/elbv2", To: "domains/iam", Debt: "IAM implementation consumers in STS, Ochre and ELBv2", Reason: "system-instance role ensurer; needs an IAM capability"},
|
||||
{From: "handlers/elbv2", To: "lbagent", Debt: "handlers/elbv2/{agent_types,haproxy,health_checker,nginx}.go import lbagent", Reason: "health-report and cert/PID-path vocabulary; needs a guest/controller contract"},
|
||||
|
||||
{From: "domains/rds/instance", To: "domains/dns", Debt: "close-out: handlers/rds (ADR-0004 blocked-dependency list; inventory-rds boundary imports)", Reason: "legacy RDS calls another domain's or runtime implementation directly"},
|
||||
{From: "domains/rds/instance", To: "domains/ec2/instance", Debt: "close-out: handlers/rds (ADR-0004 blocked-dependency list; inventory-rds boundary imports)", Reason: "legacy RDS calls another domain's or runtime implementation directly"},
|
||||
@@ -159,5 +160,7 @@ var allowlist = []allowance{
|
||||
{From: "domains/sts", To: "domains/iam", Debt: "IAM implementation consumers in STS, Ochre and ELBv2", Reason: "STS reads IAM service, trust-policy and OIDC-provider state; needs an IAM capability"},
|
||||
|
||||
{From: "operator/cli", To: "accountteardown", Debt: "operator/cli (dependency)", Reason: "command tree imports implementations; needs public capabilities"},
|
||||
{From: "operator/cli", To: "daemon", Debt: "operator/cli (dependency)", Reason: "command tree imports implementations; needs public capabilities"},
|
||||
{From: "operator/cli", To: "gateway", Debt: "operator/cli (dependency)", Reason: "command tree imports implementations; needs public capabilities"},
|
||||
{From: "operator/cli", To: "gateway/bedrock", Debt: "operator/cli (dependency)", Reason: "command tree imports implementations; needs public capabilities"},
|
||||
}
|
||||
|
||||
@@ -191,25 +191,3 @@ func TestADR0001_LayeringCheckerDetectsFaults(t *testing.T) {
|
||||
t.Errorf("allow-listed cross-domain edge %s was not detected by R3, so the allowlist entry would read as stale", allowedEdge)
|
||||
}
|
||||
}
|
||||
|
||||
// TestADR0001_InterimDaemonDirectionCoversRuntimeDaemon: the daemon left its
|
||||
// legacy root, so gateway importing runtime/daemon must still be rejected, and
|
||||
// the former handlers packages are held to it by R5 at their domain homes.
|
||||
func TestADR0001_InterimDaemonDirectionCoversRuntimeDaemon(t *testing.T) {
|
||||
daemon, _ := classify("spinifex/runtime/daemon")
|
||||
checks := []struct{ rule, from string }{
|
||||
{"Interim_DaemonGatewayHandlersDirection", "spinifex/gateway"},
|
||||
{"R5_DomainImportsNoRuntimeOrOperator", "spinifex/domains/ecs"},
|
||||
}
|
||||
for _, c := range checks {
|
||||
r := ruleByID(c.rule)
|
||||
f, _ := classify(c.from)
|
||||
if !r.subject(f, c.from) || !r.forbids(f, daemon, c.from) {
|
||||
t.Errorf("%s -> runtime/daemon is not rejected by %s", c.from, c.rule)
|
||||
}
|
||||
}
|
||||
other, _ := classify("spinifex/runtime/host/process")
|
||||
if gw, _ := classify("spinifex/gateway"); ruleByID("Interim_DaemonGatewayHandlersDirection").forbids(gw, other, "spinifex/gateway") {
|
||||
t.Error("gateway -> runtime/host/process is rejected, but it is not the daemon")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -42,7 +42,6 @@ type class struct {
|
||||
legacy string // legacy root name when layer is layerLegacy
|
||||
domain string // owning domain for domain-shaped and agent packages
|
||||
kind domainKind
|
||||
daemon bool // runtime/daemon, the interim daemon -> gateway -> handlers root
|
||||
}
|
||||
|
||||
func (c class) domainShaped() bool { return c.kind != notDomain }
|
||||
@@ -60,8 +59,8 @@ func (c class) isLegacy(roots ...string) bool {
|
||||
|
||||
// legacyRoots are pre-ADR-0001 application roots still standing.
|
||||
var legacyRoots = map[string]bool{
|
||||
"gateway": true,
|
||||
"accountteardown": true,
|
||||
"handlers": true, "gateway": true, "daemon": true,
|
||||
"accountteardown": true, "lbagent": true,
|
||||
}
|
||||
|
||||
// gatewayDomains names the domain each gateway/<service> adapter belongs to;
|
||||
@@ -71,8 +70,12 @@ var gatewayDomains = map[string]string{
|
||||
"rds": "rds", "sts": "sts",
|
||||
}
|
||||
|
||||
// operatorLegacy are the legacy roots whose ADR-0001 target is operator/.
|
||||
var operatorLegacy = []string{"accountteardown"}
|
||||
// runtimeLegacy and operatorLegacy are the legacy roots whose ADR-0001 target
|
||||
// is runtime/ and operator/ respectively.
|
||||
var (
|
||||
runtimeLegacy = []string{"daemon"}
|
||||
operatorLegacy = []string{"accountteardown"}
|
||||
)
|
||||
|
||||
// classify maps a module-relative package directory to its layer. ok is false
|
||||
// for a package that has no deliberate classification.
|
||||
@@ -109,7 +112,7 @@ func classifyApp(app []string) (class, bool) {
|
||||
case "ingress":
|
||||
return class{layer: layerIngress}, true
|
||||
case "runtime":
|
||||
return class{layer: layerRuntime, daemon: len(app) >= 2 && app[1] == "daemon"}, true
|
||||
return class{layer: layerRuntime}, true
|
||||
case "operator":
|
||||
return class{layer: layerOperator}, true
|
||||
case "providers":
|
||||
|
||||
@@ -11,15 +11,20 @@ import (
|
||||
// completed move whose line must be deleted.
|
||||
var frozenLegacyPackages = []string{
|
||||
"accountteardown",
|
||||
"daemon",
|
||||
"gateway",
|
||||
"gateway/bedrock",
|
||||
"gateway/ecs",
|
||||
"gateway/elbv2",
|
||||
"gateway/spx",
|
||||
"handlers/ecs",
|
||||
"handlers/elbv2",
|
||||
"lbagent",
|
||||
}
|
||||
|
||||
// retiredLegacyRoots were legacy roots whose every package has moved to its
|
||||
// ADR-0001 target; recreating one would reopen a closed migration row.
|
||||
var retiredLegacyRoots = []string{"admin", "daemon", "handlers", "lbagent", "services", "utils", "vpcd"}
|
||||
var retiredLegacyRoots = []string{"admin", "services", "utils", "vpcd"}
|
||||
|
||||
// TestADR0001_LegacyRootsFrozen: the current-to-target map gives every legacy
|
||||
// root a destination, so none may gain a package and no retired root may return.
|
||||
|
||||
@@ -73,7 +73,7 @@ var rules = []rule{
|
||||
subject: func(f class, _ string) bool { return f.domainShaped() },
|
||||
forbids: func(f, t class, _ string) bool {
|
||||
return f.domainShaped() && (isLayer(t, layerRuntime, layerOperator) ||
|
||||
t.isLegacy(operatorLegacy...))
|
||||
t.isLegacy(runtimeLegacy...) || t.isLegacy(operatorLegacy...))
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -121,7 +121,7 @@ var rules = []rule{
|
||||
return false
|
||||
}
|
||||
switch {
|
||||
case t.layer == layerAgents:
|
||||
case t.layer == layerAgents || t.isLegacy("lbagent"):
|
||||
return true
|
||||
case t.layer == layerIngress:
|
||||
return f.kind != domainAWSAPI && f.kind != legacyAdapter
|
||||
@@ -137,9 +137,9 @@ var rules = []rule{
|
||||
id: "Agents_ImportNoDomainOrLegacyImplementation",
|
||||
clause: "area table: agents/<domain> own in-guest and service-role behaviour, not tenant API authority or another domain's private state",
|
||||
hint: "move the shared wire vocabulary behind a guest/controller contract under contracts/",
|
||||
subject: func(f class, _ string) bool { return f.layer == layerAgents },
|
||||
subject: func(f class, _ string) bool { return f.layer == layerAgents || f.isLegacy("lbagent") },
|
||||
forbids: func(f, t class, _ string) bool {
|
||||
return f.layer == layerAgents && (t.domainShaped() || t.isLegacy())
|
||||
return (f.layer == layerAgents || f.isLegacy("lbagent")) && (t.domainShaped() || t.isLegacy())
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -160,9 +160,9 @@ var rules = []rule{
|
||||
forbids: func(f, t class, _ string) bool {
|
||||
switch {
|
||||
case f.isLegacy("handlers"):
|
||||
return t.isLegacy("gateway") || t.daemon
|
||||
return t.isLegacy("gateway", "daemon")
|
||||
case f.isLegacy("gateway"):
|
||||
return t.daemon
|
||||
return t.isLegacy("daemon")
|
||||
}
|
||||
return false
|
||||
},
|
||||
|
||||
@@ -42,9 +42,16 @@ RUN COMMIT=$(cd spinifex && git rev-parse --short HEAD 2>/dev/null || echo "unkn
|
||||
# Build nbdkit viperblock plugin (.so)
|
||||
RUN cd viperblock && GOFIPS140=v1.0.0 go build -o /out/nbdkit-viperblock-plugin.so -buildmode=c-shared nbd/viperblock.go
|
||||
|
||||
# One tarball ships to every OS. The ceiling is what both binaries need today,
|
||||
# so a rise is a decision rather than an accident; every supported OS has 2.39+.
|
||||
RUN spinifex/scripts/check-glibc-ceiling.sh 2.34 /out/spx /out/nbdkit-viperblock-plugin.so
|
||||
|
||||
# Copy setup scripts and systemd units
|
||||
RUN cp spinifex/scripts/setup.sh /out/setup.sh 2>/dev/null || true
|
||||
RUN cp spinifex/scripts/setup-ovn.sh /out/setup-ovn.sh
|
||||
RUN cd spinifex/scripts && cp install-node.sh smoke-test.sh node-reset.sh uninstall-spx.sh /out/
|
||||
RUN mkdir -p /out/udev && cp spinifex/build/udev/* /out/udev/
|
||||
RUN mkdir -p /out/lib && cp spinifex/scripts/lib/distro.sh /out/lib/distro.sh
|
||||
RUN mkdir -p /out/systemd && cp spinifex/build/systemd/* /out/systemd/
|
||||
RUN mkdir -p /out/scripts && cp spinifex/build/scripts/* /out/scripts/
|
||||
RUN cp spinifex/build/logrotate/spinifex /out/logrotate-spinifex
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
# spinifex-unit-version: 3
|
||||
# spinifex-unit-version: 5
|
||||
[Unit]
|
||||
Description=Spinifex Daemon (VM Orchestration)
|
||||
PartOf=spinifex.target
|
||||
After=spinifex-nats.service spinifex-viperblock.service openvswitch-switch.service ovn-controller.service openvswitch-ipsec.service
|
||||
# Wants (not Requires): stay bootable when openvswitch-ipsec is absent
|
||||
Wants=openvswitch-ipsec.service
|
||||
After=spinifex-nats.service spinifex-viperblock.service ovsdb-server.service ovs-vswitchd.service ovn-controller.service openvswitch-ipsec.service spinifex-hostnet.service
|
||||
# Wants (not Requires): boot without openvswitch-ipsec, survive OVS restarts via hostnet
|
||||
Wants=openvswitch-ipsec.service spinifex-hostnet.service
|
||||
|
||||
[Service]
|
||||
EnvironmentFile=-/etc/spinifex/telemetry.env
|
||||
@@ -14,6 +14,7 @@ Type=simple
|
||||
Slice=spinifex-system.slice
|
||||
User=spinifex-daemon
|
||||
Group=spinifex
|
||||
ExecStartPre=/usr/local/bin/spx admin hostnet check
|
||||
ExecStart=/usr/local/bin/spx service spinifex start
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
# spinifex-unit-version: 1
|
||||
[Unit]
|
||||
Description=Spinifex host network wiring (veth pairs, mgmt address, encap route)
|
||||
# PartOf/WantedBy ovs-vswitchd: re-applied whenever OVS starts; daemon and vpcd only Want it
|
||||
Requires=ovs-vswitchd.service
|
||||
After=ovs-vswitchd.service NetworkManager-wait-online.service
|
||||
PartOf=ovs-vswitchd.service
|
||||
Before=spinifex.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
ExecStart=/usr/local/bin/spx admin hostnet apply
|
||||
# Covers the 120s wait behind setup-ovn.sh plus the 60s wait for links
|
||||
TimeoutStartSec=240
|
||||
TimeoutStopSec=10
|
||||
ProtectSystem=full
|
||||
ProtectHome=yes
|
||||
PrivateTmp=yes
|
||||
|
||||
[Install]
|
||||
WantedBy=spinifex.target ovs-vswitchd.service
|
||||
@@ -1,9 +1,9 @@
|
||||
# spinifex-unit-version: 1
|
||||
# spinifex-unit-version: 3
|
||||
[Unit]
|
||||
Description=Spinifex VPC Daemon (OVN Networking)
|
||||
PartOf=spinifex.target
|
||||
After=spinifex-nats.service spinifex-daemon.service systemd-networkd.service openvswitch-switch.service ovn-controller.service
|
||||
Wants=spinifex-nats.service ovn-controller.service
|
||||
After=spinifex-nats.service spinifex-daemon.service systemd-networkd.service ovsdb-server.service ovs-vswitchd.service ovn-controller.service spinifex-hostnet.service
|
||||
Wants=spinifex-nats.service ovn-controller.service spinifex-hostnet.service
|
||||
|
||||
[Service]
|
||||
EnvironmentFile=-/etc/spinifex/telemetry.env
|
||||
@@ -11,6 +11,7 @@ Type=simple
|
||||
Slice=spinifex-system.slice
|
||||
User=spinifex-vpcd
|
||||
Group=spinifex
|
||||
ExecStartPre=/usr/local/bin/spx admin hostnet check
|
||||
ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
|
||||
ExecStart=/usr/local/bin/spx service vpcd start
|
||||
Restart=on-failure
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
# spinifex-unit-version: 1
|
||||
# spinifex-unit-version: 2
|
||||
[Unit]
|
||||
Description=Spinifex Infrastructure Platform
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
Wants=spinifex-nats.service spinifex-predastore.service spinifex-northstar.service spinifex-viperblock.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service spinifex-ui.service spinifex-qmp-collector.service spinifex-shutdown.service spinifex-nats-watchdog.timer openvswitch-switch.service ovn-controller.service
|
||||
Wants=spinifex-nats.service spinifex-predastore.service spinifex-northstar.service spinifex-viperblock.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service spinifex-ui.service spinifex-qmp-collector.service spinifex-shutdown.service spinifex-nats-watchdog.timer ovsdb-server.service ovs-vswitchd.service ovn-controller.service
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
||||
@@ -19,7 +19,7 @@ import (
|
||||
"syscall"
|
||||
|
||||
"github.com/mulgadc/bluebottle/pkg/otelsetup"
|
||||
lbagent "github.com/mulgadc/spinifex/spinifex/agents/elbv2"
|
||||
"github.com/mulgadc/spinifex/spinifex/lbagent"
|
||||
|
||||
_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
|
||||
)
|
||||
|
||||
@@ -17,7 +17,7 @@ func TestVolumeAbandonSubject_BothForms(t *testing.T) {
|
||||
assert.Equal(t, "ebs.abandon", viperblocklegacyv1.VolumeAbandonSubject(""))
|
||||
}
|
||||
|
||||
// TestVolumeAbandonRequest_JSON pins the request runtime/daemon/vm_adapters.go sends
|
||||
// TestVolumeAbandonRequest_JSON pins the request daemon/vm_adapters.go sends
|
||||
// on the abandon route.
|
||||
func TestVolumeAbandonRequest_JSON(t *testing.T) {
|
||||
golden := `{"volume":"vol-1","reason":"instance owned by another node"}`
|
||||
|
||||
+3
-3
@@ -95,7 +95,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
|
||||
|
||||
| Command | Flags | Description |
|
||||
|---------|-------|-------------|
|
||||
| `spx admin init` | `--nodes`, `--node`, `--bind`, `--advertise` (external IP off-host clients dial; auto-detected from the default route), `--port`, `--region`, `--az`, `--cluster-name`, `--cluster-bind`, `--cluster-routes`, `--predastore-nodes`, `--predastore-compaction-interval` (seconds; test clusters set a short one), `--services`, `--formation-timeout`, `--token-ttl`, `--force`, `--discard-jetstream` (with `--nodes` >= 2, remove this node's existing JetStream streams once every node has joined; without it a multi-node init over a store that holds streams is refused, and it never proceeds while NATS is running), `--skip-host-dns`, `--email` (operator address for update and security notices), `--no-telemetry`, `--gpu-passthrough`, `--ipsec` (default true; encrypt intra-AZ Geneve via OVN native IPsec cluster-wide — disable only for a trusted single-rack lab), `--host` (leader to join; falls back to multicast discovery), `--external-mode` (`pool` \| `nat` routed NAT for non-bridgeable uplinks, single-node; pair with `setup-ovn.sh --nat-uplink`; note: system instances (ECS/EKS/load-balancer agents) are unsupported in nat v1 — they require `pool` mode), `--external-iface` (WAN NIC for br-external, auto-detected), `--external-source` (`dhcp` \| `static`), `--external-pool` (`start-end`), `--external-prefix-len` (default 24), `--external-gateway` (auto-detected), `--external-dns` (comma-separated IPv4 addresses, at most 3; the DNS servers written to the external pool's `dns_servers` and handed to guests, verbatim; when omitted they are detected from the host resolver at init time, so set it to give every node of a cluster the same value), `--external-bind-bridge` (default `br-wan` when `--external-source=dhcp`), `--gateway-ip` (OVN gateway router's external IP for SNAT; defaults to the pool's range start, required for `nat` mode without DHCP) | Generates root IAM credentials (AKIA-prefixed access key + secret) → creates master.key (AES-256, 32 bytes, 0600) → writes bootstrap.json (consumed on first start) → generates CA + server TLS certificates → generates join token (written to `join-token` file, displayed in join command) → creates NATS config with auth token → writes spinifex.toml, awsgw.toml, predastore.toml → configures AWS CLI `spx` profile → creates directory structure under `~/spinifex/` → points the host resolver at this node's local northstar so the Spinifex zones (ELB/EKS names) resolve from the node itself (opt out with `--skip-host-dns`). `--force` on an already-initialized node is idempotent for crypto: it preserves the existing keys (master.key, predastore/viperblock encryption keys), system + admin credentials, and CA, and refreshes only the config files and the CA-signed server certificate (so a changed bind IP / SANs is picked up without breaking already-joined nodes or CA-baked AMIs). A genuine clean slate requires removing the data dirs first. |
|
||||
| `spx admin init` | `--nodes`, `--node`, `--bind`, `--advertise` (external IP off-host clients dial; auto-detected from the default route), `--port`, `--region`, `--az`, `--cluster-name`, `--cluster-bind`, `--cluster-routes`, `--predastore-nodes`, `--predastore-compaction-interval` (seconds; test clusters set a short one), `--services`, `--formation-timeout`, `--token-ttl`, `--force`, `--discard-jetstream` (with `--nodes` >= 2, remove this node's existing JetStream streams once every node has joined; without it a multi-node init over a store that holds streams is refused, and it never proceeds while NATS is running), `--skip-host-dns`, `--email` (operator address for update and security notices), `--no-telemetry`, `--gpu-passthrough`, `--ipsec` (default true; encrypt intra-AZ Geneve via OVN native IPsec cluster-wide — disable only for a trusted single-rack lab; on a host whose IPsec stack is libreswan it defaults to false and `--ipsec=true` is refused), `--host` (leader to join; falls back to multicast discovery), `--external-mode` (`pool` \| `nat` routed NAT for non-bridgeable uplinks, single-node; pair with `setup-ovn.sh --nat-uplink`; note: system instances (ECS/EKS/load-balancer agents) are unsupported in nat v1 — they require `pool` mode), `--external-iface` (WAN NIC for br-external, auto-detected), `--external-source` (`dhcp` \| `static`), `--external-pool` (`start-end`), `--external-prefix-len` (default 24), `--external-gateway` (auto-detected), `--external-dns` (comma-separated IPv4 addresses, at most 3; the DNS servers written to the external pool's `dns_servers` and handed to guests, verbatim; when omitted they are detected from the host resolver at init time, so set it to give every node of a cluster the same value), `--external-bind-bridge` (default `br-wan` when `--external-source=dhcp`), `--gateway-ip` (OVN gateway router's external IP for SNAT; defaults to the pool's range start, required for `nat` mode without DHCP) | Generates root IAM credentials (AKIA-prefixed access key + secret) → creates master.key (AES-256, 32 bytes, 0600) → writes bootstrap.json (consumed on first start) → generates CA + server TLS certificates → generates join token (written to `join-token` file, displayed in join command) → creates NATS config with auth token → writes spinifex.toml, awsgw.toml, predastore.toml → configures AWS CLI `spx` profile → creates directory structure under `~/spinifex/` → points the host resolver at this node's local northstar so the Spinifex zones (ELB/EKS names) resolve from the node itself (opt out with `--skip-host-dns`). `--force` on an already-initialized node is idempotent for crypto: it preserves the existing keys (master.key, predastore/viperblock encryption keys), system + admin credentials, and CA, and refreshes only the config files and the CA-signed server certificate (so a changed bind IP / SANs is picked up without breaking already-joined nodes or CA-baked AMIs). A genuine clean slate requires removing the data dirs first. |
|
||||
| `spx admin join` | `--host` (required), `--node` (required), `--token` (required), `--bind`, `--advertise` (defaults to `--bind`, or the auto-detected WAN IP when `--bind` is 0.0.0.0), `--port`, `--region`, `--az`, `--cluster-bind`, `--cluster-routes`, `--data-dir`, `--services`, `--skip-host-dns`, `--email`, `--no-telemetry`, `--predastore-compaction-interval`, `--join-timeout` (how long to keep retrying while the formation server is unreachable, default 20m), `--force` (join even though this node is already initialized or holds JetStream streams, **discarding its own CA, master key and JetStream store**; the store is removed only after formation succeeds, and the join is refused while NATS is running even with `--force`) | Connects to leader node with join token (Authorization: Bearer header) → retrieves cluster configuration → configures local node to join cluster and participate in distributed operations → points the host resolver at this node's local northstar so the Spinifex zones resolve from the node itself (opt out with `--skip-host-dns`) |
|
||||
|
||||
### Version
|
||||
@@ -126,7 +126,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
|
||||
|
||||
| Command | Flags | Description |
|
||||
|---------|-------|-------------|
|
||||
| `spx admin preflight` | `--quiet` (print only non-OK rows) | Reports whether this host has the privileged helper scripts and sudoers grants the installed build depends on. `spx version` knows the build but not the host: `setup.sh` and `setup-ovn.sh` install those assets, and a binary-swap deploy (`make deploy`, or a manual copy) skips them. It also reports any running service still executing a replaced binary — overwriting `/usr/local/bin/spx` while `spinifex.target` is up does not move services onto it, so each keeps executing the replaced inode until it restarts and a node can serve a mixture of two builds indefinitely. Checks compare content, not existence, so a present-but-stale asset is caught. Exits non-zero if any asset is Missing, Stale or Ungranted. |
|
||||
| `spx admin preflight` | `--quiet` (print only non-OK rows) | Reports whether this host has the privileged helper scripts and sudoers grants the installed build depends on. `spx version` knows the build but not the host: `setup.sh` and `setup-ovn.sh` install those assets, and a binary-swap deploy (`make deploy`, or a manual copy) skips them. It also reports any running service still executing a replaced binary — overwriting `/usr/local/bin/spx` while `spinifex.target` is up does not move services onto it, so each keeps executing the replaced inode until it restarts and a node can serve a mixture of two builds indefinitely. Checks compare content, not existence, so a present-but-stale asset is caught. On x86_64 it also checks that the QEMU binary the daemon will run lists the `microvm` machine type and that the microvm firmware `/usr/share/spinifex/microvm/qboot.rom` is installed, since load balancers boot on both. It runs the host network check `spinifex-daemon` and `spinifex-vpcd` run before they start, and fails while firewalld or ufw is active, since the Spinifex nftables policy is the only supported node firewall. It reports a Warning, without failing, when systemd-resolved is active but `/etc/resolv.conf` is not its stub, since lookups then bypass the route that sends Spinifex names to northstar. Exits non-zero if any asset is Missing, Stale, Ungranted, Unsupported or Failed. |
|
||||
| `spx admin node js-probe` | `--timeout` (default: 10s) | Round-trips a canary key through the cluster-state KV bucket and exits non-zero if the write or the read-back fails. JetStream can latch a write error per message block and keep serving reads — nats-server's filestore records the failure on the block and returns it on every later write, and nothing clears it short of a restart. The HTTP health endpoint reports the server-wide JetStream state, which that latch never touches, so only an actual write observes it. |
|
||||
| `spx admin banner` | `--boot-check` | Writes the node information banner to `/etc/issue` (shown before login on the physical/serial console) and `/etc/motd` (shown after SSH login). With `--boot-check` it also detects a management IP that has changed since last boot and updates `/etc/spinifex/node.conf` accordingly. |
|
||||
|
||||
@@ -180,7 +180,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
|
||||
| `spx admin gpu status` | `--node` (default: local node) | Queries `spinifex.node.status` fan-out → finds the target node response → prints Node, GPU hardware (model list or "none detected"), IOMMU state, vfio-pci state, passthrough enabled/disabled, and GPU pool allocation (`allocated/total`). Also lists GPU-capable instance types when passthrough is active. |
|
||||
| `spx admin gpu enable` | — | Checks current passthrough state via NATS → errors if already enabled or prerequisites not met (directs to `setup`) → writes `gpu_passthrough = true` to `spinifex.toml` via `admin.SetGPUPassthrough` → sends SIGHUP to `spinifex-daemon` via `systemctl kill -s HUP` → polls node status for up to 30 s until daemon confirms new state → prints final `gpu status` output. Must be run directly on the target host. |
|
||||
| `spx admin gpu disable` | — | Checks current passthrough state via NATS → errors if already disabled or if `AllocGPUs > 0` (must terminate GPU instances first) → writes `gpu_passthrough = false` → sends SIGHUP to `spinifex-daemon` → polls for up to 30 s → prints final `gpu status` output. Must be run directly on the target host. |
|
||||
| `spx admin gpu setup` | — | Idempotent host configuration for GPU passthrough. Steps (skipped if already applied): detect GPUs via `gpu.Discover` → collect PCI IDs for all IOMMU-group siblings → check/enable IOMMU in GRUB (`intel_iommu=on iommu=pt` or `amd_iommu=on iommu=pt`) → write vfio udev rule (`/etc/udev/rules.d/99-spinifex-vfio.rules`) → blacklist nouveau (`/etc/modprobe.d/blacklist-nouveau.conf`) → blacklist amdgpu if AMD GPU present (`/etc/modprobe.d/blacklist-amdgpu.conf`) → write vfio-pci early binding config (`/etc/modprobe.d/vfio-pci.conf`) → add vfio modules to initramfs. If any change requires a reboot: runs `update-initramfs -u` and exits with reboot instructions. After reboot: verifies `vfio_pci` module is loaded → verifies each GPU is bound to `vfio-pci` (binds explicitly via `driver_override` if unbound) → calls `gpu enable` to activate passthrough. |
|
||||
| `spx admin gpu setup` | — | Idempotent host configuration for GPU passthrough. Resolves its boot tools first (`update-grub` or `grubby`, `update-initramfs` or `dracut`) and stops before any change if either pair has neither or both. Steps (skipped if already applied): detect GPUs via `gpu.Discover` → collect PCI IDs for all IOMMU-group siblings → if IOMMU is inactive, add `intel_iommu=on iommu=pt` or `amd_iommu=on iommu=pt` to `GRUB_CMDLINE_LINUX_DEFAULT` and run `update-grub`, or run `grubby --update-kernel=ALL --args=...` and check the default entry and `/etc/kernel/cmdline` carry them → write vfio udev rule (`/etc/udev/rules.d/99-spinifex-vfio.rules`) → blacklist nouveau (`/etc/modprobe.d/blacklist-nouveau.conf`) → blacklist amdgpu if AMD GPU present (`/etc/modprobe.d/blacklist-amdgpu.conf`) → write vfio-pci early binding config (`/etc/modprobe.d/vfio-pci.conf`) → add vfio modules to `/etc/initramfs-tools/modules`, or write `/etc/dracut.conf.d/spinifex-vfio.conf` (`force_drivers` for the vfio modules, `install_items` for the `modprobe.d` files). If any change requires a reboot: runs `update-initramfs -u`, or `dracut -f --regenerate-all` and checks every kernel's image with `lsinitrd`, then exits with reboot instructions. After reboot: verifies `vfio_pci` module is loaded → verifies every PCI function in each GPU's slot is bound to `vfio-pci` (binds explicitly via `driver_override` if unbound, fails naming any function on another driver) → calls `gpu enable` to activate passthrough. |
|
||||
| `spx admin gpu mig status` | — (must be run on target host) | Runs `gpu.Discover()` locally → for each GPU prints: PCI address, model, MIG capability, MIG mode (enabled/disabled/N/A); for GPUs with MIG enabled lists active MIG slices with GI ID, profile name, and mdev path. |
|
||||
| `spx admin gpu mig enable` | `--profile <name>` (required, e.g. `1g.10gb`), `--gpu <pci-addr>` (optional, default: all MIG-capable GPUs) | Checks no GPU instances running (NATS) → discovers MIG-capable GPUs via `gpu.Discover()` (filtered by `--gpu` if set) → enables MIG mode on each target (`gpu.EnableMIGMode`) → lists available profiles (`gpu.ListProfiles`) and validates requested profile name → destroys any existing instances (`gpu.DestroyAllInstances`) → creates new instances filling GPU capacity (`gpu.CreateInstances`) → writes `mig_profile` to `spinifex.toml` via `admin.SetMIGProfile` → sends SIGHUP to `spinifex-daemon`. Must be run directly on the target host. |
|
||||
| `spx admin gpu mig disable` | `--gpu <pci-addr>` (optional, default: all MIG-capable GPUs) | Checks no GPU instances running (NATS) → discovers MIG-capable GPUs via `gpu.Discover()` (filtered by `--gpu` if set) → destroys all GPU instances (`gpu.DestroyAllInstances`) → disables MIG mode (`gpu.DisableMIGMode`) → clears `mig_profile` in `spinifex.toml` via `admin.SetMIGProfile` → sends SIGHUP to `spinifex-daemon`. Must be run directly on the target host. |
|
||||
|
||||
+53
-36
@@ -50,12 +50,14 @@ Request routing (`spinifex/gateway/gateway.go`):
|
||||
1. **Authentication**: SigV4 middleware validates AWS credentials and resolves the account ID
|
||||
2. **Throttling**: Per-account+action token bucket rejects bursts post-auth
|
||||
3. **Service Detection**: Extracts service name from the Authorization header
|
||||
4. **Action Dispatch**: Routes to service-specific handler. A service registered through the ingress dispatch registry (`ingress/aws/dispatch`, wired by `runtime/roles/awsgw`) is dispatched by its registration; currently those are the ECR and ACM control planes, registered from `domains/ecr/awsapi` and `domains/acm/awsapi`. Every other service uses the legacy switch:
|
||||
4. **Action Dispatch**: Routes to service-specific handler. A service registered through the ingress dispatch registry (`ingress/aws/dispatch`, wired by `runtime/roles/awsgw`) is dispatched by its registration; currently that is ECR's control plane, registered from `domains/ecr/awsapi`. Every other service uses the legacy switch:
|
||||
|
||||
```go
|
||||
switch svc {
|
||||
case "ec2":
|
||||
err = gw.EC2_Request(w, r)
|
||||
case "account":
|
||||
err = gw.Account_Request(w, r)
|
||||
case "iam":
|
||||
err = gw.IAM_Request(w, r)
|
||||
case "sts":
|
||||
@@ -64,13 +66,10 @@ case "elasticloadbalancing":
|
||||
err = gw.ELBv2_Request(w, r)
|
||||
case "eks":
|
||||
err = gw.EKS_Request(w, r)
|
||||
case "bedrock":
|
||||
err = gw.Bedrock_Request(w, r)
|
||||
// ... bedrock-runtime, bedrock-agent, bedrock-agent-runtime
|
||||
case "ecs":
|
||||
err = gw.ECS_Request(w, r)
|
||||
case "rds":
|
||||
err = gw.RDS_Request(w, r)
|
||||
case "acm":
|
||||
err = gw.ACM_Request(w, r)
|
||||
case "tagging":
|
||||
err = gw.Tagging_Request(w, r)
|
||||
case "spinifex":
|
||||
@@ -80,15 +79,14 @@ case "spinifex":
|
||||
|
||||
### 3. EC2 Handler
|
||||
|
||||
The EC2 handler (`spinifex/gateway/ec2.go`) parses the `Action` parameter and delegates to specific handlers via the generic `ec2Handler` and `ec2HandlerWithReq` wrappers. The account ID resolved by SigV4 auth is threaded into every handler so requests are scoped per IAM principal:
|
||||
The EC2 handler (`spinifex/gateway/ec2.go`) parses the `Action` parameter and delegates to specific handlers via a generic `ec2Handler` wrapper. The account ID resolved by SigV4 auth is threaded into every handler so requests are scoped per IAM principal:
|
||||
|
||||
```go
|
||||
"RunInstances": ec2HandlerWithReq(func(ctx context.Context, input *ec2.RunInstancesInput, gw *GatewayConfig, accountID string, r *http.Request) (any, error) {
|
||||
// ... PassRole and launch-quota checks, then the vCPU quota charge
|
||||
return ec2instanceapi.RunInstances(ctx, input, gw.NATSConn, gw.IAMService, accountID, passRoleCheck, launchQuotaCheck, gw.ExpectedNodes)
|
||||
"RunInstances": ec2Handler(func(input *ec2.RunInstancesInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2instanceapi.RunInstances(input, gw.NATSConn, accountID)
|
||||
}),
|
||||
"DescribeInstances": ec2Handler(func(ctx context.Context, input *ec2.DescribeInstancesInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2instanceapi.DescribeInstancesChecked(ctx, input, gw.NATSConn, 0, gw.NodeIDs, accountID)
|
||||
"DescribeInstances": ec2Handler(func(input *ec2.DescribeInstancesInput, gw *GatewayConfig, accountID string) (any, error) {
|
||||
return ec2instanceapi.DescribeInstances(input, gw.NATSConn, gw.DiscoverActiveNodes(), accountID)
|
||||
}),
|
||||
// ... + volumes, snapshots, VPCs, subnets, route tables, IGWs, NAT gateways,
|
||||
// security groups, network interfaces, elastic IPs, placement groups, key pairs,
|
||||
@@ -100,13 +98,13 @@ The EC2 handler (`spinifex/gateway/ec2.go`) parses the `Action` parameter and de
|
||||
The gateway communicates with daemons via NATS request/response. Most calls go through `natsmsg.NATSRequest`, which marshals the input, attaches the account ID as a NATS header, and unmarshals the typed response:
|
||||
|
||||
```go
|
||||
func (s *NATSInstanceService) RunInstances(ctx context.Context, input *ec2.RunInstancesInput, accountID string) (*ec2.Reservation, error) {
|
||||
func (s *NATSInstanceService) RunInstances(input *ec2.RunInstancesInput, accountID string) (*ec2.Reservation, error) {
|
||||
topic := fmt.Sprintf("ec2.RunInstances.%s", aws.StringValue(input.InstanceType))
|
||||
return natsmsg.NATSRequest[ec2.Reservation](ctx, s.natsConn, topic, input, 5*time.Minute, accountID)
|
||||
return natsmsg.NATSRequest[ec2.Reservation](s.natsConn, topic, input, 5*time.Minute, accountID)
|
||||
}
|
||||
```
|
||||
|
||||
The gateway's `RunInstances` places instances itself (`domains/ec2/awsapi/instance/placement.go`): it gathers capacity from every node on `spinifex.node.status`, spreads the launch across eligible nodes, and sends each node its share on the node-targeted subject `ec2.RunInstances.{type}.{nodeId}`, rolling back on partial failure.
|
||||
`RunInstances` uses a per-instance-type subject so NATS only delivers the request to a node with spare capacity for that type — no application-level reject-and-retry.
|
||||
|
||||
### Cross-Process Contract Ownership
|
||||
|
||||
@@ -126,14 +124,14 @@ refactor. It needs a versioned successor or an explicit compatibility plan.
|
||||
|
||||
### 5. Daemon Processing
|
||||
|
||||
Daemons (`spinifex/runtime/daemon/daemon.go`) subscribe to NATS topics and handle requests. A table-driven `subscribeAll()` registers the static EC2/ELBv2 surface at startup:
|
||||
Daemons (`spinifex/daemon/daemon.go`) subscribe to NATS topics and handle requests. A table-driven `subscribeAll()` registers the static EC2/ELBv2 surface at startup:
|
||||
|
||||
```go
|
||||
// Queue group → load-balanced (one daemon handles each request)
|
||||
{"ec2.CreateKeyPair", handleNATSRequest(d.node, d.keyService.CreateKeyPair), "spinifex-workers"},
|
||||
{"ec2.CreateKeyPair", d.handleEC2CreateKeyPair, "spinifex-workers"},
|
||||
|
||||
// No queue group → fan-out (every daemon responds)
|
||||
{"ec2.DescribeInstanceTypes", handleNATSRequest(d.node, d.instanceService.DescribeInstanceTypes), ""},
|
||||
{"ec2.DescribeInstanceTypes", d.handleEC2DescribeInstanceTypes, ""},
|
||||
```
|
||||
|
||||
`ec2.RunInstances` is dynamic: `ResourceManager` subscribes to `ec2.RunInstances.{type}` only while the node has capacity for that type, and unsubscribes when full. The node-targeted subject `ec2.RunInstances.{type}.{nodeId}` stays subscribed even at full capacity — the launch-time `allocate()` enforces limits there. Per-instance commands flow over `ec2.cmd.{instanceID}`, subscribed only by the owning node.
|
||||
@@ -146,17 +144,20 @@ When a daemon handles `RunInstances`:
|
||||
|
||||
1. **Resource Check**: Validates CPU/memory availability via `ResourceManager`
|
||||
2. **Volume Generation**: Creates boot and EFI volumes via Viperblock
|
||||
3. **Volume Mount**: Sends a mount request to the colocated Viperblock worker on `ebs.{node}.mount`, receives NBD URI
|
||||
3. **Volume Mount**: Sends `ebs.mount` request to Viperblock, receives NBD URI
|
||||
4. **QEMU Launch**: Builds and executes QEMU command with NBD-backed disks
|
||||
5. **QMP Monitoring**: Establishes QEMU Machine Protocol connection for VM management
|
||||
6. **Command subscription**: Subscribes to `ec2.cmd.{instanceID}` for subsequent commands on this VM
|
||||
|
||||
```go
|
||||
// Volume mounting via NATS (runtime/daemon/vm_adapters.go)
|
||||
reply, err := ebsRequestWithTrace(ctx, a.nc, instance.AccountID, viperblocklegacyv1.MountSubject(a.node), ebsMountRequest, 30*time.Second)
|
||||
// Volume mounting via NATS
|
||||
msg, err := d.natsConn.Request("ebs.mount", ebsMountRequest, 10*time.Second)
|
||||
|
||||
// QEMU launch with NBD-backed drives in args (runtime/compute/vm/vm.go)
|
||||
cmd := exec.Command(qemuArchitecture, args...) // qemu-system-x86_64 or qemu-system-aarch64
|
||||
// QEMU launch with NBD storage
|
||||
cmd := exec.Command("qemu-system-x86_64",
|
||||
"-drive", fmt.Sprintf("file=nbd:%s,format=raw", nbdURI),
|
||||
// ... additional QEMU args
|
||||
)
|
||||
```
|
||||
|
||||
## Key Components
|
||||
@@ -198,16 +199,32 @@ type ResourceManager struct {
|
||||
|
||||
### Multi-Node Aggregation
|
||||
|
||||
For operations that need data from all nodes (like `DescribeInstances`), the gateway uses the scatter-gather helper `natsmsg.Gather` (`spinifex/foundation/messaging/nats/nats.go`), called from `spinifex/domains/ec2/awsapi/instance/DescribeInstances.go`:
|
||||
For operations that need data from all nodes (like `DescribeInstances`), the gateway uses inbox-based fan-out (`spinifex/domains/ec2/awsapi/instance/DescribeInstances.go`):
|
||||
|
||||
```go
|
||||
// Publish to every node (no queue group = all daemons respond) and collect
|
||||
// the replies. The account ID travels as a NATS header so each daemon scopes
|
||||
// its response.
|
||||
frames, sum, err := natsmsg.Gather(ctx, natsConn, "ec2.DescribeInstances", jsonData, gatherOpts)
|
||||
```
|
||||
func DescribeInstances(...) {
|
||||
// Create unique inbox for collecting responses
|
||||
inbox := nats.NewInbox()
|
||||
sub, _ := natsConn.SubscribeSync(inbox)
|
||||
|
||||
Given the configured node IDs, `Gather` checks each reply's `X-Node-ID` header against them, and `DescribeInstancesChecked` fails closed rather than returning a partial listing when a node is missing, duplicated or unidentified.
|
||||
// Publish to all nodes (no queue group = all daemons respond).
|
||||
// Account ID is propagated as a NATS header so each daemon scopes its response.
|
||||
pubMsg := nats.NewMsg("ec2.DescribeInstances")
|
||||
pubMsg.Reply = inbox
|
||||
pubMsg.Data = jsonData
|
||||
pubMsg.Header.Set(natsmsg.AccountIDHeader, accountID)
|
||||
_ = natsConn.PublishMsg(pubMsg)
|
||||
|
||||
// Collect responses, returning early once expectedNodes have replied
|
||||
// or the 3s timeout is hit.
|
||||
for {
|
||||
msg, err := sub.NextMsg(3 * time.Second)
|
||||
allReservations = append(allReservations, nodeOutput.Reservations...)
|
||||
}
|
||||
|
||||
return allReservations
|
||||
}
|
||||
```
|
||||
|
||||
## Storage Integration
|
||||
|
||||
@@ -223,7 +240,6 @@ type EBSRequest struct {
|
||||
Boot bool // Boot volume flag
|
||||
EFI bool // EFI boot volume
|
||||
NBDURI string // NBD URI returned from mount
|
||||
// ... + DeviceName, DeleteOnTermination, hotplug port and host block-device fields
|
||||
}
|
||||
```
|
||||
|
||||
@@ -282,7 +298,6 @@ type Config struct {
|
||||
Predastore PredastoreConfig // S3 endpoint + service credentials
|
||||
Viperblock ViperblockConfig
|
||||
VPCD VPCDConfig // OVN/OVS endpoints
|
||||
// ... + EBS, Northstar, RDS, ACM, Bedrock, OchreVector
|
||||
}
|
||||
```
|
||||
|
||||
@@ -305,13 +320,15 @@ spinifex/
|
||||
│ ├── ingress/ # AWS protocol ingress: query/REST parsing, dispatch registry, rate limiting
|
||||
│ ├── runtime/
|
||||
│ │ ├── roles/ # Long-running service roles (awsgw, spinifex, vpcd, nats, predastore, ...)
|
||||
│ │ ├── daemon/ # Per-node daemon and its NATS subscription table
|
||||
│ │ └── compute/ # VM lifecycle, QMP, NBD, GPU
|
||||
│ ├── bootstrap/ # Cluster and node configuration structures, preflight checks
|
||||
│ ├── operator/ # spx CLI commands, admin use cases, installer, upgrade
|
||||
│ ├── agents/ # In-guest agent libraries (ecs, eks, elbv2, rds)
|
||||
│ ├── agents/ # In-guest agent libraries (ecs, eks, rds)
|
||||
│ ├── gateway/ # Legacy root: HTTP route assembly, SigV4 auth, legacy per-service dispatch
|
||||
│ └── accountteardown/ # Legacy root: account deletion cleanup
|
||||
│ ├── handlers/ # Legacy root: ecs, eks, elbv2, iam, rds, sts service implementations
|
||||
│ ├── daemon/ # Legacy root: per-node daemon and its NATS subscription table
|
||||
│ ├── accountteardown/ # Legacy root: account deletion cleanup
|
||||
│ └── lbagent/ # Legacy root: load-balancer agent
|
||||
├── internal/ # In-guest SigV4 clients, AWS Smithy models and test kits
|
||||
├── architecture/ # Executable architecture rules (package layering)
|
||||
├── build/ # systemd units, node helper scripts, microVM build
|
||||
@@ -321,7 +338,7 @@ spinifex/
|
||||
└── e2e/ # Live-cluster suites (see docs/testing/e2e-coverage.md)
|
||||
```
|
||||
|
||||
The two remaining legacy roots predate ADR-0001 and are frozen: they may shrink but not gain packages.
|
||||
The five legacy roots predate ADR-0001 and are frozen: they may shrink but not gain packages.
|
||||
[`PACKAGE_BOUNDARY_MIGRATION.md`](PACKAGE_BOUNDARY_MIGRATION.md) records what remains in each and what blocks its move.
|
||||
|
||||
## Development
|
||||
|
||||
@@ -345,11 +345,6 @@ while the branch is in flight; they do not describe the umbrella repository's
|
||||
| `spinifex#1251` | `spinifex/handlers/rds` | `spinifex/domains/rds/instance` (`instance`, imported as `rdsinstance`) | A mechanical relocation of the remaining RDS instance resource to its ADR-0001 resource package; APIs, NATS subjects, KV buckets and behaviour are unchanged and only the package name changed from `handlers_rds`. The frozen-legacy entry is removed and the allowlist entries naming `handlers/rds` now name `domains/rds/instance`. |
|
||||
| `spinifex#1264` | `spinifex/handlers/eks` | `spinifex/domains/eks/cluster` (`cluster`, imported as `ekscluster`) | A mechanical relocation of the remaining EKS implementation, nodegroups included; APIs, NATS subjects, KV buckets and behaviour are unchanged and only the package name changed from `handlers_eks`. The frozen-legacy entry is removed, the allowlist entries naming `handlers/eks` now name `domains/eks/cluster`, and the `operator/cli` entry is removed because an operator importing a domain package no longer breaks the legacy rule. Nodegroup service extraction is deferred. |
|
||||
| `spinifex#1265` | `spinifex/gateway/{iam,sts,eks,rds}` | `spinifex/domains/{iam,sts,eks,rds}/awsapi` (`awsapi`, imported as `iamawsapi`, `stsawsapi`, `eksawsapi`, `rdsawsapi`) | A mechanical relocation of the IAM, STS, EKS and RDS AWS action adapters to their ADR-0001 domain homes; APIs, NATS subjects, KV buckets and behaviour are unchanged. The root `gateway` dispatch still imports each package, recorded as `gateway` (service registration) debt. `gateway/bedrock` stays: ADR-0001 moves only its action adapters, and the package also holds the Ochre domain core, stores and inference engine. |
|
||||
| `spinifex#1266` | `spinifex/gateway/bedrock` | `spinifex/domains/ochre` (merged into package `ochre`) | A mechanical whole-package relocation: the Ochre domain core, stores, inference engine and Bedrock action layer join the existing `domains/ochre` package with their package-private code and tests intact. No identifier, API, NATS subject, KV bucket, metric, error string or behaviour changes; `domains/ochre` no longer imports a legacy package. Separating the action adapters into `domains/ochre/awsapi` is deferred, recorded as `domains/ochre` (action adapters) debt. |
|
||||
| `spinifex#1267` | `spinifex/daemon` | `spinifex/runtime/daemon` | A mechanical whole-package relocation under the 2026-10-26 ADR-0001 amendment: the per-node daemon keeps package name `daemon`, its files, tests and current dependencies. No identifier, API, NATS subject, KV bucket, metric, error string or behaviour changes. The interim daemon → gateway → handlers guardrail now matches `runtime/daemon`. Q-28's ownership and deployment refactor is deferred, recorded as `runtime/daemon` (Q-28 ownership and deployment) debt. |
|
||||
| `spinifex#1268` | `spinifex/lbagent` | `spinifex/agents/elbv2` (package `elbv2`) | A mechanical whole-package relocation under the 2026-10-26 ADR-0001 amendment: the load-balancer agent keeps its files, tests and current dependencies, and importers alias it `lbagent` so call sites are unchanged. No identifier, API, NATS subject, file path on the guest, metric, error string or behaviour changes; `cmd/lb-agent` stays the thin main. Q-30's data-plane boundary is deferred, recorded as `agents/elbv2` (Q-30 data-plane boundary) debt. |
|
||||
| `spinifex#1269` | `spinifex/handlers/elbv2` | `spinifex/domains/elbv2` (package `elbv2`) | A mechanical whole-package relocation under the 2026-10-26 ADR-0001 amendment: the ELBv2 control plane keeps its files, tests and current dependencies, and importers alias it `elbv2domain`. No identifier, API, NATS subject, KV bucket, metric, error string or behaviour changes. Its allowlist entries are now sourced from `domains/elbv2`. Q-30's data-plane boundary stays deferred, recorded as `agents/elbv2` (Q-30 data-plane boundary) debt. |
|
||||
| `spinifex#1271` | `spinifex/handlers/ecs` | `spinifex/domains/ecs` (package `ecs`) | A mechanical whole-package relocation: ADR-0001's `handlers/ecs` row targets `domains/ecs`, the files are already organised by resource, and resource packages are introduced only where callers need a compiler boundary. The ECS control plane keeps its files, tests and current dependencies alongside the already-extracted `domains/ecs/taskdefinition`, and importers alias it `ecsdomain`. No identifier, API, NATS subject, KV bucket, metric, error string or behaviour changes. Its allowlist entries are now sourced from `domains/ecs`. `handlers` has no package left and is a retired legacy root; R5 holds `domains/ecs` to the interim rule's ban on importing the daemon. |
|
||||
The EC2 contract row is intentionally different from the directory moves: it creates
|
||||
a compatibility boundary. `ec2.cmd.*` retains its deployed one-token NATS
|
||||
subscription and permission shape, while callers use
|
||||
@@ -423,15 +418,14 @@ These residuals were recorded by individual slices and must be resolved once the
|
||||
- The public `/keys` endpoint (root `gateway`, `oidc_discovery.go`) serves malformed stored JWKS bytes with HTTP 200. Removal condition: validate before serving, or record it as accepted.
|
||||
- The public discovery and `/keys` endpoints return 404 for KV read errors, including timeouts; only a bucket-open failure returns 500. Removal condition: distinguish dependency failure from not-found.
|
||||
- The root `gateway` package imports `domains/eks/cluster` directly (`gateway/oidc_discovery.go`: `AccountBucketName`, `GetClusterMeta`). The layering rules don't see it because the root gateway has no domain classification, so it has no allowlist entry. Removal condition: the public OIDC discovery and JWKS endpoints consume an EKS-owned resolver injected at composition, the same pattern as STS, leaving the root gateway with no `domains/eks/cluster` import.
|
||||
- `domains/elbv2/{agent_types,haproxy,health_checker,nginx}.go` import `agents/elbv2` (`HealthReport`, `ServerStatus`, `CertDir`, `NginxPIDPath`, `NginxCertDir`), a domain reading an agent's implementation package rather than a declared wire contract. Removal condition: the health-report and cert/PID-path vocabulary moves behind a guest/controller contract the domain and the agent both consume.
|
||||
- `agents/elbv2` (Q-30 data-plane boundary): the agent moved whole, keeping its current dependencies and the `domains/elbv2` import above. Q-30's data-plane boundary and the common agent layout remain deferred by ADR-0001 and are not paid by the move.
|
||||
- `domains/ec2/instance/state/imds_readers.go` (dependency, IMDS readers) imports `runtime/daemon` for `LocalState`, `ReadLocalState`, `LocalStatePath`, `NewJetStreamManager` and `InstanceStateBucket`. The local-file reader and the instance-record loader's `JetStreamManager` open, which `vpcd/imds_instance_state.go` held directly, moved here unchanged and are now injected into `vpcd` through `guestmetadata.LocalStateReader` and `guestmetadata.RecordLoader`. The readers already live in `domains/ec2/instance/state`. Removal condition: the legacy dependencies they call are extracted from `daemon` — `LocalStatePath`, `ReadLocalState` and `LocalState`, the instance bucket constants, and `JetStreamManager` with its bucket initialisation (`InitKVBucket`) — leaving `domains/ec2/instance/state` with no `daemon` import.
|
||||
- `handlers/elbv2/{agent_types,haproxy,health_checker,nginx}.go` import `lbagent` (`HealthReport`, `ServerStatus`, `CertDir`, `NginxPIDPath`, `NginxCertDir`), a domain reading an agent's implementation package rather than a declared wire contract. Removal condition: the health-report and cert/PID-path vocabulary moves behind a guest/controller contract the domain and the agent both consume.
|
||||
- `domains/ec2/instance/state/imds_readers.go` (dependency, IMDS readers) imports `daemon` for `LocalState`, `ReadLocalState`, `LocalStatePath`, `NewJetStreamManager` and `InstanceStateBucket`. The local-file reader and the instance-record loader's `JetStreamManager` open, which `vpcd/imds_instance_state.go` held directly, moved here unchanged and are now injected into `vpcd` through `guestmetadata.LocalStateReader` and `guestmetadata.RecordLoader`. The readers already live in `domains/ec2/instance/state`. Removal condition: the legacy dependencies they call are extracted from `daemon` — `LocalStatePath`, `ReadLocalState` and `LocalState`, the instance bucket constants, and `JetStreamManager` with its bucket initialisation (`InitKVBucket`) — leaving `domains/ec2/instance/state` with no `daemon` import.
|
||||
- Advisory: 27 production files across `domains/{acm,dns,ec2/*,network/host,network/ovn,ochre}` import `bootstrap/config` directly rather than through typed constructor inputs, which ADR-0001's `config` row expects. `bootstrap/preflight/preflight.go:15` imports `runtime/host/command` and `services.go:13` imports `operator/host/systemd`; both are expected bootstrap-to-runtime/operator edges for startup validation and are not domain-rule violations.
|
||||
- Operator bootstrap credential generation: `operator/cli/admin.go` mints the bootstrap admin and Northstar access keys and secrets directly with `awscredentials` and writes them into config, rather than asking an IAM bootstrap capability to issue them. `awscredentials` only formats the values. Removal condition: a real IAM bootstrap capability issues these credentials and `operator/cli` calls it.
|
||||
- `domains/dns`: imports `runtime/compute/vm` (domain-to-runtime), `domains/network/reconcile` for the reconcile lock and `bootstrap/config` for node configuration.
|
||||
- ACM: the IAM AES-GCM helper import (ACM relocation residual).
|
||||
- `runtime/roles/awsgw`: the quota reconcile loop belongs to `domains/admission/quota`, and the EKS token-verify responder belongs with STS or EKS identity.
|
||||
- `runtime/roles/awsgw` → `domains/elbv2` (inherited from `gateway/tagging`): this is a new physical import in AWSGW that inherits the former `gateway/tagging` dependency, because the `elbv2Lister` adapter builds its client with `elbv2domain.NewNATSELBv2Service` and calls the `ELBv2Service` interface. AWSGW is the composition root, so the layering test needs no allowlist row. Removal condition: it must clear when ELBv2 exposes a domain-owned boundary (its NATS client or contract under its domain owner) and the adapter consumes that instead.
|
||||
- `runtime/roles/awsgw` → `handlers/elbv2` (inherited from `gateway/tagging`): this is a new physical import in AWSGW that inherits the former `gateway/tagging` dependency, because the `elbv2Lister` adapter builds its client with `handlers_elbv2.NewNATSELBv2Service` and calls the `ELBv2Service` interface. It is permitted temporarily because AWSGW is the composition root, which may wire legacy packages, so the layering test needs no allowlist row. Removal condition: it must clear when ELBv2 exposes a domain-owned boundary (its NATS client or contract under its domain owner) and the adapter consumes that instead.
|
||||
- `runtime/roles/vpcd` (placement): the role still mixes process/role supervision with network-controller responsibilities. `lease_reaper.go` (orphaned external-pool DHCP lease sweep), `ovn_controller_watchdog.go` (OVN controller liveness and restart), `host_eip_loop.go` (host-EIP reconcile loop) and `lease_datapath_reconcile.go` (lease datapath reconcile) are periodic network-control loops, not role wiring; `vpcd.go` itself also carries bridge-mode detection and verification, chassis-list parsing, external-CIDR resolution, DHCP manager startup and lease/IP-change handling, and gateway/host-EIP allocator and neighbour flush/prime construction alongside `Service.Start`. The future placement question is whether these belong in `domains/network`. Removal condition: an accepted decision on network-controller placement, and extraction of those loops and helpers behind the role.
|
||||
- ECR control-plane registration: `awsapi.Deps.NATS` is a raw `*nats.Conn`, not a narrow capability. Every NATS-backed `Handler` passes it to `domains/ecr.NewNATSMetaStore`, which calls `natsmsg.NATSRequest`, so narrowing it means changing the exported `Handler` type, every handler and the meta store; it must not be broadened further meanwhile.
|
||||
- ACM control-plane registration: `domains/acm/awsapi.Deps.NATS` (`registration.go`) is likewise a raw `*nats.Conn` carried by every `Handler`, for the same reason; it must not be broadened further meanwhile.
|
||||
@@ -443,17 +437,17 @@ These residuals were recorded by individual slices and must be resolved once the
|
||||
- `foundation/messaging/nats/nats_trace.go` still names its OpenTelemetry tracer `github.com/mulgadc/spinifex/spinifex/utils`. It is an emitted instrumentation-scope name that dashboards may filter on, so renaming it is a telemetry change, not a path fix.
|
||||
- `runtime/compute/cache.New`: two construction-time panics carry a lint exemption pending a constructor that returns an error.
|
||||
- `domains/ec2/systeminstance`: managed-service parent metadata is only the `ManagedBy` tag value.
|
||||
- `domains/ochre` (action adapters): since `spinifex#1266` the Bedrock action layer (handlers, action authorization, event-stream framing and the CRUD, catalogue and Converse/Invoke entry points) shares package `ochre` with the domain core, stores and inference engine, and imports `ingress/aws/bodyscope`. Moving it to `domains/ochre/awsapi` is not mechanical: the adapters reach package-private store internals (the guardrail and provisioned-throughput store handles and get/put/update methods), and adapter tests override the package-private model catalogue. It needs core capabilities first, then the move; the top-level `gateway/bedrock*.go` HTTP handlers additionally wait for the ingress registration seam.
|
||||
- `gateway/bedrock`: holds the Ochre domain core (model catalogue, provisioned throughput, guardrail, access, credential, usage, price, logging and weights stores), the inference engine (provider, vLLM, Anthropic, embeddings, reranker, admission) and the Bedrock action layer in one package with internal cycles. `domains/ochre` imports it, which is the ADR step 7 Ochre gateway import. Splitting it needs a plan: domain core and stores to `domains/ochre/<resource>`, inference to an Ochre-owned package, and only the action layer to `domains/ochre/awsapi`. The top-level `gateway/bedrock*.go` HTTP handlers additionally wait for the ingress registration seam.
|
||||
- `domains/network`: `reconcile/intent.go` builds desired network state by reading EC2's KV buckets and decoding EC2's private VPC, subnet, ENI, route-table, gateway, EIP, NAT and security-group records, and `reconcile/drift.go` watches those buckets. ADR-0001 names this pair explicitly: EC2 and network communicate through a capability/projection or versioned contract, never by one being the other's KV reader. Step 7 fix: an EC2-owned `domains/ec2/projection` exposing network intent with list and watch (it may stay KV-backed, so no data moves); a narrow intent interface declared by `reconcile`; wiring at the composition root; and the duplicated `PurposeEIP`/`PurposeIGWLRP` constants and mirrored `SGRule`/`SGEvent` folded into the projection or `contracts/ec2/v1`. The drift-watch coverage test and reconcile scenario tests are the evidence the projection must keep. `domains/network` also imports `runtime/compute/vm` and `runtime/host/command` (domain-to-runtime).
|
||||
- `domains/ec2/instance`: imports `runtime/compute/vm` and `runtime/compute/gpu` (domain-to-runtime) and `domains/dns` (cross-domain implementation import).
|
||||
- `domains/ec2/instance/state` (dependency): the vpcd IMDS local reader imports `runtime/compute/vm` to return `vm.VM`. Removal condition: that type moves into a domain-owned package that `runtime/compute/vm` consumes (runtime to domain is allowed).
|
||||
- `agents/eks/tokenwebhook`: imports `domains/eks/cluster` for `WebhookTokenReviewResult`, an agent depending on a domain implementation. Temporary: the eventual EKS contract or projection for the token-review reply needs its own design, and no contract is invented to complete the move.
|
||||
- `agents/ecs/agent`: takes its wire types from `contracts/ecs/v1`, and still imports `domains/ecs` for the poll-assignment and GPU-report inputs and outputs and the instance-type and availability-zone attribute names. Temporary agent-to-domain debt; no contract is created during the relocation.
|
||||
- `agents/ecs/agent`: takes its wire types from `contracts/ecs/v1`, and still imports `handlers/ecs` for the poll-assignment and GPU-report inputs and outputs and the instance-type and availability-zone attribute names. Temporary agent-to-domain debt; no contract is created during the relocation.
|
||||
- `agents/rds/agent` (wire contract): imports `domains/rds/instance` for the command, reply, parameter, health and bootstrap types exchanged with the control plane. Temporary agent-to-domain debt; no contract is created during the relocation. The earlier `LookupEngine` dependency on the RDS domain's engine registry has since cleared: `controlPlaneRulesFrom` and both engine constructors now take the agent-declared `EngineCatalog` (see the narrowed entry below), with zero production uses of `LookupEngine` remaining in `agents/rds/agent`.
|
||||
- `operator/installer/firstboot` (security): `Config.InstallCallback` is rendered into `spinifex-firstboot.sh`, which is world-readable (0755). The field is currently unused (`fireInstallCallback` reads `SPINIFEX_INSTALL_CALLBACK` from the environment instead), but any future wiring must not place secrets or tokenised callback URLs in the generated script; sensitive callback credentials should reach first boot through a protected channel such as a 0600 file or a credential store.
|
||||
- `operator/cli` (known security risk): `spx admin join` contacts the formation server with `InsecureSkipVerify`, sends the join token as a bearer credential and receives the cluster CA certificate and key, master key, Viperblock key and shared credentials, so an on-path attacker during formation can capture the token and the cluster root secrets or substitute a CA. This predates the move; lint surfaced it when the tree left `cmd/`. It is accepted for now and carries a line-level gosec annotation stating it is not a false positive. Removal condition: authenticate formation TLS with a pinned key or certificate delivered with the join material, or an out-of-band CA, with tests proving a mismatched server certificate fails before credentials are sent.
|
||||
- `domains/eks/cluster` (potential security risk): `ClusterMeta.ControlPlaneTemplate` (`domains/eks/cluster/service_impl.go`) clears the template's credentials but keeps `OIDCPrivateKeyPEM` and the k3s `JoinToken`, so both are persisted in the `eks-account-{acct}` cluster record, which the gateway also reads. Whether that record is encrypted at rest is unconfirmed; if it is not, a reader of the bucket can forge service-account tokens for the cluster's OIDC issuer and join nodes to it. This predates the refactor and is not fixed in it. Removal condition: the key and token are held by a dedicated secret store (a future KMS or Secrets Manager capability is the likely owner) and the cluster record keeps only a reference, with a test proving the persisted record contains neither secret.
|
||||
- `operator/cli` (dependency): the command tree still imports domain and service implementations directly (`handlers/{iam,eks}`, `domains/admission/quota`, `gateway`, `domains/ochre`, `vpcd`, `accountteardown`, the Viperblock adapter and runtime roles). The ADR's `cmd/spinifex` current-to-target map row requires command packages to reach domains through public capabilities; splitting supported use cases into `operator/*` packages is later work.
|
||||
- `operator/cli` (dependency): the command tree still imports domain and service implementations directly (`handlers/{iam,eks}`, `domains/admission/quota`, `gateway`, `gateway/bedrock`, `daemon`, `vpcd`, `accountteardown`, the Viperblock adapter and runtime roles). The ADR's `cmd/spinifex` current-to-target map row requires command packages to reach domains through public capabilities; splitting supported use cases into `operator/*` packages is later work.
|
||||
- `providers/ebs/viperblock` → `runtime/compute/nbd` and `runtime/host/process` (dependency, moved unchanged from `services/viperblockd`): the adapter formats NBD endpoints and manages nbdkit processes through runtime packages. ADR-0001 does not list providers → runtime as forbidden and the layering test does not enforce it (same category as `providers/ebs/conformance` → `runtime/compute/nbd`), but a provider should not depend on node runtime. Clears when NBD endpoint naming has a provider-facing home (a candidate for `contracts/ebs/v1`) and process supervision is injected by the role. The adapter also imports `providers/objectstore`, a provider-to-provider edge that is allowed.
|
||||
- `gateway/ec2.go` (registration): the EC2 dispatch map stays a `GatewayConfig` method in `gateway` and imports every `domains/ec2/awsapi/*` package; `gateway/ec2_idempotency.go` also imports `domains/ec2/awsapi/idem` (`KVBucket`, `TTL`, `TokenAndParams`) for the dispatch-time idempotency wrapper. ADR-0001 makes the domain own its action registration; moving the map waits on the generic ingress registration seam and must not be done as a package move.
|
||||
- `gateway` (service registration): the root `gateway` dispatch for IAM, STS, EKS and RDS imports `domains/{iam,sts,eks,rds}/awsapi`, which only AWSGW may wire. It clears when each service registers through the ADR-0002 ingress registration seam; as with EC2, moving the dispatch must not be done as a package move.
|
||||
@@ -462,23 +456,22 @@ These residuals were recorded by individual slices and must be resolved once the
|
||||
- `domains/admission/quota` (dependency, moved from `handlers/quota` unchanged): `live.go` imports `gateway/elbv2` and `domains/rds/instance` for its live ELBv2 and RDS counters; `records.go` imports `runtime/compute/vm` for the instance-record vCPU sweep; `live.go`, `vcpu.go` and `storage.go` call `domains/ec2/awsapi/{vpc,eip,volume}` describe adapters to count VPCs, subnets, addresses and volumes, as they did under `gateway/ec2`. The ADR's `handlers/quota` current-to-target map row replaces these with reader capabilities once their final locations exist.
|
||||
- `domains/ec2/awsapi/instance` (dependency): imports `domains/iam` (`IAMService` in `DescribeInstances.go`, `RunInstances.go` and `IamInstanceProfileAssociation.go`; `InstanceProfile` in the latter two), `runtime/compute/cache` (`NodeState`, `NodeUnknown`, `NodeLive`) and `runtime/compute/vm` (`VM`), both in `status_synthesis.go`. The IAM edge needs an IAM capability; the runtime edges are domain-to-runtime and need the instance record and node liveness exposed through an EC2-owned projection.
|
||||
- `domains/ec2/awsapi/spotinstance` (dependency): imports `domains/iam.IAMService` and `domains/admission/quota.Service` in `RequestSpotInstances.go`.
|
||||
- Instance adapter consumers outside EC2: `domains/rds/awsapi/enginecatalog.go`, `runtime/daemon/eks_cp_control.go`, `runtime/daemon/rds_deps.go`, `domains/admission/quota`'s `vcpu.go` and `gateway/iam.go` (`CountInstanceProfileAssociations`) call `domains/ec2/awsapi/instance` directly; each needs an EC2 capability rather than the AWS action adapter.
|
||||
- `runtime/daemon` (Q-28 ownership and deployment): the daemon moved whole, keeping its current ownership, deployment and `daemon → gateway → handlers` dependencies. Q-28's write-side consolidation and any daemon decomposition or split remain deferred by ADR-0001 and are not paid by the move.
|
||||
- Instance adapter consumers outside EC2: `domains/rds/awsapi/enginecatalog.go`, `daemon/eks_cp_control.go`, `daemon/rds_deps.go`, `domains/admission/quota`'s `vcpu.go` and `gateway/iam.go` (`CountInstanceProfileAssociations`) call `domains/ec2/awsapi/instance` directly; each needs an EC2 capability rather than the AWS action adapter.
|
||||
|
||||
Enforcement: the owner decided the layering test is blocking now, with a ratchet. Its allowlist mirrors this debt list exactly, each edge carrying its owner and removal condition; any new unrecorded forbidden edge fails local preflight and CI, and an allowlisted edge that no longer exists must be removed from the list.
|
||||
- `domains/rds/instance` → `domains/rds/subnetgroup` (legacy edges): `instanceReferences` binds `Dependants` by scanning instance records, and `tags.go` writes the group record through its generic tag registry. The first clears when the RDS instance gains its own owner and answers `Dependants` from it. The second clears when tagging invokes a subnet-group-owned tag/untag capability rather than writing the record; moving the tagging adapter to `domains/rds/awsapi` alone does not clear it.
|
||||
- `domains/rds/instance` → `domains/rds/parametergroup` (legacy edges): `parameterGroupReferences` binds `Dependants` by scanning instance records; this clears when the RDS instance gains its own owner and answers `Dependants` from it. `tags.go` writes the meta record through its generic tag registry; this clears when tagging invokes a parameter-group-owned tag mutation capability, and moving the tagging adapter to `domains/rds/awsapi` alone does not clear it. The Modify handler still calls `propagateParameterGroup` after the owner's write; propagation is instance-side realization and stays legacy until it is designed as asynchronous realization (group generation, instance staleness, retry). `parameterGroupValues` returns the persisted `Record` and overrides to instance code; when the instance owner is extracted, replace them with a consumer-owned resolved-parameters projection.
|
||||
- `domains/ecs` → `domains/ecs/taskdefinition` (owner bypass): `tags.go` reads and writes the revision record through its generic tag registry; this clears when ECS tagging invokes an owner-provided tag mutation capability, and moving `tags.go` to `domains/ecs/awsapi` alone does not clear it. `taskDefinitionResolver` returns the persisted `Record` to task and service launch code; when the task and service owners are extracted, replace the record crossing that boundary with a consumer-owned resolved task-definition (launch-spec) projection. Moving packages has not made the record private.
|
||||
- `domains/ecs/taskdefinition` (deliberate duplication, not debt): private `getJSON`, `putJSON` and `keysWithPrefix` copy the raw-KV helpers in `domains/ecs` so the extraction keeps ECS's get/put/list error and recovery semantics. Revisit only if a later, tested state abstraction proves it preserves those semantics; do not migrate ECS to `kvstore` merely to remove them.
|
||||
- `handlers/ecs` → `domains/ecs/taskdefinition` (legacy edges): `tags.go` reads and writes the revision record through its generic tag registry; this clears when ECS tagging invokes an owner-provided tag mutation capability, and moving `tags.go` to `domains/ecs/awsapi` alone does not clear it. `taskDefinitionResolver` returns the persisted `Record` to task and service launch code; when the task and service owners are extracted, replace the record crossing that boundary with a consumer-owned resolved task-definition (launch-spec) projection. Moving packages has not made the record private.
|
||||
- `domains/ecs/taskdefinition` (deliberate duplication, not debt): private `getJSON`, `putJSON` and `keysWithPrefix` copy the raw-KV helpers in `handlers/ecs` so the extraction keeps ECS's get/put/list error and recovery semantics. Revisit only if a later, tested state abstraction proves it preserves those semantics; do not migrate ECS to `kvstore` merely to remove them.
|
||||
- `domains/eks/cluster` → `domains/eks/access` (legacy edges): `ResolveTokenReview` opens the EKS bucket in the gateway process and reads records through the exported `access.Get`; this clears when token review asks the owner across a contract rather than opening the bucket itself. `DeleteClusterPrefix` erases access-entry keys as part of its blanket cluster sweep; this clears when cluster delete invokes an owner-provided removal of the cluster's entries. The owner returns the persisted `Record` to the AWS adapter and token review; replace it with consumer-owned projections when those consumers are extracted.
|
||||
- `domains/eks/cluster` → `domains/eks/addon` (legacy edges): `DeleteClusterPrefix` erases add-on records and manifests in its blanket cluster sweep; this clears when cluster delete invokes an add-on-owned teardown capability. `stageGPUDeviceAddon` is a temporary node-group-to-add-on orchestration edge: best-effort, errors logged and swallowed, never undone on node-group delete; it clears when the node-group owner is extracted and the dependency is designed. `ClusterReconciler` hosts the add-on status subscription; it clears when add-on delivery observation has its own host under the lifecycle correction. Cluster existence is checked in the handler, not through an owner-declared capability; it moves into the owner when cluster admission is defined (today status is ignored). Guest wire types (`AddonStatusReport`, `AddonDeliveryPhase`, `AddonStatusSubject`, `StagedAddonManifest`, `InternalAddonsResponse`) are owned by `contracts/eks/v1`, which `domains/eks/cluster`, `gateway/eks` and `agents/eks/gatewayfetch` import directly.
|
||||
- `domains/eks/cluster` and `domains/eks/awsapi` → `domains/eks/addon` (further edges): the AWS adapter reads the persisted `Record` through `addon.Get`/`addon.List`; this clears when the owner returns a view and the add-on adapter moves to `domains/eks/awsapi`. The guest route reaches the owner through the `handlers_eks` NATS RPC (`ListStagedAddonManifests`); this clears with per-resource commands behind EKS registration. The add-on ARN is derived in both the owner and `domains/eks/awsapi/authz.go`; this clears when authorization obtains the ARN from the owner, which an incarnation suffix would require. The owner's `Manifest`/`Report` beside the `contracts/eks/v1` types is deliberate duplication, not debt.
|
||||
- `agents/rds/agent` (dependency, narrowed): engine metadata now arrives through the agent-declared `EngineCatalog`, but the agent still imports `domains/rds/instance` for the command protocol, wire and record types (`Command*`, `Parameter`, `EngineHealth*`, `BootstrapMode*`) and API output types. This clears when those move behind a guest/controller wire contract.
|
||||
- `domains/rds/awsapi` → `domains/rds/instance` (`InstanceSizing`): the gateway obtains its `Sizing` through the RDS handler package's adapter rather than its own composition root. This clears when the gateway's RDS composition moves to `runtime/roles/awsgw` and builds `Sizing` there.
|
||||
- `domains/admission/quota` → `domains/ec2/instancetypes`: `reconcile.go`, `records.go` and `vcpu.go` read `DefaultVCPUs` to size instances without a recorded vCPU count. This clears when quota receives vCPU sizing through a consumer-owned capability or an EC2 projection.
|
||||
- IAM implementation consumers in STS, Ochre and ELBv2: `domains/sts` imports `domains/iam` across five files (`IAMService`, `SessionRevoker`, `EncryptSecret`/`DecryptSecret`, `AccessKeyStatusActive`, `ValidateTrustPolicyDocument`, `PolicyEffectAllow`/`PolicyEffectDeny`, `IAMAccountBucketName`, `OIDCProviderARN`, `OIDCProviderKey`), not only `oidc_jwks.go`; `domains/sts/awsapi/GetCallerIdentity.go` takes `IAMService`; `domains/ochre/credentials.go` uses `EncryptSecret`/`DecryptSecret`; and `domains/elbv2/{lb_role,service_impl}.go` use `EnsureSystemInstanceProfile` and `SystemInstanceRoleEnsurer`. Each clears when its consumer receives a narrow IAM capability, or a shared crypto boundary for the secret helpers, rather than importing the IAM implementation.
|
||||
- `domains/elbv2` (cross-domain): `service_impl.go` imports `domains/acm` (`Store`, `NewStore`), `domains/dns` (`ELBChanges`, `ELBName`, `Change`, `ActionUpsert`, `ResolveBaseDomain`), `domains/ec2/vpc` (`VPCServiceImpl`) and `domains/network/topology` (`Port`), and `launcher.go` re-exports `domains/ec2/systeminstance` launch types as aliases. These clear when each edge becomes a consumer-owned capability.
|
||||
- `docs/service-interfaces.yaml` (ECS coverage gap, closed): the traced `ecs.bus.*` family and the ECS AWS-API action subjects are both listed. The action subjects are declared as `ecs.*`, the same wildcard convention as `rds.*`: `domains/ecs/service_nats.go` publishes them from the gateway and `runtime/daemon/daemon.go` subscribes to them. Declaring `ecs.bus.*` alone had made `ecs` a managed prefix, so `manifest-lint` failed CI on all 34 undeclared action subjects until this was added.
|
||||
- IAM implementation consumers in STS, Ochre and ELBv2: `domains/sts` imports `domains/iam` across five files (`IAMService`, `SessionRevoker`, `EncryptSecret`/`DecryptSecret`, `AccessKeyStatusActive`, `ValidateTrustPolicyDocument`, `PolicyEffectAllow`/`PolicyEffectDeny`, `IAMAccountBucketName`, `OIDCProviderARN`, `OIDCProviderKey`), not only `oidc_jwks.go`; `domains/sts/awsapi/GetCallerIdentity.go` takes `IAMService`; `gateway/bedrock/credentials.go` uses `EncryptSecret`/`DecryptSecret`; and `handlers/elbv2/{lb_role,service_impl}.go` use `EnsureSystemInstanceProfile` and `SystemInstanceRoleEnsurer`. Each clears when its consumer receives a narrow IAM capability, or a shared crypto boundary for the secret helpers, rather than importing the IAM implementation.
|
||||
- `handlers/elbv2` (cross-domain): `service_impl.go` imports `domains/acm` (`Store`, `NewStore`), `domains/dns` (`ELBChanges`, `ELBName`, `Change`, `ActionUpsert`, `ResolveBaseDomain`), `domains/ec2/vpc` (`VPCServiceImpl`) and `domains/network/topology` (`Port`), and `launcher.go` re-exports `domains/ec2/systeminstance` launch types as aliases. These clear when ELBv2 is extracted and each edge becomes a consumer-owned capability, which ADR-0001 sequences after the ELBv2 data-plane boundary is settled.
|
||||
- `docs/service-interfaces.yaml` (ECS coverage gap, closed): the traced `ecs.bus.*` family and the ECS AWS-API action subjects are both listed. The action subjects are declared as `ecs.*`, the same wildcard convention as `rds.*`: `handlers/ecs/service_nats.go` publishes them from the gateway and `daemon/daemon.go` subscribes to them. Declaring `ecs.bus.*` alone had made `ecs` a managed prefix, so `manifest-lint` failed CI on all 34 undeclared action subjects until this was added.
|
||||
- `contracts/ecs/v1` (dead routes): four of five bus subjects have no producer. The scheduler leader still subscribes to `instance-register`, `instance-heartbeat` and `task-state`, which can never fire, and `assign` has neither producer nor consumer; only `service-reconcile` is live. Removal condition: an owner decision to retire these routes (a successor contract under the compatibility rules) or to adopt them in the agent.
|
||||
|
||||
The layering test enforcing these rules (`architecture/layering`, a module-root repository governance test, not Spinifex product code; run by `make test`, `test-cover` and `test-race`) is blocking, with a ratchet: the debt recorded in this section is narrowly allow-listed by owner and removal condition, and any new, unrecorded forbidden edge fails local preflight and CI. The allow-list mirrors the edges in this section that break an import rule; same-domain legacy edges, placement residuals and security notes are not import-rule violations and have no allow-list entry.
|
||||
@@ -663,40 +656,40 @@ series above; "needs decision" is open.
|
||||
|
||||
| Legacy path | What remains | What blocks it |
|
||||
|---|---|---|
|
||||
| `handlers/iam` | — | Done. The implementation moved to `domains/iam` (package `iam`) in `spinifex#1249` with no API, NATS subject, storage or behaviour change. ADR-0001's row says a shared IAM contract is not inferred from current KV readers, and the move adds none. The cross-domain readers (the EC2 instance and spot-instance AWS adapters, `domains/sts`, `domains/sts/awsapi`, `domains/ochre`, the ELBv2 system-instance role path and ECR/Ochre signing-key crypto) still import the implementation and each still needs an IAM capability; that is recorded dependency-rule debt, with its allowlist entries now targeting `domains/iam`, not a prerequisite to relocating IAM's implementation. |
|
||||
| `handlers/iam` | — | Done. The implementation moved to `domains/iam` (package `iam`) in `spinifex#1249` with no API, NATS subject, storage or behaviour change. ADR-0001's row says a shared IAM contract is not inferred from current KV readers, and the move adds none. The cross-domain readers (the EC2 instance and spot-instance AWS adapters, `domains/sts`, `domains/sts/awsapi`, `gateway/bedrock`, the ELBv2 system-instance role path and ECR/Ochre signing-key crypto) still import the implementation and each still needs an IAM capability; that is recorded dependency-rule debt, with its allowlist entries now targeting `domains/iam`, not a prerequisite to relocating IAM's implementation. |
|
||||
| `handlers/sts` | — | Done. The implementation moved to `domains/sts` (package `sts`) in `spinifex#1250` with no API, NATS subject, storage or behaviour change. ADR-0001's condition, cutting the EKS dependency, was already met through the injected `ClusterJWKSResolver`. `domains/sts` still imports `domains/iam` directly; per-consumer IAM capabilities remain dependency-rule debt, with its allowlist entry now sourced from `domains/sts`. |
|
||||
| `handlers/ecs` | — | Done. The ECS control plane moved to `domains/ecs` (package `ecs`) with no API, NATS subject, storage or behaviour change, alongside the already-extracted `domains/ecs/taskdefinition`. The ADR-0004 blocked-dependency list (`iam:PassRole` in the gateway adapter and EC2/ELBv2/EIP adapters called as internal APIs) remains dependency-rule debt, with its allowlist entries now sourced from `domains/ecs`. |
|
||||
| `handlers/ecs` | Task, service and capacity resources (`taskdefinition` already extracted to `domains/ecs/taskdefinition`). | Deferred: ADR-0004 blocked-dependency list — `iam:PassRole` in the gateway adapter and EC2/ELBv2/EIP adapters called as internal APIs. |
|
||||
| `handlers/eks` | — | Done for the relocation: the remaining EKS implementation moved to `domains/eks/cluster` (package `cluster`) in `spinifex#1264` with no API, NATS subject, storage or behaviour change, alongside the already-extracted `access` and `addon` packages. Deferred follow-on: nodegroup service extraction. The nodegroup operations are methods on the shared `EKSServiceImpl` and use its private state, so moving them to their own resource package needs a nodegroup owner and interfaces, which is decomposition work, not a move. The ADR-0004 blocked-dependency imports (IAM role/instance-profile creation, Bluebottle `auth`, the NATS CA, and EKS identity: token review, OIDC) remain dependency-rule debt, with their allowlist entries now sourced from `domains/eks/cluster`. |
|
||||
| `handlers/rds` | — | Done. The remaining RDS instance resource moved to `domains/rds/instance` (package `instance`) in `spinifex#1251` with no API, NATS subject, storage or behaviour change, fulfilling ADR-0001's organise-by-resource condition alongside the already-extracted subnet group, parameter group and engine catalogue. The ADR-0004 blocked-dependency imports (quota admission, the duplicated expected-node fan-out, the IAM ensurer and the Bluebottle master key) remain dependency-rule debt, with their allowlist entries now sourced from `domains/rds/instance`. |
|
||||
| `handlers/elbv2` | — | Done. The implementation moved to `domains/elbv2` (package `elbv2`) with no API, NATS subject, storage or behaviour change. Q-30's data-plane boundary remains deferred debt; the amended ADR-0001 row makes it not a relocation prerequisite. Its cross-domain, IAM and `agents/elbv2` imports remain dependency-rule debt, with their allowlist entries now sourced from `domains/elbv2`. |
|
||||
| `handlers/elbv2` | Everything; not yet touched. | Deferred: ADR-0001 sequences this after the ELBv2 data-plane boundary is settled. Needs decision: what evidence establishes that boundary is settled. |
|
||||
| `handlers/iam/mock` | — | Done. Moved with the implementation to `domains/iam/mock`. |
|
||||
| `gateway` (root) | Generic HTTP route assembly, SigV4/registry authentication, the policy gate, request-audit middleware, request-context keys, and the legacy per-service dispatch switch for every service not yet registered. | Deferred: the ADR-0002 phases above (auth/IAM and request-context boundary design) gate moving the audit, context-key and legacy-dispatch mechanics. |
|
||||
| `gateway/bedrock` | — | Done. The whole package moved into `domains/ochre` in `spinifex#1266` with its package-private core and tests intact and no behaviour change. Separating the action adapters into `domains/ochre/awsapi` remains deferred, recorded as `domains/ochre` (action adapters) debt above. |
|
||||
| `gateway/bedrock` | The Ochre domain core, its stores and the inference engine, together with the Bedrock action layer, in one package with internal cycles. | Deferred: needs the split plan recorded above — domain core and stores to `domains/ochre/<resource>`, inference to an Ochre-owned package, and only the action layer to `domains/ochre/awsapi`. |
|
||||
| `gateway/{iam,sts,eks,rds}` | — | Done. The action adapters moved to `domains/{iam,sts,eks,rds}/awsapi` in `spinifex#1265` with no API, NATS subject, storage or behaviour change. ADR-0001 does not make ADR-0002 registration a prerequisite for relocating a domain's adapters, as the EC2 adapter moves already showed; the root `gateway` dispatch that still imports them is recorded `gateway` (service registration) debt. |
|
||||
| `gateway/ecs` | The ECS AWS action adapters and dispatch, not yet registered through ADR-0002. | Not moved. Its handler is now `domains/ecs`, so the follows-its-handler blocker no longer applies. |
|
||||
| `gateway/elbv2` | The ELBv2 AWS action adapters and dispatch, not yet registered through ADR-0002. | Not moved. Its handler is now `domains/elbv2`, so the follows-its-handler blocker no longer applies. |
|
||||
| `gateway/{ecs,elbv2}` | Each service's AWS action adapters and dispatch, not yet registered through ADR-0002. | Deferred: each follows its `handlers/<svc>`, because a new-path `awsapi` cannot import a legacy handler. |
|
||||
| `gateway/spx` | `promote_image.go` and `spx.go` dispatch (`storage.go` already consumes `contracts/operator/v1`). | Needs decision: which package owns operator-plane action dispatch registration — an `ingress/aws`-style seam for the operator plane, or `gateway` stays its home. |
|
||||
| `gateway/tagging` | — | Done. `domains/tagging` owns the read-only cross-resource `GetResources` aggregation and its API semantics (`0d830e901`, `aedd0b2c4`); AWSGW composes the owner listers, the EC2 tag index and ELBv2 (`f3545e3ea`); the root `gateway` keeps only the legacy dispatch in `gateway/tagging.go`. The package is removed, and its two allowlist rows and frozen-legacy entry are gone. Owner tag mutation capabilities remain future direction, so the RDS and ECS tag-writing edges above stay open. |
|
||||
| `services/viperblockd` | — | Done. The adapter is `providers/ebs/viperblock` (with the read-only `dirty` reader), the shared vocabulary is `contracts/viperblockd/legacy/v1`, and the role is `runtime/roles/viperblockd`. `services/` is empty. |
|
||||
| `utils` | — | Done and deleted. `ec2_subject.go` went to `contracts/ec2/v1`, the vpcd protocol to `contracts/network/v1` with the typed client in `domains/network/projection`, and the master-key loader to `domains/ec2/ebs/encryption`. |
|
||||
| `vpcd` | — | Done. The IMDS local-file reader and record-space opener moved to `domains/ec2/instance/state` and are injected through `guestmetadata.LocalStateReader`/`RecordLoader` (recorded above), so `vpcd` no longer imports `daemon` directly; its inter-process boundary now exists (`contracts/network/v1`, with `domains/network/subscribers` as the consumer and `domains/network/projection` as the producer client). The package moved unchanged to `runtime/roles/vpcd` (recorded below), and the legacy directory is removed. |
|
||||
| `daemon` | `runtime/daemon` (whole-package move, above) | Done. Q-28's ownership and deployment refactor remains deferred debt; the amended ADR-0001 row makes it not a relocation prerequisite. |
|
||||
| `daemon` | Reserved as `runtime/daemon`'s home only; not split or moved. | Deferred: Q-28's write-side consolidation is a prerequisite ADR-0001 names explicitly. |
|
||||
| `admin` | — | Done and deleted. The operator use cases moved to `operator/admin`, image promotion to `domains/ec2/image`, the bootstrap account identifiers to `foundation/aws/identifiers`, and the credential-format generators to `foundation/aws/credentials` (recorded above). Operator bootstrap's direct credential generation stays recorded as debt. |
|
||||
| `accountteardown` | The whole package, a frozen legacy root. Its direct cleanup calls into `domains/ec2/awsapi/*` action adapters, other domain and handler services, the object-store provider, raw NATS subjects and a raw KV bucket stay as recorded debt. | Assessed for relocation and not moved. ADR-0001's `accountteardown` row requires conversion to domain-owned cleanup capabilities before moving, and ADR-0008 does not authorise a mechanical rename. It remains a legacy root until its direct cleanup calls are converted to domain-owned capabilities; its destination after that conversion is `operator/accountlifecycle`. |
|
||||
| `lbagent` | `agents/elbv2` (whole-package move, above) | Done. Q-30's data-plane boundary remains deferred debt; the amended ADR-0001 row makes it not a relocation prerequisite. The `domains/elbv2` import stays recorded debt. |
|
||||
| `lbagent` | The whole package; `handlers/elbv2` imports it as a domain-to-agent implementation edge (recorded above). | Deferred: Q-30 reserves `agents/elbv2` as its destination but forbids moving it as an isolated rename ahead of the common-agent design. |
|
||||
| `migrate` | Nothing at the old path. | Split, not an exact ADR row: ADR-0001 maps `migrate` to `foundation/state/migrate`, but the package held two responsibilities. The KV migration engine and its tests are in `foundation/state/migrate` (`8cd9cff75`); the config-file and object-store upgrade workflow moved to `operator/upgrade` (`919d5bfa6`) by owner decision. Needs an ADR-0001 map clarification before this row can be claimed as target conformance. |
|
||||
|
||||
### Remaining legacy roots by blocker class
|
||||
|
||||
Two legacy roots remain: `accountteardown` and `gateway`.
|
||||
Five legacy roots remain: `accountteardown`, `daemon`, `gateway`, `handlers` and `lbagent`.
|
||||
The rows above give each path's blocker; this summary groups them so a deferral is not mistaken for unfinished mechanical work.
|
||||
ADR-0001 completion is structural, so none of these is cleared by remediating service lifecycle behaviour.
|
||||
|
||||
| Class | Paths | Status |
|
||||
|---|---|---|
|
||||
| Explicitly deferred by ADR-0001 | `daemon` (Q-28), `lbagent` (Q-30) | Closed for this refactor; the ADR names the prerequisite. |
|
||||
| No missing boundary (candidate moves) | `handlers/ecs/bus` → `contracts/ecs/v1` (`a130f6b8e`); `admin/images_promote.go` → `domains/ec2/image` (`263c58167`); `admin` bootstrap account identifiers → `foundation/aws/identifiers` (`3fdaa54d9`); `admin` credential-format generators → `foundation/aws/credentials` `0368e861a`, retiring `admin` | Done. |
|
||||
| One narrow missing boundary | Done: `handlers/sts` EKS JWKS lookup → `handlers/eks` `ClusterJWKSResolver` (`254d898a3`); `gateway/tagging` → `domains/tagging` with AWSGW-composed listers (`0d830e901`, `f3545e3ea`, `aedd0b2c4`). Open: `gateway/spx` (owner of operator-plane dispatch registration) | Each needs one decision or one capability, then a mechanical move. |
|
||||
| Broader boundary or follows another package | `accountteardown` (domain cleanup capabilities); `gateway` root (ADR-0002 auth and request-context phase) | Recorded debt with removal conditions above. |
|
||||
| Broader boundary or follows another package | `handlers/{ecs,eks,rds}` (ADR-0004 blocked-dependency lists); `handlers/elbv2` (data-plane boundary evidence); `accountteardown` (domain cleanup capabilities); `gateway` root (ADR-0002 auth and request-context phase); `gateway/{ecs,elbv2}` (each follows its `handlers/<svc>`, because a new-path `awsapi` cannot import a legacy handler); `gateway/bedrock` (the recorded Ochre split plan) | Recorded debt with removal conditions above. |
|
||||
|
||||
Per-consumer IAM capabilities are the widest unblocker: they gate STS's IAM reads, Ochre's IAM reads, the ELBv2 role path and the ECS, EKS and RDS ensurers.
|
||||
|
||||
|
||||
@@ -51,12 +51,24 @@ sudo spx admin gpu setup
|
||||
What it does:
|
||||
|
||||
1. Detects installed GPUs and their PCI IDs.
|
||||
2. Enables IOMMU in GRUB (`intel_iommu=on iommu=pt` or `amd_iommu=on iommu=pt`).
|
||||
2. Adds the IOMMU kernel arguments (`intel_iommu=on iommu=pt` or `amd_iommu=on iommu=pt`): to `GRUB_CMDLINE_LINUX_DEFAULT` with `update-grub`, or with `grubby` to every boot entry and to `/etc/kernel/cmdline`, which kernels installed later inherit.
|
||||
3. Writes the vfio udev rule (`/etc/udev/rules.d/99-spinifex-vfio.rules`).
|
||||
4. Blacklists `nouveau` (NVIDIA) and `amdgpu` (AMD) so the host kernel does not claim the device.
|
||||
5. Configures `vfio-pci` early binding in `/etc/modprobe.d/vfio-pci.conf`.
|
||||
6. Adds vfio modules to initramfs.
|
||||
6. Loads the vfio modules from the initramfs: through `/etc/initramfs-tools/modules` and `update-initramfs -u`, or through `/etc/dracut.conf.d/spinifex-vfio.conf` and `dracut -f --regenerate-all`. With dracut, each rebuilt image is checked with `lsinitrd` for `vfio_pci`, its early-load line and the `modprobe.d` files, and setup stops naming whatever is missing.
|
||||
|
||||
The command finds its boot tools before changing anything, and stops naming them if the host has neither or both of `update-grub` and `grubby`, or of `update-initramfs` and `dracut`.
|
||||
|
||||
**Oracle Linux hosts** take the `grubby` and `dracut` path: Oracle Linux keeps kernel arguments in BLS boot entries and has no `GRUB_CMDLINE_LINUX_DEFAULT`.
|
||||
`grubby --update-kernel=ALL` writes the IOMMU arguments to every entry and to `/etc/kernel/cmdline`, from which a kernel installed later builds its own entry, and dracut applies `spinifex-vfio.conf` to every initramfs it builds.
|
||||
After installing a new `kernel-uek`, check its entry before rebooting into it:
|
||||
|
||||
```bash
|
||||
sudo grubby --info=ALL | grep -E '^(kernel|args)'
|
||||
sudo lsinitrd -k <new-kernel-version> | grep vfio
|
||||
```
|
||||
|
||||
Whole-GPU passthrough is supported on Oracle Linux; NVIDIA MIG is not.
|
||||
If any change requires a reboot, the command prints instructions and exits:
|
||||
|
||||
```
|
||||
@@ -175,7 +187,7 @@ The command signals the daemon and waits for confirmation. The GPU is released b
|
||||
|
||||
### IOMMU not active after reboot
|
||||
|
||||
Verify IOMMU is enabled in BIOS/UEFI (`Intel VT-d` or `AMD-Vi`). Then check GRUB picked up the parameters:
|
||||
Verify IOMMU is enabled in BIOS/UEFI (`Intel VT-d` or `AMD-Vi`). Then check the boot loader picked up the parameters (on Oracle Linux, `sudo grubby --info=DEFAULT` lists them for the default entry):
|
||||
|
||||
```bash
|
||||
cat /proc/cmdline | grep iommu
|
||||
|
||||
@@ -207,7 +207,7 @@ Spinifex's own **host** firewall is separate, and already carries this policy
|
||||
|
||||
Choose one method and apply it to **every** server in the cluster.
|
||||
|
||||
**Option A — existing OS.** On a server already running Ubuntu 26.04 or Debian 13:
|
||||
**Option A — existing OS.** On a server already running Ubuntu 26.04 or Debian 13 (Oracle Linux runs a single node only for now, see [Oracle Linux Hosts](/docs/install-oracle-linux)):
|
||||
|
||||
```bash
|
||||
curl -fsSL https://install.mulgadc.com | bash
|
||||
|
||||
@@ -0,0 +1,234 @@
|
||||
---
|
||||
title: "Oracle Linux Hosts"
|
||||
seoTitle: "Install Spinifex on Oracle Linux 10 — Spinifex Docs"
|
||||
description: "Prepare an Oracle Linux 10 host for Spinifex: UEK R8, SELinux, a NetworkManager br-wan bridge, the CentOS NFV SIG repository, firewalld handover and updates."
|
||||
category: "Install"
|
||||
tags:
|
||||
- install
|
||||
- oracle linux
|
||||
- single node
|
||||
resources:
|
||||
- title: "Oracle Linux 10 Documentation"
|
||||
url: "https://docs.oracle.com/en/operating-systems/oracle-linux/10/"
|
||||
- title: "CentOS NFV SIG"
|
||||
url: "https://sigs.centos.org/nfv/"
|
||||
---
|
||||
|
||||
# Oracle Linux Hosts
|
||||
|
||||
> What differs when the Spinifex host runs Oracle Linux 10 instead of Debian or Ubuntu.
|
||||
|
||||
## Table of Contents
|
||||
|
||||
- [Overview](#overview)
|
||||
- [Prerequisites](#prerequisites)
|
||||
- [Instructions](#instructions)
|
||||
- [Troubleshooting](#troubleshooting)
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
Spinifex installs on Oracle Linux 10.2 or a later 10.x release, running the UEK R8 kernel, on x86_64.
|
||||
The installer, the release tarball and the install steps are the same as on Debian; this page covers what the host needs first and what the installer changes on it.
|
||||
Follow it alongside [Single-Node Install](/docs/install).
|
||||
|
||||
**Supported on Oracle Linux:**
|
||||
|
||||
- A single node.
|
||||
- Guests, VPCs, routed NAT, Elastic IPs, and Application and Network Load Balancers.
|
||||
- Whole-GPU passthrough, on x86_64 GPU hosts (see [GPU Passthrough](/docs/gpu-passthrough)).
|
||||
|
||||
**Not supported on Oracle Linux yet:**
|
||||
|
||||
- Clusters of more than one node. They wait for IPsec on libreswan, so `spx admin init` turns IPsec off on Oracle Linux and refuses `--ipsec=true`.
|
||||
- Clusters that mix Oracle Linux with Debian or Ubuntu nodes, on any release.
|
||||
- SELinux enforcing, aarch64, Oracle Linux 9 or earlier, the Red Hat Compatible Kernel, a ZFS root, and NVIDIA MIG.
|
||||
|
||||
Open vSwitch and OVN come from the CentOS NFV SIG, because Oracle ships neither for Oracle Linux 10.
|
||||
The NFV SIG is a community repository, and Oracle does not support the packages it publishes.
|
||||
Spinifex installs `openvswitch3.5` and `ovn25.03`, the versions Debian 13 ships, and its repository file lets the SIG supply nothing else.
|
||||
QEMU comes from Oracle's KVM Utilities repository, because only that build has the `microvm` machine type Spinifex load balancers boot on.
|
||||
|
||||
An Oracle Linux node must be a dedicated Spinifex host.
|
||||
Spinifex claims every `tap*` device and stops libreswan's `ipsec.service`, so VPNs, hand-run QEMU guests and libreswan tunnels of your own cannot share the host.
|
||||
Crypto policies other than `DEFAULT` and `FIPS` are untested, and so is Secure Boot.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
### Operating system and kernel
|
||||
|
||||
Install Oracle Linux 10 with Oracle's ISO or a KVM template, then update it to the current minor release and reboot:
|
||||
|
||||
```bash
|
||||
sudo dnf update -y
|
||||
sudo reboot
|
||||
```
|
||||
|
||||
After the reboot, check the release, the kernel and the CPU:
|
||||
|
||||
```bash
|
||||
cat /etc/oracle-release # Oracle Linux Server release 10.2 or later
|
||||
uname -r # 6.12.0-...el10uek.x86_64
|
||||
grep -c -w avx2 /proc/cpuinfo # non-zero
|
||||
```
|
||||
|
||||
The kernel must be UEK R8 (`6.12.0-*uek`), and it must be the running kernel: an installed but not booted `kernel-uek` is refused.
|
||||
The CPU must be x86-64-v3 or newer (Intel Haswell, AMD Excavator or later), since the NFV SIG and KVM Utilities builds target it.
|
||||
The installer names any missing CPU flag.
|
||||
|
||||
The installer adds `kernel-uek-modules` for the running kernel, which carries `openvswitch.ko` and pulls in `linux-firmware`, about 800 MB.
|
||||
Oracle publishes modules for each kernel separately, so a host running an old kernel may find its modules gone from the repository; the installer then stops and asks for a kernel update and a reboot.
|
||||
|
||||
### SELinux
|
||||
|
||||
Spinifex ships no SELinux policy yet, so SELinux must be permissive or disabled, both now and at the next boot.
|
||||
Oracle's images boot enforcing.
|
||||
|
||||
```bash
|
||||
sudo setenforce 0
|
||||
sudo sed -i 's/^SELINUX=.*/SELINUX=permissive/' /etc/selinux/config
|
||||
getenforce # Permissive
|
||||
```
|
||||
|
||||
The installer and `spx admin preflight` refuse a host that is enforcing in either place.
|
||||
|
||||
### WAN bridge with NetworkManager
|
||||
|
||||
As on every OS, the host's uplink must already sit on a Linux bridge named `br-wan`, with the host address and default route on the bridge (see [Single-Node Install](/docs/install#prerequisites)).
|
||||
On Oracle Linux, NetworkManager owns `br-wan`; Spinifex leaves the bridge and its NIC to it and marks the links it creates itself as unmanaged.
|
||||
|
||||
Create the bridge from the console, not over SSH on the NIC being moved, since the host loses its address while the profile changes.
|
||||
Replace `eno1` with the uplink NIC and `"System eno1"` with its current profile (`nmcli connection show`):
|
||||
|
||||
```bash
|
||||
sudo nmcli connection add type bridge ifname br-wan con-name br-wan \
|
||||
bridge.stp no ipv4.method auto ipv6.method disabled
|
||||
sudo nmcli connection add type bridge-slave ifname eno1 master br-wan con-name br-wan-port
|
||||
sudo nmcli connection down "System eno1"
|
||||
sudo nmcli connection up br-wan
|
||||
```
|
||||
|
||||
For a static address, give the bridge `ipv4.method manual` with `ipv4.addresses`, `ipv4.gateway` and `ipv4.dns` instead of `ipv4.method auto`.
|
||||
|
||||
`bridge.stp no` matters: a NetworkManager bridge runs STP by default, which holds the port in its forwarding delay for about 15 seconds after every link up, long enough for DHCP on the bridge to time out.
|
||||
Then verify, as on Debian:
|
||||
|
||||
```bash
|
||||
ip -br link show br-wan # UP
|
||||
ip route # default via ... dev br-wan
|
||||
```
|
||||
|
||||
### Network access during install
|
||||
|
||||
Besides `install.mulgadc.com` and the AWS CLI download, the install reaches `yum.oracle.com` for the Oracle Linux repositories, `mirror.stream.centos.org` for the NFV SIG packages, and `www.centos.org` for the NFV SIG signing key.
|
||||
An air-gapped install on Oracle Linux is not documented yet.
|
||||
|
||||
## Instructions
|
||||
|
||||
## Step 1. Install Spinifex
|
||||
|
||||
Run the installer as a user with `sudo`, exactly as on Debian:
|
||||
|
||||
```bash
|
||||
curl -fsSL https://install.mulgadc.com | bash
|
||||
```
|
||||
|
||||
Then continue with [Single-Node Install](/docs/install) from Step 2: `setup-ovn.sh --management`, `spx admin init` and `systemctl start spinifex.target`.
|
||||
`spx admin init` prints that IPsec is off on this host; on a single node there is no peer to encrypt traffic to.
|
||||
|
||||
## Step 2. Know What the Installer Changed
|
||||
|
||||
On Oracle Linux the installer makes these changes, which Debian hosts do not need:
|
||||
|
||||
- **Repositories.** It enables `ol10_UEKR8` and `ol10_kvm_utils`, and adds `qemu-kvm*` and `qemu-img*` to the `excludepkgs` of `ol10_appstream`, keeping any excludes already set, so a later AppStream build can never replace the KVM Utilities QEMU.
|
||||
- **NFV SIG.** It writes `/etc/yum.repos.d/spinifex-nfv-sig.repo` with an `includepkgs=` list of the OVS and OVN packages, and imports the SIG key to `/etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-NFV` only when its fingerprint is `35154228174901BEFA8E69A621465E289D2A76A7`.
|
||||
- **OVS and OVN user.** Both run as root, as on Debian, through `OVS_USER_ID` and `OVN_USER_ID` in `/etc/sysconfig/openvswitch` and `/etc/sysconfig/ovn`.
|
||||
- **DNS.** Where systemd-resolved is installed but not running, it saves `/etc/resolv.conf` to `/var/lib/spinifex-reset/resolv.conf.orig`, starts resolved behind NetworkManager and links `/etc/resolv.conf` to the resolved stub. It then checks that names still resolve and stops, printing the commands that put DNS back, if they do not.
|
||||
- **chrony.** It appends `confdir /etc/chrony.d` to `/etc/chrony.conf` once, and writes its step policy to `/etc/chrony.d/spinifex.conf`.
|
||||
- **`spx` under `sudo`.** Oracle Linux's `sudo` path leaves out `/usr/local/bin`, so the installer links `/usr/bin/spx` to `/usr/local/bin/spx`.
|
||||
|
||||
The kernel modules list (`/etc/modules-load.d/spinifex.conf`) and the host firewall below apply on every OS.
|
||||
|
||||
## Step 3. Replace firewalld Rules You Depend On
|
||||
|
||||
The installer stops and masks firewalld, because with it running a packet must pass both firewalld's table and the Spinifex one, which silently breaks routed NAT, Elastic IPs and instance metadata.
|
||||
When firewalld was active, the installer turns on the Spinifex nftables policy in its place; pass `--firewall=on` or `--firewall=off` to choose yourself.
|
||||
|
||||
firewalld's rules are not saved or migrated.
|
||||
The Spinifex policy filters inbound traffic only and opens 53, 443, 3000, 8443 and 9999 to any source.
|
||||
It accepts SSH from every IPv4 source until `$trusted_ssh_peers` in `/etc/spinifex/firewall/custom.nft` narrows it.
|
||||
Restate any other restriction you had, such as SSH limited to a management subnet, in that file; [Host Firewall](/docs/host-firewall) shows how.
|
||||
|
||||
From the moment firewalld stops until the daemon first loads the Spinifex policy, the host accepts every connection.
|
||||
Uninstalling Spinifex leaves firewalld unmasked but disabled; it does not turn it back on.
|
||||
|
||||
## Step 4. Keep the Host Updated
|
||||
|
||||
Do not enable `dnf-automatic`.
|
||||
An Open vSwitch package upgrade restarts OVS, which interrupts guest networking, so update `openvswitch3.5`, `ovn25.03*` and `qemu-kvm*` in a maintenance window.
|
||||
Stay on the current Oracle Linux 10 minor release, which is what Spinifex is tested on.
|
||||
|
||||
Before rebooting into a new kernel, check that its modules are installed, or OVS cannot start after the reboot:
|
||||
|
||||
```bash
|
||||
rpm -q kernel-uek-core kernel-uek-modules
|
||||
```
|
||||
|
||||
Each `kernel-uek-core` version needs a `kernel-uek-modules` of the same version; install a missing one with `sudo dnf install kernel-uek-modules-<version>`.
|
||||
|
||||
Track Oracle Linux security advisories (ELSA) for the host, the kernel and QEMU, for example with `dnf updateinfo list --security`.
|
||||
Those advisories do not cover the NFV SIG packages, so follow Open vSwitch and OVN security announcements for those.
|
||||
|
||||
## Step 5. Uninstall
|
||||
|
||||
`/usr/local/share/spinifex/uninstall-spx.sh` removes Spinifex as on Debian, and lists what it leaves in place on Oracle Linux: the repository changes and the SIG key, `OVS_USER_ID` and `OVN_USER_ID`, the OVS and OVN logrotate edits, the chrony include, `/etc/modules-load.d/spinifex.conf`, the SELinux mode, a masked `ipsec.service`, firewalld disabled, and any GPU passthrough configuration.
|
||||
`--purge-deps` is refused on Oracle Linux, because dnf cannot tell the packages Spinifex installed from the host's own; remove them with `dnf remove` yourself.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Installer stops: SELinux is enforcing
|
||||
|
||||
Set SELinux permissive now and in `/etc/selinux/config`, as in [SELinux](#selinux), then run the installer again.
|
||||
Fixing only one of the two is refused too.
|
||||
|
||||
### Installer stops: the running kernel is not UEK R8
|
||||
|
||||
Install `kernel-uek`, reboot into it, check `uname -r`, then run the installer again.
|
||||
|
||||
### Installer stops: kernel-uek-modules is no longer in the repositories
|
||||
|
||||
The running kernel is older than anything the repository still carries modules for.
|
||||
|
||||
```bash
|
||||
sudo dnf update -y kernel-uek
|
||||
sudo reboot
|
||||
```
|
||||
|
||||
### Installer stops: a repository is unreachable
|
||||
|
||||
The message names the repository.
|
||||
Check the host can reach `yum.oracle.com` and `mirror.stream.centos.org` over HTTPS, including through any proxy dnf uses, then run the installer again.
|
||||
|
||||
### DHCP on br-wan times out
|
||||
|
||||
Check STP is off on the bridge:
|
||||
|
||||
```bash
|
||||
nmcli -g bridge.stp connection show br-wan
|
||||
```
|
||||
|
||||
If it prints `yes`, run `sudo nmcli connection modify br-wan bridge.stp no` and `sudo nmcli connection up br-wan`.
|
||||
|
||||
### `sudo spx` finds no command
|
||||
|
||||
Check that `/usr/bin/spx` links to `/usr/local/bin/spx`; run the installer again if it does not, or call `sudo /usr/local/bin/spx` meanwhile.
|
||||
|
||||
### CA certificate not trusted
|
||||
|
||||
`sudo spx admin init` installs the CA automatically. To install it again by hand on Oracle Linux:
|
||||
|
||||
```bash
|
||||
sudo cp /etc/spinifex/ca.pem /etc/pki/ca-trust/source/anchors/spinifex-ca.crt
|
||||
sudo update-ca-trust
|
||||
```
|
||||
@@ -1,7 +1,7 @@
|
||||
---
|
||||
title: "Source Install"
|
||||
seoTitle: "Build and Install Spinifex from Source — Spinifex Docs"
|
||||
description: "Build Spinifex from source on Ubuntu or Debian for development, custom builds, or contributing changes, then install and run the resulting binaries locally."
|
||||
description: "Build Spinifex from source on Debian, Ubuntu or Oracle Linux for development, custom builds or contributing changes, then install and run the binaries."
|
||||
category: "Install"
|
||||
tags:
|
||||
- install
|
||||
@@ -34,6 +34,7 @@ This guide builds Spinifex from source. For production deployments, the [binary
|
||||
|
||||
- Ubuntu 26.04
|
||||
- Debian 13
|
||||
- Oracle Linux 10.2 or a later 10.x, on UEK R8, x86_64 only. Prepare the host with [Oracle Linux Hosts](/docs/install-oracle-linux) first; `make quickinstall` also enables the CodeReady Builder repository for `nbdkit-devel`.
|
||||
|
||||
## Instructions
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
---
|
||||
title: "Single-Node Install"
|
||||
seoTitle: "Install Spinifex on a Single Server — Spinifex Docs"
|
||||
description: "Install Spinifex on one Ubuntu or Debian server with the binary installer and get an AWS-compatible EC2, S3, and VPC stack running on your own hardware."
|
||||
description: "Install Spinifex on one Debian, Ubuntu or Oracle Linux server with the binary installer and run an AWS-compatible EC2, S3 and VPC stack on your hardware."
|
||||
category: "Install"
|
||||
tags:
|
||||
- install
|
||||
@@ -39,6 +39,7 @@ This guide installs Spinifex on a single server using the binary installer. For
|
||||
|
||||
- Ubuntu 26.04
|
||||
- Debian 13
|
||||
- Oracle Linux 10.2 or a later 10.x, on UEK R8, x86_64 only, as a single node. Open vSwitch and OVN come from the CentOS NFV SIG, a community repository that Oracle does not support. Prepare the host with [Oracle Linux Hosts](/docs/install-oracle-linux) first.
|
||||
|
||||
**What Gets Installed:**
|
||||
|
||||
@@ -139,19 +140,33 @@ Continue to [Setting Up Your Cluster](/docs/setting-up-your-cluster) to import a
|
||||
|
||||
### spx Command Not Found
|
||||
|
||||
The installer puts `spx` in `/usr/local/bin`.
|
||||
Where `sudo`'s `secure_path` leaves that directory out, as on Oracle Linux, the installer also links `/usr/bin/spx` to it, so `sudo spx` works; changing `PATH` in your shell does not reach commands run with `sudo`.
|
||||
Check both:
|
||||
|
||||
```bash
|
||||
export PATH=$PATH:/usr/local/bin
|
||||
ls -l /usr/local/bin/spx /usr/bin/spx
|
||||
sudo sh -c 'command -v spx'
|
||||
```
|
||||
|
||||
If `sudo` still finds no `spx`, run the installer again, or call `sudo /usr/local/bin/spx` meanwhile.
|
||||
|
||||
### CA Certificate Not Trusted
|
||||
|
||||
`sudo spx admin init` installs the CA automatically. If you need to re-install it manually:
|
||||
`sudo spx admin init` installs the CA automatically. If you need to re-install it manually on Debian or Ubuntu:
|
||||
|
||||
```bash
|
||||
sudo cp /etc/spinifex/ca.pem /usr/local/share/ca-certificates/spinifex-ca.crt
|
||||
sudo update-ca-certificates
|
||||
```
|
||||
|
||||
On Oracle Linux:
|
||||
|
||||
```bash
|
||||
sudo cp /etc/spinifex/ca.pem /etc/pki/ca-trust/source/anchors/spinifex-ca.crt
|
||||
sudo update-ca-trust
|
||||
```
|
||||
|
||||
### OVN Services Not Starting
|
||||
|
||||
```bash
|
||||
|
||||
@@ -127,8 +127,8 @@ Operators own remediation for layers Spinifex does not ship:
|
||||
| Layer | Responsibility |
|
||||
|-------|----------------|
|
||||
| Spinifex releases | Apply released patches within the correct-SLA above. |
|
||||
| Host OS + kernel | Track distribution advisories (Debian DSA, Ubuntu USN) at matching cadence. |
|
||||
| OVN / OVS | Patched via the operator's package channel alongside the OS. |
|
||||
| Host OS + kernel | Track distribution advisories (Debian DSA, Ubuntu USN, Oracle Linux ELSA) at matching cadence. |
|
||||
| OVN / OVS | Patched via the operator's package channel alongside the OS. On Oracle Linux they come from the CentOS NFV SIG, which ELSA does not cover, so track upstream Open vSwitch and OVN security announcements. |
|
||||
| QEMU / KVM / libvirt | As above — hypervisor CVEs affect tenant isolation. |
|
||||
| Host AV / EDR agent | See [Malware Protection §2](/docs/malware-protection#2-update-requirements-sil1-3144). |
|
||||
|
||||
|
||||
@@ -75,7 +75,7 @@ All Spinifex nodes must include the following in their malware protection scope.
|
||||
| `/usr/local/bin/spx` | Spinifex CLI and service entry point. All `spinifex-*.service` units invoke `spx service <name> start`, so every service binary is reached through this one path. |
|
||||
| `/usr/bin/ovn-*`, `/usr/bin/ovs-*` | OVN / OVS control-plane and datapath binaries (installed via `apt` from the `ovn-central`, `ovn-host`, `openvswitch-switch` packages). |
|
||||
| `/usr/lib/<arch>-linux-gnu/nbdkit/plugins/nbdkit-viperblock-plugin.so` | Viperblock NBD plugin loaded by `nbdkit` to expose EBS volumes as block devices. The exact path comes from `nbdkit --dump-config` (`plugindir=`); on Debian 13 amd64 it is `/usr/lib/x86_64-linux-gnu/nbdkit/plugins/`. |
|
||||
| `/usr/local/share/ca-certificates/spinifex-ca.crt` | Cluster CA cert installed into the system trust store |
|
||||
| `/usr/local/share/ca-certificates/spinifex-ca.crt` | Cluster CA cert installed into the system trust store. On Oracle Linux it is `/etc/pki/ca-trust/source/anchors/spinifex-ca.crt`. |
|
||||
|
||||
### 1.2 Configuration and Secrets
|
||||
|
||||
|
||||
@@ -145,6 +145,16 @@ Spinifex nodes initiate a small, fixed set of outbound connections.
|
||||
| `https://d2yp8ipz5jfqcw.cloudfront.net` | Alpine image for managed HAProxy load-balancer | HTTPS | TLS + checksum verification |
|
||||
| `https://install.mulgadc.com/install` | One-shot install telemetry POST on `spx admin init` / `join`. | HTTPS | TLS |
|
||||
|
||||
**During install and package updates on Oracle Linux hosts:**
|
||||
|
||||
| Destination | Purpose | Protocol | Verification |
|
||||
|-------------|---------|----------|--------------|
|
||||
| `https://yum.oracle.com` | Oracle Linux BaseOS, AppStream, UEK R8 and KVM Utilities repositories | HTTPS | TLS + Oracle RPM signatures |
|
||||
| `https://mirror.stream.centos.org/SIGs/10-stream/nfv/` | Open vSwitch and OVN from the CentOS NFV SIG, limited by `includepkgs=` | HTTPS | TLS + NFV SIG RPM signatures |
|
||||
| `https://www.centos.org/keys/RPM-GPG-KEY-CentOS-SIG-NFV` | NFV SIG signing key, fetched once at install | HTTPS | TLS + fingerprint pinned in `setup.sh` |
|
||||
|
||||
Debian and Ubuntu hosts reach their distribution's own mirrors for the same purpose.
|
||||
|
||||
**To peer nodes (cluster-internal):** NATS federation (4248), Predastore S3 (8443), OVN NB/SB (6641/6642), northstar DNS forwarding (5300), and the Geneve/IPsec overlay (6081, 500, 4500, ESP) — see [§3](#3-cross-node-internal-connections) for encryption and verification of each. The daemon also polls local NATS monitoring at `127.0.0.1:8222/varz` (loopback HTTP). It opens no connection to Predastore for status: the storage topology it reports comes from reading `predastore.toml`, and no Predastore node serves a status endpoint.
|
||||
|
||||
**Update checks and metadata.** Spinifex does not check for updates and does not consume a cloud metadata service (`169.254.169.254` is served *by* the cluster to guest VMs). Node software updates come from the operator's OS package channel. The install-telemetry endpoint above is the only vendor-operated destination contacted by a node; closed-egress deployments should disable it and record the opt-out in the security plan.
|
||||
@@ -173,7 +183,10 @@ Nodes **must** sit on a network segment that is not routed to tenant/guest VMs o
|
||||
|
||||
Default external surface is five listeners — **9999** (AWS API), **3000** (UI), **22** (SSH), **8443** (S3) and **53** (DNS) — plus **4432** transiently during bootstrap. Every other listener is cluster- or encap-scoped and the operator must enforce this with a host firewall or an upstream network ACL.
|
||||
|
||||
The nodes ship no firewall policy today. Until one is installed, this is the operator's responsibility and the reference below is the starting point.
|
||||
Every node ships a host firewall policy, `table inet spinifex_filter` in `/etc/spinifex/firewall/spinifex.nft`, written by `setup.sh`; `spinifex-daemon` loads it once cluster peers resolve, filling the peer and encap sets itself.
|
||||
It is armed by default on ISO installs; a `setup.sh` install arms it with `--firewall=on`, or when firewalld was active, and otherwise installs it unarmed, so on those nodes the scope split is still the operator's to enforce.
|
||||
`setup.sh` masks firewalld wherever it is installed, and `spx admin preflight` fails while firewalld or ufw is active, because a second ruleset silently drops routed NAT, elastic IP and metadata traffic.
|
||||
The shipped policy is the authority; the reference below explains the rules it needs and is looser than the shipped one in places, such as accepting all `ime-*` traffic where the shipped policy accepts only the metadata and DNS endpoints.
|
||||
|
||||
> **Read the notes under the ruleset before applying it.** A default-deny input policy that
|
||||
> omits any of the loopback, conntrack, `ime-*` or Geneve rules will break instance
|
||||
|
||||
@@ -50,13 +50,13 @@ services:
|
||||
# Translates VPC / EBS / S3 operations into vpc.* / ebs.* / s3.* and
|
||||
# forwards to vpcd / viperblockd / predastore.
|
||||
#
|
||||
# Source layout: spinifex/runtime/roles/spinifex (entry), spinifex/runtime/daemon
|
||||
# Source layout: spinifex/runtime/roles/spinifex (entry), spinifex/daemon
|
||||
# (subscribe registration), spinifex/domains/<domain> and the legacy
|
||||
# spinifex/handlers/{ecs,eks,elbv2,iam,rds} packages (per-subject handler
|
||||
# bodies). Selector treats these paths as belonging to this service via
|
||||
# the `additional_paths` field.
|
||||
additional_paths:
|
||||
- spinifex/runtime/daemon
|
||||
- spinifex/daemon
|
||||
- spinifex/domains/ec2/account
|
||||
- spinifex/domains/ec2/eigw
|
||||
- spinifex/domains/ec2/eip
|
||||
@@ -74,10 +74,10 @@ services:
|
||||
- spinifex/domains/ec2/volume
|
||||
- spinifex/domains/ec2/volumestate
|
||||
- spinifex/domains/ec2/vpc
|
||||
- spinifex/domains/elbv2
|
||||
- spinifex/handlers/elbv2
|
||||
- spinifex/domains/acm
|
||||
- spinifex/domains/iam
|
||||
- spinifex/domains/ecs
|
||||
- spinifex/handlers/ecs
|
||||
- spinifex/domains/rds/instance
|
||||
- spinifex/domains/rds/subnetgroup
|
||||
- spinifex/domains/rds/parametergroup
|
||||
@@ -136,7 +136,7 @@ services:
|
||||
- "spinifex.cluster.shutdown.infra"
|
||||
# Operator-plane Predastore topology contract (contracts/operator/v1).
|
||||
- "spinifex.storage.config"
|
||||
# ELBv2 control plane (domains/elbv2/service_nats.go topic map).
|
||||
# ELBv2 control plane (handlers/elbv2/service_nats.go topic map).
|
||||
- "elbv2.CreateLoadBalancer"
|
||||
- "elbv2.DeleteLoadBalancer"
|
||||
- "elbv2.DescribeLoadBalancers"
|
||||
@@ -163,7 +163,7 @@ services:
|
||||
# per-subject list would be a hand-mirror of that file with nothing
|
||||
# checking it.
|
||||
- "rds.*"
|
||||
# ECS control plane (domains/ecs/service_nats.go, registered in
|
||||
# ECS control plane (handlers/ecs/service_nats.go, registered in
|
||||
# daemon.go). One wildcard, as for RDS: every action subject is a literal
|
||||
# in those two files, so a per-subject list would only mirror them.
|
||||
- "ecs.*"
|
||||
@@ -403,7 +403,7 @@ suites:
|
||||
e2e-lb:
|
||||
path: tests/e2e/lb
|
||||
covers: [awsgw, spinifex_daemon, vpcd, viperblockd, predastore]
|
||||
# Exercises full ELBv2 control plane (domains/elbv2) + data plane
|
||||
# Exercises full ELBv2 control plane (handlers/elbv2) + data plane
|
||||
# (LB agent on the system VM). ELBv2 is part of spinifex_daemon in
|
||||
# this manifest; revisit if it splits into a standalone service.
|
||||
|
||||
@@ -436,7 +436,7 @@ suites:
|
||||
path: tests/e2e/bedrock
|
||||
covers: [awsgw]
|
||||
# Ochre (AWS-Bedrock) control plane: static catalog list/get served
|
||||
# straight from awsgw (gateway/bedrock.go, domains/ochre/) — no NATS
|
||||
# straight from awsgw (gateway/bedrock.go, gateway/bedrock/) — no NATS
|
||||
# hop to spinifex_daemon. Converse (Anthropic-direct / self-host vLLM)
|
||||
# subtests are gated on backend availability and skip by default.
|
||||
|
||||
|
||||
@@ -30,7 +30,7 @@ The largest suite, 27 top-level tests. Notable entry points:
|
||||
- **`TestIMDSDatapathSurvivesVswitchdRestart`** restarts `ovs-vswitchd` under a guest with an Elastic IP, once through systemd and once with SIGKILL. After each, the guest must answer over its EIP and fetch its instance ID through IMDSv2 within 60 seconds, and `br-imds` must hold all 7 of the guest's own flows. It is sequential because it takes the whole node's datapath down.
|
||||
- `TestStopStart`, `TestEarlyRebootLiveness`, `TestGuestChurnDurability`, `TestSnapshotLifecycle`, `TestSnapshotBackedLaunch`, `TestCreateImage`, `TestVolumeLifecycle`, `TestSpotInstanceLifecycle`, `TestLaunchTemplateBoot`.
|
||||
- `TestNegativeErrorPaths` — the `8a`–`8t` AWS error-shape cases (malformed AMI ID, invalid instance type, volume in use, detach-root forbidden, unknown or malformed instance IDs on the lifecycle verbs, and so on).
|
||||
- **`TestKVReplication`** — the single-node half of the KV replication policy, and it is not the multi-node assertion with a smaller number in it. On one node one replica per bucket is the correct answer rather than the defect, so what it proves is the opposite: that nothing declared a replica count this deployment cannot place, that `spx admin kv replicas` exits 0 and reports `WANT 1`, and that `--repair` run twice against a cluster with nothing to repair changes no bucket's replica count and leaves every bucket serving. The multi-node suites cannot cover any of that, because on three nodes the unplaceable count is the right one. It lives here rather than in its own suite so it runs in every single-node nightly cell, across all three host and network permutations.
|
||||
- **`TestKVReplication`** — the single-node half of the KV replication policy, and it is not the multi-node assertion with a smaller number in it. On one node one replica per bucket is the correct answer rather than the defect, so what it proves is the opposite: that nothing declared a replica count this deployment cannot place, that `spx admin kv replicas` exits 0 and reports `WANT 1`, and that `--repair` run twice against a cluster with nothing to repair changes no bucket's replica count and leaves every bucket serving. The multi-node suites cannot cover any of that, because on three nodes the unplaceable count is the right one. It lives here rather than in its own suite so it runs in every single-node nightly cell, across every host and network permutation.
|
||||
|
||||
### `natuplink` — routed NAT (`external_mode = "nat"`), one node or three
|
||||
|
||||
@@ -40,7 +40,7 @@ Runs **on** a node, shelling out to `ip`/`iptables`/`ovn-nbctl` locally. Skips u
|
||||
|
||||
Phases 7 and 8 are gated on a cluster by what the datapath can actually do: the transit segment is a localnet, so host-to-guest Tier 1 ingress only exists where the VPC's gateway router port and the guest are both on this host. The VPC ingress route itself is installed on every node, so phase 8 checks it unconditionally.
|
||||
|
||||
Two nightly cells run it: `nat-single` (cell 19) and `nat-multi` (cell 30). Cell 20 is the bare-metal job and is not in the permutation matrix at all, which is why the multi-node NAT cell is numbered past the end rather than filling a gap.
|
||||
Three nightly cells run it: `nat-single` (cell 19), its Oracle Linux twin (cell 36) and `nat-multi` (cell 30). Cell 20 is the bare-metal job and is not in the permutation matrix at all, which is why the multi-node NAT cell is numbered past the end rather than filling a gap.
|
||||
|
||||
### `multinode` — behaviour that only exists on more than one node
|
||||
|
||||
|
||||
+52
-7
@@ -6,22 +6,66 @@ Node installation, dev-environment lifecycle, guest image builds, and verificati
|
||||
|
||||
| Script | What it does |
|
||||
|---|---|
|
||||
| `setup.sh` | The production binary installer, served from `install.mulgadc.com`. Installs deps, users, sudoers, files, systemd units, logrotate and udev rules. `SETUP_STAGES` runs a subset; `ISO_BUILD=1` runs it inside the ISO builder's chroot. |
|
||||
| `setup-ovn.sh` | Bootstraps a compute node for OVN VPC networking: installs OVN/OVS, creates `br-int`, configures the WAN bridge (bridged, DHCP, or routed-NAT uplink), sets the chassis identity, and applies overlay sysctls. `--management` also starts the OVN central services. |
|
||||
| `install-node.sh` | Forms a multi-node cluster across hosts that are already running Spinifex: rebuilds the OVN database in clustered form, runs `spx admin init` on the first host and `spx admin join` on the rest, then verifies. The scripted form of [install-multi-node](../docs/install/install-multi-node/README.md). Destroys the joining nodes' logical network state and crypto — see its `--help`. |
|
||||
| `setup.sh` | The production binary installer, served from `install.mulgadc.com`. Installs deps, users, sudoers, the host firewall policy, files, systemd units, logrotate and udev rules, and loads `openvswitch`, `geneve`, `vhost_net` and `tun`, listing them in `/etc/modules-load.d/spinifex.conf` for later boots. It masks firewalld wherever it is installed, since the Spinifex nftables policy is the only supported node firewall. An unset `--firewall` keeps the mode recorded in `/etc/spinifex/firewall/mode`, and otherwise arms the policy when firewalld was active. Where systemd-resolved is installed but not running, resolvconf is absent and NetworkManager runs, it hands host DNS to resolved: it saves `/etc/resolv.conf` once to `/var/lib/spinifex-reset/resolv.conf.orig`, links `/etc/resolv.conf` to the resolved stub, and stops unless resolved has upstream servers and the package repository host still resolves. It links `/usr/bin/spx` to `/usr/local/bin/spx` where sudo's PATH leaves out `/usr/local/bin`, and takes the nbdkit plugin directory from `nbdkit --dump-config`, stopping when nbdkit cannot say. `SETUP_STAGES` runs a subset; `ISO_BUILD=1` runs it inside the ISO builder's chroot. |
|
||||
| `setup-ovn.sh` | Bootstraps a compute node for OVN VPC networking: installs OVN/OVS, creates `br-int`, configures the WAN bridge (bridged, DHCP, or routed-NAT uplink), sets the chassis identity, and applies overlay sysctls. It runs OVS and OVN as root, setting `OVS_USER_ID` and `OVN_USER_ID` where `/etc/sysconfig` holds them, and hooks the socket-permission helper onto `ovs-vswitchd`, `ovn-controller`, `openvswitch-ipsec` and, on EL, `ovn-northd`. `--management` also starts the OVN central services, writes their `ovn-ctl` flags to `/etc/default/ovn-central` (Debian) or `/etc/sysconfig/ovn-northd` (EL), and records the role in `/etc/spinifex/ovn-role`. A later run without `--management` refuses while that file, or the `OVN_CTL_OPTS` line older nodes carry in `/etc/default/ovn-central`, records the role. It adds `maxsize 100M` to the OVS and OVN logrotate files, plus `rotate 5` where the file sets no count, and rotates as root, matching the daemons. On a host whose IPsec stack is libreswan it stops and masks `openvswitch-ipsec.service` and `ipsec.service`, because OVN IPsec does not run on libreswan yet; elsewhere it enables and starts `openvswitch-ipsec.service`, unless the daemon has masked it to keep IPsec off. The links it creates persist through systemd-networkd where that is enabled, and otherwise through `/etc/spinifex/hostnet.json`, which `spinifex-hostnet.service` (`spx admin hostnet apply`) replays at boot and on every OVS start; the first run records the choice in `/etc/spinifex/hostnet-backend`. Where NetworkManager runs, it leaves the links Spinifex owns unmanaged. |
|
||||
| `install-node.sh` | Forms a multi-node cluster across hosts that are already running Spinifex: rebuilds the OVN database in clustered form, runs `spx admin init` on the first host and `spx admin join` on the rest, then verifies. Before touching any host it refuses hosts of different OS families (`distro_os_family`) or `spx` versions, and `--ipsec on` when any host uses libreswan. The scripted form of [install-multi-node](../docs/install/install-multi-node/README.md). Destroys the joining nodes' logical network state and crypto — see its `--help`. |
|
||||
| `dev-install.sh` | Full local dev setup through the production installer: builds from source, packs a tarball, runs `setup.sh`, initialises the cluster, and starts services under systemd. |
|
||||
| `clone-deps.sh` | Clones or updates the viperblock and predastore checkouts alongside spinifex for cross-repo development. |
|
||||
|
||||
### `lib/distro.sh`: host-OS capability probes
|
||||
|
||||
The node scripts select host-OS behaviour by probing for the capability, not by reading `/etc/os-release`; only `setup.sh`'s supported-OS gate keys on the OS ID.
|
||||
Each probe prints its answer, or names every candidate it tried on stderr and returns 1, which it also does when it finds both candidates of a pair.
|
||||
|
||||
| Probe | Answers |
|
||||
|---|---|
|
||||
| `distro_pkg_manager` | `apt` or `dnf` |
|
||||
| `distro_ovs_unit` | `openvswitch-switch.service` (Debian, Ubuntu) or `openvswitch.service` (EL) |
|
||||
| `distro_ovn_central_layout` | `split` (Debian's `ovn-central` plus separate NB and SB units) or `lone-northd` (EL's single `ovn-northd.service`) |
|
||||
| `distro_ca_trust` | the trust-store update command and its anchor directory |
|
||||
| `distro_ovs_logrotate` | `/etc/logrotate.d/openvswitch-switch` or `/etc/logrotate.d/openvswitch` |
|
||||
| `distro_ovn_logrotate` | `/etc/logrotate.d/ovn-common` or `/etc/logrotate.d/ovn` |
|
||||
| `distro_os_family` | the OS family of the `/etc/os-release` text on stdin: `debian` for Debian and Ubuntu, otherwise the `ID` |
|
||||
|
||||
`distro_has_libreswan` is a predicate rather than a probe: it succeeds when libreswan's IKE daemon, `/usr/libexec/ipsec/pluto`, is installed.
|
||||
EL's `openvswitch-ipsec` drives libreswan even beside strongSwan, so finding both is not an error.
|
||||
|
||||
`distro_missing_ovs_deps <unit-file>...` is a check rather than a probe: it prints each OVS or OVN unit the given unit files depend on or order against that is not installed, and fails only when a file cannot be read.
|
||||
`setup-ovn.sh` runs it against the installed Spinifex units and stops when one is missing, because systemd silently drops ordering against a unit that does not exist; a missing `openvswitch-ipsec.service` is only a warning.
|
||||
|
||||
`setup-ovn.sh`, `uninstall-spx.sh` and `install-node.sh` source it from `lib/` beside themselves, then from `/usr/local/share/spinifex/lib/distro.sh`, where `setup.sh` installs it, and stop naming both paths when neither exists.
|
||||
`setup.sh` is piped from the install endpoint before the tarball is downloaded, and `install-node.sh --wipe` copies `node-reset.sh` alone to each host, so those two carry inline copies of the probes they need between `# --- BEGIN distro.sh probes ---` markers.
|
||||
`lib/distro_test.sh` fails when an inline copy differs from the library.
|
||||
Go code uses `spinifex/runtime/host/hostos` for the same purpose.
|
||||
|
||||
### System packages
|
||||
|
||||
`setup.sh` installs `APT_RUNTIME_PACKAGES` with apt, or on Oracle Linux `OL_RUNTIME_PACKAGES` with dnf, choosing by `distro_pkg_manager`.
|
||||
Before the dnf install it enables `ol10_UEKR8` and `ol10_kvm_utils`, excludes `qemu-kvm*` and `qemu-img*` from `ol10_appstream` so QEMU always comes from KVM Utilities, and writes `/etc/yum.repos.d/spinifex-nfv-sig.repo` for OVS 3.5 and OVN 25.03 from the CentOS NFV SIG.
|
||||
That repo carries an `includepkgs=` allowlist, and its key is refused unless its fingerprint matches `NFV_SIG_KEY_FPR`.
|
||||
It then installs `kernel-uek-modules` for the running kernel, stopping with a reboot instruction if that version has left the repositories, and loads `openvswitch`, `geneve`, `vhost_net` and `tun`.
|
||||
`make install-system` takes the same dnf path, adding the build packages from CodeReady Builder.
|
||||
|
||||
| Script | What it does |
|
||||
|---|---|
|
||||
| `lint-apt-packages.sh <image>` | Resolves the apt list against a Debian or Ubuntu image's real indices with `apt-get install -s`. |
|
||||
| `lint-dnf-packages.sh [image]` | Runs `setup.sh`'s Oracle Linux repository setup in an `oraclelinux:10` container, resolves the dnf list, and fails unless QEMU comes from KVM Utilities and OVS and OVN from the NFV SIG. |
|
||||
| `check-glibc-ceiling.sh <ceiling> <elf>...` | Fails when an ELF file needs a glibc symbol version newer than the ceiling, naming the symbols. `build/Dockerfile.distro` runs it on `spx` and the nbdkit plugin with 2.34, so the one release tarball keeps loading on every supported OS. |
|
||||
|
||||
## Teardown
|
||||
|
||||
Both ship to `/usr/local/share/spinifex/` on every node, so they are available on a host installed with `curl … | bash` and not only from a checkout. They differ in what they take away:
|
||||
|
||||
| Script | What it does |
|
||||
|---|---|
|
||||
| `node-reset.sh` | Clears **state** and leaves the node installed: stops services, destroys guests, wipes `/etc/spinifex`, `/var/lib/spinifex`, `/var/log/spinifex` and the OVN databases, and removes the OVS bridges. The node is then ready for a fresh `spx admin init`. `--keep-data` preserves `/var/lib/spinifex`; `SPX_RESET_CONFIRM=reset` runs it unattended. |
|
||||
| `uninstall-spx.sh` | Removes the **software** — the inverse of `setup.sh`. Delegates the state teardown above to `node-reset.sh` rather than reimplementing it, then removes the binary, the nbdkit plugin, the systemd units and slices, the service users, the sudoers grant, the sysctl/logrotate/chrony/udev drop-ins, the `systemd-networkd` units, the host firewall policy and its loaded nft table, and the Spinifex CA from the host trust store. Verifies afterwards and exits non-zero naming anything that survived. |
|
||||
| `node-reset.sh` | Clears **state** and leaves the node installed: stops services, destroys guests, wipes `/etc/spinifex`, `/var/lib/spinifex`, `/var/log/spinifex` and the OVN databases, and removes the OVS bridges. It stops OVS and OVN central by the unit names the host's packages use. It leaves the NetworkManager unmanaged-device config, which the next `setup-ovn.sh` run rewrites, so NetworkManager cannot take a NIC back in between. The node is then ready for a fresh `spx admin init`. `--keep-data` preserves `/var/lib/spinifex`; `SPX_RESET_CONFIRM=reset` runs it unattended. |
|
||||
| `uninstall-spx.sh` | Removes the **software** — the inverse of `setup.sh`. Delegates the state teardown above to `node-reset.sh` rather than reimplementing it, then removes the binary, the nbdkit plugin, the systemd units and slices, the service users, the sudoers grant, the sysctl/logrotate/chrony/udev drop-ins, the `systemd-networkd` units, the host firewall policy and its loaded nft table, the Spinifex systemd-resolved drop-in, and the Spinifex CA from the host trust store. It unmasks firewalld and leaves it disabled. Where `setup.sh` handed DNS to systemd-resolved, it restores the saved `/etc/resolv.conf` and turns resolved off. Verifies afterwards and exits non-zero naming anything that survived, then lists the host settings `setup.sh` changed that it leaves in place, such as the dnf repositories, `OVS_USER_ID`, masked IPsec units and the SELinux mode. |
|
||||
|
||||
`uninstall-spx.sh` keeps `/var/lib/spinifex` and every apt package by default — uninstalling software should not destroy data, and a `curl … | bash` install lands on hosts that were already using openvswitch, libvirt and chrony. `--purge-data` and `--purge-deps` opt into each, and both confirm separately even under `--yes` (`SPX_PURGE_CONFIRM=destroy` for unattended data removal). `--dry-run` prints the whole plan and touches nothing.
|
||||
`node-reset.sh` finds guests by process name, `qemu-system-*` or `qemu-kvm`, owned by the daemon's service user, and waits for them to exit before it deletes any state.
|
||||
`node-reset_test.sh` checks that match against real processes, and that `reset-dev-env.sh` uses the same pattern.
|
||||
|
||||
`uninstall-spx.sh` keeps `/var/lib/spinifex` and every package by default — uninstalling software should not destroy data, and a `curl … | bash` install lands on hosts that were already using openvswitch, libvirt and chrony. `--purge-data` and `--purge-deps` opt into each, and both confirm separately even under `--yes` (`SPX_PURGE_CONFIRM=destroy` for unattended data removal). `--dry-run` prints the whole plan and touches nothing.
|
||||
`--purge-deps` removes with apt and is refused on dnf hosts, where its fixed list cannot tell what `setup.sh` installed from what the host already had, and dnf would also remove every package that requires them.
|
||||
|
||||
**It refuses on a multi-node cluster, and `--yes` does not override that.** Uninstalling one node of a running cluster strands its guests and leaves the survivors reconciling against a chassis that will never answer. Run `spx admin cluster shutdown` first, then the uninstaller on each node.
|
||||
|
||||
@@ -36,7 +80,7 @@ Both ship to `/usr/local/share/spinifex/` on every node, so they are available o
|
||||
|
||||
| Script | What it does |
|
||||
|---|---|
|
||||
| `smoke-test.sh` | End-to-end sanity check on a running node: imports an SSH keypair and the Ubuntu AMI, then launches an instance. `--create-vpc` builds the VPC, gateway, subnet and routes first and tears them down after; `--nodes N` launches through a spread placement group so each instance lands on a different physical server. `TEST_GPU=1` adds GPU passthrough checks. |
|
||||
| `smoke-test.sh` | End-to-end sanity check on a running node: imports an SSH keypair and the Ubuntu AMI, then launches an instance. `--create-vpc` builds the VPC, gateway, subnet and routes first and tears them down after; `--nodes N` launches through a spread placement group so each instance lands on a different physical server. `TEST_GPU=1` adds GPU passthrough checks. It runs the AWS CLI by absolute path, since sudo's `secure_path` leaves out `/usr/local/bin` on Oracle Linux. |
|
||||
| `verify-network.sh` | Asserts a node's datapath matches its declared planes — EIP reachability, which plane carries Geneve, and which interfaces services bind. Catches what `smoke-test.sh` cannot, since none of its traffic leaves the box. |
|
||||
| `check-coverage.sh` | Fails the build if total Go coverage falls below the minimum threshold, excluding packages exempt from the requirement. |
|
||||
| `diff-coverage.sh` | Coverage check scoped to changed lines only, against an auto-detected base ref (`HEAD~1` on main, `origin/main` on dev, otherwise `origin/dev`). |
|
||||
@@ -52,6 +96,7 @@ Both ship to `/usr/local/share/spinifex/` on every node, so they are available o
|
||||
| `build-system-image.sh` | Builds a minimal system AMI (Alpine or Ubuntu) from a manifest. All customisation happens inside the libguestfs appliance, so the build touches no host block device, no host mount, and needs no sudo. `--import` also registers it as an AMI. |
|
||||
| `publish-system-image.sh` | Converts a built raw image to compressed qcow2, writes the `.sha256` sidecar the catalog verifier expects, and uploads both to the R2 bucket behind `iso.mulgadc.com/system-ami/`. |
|
||||
| `build-microvm-image.sh` | Builds the minimal Alpine kernel + initramfs used by the QEMU microvm machine type and the lb-agent, carrying haproxy (ALB) and nginx-stream (NLB) data planes. |
|
||||
| `build-qboot.sh` | Builds qboot, the firmware the daemon passes to every microvm with `-bios`, from a pinned upstream commit in a digest-pinned Debian container. Fails unless the output matches the recorded sha256, which is also Debian 13's `qboot.rom`. Writes `qboot.rom` and its GPL-2.0 licence beside the microvm kernel, and the matching source tarball, which releases publish, under `build/qboot/`. `make build-microvm-image` runs it. |
|
||||
| `mkosi-build.sh` | Runs an mkosi system-image build inside the pinned builder container, so a runner needs only Docker and the toolchain cannot drift. `--image` names an output and expands to the right profile list. |
|
||||
| `mkosi-publish.sh` | Publishes an mkosi-built image to R2 from inside the same pinned container — qcow2 conversion, `.sha256` sidecar, and upload. |
|
||||
| `mkosi-common.sh` | Shared plumbing sourced by both mkosi scripts. Builds the toolchain image once, with the builder account matched to the invoking user so bind-mounted output is not written back as root. |
|
||||
|
||||
Executable
+75
@@ -0,0 +1,75 @@
|
||||
#!/bin/bash
|
||||
# build-qboot.sh — build qboot, the firmware the QEMU microvm machine boots
|
||||
# system instances (ALB/NLB) with, from a pinned upstream commit.
|
||||
#
|
||||
# Outputs:
|
||||
# $REPO_ROOT/build/microvm/qboot.rom firmware, shipped beside the microvm kernel
|
||||
# $REPO_ROOT/build/microvm/qboot.LICENSE qboot's GPL-2.0 licence text
|
||||
# $REPO_ROOT/build/qboot/qboot-<commit>.tar.gz the matching source, published with releases
|
||||
#
|
||||
# The build runs in a digest-pinned Debian container, so its output is
|
||||
# reproducible and is checked against a recorded hash. That hash is also the
|
||||
# sha256 of Debian 13's /usr/share/qemu/qboot.rom, which is built from the same commit.
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
OUT_DIR="$REPO_ROOT/build/microvm"
|
||||
SRC_DIR="$REPO_ROOT/build/qboot"
|
||||
|
||||
QBOOT_REPO="${QBOOT_REPO:-https://gitlab.com/qemu-project/qboot.git}"
|
||||
QBOOT_COMMIT="8ca302e86d685fa05b16e2b208888243da319941"
|
||||
QBOOT_SHA256="5c9c4b12c4438e3cf11d087263afe6519c8c05dd5896727b8b169b8f5dbf9877"
|
||||
BUILD_IMAGE="debian:trixie@sha256:4c25275446ca371766b8b093c696782eabce06263fccacb8d5d813b3dc44b308"
|
||||
|
||||
log() { echo "[build-qboot] $*"; }
|
||||
fail() { echo "ERROR: $*" >&2; exit 1; }
|
||||
|
||||
CONTAINER_TOOL=""
|
||||
for tool in docker podman; do
|
||||
if command -v "$tool" >/dev/null 2>&1 && "$tool" info >/dev/null 2>&1; then
|
||||
CONTAINER_TOOL=$tool
|
||||
break
|
||||
fi
|
||||
done
|
||||
[ -n "$CONTAINER_TOOL" ] || fail "qboot builds in a container; install docker or podman"
|
||||
command -v git >/dev/null 2>&1 || fail "required tool not found: git"
|
||||
|
||||
WORK=$(mktemp -d)
|
||||
trap 'rm -rf "$WORK"' EXIT
|
||||
|
||||
log "fetching qboot $QBOOT_COMMIT from $QBOOT_REPO"
|
||||
git init -q "$WORK/src"
|
||||
git -C "$WORK/src" fetch -q --depth 1 "$QBOOT_REPO" "$QBOOT_COMMIT"
|
||||
git -C "$WORK/src" checkout -q FETCH_HEAD
|
||||
got_commit=$(git -C "$WORK/src" rev-parse HEAD)
|
||||
[ "$got_commit" = "$QBOOT_COMMIT" ] || fail "fetched qboot $got_commit, not the pinned $QBOOT_COMMIT"
|
||||
|
||||
mkdir -p "$SRC_DIR"
|
||||
src_tarball="$SRC_DIR/qboot-${QBOOT_COMMIT}.tar.gz"
|
||||
git -C "$WORK/src" archive --format=tar.gz --prefix="qboot-${QBOOT_COMMIT}/" -o "$src_tarball" "$QBOOT_COMMIT"
|
||||
|
||||
log "building in $BUILD_IMAGE with $CONTAINER_TOOL"
|
||||
# The firmware comes back on stdout, so nothing the container writes is left
|
||||
# root-owned on the host.
|
||||
"$CONTAINER_TOOL" run --rm --platform linux/amd64 -v "$WORK/src:/src:ro" \
|
||||
-e DEBIAN_FRONTEND=noninteractive "$BUILD_IMAGE" sh -euc '
|
||||
{
|
||||
apt-get update -qq
|
||||
apt-get install -y -qq --no-install-recommends gcc libc6-dev meson ninja-build binutils
|
||||
meson setup /tmp/build /src
|
||||
ninja -C /tmp/build
|
||||
} >/dev/null
|
||||
cat /tmp/build/bios.bin
|
||||
' > "$WORK/qboot.rom"
|
||||
|
||||
got_sha=$(sha256sum "$WORK/qboot.rom" | cut -d' ' -f1)
|
||||
if [ "$got_sha" != "$QBOOT_SHA256" ]; then
|
||||
fail "qboot.rom has sha256 $got_sha, not the recorded $QBOOT_SHA256.
|
||||
The pinned commit or build image changed; rebuild, boot an ALB on it, then update QBOOT_SHA256."
|
||||
fi
|
||||
|
||||
mkdir -p "$OUT_DIR"
|
||||
install -m 0644 "$WORK/qboot.rom" "$OUT_DIR/qboot.rom"
|
||||
install -m 0644 "$WORK/src/LICENSE" "$OUT_DIR/qboot.LICENSE"
|
||||
log "qboot.rom sha256 $got_sha"
|
||||
log "source: $src_tarball"
|
||||
Executable
+46
@@ -0,0 +1,46 @@
|
||||
#!/bin/bash
|
||||
# check-glibc-ceiling.sh — fail when an ELF file needs a glibc symbol version
|
||||
# newer than a ceiling, so the one release tarball keeps loading on the oldest
|
||||
# glibc of every OS it ships to.
|
||||
#
|
||||
# Usage: scripts/check-glibc-ceiling.sh <ceiling, e.g. 2.34> <elf file>...
|
||||
set -euo pipefail
|
||||
|
||||
if [ $# -lt 2 ]; then
|
||||
echo "usage: $0 <ceiling> <elf file>..." >&2
|
||||
exit 2
|
||||
fi
|
||||
ceiling=$1
|
||||
shift
|
||||
|
||||
failed=0
|
||||
for f in "$@"; do
|
||||
if ! syms=$(objdump -T "$f" 2>&1); then
|
||||
case "$syms" in
|
||||
*"not a dynamic object"*)
|
||||
echo "$f: statically linked, needs no glibc"
|
||||
continue
|
||||
;;
|
||||
esac
|
||||
echo "ERROR: cannot read the dynamic symbols of $f: $syms" >&2
|
||||
failed=1
|
||||
continue
|
||||
fi
|
||||
need=$(grep -o 'GLIBC_[0-9][0-9.]*' <<<"$syms" | sed 's/^GLIBC_//' | sort -uV | tail -1 || true)
|
||||
if [ -z "$need" ]; then
|
||||
echo "$f: needs no versioned glibc symbol"
|
||||
continue
|
||||
fi
|
||||
if [ "$(printf '%s\n%s\n' "$need" "$ceiling" | sort -V | tail -1)" = "$ceiling" ]; then
|
||||
echo "$f: needs at most GLIBC_$need (ceiling GLIBC_$ceiling)"
|
||||
continue
|
||||
fi
|
||||
echo "ERROR: $f needs GLIBC_$need, newer than the GLIBC_$ceiling ceiling. Symbols above it:" >&2
|
||||
while read -r ver sym; do
|
||||
if [ "$(printf '%s\n%s\n' "$ver" "$ceiling" | sort -V | tail -1)" != "$ceiling" ]; then
|
||||
echo " $sym@GLIBC_$ver" >&2
|
||||
fi
|
||||
done < <(awk '{ for (i = 1; i <= NF; i++) if ($i ~ /^\(?GLIBC_[0-9]/) { v = $i; gsub(/[()]|GLIBC_/, "", v); print v, $NF } }' <<<"$syms" | sort -u)
|
||||
failed=1
|
||||
done
|
||||
exit "$failed"
|
||||
@@ -31,6 +31,9 @@ cp "$MULGA_ROOT/viperblock/lib/nbdkit-viperblock-plugin.so" "$STAGING/"
|
||||
cp "$PROJECT_ROOT/scripts/setup-ovn.sh" "$STAGING/"
|
||||
cp "$PROJECT_ROOT/scripts/node-reset.sh" "$STAGING/"
|
||||
cp "$PROJECT_ROOT/scripts/uninstall-spx.sh" "$STAGING/"
|
||||
cp "$PROJECT_ROOT/scripts/install-node.sh" "$PROJECT_ROOT/scripts/smoke-test.sh" "$STAGING/"
|
||||
mkdir -p "$STAGING/lib"
|
||||
cp "$PROJECT_ROOT/scripts/lib/distro.sh" "$STAGING/lib/"
|
||||
mkdir -p "$STAGING/systemd"
|
||||
cp "$PROJECT_ROOT/build/systemd/"* "$STAGING/systemd/"
|
||||
mkdir -p "$STAGING/scripts"
|
||||
|
||||
+46
-8
@@ -7,7 +7,7 @@
|
||||
# `spx admin join` on the rest, restarts services and verifies the result.
|
||||
#
|
||||
# It is the scripted form of docs/install/install-multi-node. It installs no
|
||||
# software — every host must already be running the version you want. Use
|
||||
# software — every host must already run the version you want, on one OS. Use
|
||||
# mulga's scripts/update-nodes.sh, or the release installer, for that first.
|
||||
#
|
||||
# Usage:
|
||||
@@ -69,6 +69,19 @@ set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
# Host-OS capability probes: beside this script in a checkout and in the
|
||||
# installed share dir, else where setup.sh installs them.
|
||||
DISTRO_LIB=""
|
||||
for candidate in "$SCRIPT_DIR/lib/distro.sh" /usr/local/share/spinifex/lib/distro.sh; do
|
||||
if [ -f "$candidate" ]; then DISTRO_LIB="$candidate"; break; fi
|
||||
done
|
||||
if [ -z "$DISTRO_LIB" ]; then
|
||||
echo "ERROR: distro.sh not found at $SCRIPT_DIR/lib/distro.sh or /usr/local/share/spinifex/lib/distro.sh; re-run setup.sh to install it" >&2
|
||||
exit 1
|
||||
fi
|
||||
# shellcheck source=lib/distro.sh
|
||||
. "$DISTRO_LIB"
|
||||
|
||||
SSH_USER="${SSH_USER:-spinifex}"
|
||||
IDENTITY=""
|
||||
HOSTS=()
|
||||
@@ -284,6 +297,8 @@ resolve_plane() {
|
||||
log "preflight on $N host(s)"
|
||||
|
||||
versions=""
|
||||
families=""
|
||||
libreswan_hosts=""
|
||||
for i in $(seq 0 $((N - 1))); do
|
||||
host="${HOSTS[$i]}"
|
||||
|
||||
@@ -303,6 +318,16 @@ for i in $(seq 0 $((N - 1))); do
|
||||
version="${version//[$'\r\n']/}"
|
||||
versions+="${version}"$'\n'
|
||||
|
||||
family=$(out "$host" "cat /etc/os-release" | distro_os_family) ||
|
||||
fail "$host: cannot tell its OS from /etc/os-release"
|
||||
families+="${host}: ${family}"$'\n'
|
||||
|
||||
# libreswan's pluto: Spinifex cannot run OVN IPsec on it yet, and spx admin
|
||||
# init refuses --ipsec=true there, which would stop formation part way.
|
||||
if [ "$(out "$host" "test -x /usr/libexec/ipsec/pluto && echo yes")" = "yes" ]; then
|
||||
libreswan_hosts+=" $host"
|
||||
fi
|
||||
|
||||
# A node that is already in a multi-node cluster is one whose keys and
|
||||
# logical network state are load-bearing. Refusing here is deliberate:
|
||||
# forming a cluster must never be the command that destroys a live one.
|
||||
@@ -353,6 +378,20 @@ $(sort -u <<<"$versions" | sed 's/^/ /')
|
||||
Bring them all to the same build before forming a cluster."
|
||||
fi
|
||||
|
||||
# A cluster runs one OS. Debian and Oracle Linux ship different OVN builds and
|
||||
# IKE daemons, and nothing qualifies them against each other.
|
||||
if [ "$(cut -d' ' -f2 <<<"$families" | sort -u | grep -c .)" -gt 1 ]; then
|
||||
fail "hosts run different operating systems:
|
||||
$(sed '/^$/d; s/^/ /' <<<"$families")
|
||||
A cluster must run one OS on every host."
|
||||
fi
|
||||
|
||||
if $IPSEC_ENABLED && [ -n "$libreswan_hosts" ]; then
|
||||
fail "--ipsec on, but these hosts use libreswan for IPsec:$libreswan_hosts
|
||||
Spinifex cannot run OVN IPsec on libreswan yet. Re-run with --ipsec off
|
||||
only if the nodes share a trusted private link."
|
||||
fi
|
||||
|
||||
# --- Confirm ---------------------------------------------------------------
|
||||
|
||||
echo ""
|
||||
@@ -489,10 +528,10 @@ fi
|
||||
|
||||
# --- Wipe ------------------------------------------------------------------
|
||||
#
|
||||
# node-reset.sh is not part of the release tarball, so it is pushed from this
|
||||
# checkout the same way smoke-test.sh is. After this a node has no CA, master
|
||||
# key, OVN database or chassis identity left for the formation phase to collide
|
||||
# with — which is the whole reason --wipe exists.
|
||||
# Pushed from beside this script, as smoke-test.sh is, so the reset matches this
|
||||
# script whatever the host has installed; it carries its own probes. After this
|
||||
# a node has no CA, master key, OVN database or chassis identity left for the
|
||||
# formation phase to collide with — which is the whole reason --wipe exists.
|
||||
if $WIPE; then
|
||||
log "wiping every host back to its pre-install state"
|
||||
for host in "${HOSTS[@]}"; do
|
||||
@@ -964,9 +1003,8 @@ out "${HOSTS[0]}" "sudo sed -n '/^\[\[network.external_pools\]\]/,/^$/p' /etc/sp
|
||||
if $RUN_SMOKE; then
|
||||
echo ""
|
||||
log "running the smoke test on ${HOSTS[0]}"
|
||||
# smoke-test.sh is not part of the release tarball, so it is pushed from
|
||||
# this checkout — which also guarantees the test and this script are the
|
||||
# same version.
|
||||
# Pushed from beside this script, so the test and this script are the same
|
||||
# version whatever the host has installed.
|
||||
scp "${SSH_OPTS[@]}" -q "$SCRIPT_DIR/smoke-test.sh" "$SSH_USER@${HOSTS[0]}:/tmp/spx-smoke-test.sh"
|
||||
on "${HOSTS[0]}" "chmod 0755 /tmp/spx-smoke-test.sh && /tmp/spx-smoke-test.sh --create-vpc --nodes $N" ||
|
||||
fail "smoke test failed on ${HOSTS[0]}"
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
# shellcheck shell=bash
|
||||
# distro.sh — host-OS capability probes. Each prints its answer, or names every
|
||||
# candidate it tried on stderr and returns 1, also when it finds both of a pair.
|
||||
# setup.sh and node-reset.sh carry inline copies; distro_test.sh keeps them equal.
|
||||
|
||||
# Reads unit files off disk, not through systemctl, so it answers in a chroot.
|
||||
# DISTRO_ROOT prefixes every host path the probes read; tests set it.
|
||||
distro_unit_exists() {
|
||||
local dir
|
||||
for dir in /etc/systemd/system /run/systemd/system /usr/lib/systemd/system /lib/systemd/system; do
|
||||
if [ -e "${DISTRO_ROOT:-}$dir/$1" ] || [ -L "${DISTRO_ROOT:-}$dir/$1" ]; then
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
distro_pkg_manager() {
|
||||
local apt=false dnf=false
|
||||
command -v apt-get >/dev/null 2>&1 && apt=true
|
||||
command -v dnf >/dev/null 2>&1 && dnf=true
|
||||
if $apt && $dnf; then
|
||||
echo "distro.sh: package manager: found both apt-get and dnf; cannot tell which one manages this host" >&2
|
||||
return 1
|
||||
fi
|
||||
if $apt; then echo apt; return 0; fi
|
||||
if $dnf; then echo dnf; return 0; fi
|
||||
echo "distro.sh: package manager: found neither apt-get nor dnf on PATH" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
# The OVS umbrella unit: openvswitch-switch.service on Debian, openvswitch.service on EL.
|
||||
distro_ovs_unit() {
|
||||
local deb=false el=false
|
||||
distro_unit_exists openvswitch-switch.service && deb=true
|
||||
distro_unit_exists openvswitch.service && el=true
|
||||
if $deb && $el; then
|
||||
echo "distro.sh: OVS unit: found both openvswitch-switch.service and openvswitch.service; cannot tell which one owns OVS" >&2
|
||||
return 1
|
||||
fi
|
||||
if $deb; then echo openvswitch-switch.service; return 0; fi
|
||||
if $el; then echo openvswitch.service; return 0; fi
|
||||
echo "distro.sh: OVS unit: found neither openvswitch-switch.service nor openvswitch.service; is Open vSwitch installed?" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
# "split": Debian's ovn-central plus separate NB and SB DB units.
|
||||
# "lone-northd": EL's single ovn-northd.service, which starts both DB servers.
|
||||
distro_ovn_central_layout() {
|
||||
local split=false lone=false unit missing=""
|
||||
for unit in ovn-central.service ovn-ovsdb-server-nb.service ovn-ovsdb-server-sb.service; do
|
||||
if distro_unit_exists "$unit"; then
|
||||
split=true
|
||||
else
|
||||
missing="${missing:+$missing, }$unit"
|
||||
fi
|
||||
done
|
||||
if distro_unit_exists ovn-northd.service && [ -f "${DISTRO_ROOT:-}/etc/sysconfig/ovn" ]; then
|
||||
lone=true
|
||||
fi
|
||||
if $split && $lone; then
|
||||
echo "distro.sh: OVN central layout: found both the split units (ovn-central.service) and a lone ovn-northd.service with /etc/sysconfig/ovn; cannot tell which one owns the OVN databases" >&2
|
||||
return 1
|
||||
fi
|
||||
if $split; then
|
||||
if [ -n "$missing" ]; then
|
||||
echo "distro.sh: OVN central layout: ovn-central is installed but $missing is missing" >&2
|
||||
return 1
|
||||
fi
|
||||
echo split
|
||||
return 0
|
||||
fi
|
||||
if $lone; then echo lone-northd; return 0; fi
|
||||
echo "distro.sh: OVN central layout: found neither ovn-central.service nor ovn-northd.service with /etc/sysconfig/ovn; is OVN central installed?" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
# The trust-store update command and its local anchor directory, space-separated.
|
||||
distro_ca_trust() {
|
||||
local deb=false el=false
|
||||
command -v update-ca-certificates >/dev/null 2>&1 && deb=true
|
||||
command -v update-ca-trust >/dev/null 2>&1 && el=true
|
||||
if $deb && $el; then
|
||||
echo "distro.sh: CA trust: found both update-ca-certificates and update-ca-trust; cannot tell which trust store TLS clients read" >&2
|
||||
return 1
|
||||
fi
|
||||
if $deb; then echo "update-ca-certificates /usr/local/share/ca-certificates"; return 0; fi
|
||||
if $el; then echo "update-ca-trust /etc/pki/ca-trust/source/anchors"; return 0; fi
|
||||
echo "distro.sh: CA trust: found neither update-ca-certificates nor update-ca-trust on PATH" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
# chrony's main config file: /etc/chrony/chrony.conf on Debian, /etc/chrony.conf on EL.
|
||||
distro_chrony_conf() {
|
||||
local deb=false el=false
|
||||
[ -f "${DISTRO_ROOT:-}/etc/chrony/chrony.conf" ] && deb=true
|
||||
[ -f "${DISTRO_ROOT:-}/etc/chrony.conf" ] && el=true
|
||||
if $deb && $el; then
|
||||
echo "distro.sh: chrony config: found both /etc/chrony/chrony.conf and /etc/chrony.conf; cannot tell which one chronyd reads" >&2
|
||||
return 1
|
||||
fi
|
||||
if $deb; then echo /etc/chrony/chrony.conf; return 0; fi
|
||||
if $el; then echo /etc/chrony.conf; return 0; fi
|
||||
echo "distro.sh: chrony config: found neither /etc/chrony/chrony.conf nor /etc/chrony.conf; is chrony installed?" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
distro_ovn_logrotate() {
|
||||
local deb=false el=false
|
||||
[ -f "${DISTRO_ROOT:-}/etc/logrotate.d/ovn-common" ] && deb=true
|
||||
[ -f "${DISTRO_ROOT:-}/etc/logrotate.d/ovn" ] && el=true
|
||||
if $deb && $el; then
|
||||
echo "distro.sh: OVN logrotate: found both /etc/logrotate.d/ovn-common and /etc/logrotate.d/ovn; cannot tell which one the OVN packages own" >&2
|
||||
return 1
|
||||
fi
|
||||
if $deb; then echo /etc/logrotate.d/ovn-common; return 0; fi
|
||||
if $el; then echo /etc/logrotate.d/ovn; return 0; fi
|
||||
echo "distro.sh: OVN logrotate: found neither /etc/logrotate.d/ovn-common nor /etc/logrotate.d/ovn; is OVN installed?" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
distro_ovs_logrotate() {
|
||||
local deb=false el=false
|
||||
[ -f "${DISTRO_ROOT:-}/etc/logrotate.d/openvswitch-switch" ] && deb=true
|
||||
[ -f "${DISTRO_ROOT:-}/etc/logrotate.d/openvswitch" ] && el=true
|
||||
if $deb && $el; then
|
||||
echo "distro.sh: OVS logrotate: found both /etc/logrotate.d/openvswitch-switch and /etc/logrotate.d/openvswitch; cannot tell which one the OVS packages own" >&2
|
||||
return 1
|
||||
fi
|
||||
if $deb; then echo /etc/logrotate.d/openvswitch-switch; return 0; fi
|
||||
if $el; then echo /etc/logrotate.d/openvswitch; return 0; fi
|
||||
echo "distro.sh: OVS logrotate: found neither /etc/logrotate.d/openvswitch-switch nor /etc/logrotate.d/openvswitch; is Open vSwitch installed?" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
# libreswan's IKE daemon. EL's openvswitch-ipsec drives libreswan even beside
|
||||
# strongSwan, so this is a presence check rather than one of a pair.
|
||||
distro_has_libreswan() {
|
||||
[ -x "${DISTRO_ROOT:-}/usr/libexec/ipsec/pluto" ]
|
||||
}
|
||||
|
||||
# The OS family of the os-release text on stdin, for install-node.sh's one-OS
|
||||
# cluster check: "debian" for Debian and Ubuntu, so mixing those is not newly
|
||||
# refused, and the ID itself for anything else.
|
||||
distro_os_family() {
|
||||
local line id="" like=""
|
||||
while IFS= read -r line || [ -n "$line" ]; do
|
||||
case "$line" in
|
||||
ID=*) id=${line#ID=} ;;
|
||||
ID_LIKE=*) like=${line#ID_LIKE=} ;;
|
||||
esac
|
||||
done
|
||||
id=${id//[\"\']/}
|
||||
like=${like//[\"\']/}
|
||||
if [ -z "$id" ]; then
|
||||
echo "distro.sh: OS family: /etc/os-release has no ID" >&2
|
||||
return 1
|
||||
fi
|
||||
case " $id $like " in
|
||||
*" debian "*) echo debian ;;
|
||||
*) echo "$id" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# The OVS and OVN units the given unit files depend on or order against that
|
||||
# are not installed, one per line; prints nothing when all are. Fails only
|
||||
# when a unit file cannot be read.
|
||||
distro_missing_ovs_deps() {
|
||||
local file line dep missing=""
|
||||
for file in "$@"; do
|
||||
if [ ! -r "${DISTRO_ROOT:-}$file" ]; then
|
||||
echo "distro.sh: OVS dependencies: cannot read $file" >&2
|
||||
return 1
|
||||
fi
|
||||
while IFS= read -r line || [ -n "$line" ]; do
|
||||
case "$line" in
|
||||
After=* | Before=* | Wants=* | Requires=* | Requisite=* | BindsTo=* | PartOf=* | Upholds=*) ;;
|
||||
*) continue ;;
|
||||
esac
|
||||
for dep in ${line#*=}; do
|
||||
case "$dep" in
|
||||
ovs* | ovn* | openvswitch*) ;;
|
||||
*) continue ;;
|
||||
esac
|
||||
case " $missing " in
|
||||
*" $dep "*) continue ;;
|
||||
esac
|
||||
distro_unit_exists "$dep" || missing="${missing:+$missing }$dep"
|
||||
done
|
||||
done <"${DISTRO_ROOT:-}$file"
|
||||
done
|
||||
for dep in $missing; do echo "$dep"; done
|
||||
}
|
||||
@@ -0,0 +1,300 @@
|
||||
#!/bin/sh
|
||||
# distro_test.sh — tests for the host-OS probes and the inline copies of them.
|
||||
#
|
||||
# Every probe runs against a fixture tree (DISTRO_ROOT) and a PATH holding only
|
||||
# stub commands, so the result never depends on the distro running the test.
|
||||
set -eu
|
||||
|
||||
LIB_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
|
||||
SCRIPTS_DIR=$(dirname -- "$LIB_DIR")
|
||||
DISTRO="$LIB_DIR/distro.sh"
|
||||
TMP=$(mktemp -d)
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
|
||||
fails=0
|
||||
pass() { echo " ok - $1"; }
|
||||
fail() { echo " FAIL - $1"; fails=$((fails + 1)); }
|
||||
|
||||
echo "-- lib/distro.sh"
|
||||
|
||||
# fixture <name> — a fresh empty host root and stub bin dir.
|
||||
fixture() {
|
||||
ROOT="$TMP/$1/root"
|
||||
BIN="$TMP/$1/bin"
|
||||
mkdir -p "$ROOT" "$BIN"
|
||||
}
|
||||
unit() { mkdir -p "$ROOT/usr/lib/systemd/system" && : > "$ROOT/usr/lib/systemd/system/$1"; }
|
||||
file() { mkdir -p "$ROOT$(dirname "$1")" && : > "$ROOT$1"; }
|
||||
cmd() { printf '#!/bin/sh\nexit 0\n' > "$BIN/$1" && chmod +x "$BIN/$1"; }
|
||||
|
||||
# probe <fn> [arg...] — runs one probe in bash against the current fixture,
|
||||
# with only the stub dir on PATH. Sets OUT, ERR and RC.
|
||||
probe() {
|
||||
RC=0
|
||||
OUT=$(DISTRO_ROOT="$ROOT" PATH="$BIN" /bin/bash -c '. "$1"; shift; "$@"' _ "$DISTRO" "$@" 2>"$TMP/err") || RC=$?
|
||||
ERR=$(cat "$TMP/err")
|
||||
}
|
||||
|
||||
# expect_ok <desc> <want>; expect_fail <desc> <word>... — every word must appear
|
||||
# in the error, which is how "names every candidate" is checked.
|
||||
expect_ok() {
|
||||
if [ "$RC" = 0 ] && [ "$OUT" = "$2" ]; then pass "$1"; else fail "$1 (rc=$RC out='$OUT' err='$ERR')"; fi
|
||||
}
|
||||
expect_fail() {
|
||||
desc=$1; shift
|
||||
if [ "$RC" = 0 ]; then fail "$desc (succeeded with '$OUT')"; return; fi
|
||||
for word in "$@"; do
|
||||
case "$ERR" in
|
||||
*"$word"*) ;;
|
||||
*) fail "$desc (error does not name $word: '$ERR')"; return ;;
|
||||
esac
|
||||
done
|
||||
pass "$desc"
|
||||
}
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Package manager.
|
||||
# --------------------------------------------------------------------------
|
||||
fixture pm-apt; cmd apt-get; probe distro_pkg_manager
|
||||
expect_ok "package manager: apt-get alone is apt" apt
|
||||
fixture pm-dnf; cmd dnf; probe distro_pkg_manager
|
||||
expect_ok "package manager: dnf alone is dnf" dnf
|
||||
fixture pm-both; cmd apt-get; cmd dnf; probe distro_pkg_manager
|
||||
expect_fail "package manager: both fails naming both" apt-get dnf
|
||||
fixture pm-none; probe distro_pkg_manager
|
||||
expect_fail "package manager: neither fails naming both" apt-get dnf
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# OVS umbrella unit. Unit files are found under any systemd unit directory, and
|
||||
# a masked unit (a symlink to /dev/null) still counts as installed.
|
||||
# --------------------------------------------------------------------------
|
||||
fixture ovs-deb; unit openvswitch-switch.service; probe distro_ovs_unit
|
||||
expect_ok "OVS unit: Debian name" openvswitch-switch.service
|
||||
fixture ovs-el; unit openvswitch.service; probe distro_ovs_unit
|
||||
expect_ok "OVS unit: EL name" openvswitch.service
|
||||
fixture ovs-masked; mkdir -p "$ROOT/etc/systemd/system"
|
||||
ln -s /dev/null "$ROOT/etc/systemd/system/openvswitch.service"; probe distro_ovs_unit
|
||||
expect_ok "OVS unit: a masked unit in /etc counts" openvswitch.service
|
||||
fixture ovs-lib; mkdir -p "$ROOT/lib/systemd/system"
|
||||
: > "$ROOT/lib/systemd/system/openvswitch-switch.service"; probe distro_ovs_unit
|
||||
expect_ok "OVS unit: found under /lib on an unmerged-/usr host" openvswitch-switch.service
|
||||
fixture ovs-both; unit openvswitch-switch.service; unit openvswitch.service; probe distro_ovs_unit
|
||||
expect_fail "OVS unit: both fails naming both" openvswitch-switch.service openvswitch.service
|
||||
fixture ovs-none; probe distro_ovs_unit
|
||||
expect_fail "OVS unit: neither fails naming both" openvswitch-switch.service openvswitch.service
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# OVN central layout. Both distros ship ovn-northd.service, so the unit alone
|
||||
# must not decide it: Debian's split layout carries one too.
|
||||
# --------------------------------------------------------------------------
|
||||
split_units() { unit ovn-central.service; unit ovn-ovsdb-server-nb.service; unit ovn-ovsdb-server-sb.service; unit ovn-northd.service; }
|
||||
|
||||
fixture ovn-split; split_units; probe distro_ovn_central_layout
|
||||
expect_ok "OVN layout: Debian split units" split
|
||||
fixture ovn-lone; unit ovn-northd.service; file /etc/sysconfig/ovn; probe distro_ovn_central_layout
|
||||
expect_ok "OVN layout: EL lone ovn-northd" lone-northd
|
||||
fixture ovn-both; split_units; file /etc/sysconfig/ovn; probe distro_ovn_central_layout
|
||||
expect_fail "OVN layout: both fails naming both" ovn-central.service ovn-northd.service /etc/sysconfig/ovn
|
||||
fixture ovn-partial; unit ovn-central.service; unit ovn-ovsdb-server-nb.service; probe distro_ovn_central_layout
|
||||
expect_fail "OVN layout: a partial split install names the missing unit" ovn-ovsdb-server-sb.service
|
||||
fixture ovn-northd-only; unit ovn-northd.service; probe distro_ovn_central_layout
|
||||
expect_fail "OVN layout: ovn-northd without /etc/sysconfig/ovn is not EL" ovn-central.service ovn-northd.service /etc/sysconfig/ovn
|
||||
fixture ovn-none; probe distro_ovn_central_layout
|
||||
expect_fail "OVN layout: neither fails naming both" ovn-central.service ovn-northd.service
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# CA trust.
|
||||
# --------------------------------------------------------------------------
|
||||
fixture ca-deb; cmd update-ca-certificates; probe distro_ca_trust
|
||||
expect_ok "CA trust: Debian command and anchor dir" "update-ca-certificates /usr/local/share/ca-certificates"
|
||||
fixture ca-el; cmd update-ca-trust; probe distro_ca_trust
|
||||
expect_ok "CA trust: EL command and anchor dir" "update-ca-trust /etc/pki/ca-trust/source/anchors"
|
||||
fixture ca-both; cmd update-ca-certificates; cmd update-ca-trust; probe distro_ca_trust
|
||||
expect_fail "CA trust: both fails naming both" update-ca-certificates update-ca-trust
|
||||
fixture ca-none; probe distro_ca_trust
|
||||
expect_fail "CA trust: neither fails naming both" update-ca-certificates update-ca-trust
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# chrony config file.
|
||||
# --------------------------------------------------------------------------
|
||||
fixture chrony-deb; file /etc/chrony/chrony.conf; probe distro_chrony_conf
|
||||
expect_ok "chrony config: Debian file" /etc/chrony/chrony.conf
|
||||
fixture chrony-el; file /etc/chrony.conf; probe distro_chrony_conf
|
||||
expect_ok "chrony config: EL file" /etc/chrony.conf
|
||||
fixture chrony-both; file /etc/chrony/chrony.conf; file /etc/chrony.conf; probe distro_chrony_conf
|
||||
expect_fail "chrony config: both fails naming both" /etc/chrony/chrony.conf /etc/chrony.conf
|
||||
fixture chrony-none; probe distro_chrony_conf
|
||||
expect_fail "chrony config: neither fails naming both" /etc/chrony/chrony.conf /etc/chrony.conf
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# OVN logrotate file.
|
||||
# --------------------------------------------------------------------------
|
||||
fixture lr-deb; file /etc/logrotate.d/ovn-common; probe distro_ovn_logrotate
|
||||
expect_ok "OVN logrotate: Debian file" /etc/logrotate.d/ovn-common
|
||||
fixture lr-el; file /etc/logrotate.d/ovn; probe distro_ovn_logrotate
|
||||
expect_ok "OVN logrotate: EL file" /etc/logrotate.d/ovn
|
||||
fixture lr-both; file /etc/logrotate.d/ovn-common; file /etc/logrotate.d/ovn; probe distro_ovn_logrotate
|
||||
expect_fail "OVN logrotate: both fails naming both" /etc/logrotate.d/ovn-common /etc/logrotate.d/ovn
|
||||
fixture lr-none; probe distro_ovn_logrotate
|
||||
expect_fail "OVN logrotate: neither fails naming both" /etc/logrotate.d/ovn-common /etc/logrotate.d/ovn
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# OVS logrotate file.
|
||||
# --------------------------------------------------------------------------
|
||||
fixture ovslr-deb; file /etc/logrotate.d/openvswitch-switch; probe distro_ovs_logrotate
|
||||
expect_ok "OVS logrotate: Debian file" /etc/logrotate.d/openvswitch-switch
|
||||
fixture ovslr-el; file /etc/logrotate.d/openvswitch; probe distro_ovs_logrotate
|
||||
expect_ok "OVS logrotate: EL file" /etc/logrotate.d/openvswitch
|
||||
fixture ovslr-both; file /etc/logrotate.d/openvswitch-switch; file /etc/logrotate.d/openvswitch; probe distro_ovs_logrotate
|
||||
expect_fail "OVS logrotate: both fails naming both" /etc/logrotate.d/openvswitch-switch /etc/logrotate.d/openvswitch
|
||||
fixture ovslr-none; probe distro_ovs_logrotate
|
||||
expect_fail "OVS logrotate: neither fails naming both" /etc/logrotate.d/openvswitch-switch /etc/logrotate.d/openvswitch
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# libreswan.
|
||||
# --------------------------------------------------------------------------
|
||||
# expect_rc <desc> <want> — for the predicates, which print nothing.
|
||||
expect_rc() {
|
||||
if [ "$RC" = "$2" ]; then pass "$1"; else fail "$1 (rc=$RC, want $2)"; fi
|
||||
}
|
||||
fixture ls-pluto; file /usr/libexec/ipsec/pluto; chmod +x "$ROOT/usr/libexec/ipsec/pluto"; probe distro_has_libreswan
|
||||
expect_rc "libreswan: an executable pluto" 0
|
||||
fixture ls-noexec; file /usr/libexec/ipsec/pluto; probe distro_has_libreswan
|
||||
expect_rc "libreswan: a pluto that cannot run is not libreswan" 1
|
||||
fixture ls-charon; file /usr/lib/ipsec/charon; chmod +x "$ROOT/usr/lib/ipsec/charon"; probe distro_has_libreswan
|
||||
expect_rc "libreswan: strongSwan's charon is not libreswan" 1
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# OS family, read from os-release text on stdin.
|
||||
# --------------------------------------------------------------------------
|
||||
# family <os-release text> — runs distro_os_family on it. Sets OUT, ERR and RC.
|
||||
family() {
|
||||
RC=0
|
||||
OUT=$(printf '%s\n' "$1" | /bin/bash -c '. "$1"; distro_os_family' _ "$DISTRO" 2>"$TMP/err") || RC=$?
|
||||
ERR=$(cat "$TMP/err")
|
||||
}
|
||||
family 'PRETTY_NAME="Debian GNU/Linux 13 (trixie)"
|
||||
ID=debian
|
||||
VERSION_ID="13"'
|
||||
expect_ok "OS family: Debian" debian
|
||||
family 'NAME="Ubuntu"
|
||||
ID=ubuntu
|
||||
ID_LIKE=debian'
|
||||
expect_ok "OS family: Ubuntu is Debian's family" debian
|
||||
family 'NAME="Oracle Linux Server"
|
||||
ID="ol"
|
||||
ID_LIKE="fedora"
|
||||
VERSION_ID="10.2"'
|
||||
expect_ok "OS family: Oracle Linux, quoted, is its own" ol
|
||||
family 'ID=rocky
|
||||
ID_LIKE="rhel centos fedora"'
|
||||
expect_ok "OS family: another EL is not Oracle Linux" rocky
|
||||
family 'NAME=Something
|
||||
VERSION_ID=1'
|
||||
expect_fail "OS family: no ID fails naming the file" /etc/os-release ID
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# OVS and OVN units the shipped Spinifex units name. Run against the real unit
|
||||
# files, so a unit that names a Debian-only OVS unit fails on the EL fixture.
|
||||
# --------------------------------------------------------------------------
|
||||
UNITS_SRC="$(dirname -- "$SCRIPTS_DIR")/build/systemd"
|
||||
spinifex_units() {
|
||||
mkdir -p "$ROOT/etc/systemd/system"
|
||||
cp "$UNITS_SRC"/spinifex.target "$UNITS_SRC"/spinifex-*.service "$ROOT/etc/systemd/system/"
|
||||
}
|
||||
deps_probe() {
|
||||
set --
|
||||
for f in "$ROOT"/etc/systemd/system/spinifex*; do set -- "$@" "${f#"$ROOT"}"; done
|
||||
probe distro_missing_ovs_deps "$@"
|
||||
}
|
||||
el_ovs_units() {
|
||||
for u in openvswitch.service ovsdb-server.service ovs-vswitchd.service ovn-controller.service ovn-northd.service openvswitch-ipsec.service; do unit "$u"; done
|
||||
}
|
||||
|
||||
fixture deps-el; spinifex_units; el_ovs_units; deps_probe
|
||||
expect_ok "OVS dependencies: every one the shipped units name is installed on EL" ""
|
||||
fixture deps-no-ipsec; spinifex_units; el_ovs_units; rm "$ROOT/usr/lib/systemd/system/openvswitch-ipsec.service"; deps_probe
|
||||
expect_ok "OVS dependencies: a missing unit is printed once" openvswitch-ipsec.service
|
||||
fixture deps-debian-only; el_ovs_units
|
||||
mkdir -p "$ROOT/etc/systemd/system"
|
||||
printf '[Unit]\n# After=ovs-old.service\nAfter=openvswitch-switch.service ovn-controller.service\nWants=openvswitch-switch.service\n' >"$ROOT/etc/systemd/system/spinifex-x.service"
|
||||
printf '[Unit]\nRequires=ovn-central.service' >"$ROOT/etc/systemd/system/spinifex-y.service"
|
||||
deps_probe
|
||||
expect_ok "OVS dependencies: Debian-only names are missing on EL, comments are not read" "openvswitch-switch.service
|
||||
ovn-central.service"
|
||||
fixture deps-unreadable; probe distro_missing_ovs_deps /etc/systemd/system/spinifex.target
|
||||
expect_fail "OVS dependencies: an unreadable unit file fails naming it" "cannot read /etc/systemd/system/spinifex.target"
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Inline copies. setup.sh and node-reset.sh cannot source the library, so they
|
||||
# carry copies between markers; each copied function must match the library's
|
||||
# byte for byte, or a fix to one silently misses the other.
|
||||
# --------------------------------------------------------------------------
|
||||
# fn_body <file> <name> — the function from "name() {" to the first "}" line.
|
||||
fn_body() {
|
||||
awk -v name="$2" '$0 == name "() {" { on = 1 } on { print } on && $0 == "}" { exit }' "$1"
|
||||
}
|
||||
|
||||
for copy in setup.sh node-reset.sh; do
|
||||
path="$SCRIPTS_DIR/$copy"
|
||||
block=$(sed -n '/^# --- BEGIN distro.sh probes ---$/,/^# --- END distro.sh probes ---$/p' "$path")
|
||||
if [ -z "$block" ]; then
|
||||
fail "$copy: no distro.sh probe block"
|
||||
continue
|
||||
fi
|
||||
names=$(printf '%s\n' "$block" | sed -n 's/^\([a-z_][a-z0-9_]*\)() {$/\1/p')
|
||||
if [ -z "$names" ]; then
|
||||
fail "$copy: the distro.sh probe block holds no functions"
|
||||
continue
|
||||
fi
|
||||
for name in $names; do
|
||||
want=$(fn_body "$DISTRO" "$name")
|
||||
got=$(fn_body "$path" "$name")
|
||||
if [ -z "$want" ]; then
|
||||
fail "$copy: $name is not in lib/distro.sh"
|
||||
elif [ "$got" = "$want" ]; then
|
||||
pass "$copy: $name matches lib/distro.sh"
|
||||
else
|
||||
fail "$copy: $name differs from lib/distro.sh"
|
||||
fi
|
||||
done
|
||||
|
||||
# A copied probe calling a probe helper the block lacks fails only when run.
|
||||
for called in $(printf '%s\n' "$block" | grep -v '^ *#' | grep -o 'distro_[a-z_]*' | sort -u); do
|
||||
case " $(echo $names) " in
|
||||
*" $called "*) ;;
|
||||
*) fail "$copy: copies a probe that calls $called without copying it" ;;
|
||||
esac
|
||||
done
|
||||
done
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# A consumer with no library beside it and none installed must stop naming
|
||||
# both paths, not run on with the probes undefined. Skipped on a host where
|
||||
# setup.sh has installed the library, since the fallback then finds it.
|
||||
# --------------------------------------------------------------------------
|
||||
if [ -e /usr/local/share/spinifex/lib/distro.sh ]; then
|
||||
echo " skip - consumers without distro.sh: /usr/local/share/spinifex/lib/distro.sh is installed here"
|
||||
else
|
||||
# uninstall-spx.sh sources the library after its --help, so it is run into
|
||||
# its multi-node refusal, which exits 1 without touching the host either way.
|
||||
mkdir -p "$TMP/alone/bin" "$TMP/alone/etc"
|
||||
printf '#!/bin/sh\nexec "$@"\n' > "$TMP/alone/bin/sudo"
|
||||
chmod +x "$TMP/alone/bin/sudo"
|
||||
printf '[nodes.node1]\n[nodes.node2]\n' > "$TMP/alone/etc/spinifex.toml"
|
||||
for consumer in "setup-ovn.sh --help" "uninstall-spx.sh --dry-run"; do
|
||||
script=${consumer%% *}
|
||||
cp "$SCRIPTS_DIR/$script" "$TMP/alone/"
|
||||
PATH="$TMP/alone/bin:$PATH" SPX_ETC_DIR="$TMP/alone/etc" "$TMP/alone/$script" "${consumer#* }" >"$TMP/out" 2>&1 && rc=0 || rc=$?
|
||||
if [ "$rc" != 0 ] && grep -q "distro.sh not found at $TMP/alone/lib/distro.sh or /usr/local/share/spinifex/lib/distro.sh" "$TMP/out"; then
|
||||
pass "$script: stops naming both paths when distro.sh is missing"
|
||||
else
|
||||
fail "$script: without distro.sh exited $rc: $(head -3 "$TMP/out")"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
if [ "$fails" -gt 0 ]; then
|
||||
echo "$fails failure(s)"
|
||||
exit 1
|
||||
fi
|
||||
Executable
+45
@@ -0,0 +1,45 @@
|
||||
#!/usr/bin/env bash
|
||||
# Runs setup.sh's Oracle Linux repository setup in a container, then resolves
|
||||
# its runtime dnf list against the real repositories with dnf install --assumeno.
|
||||
# Catches a renamed or dropped package, a SIG dependency the includepkgs
|
||||
# allowlist misses, a changed SIG key, and QEMU or OVS resolving from the wrong
|
||||
# repository.
|
||||
set -euo pipefail
|
||||
|
||||
IMAGE="${1:-oraclelinux:10}"
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
|
||||
echo "Resolving ${IMAGE}..."
|
||||
# The repository setup and the list both come from setup.sh itself, so the lint
|
||||
# cannot drift from what the installer does.
|
||||
docker run --rm -v "${SCRIPT_DIR}/setup.sh:/setup.sh:ro" "${IMAGE}" bash -c '
|
||||
set -euo pipefail
|
||||
INSTALL_SPINIFEX_LIB_ONLY=1 . /setup.sh
|
||||
SUDO=""
|
||||
ensure_ol_repos
|
||||
|
||||
# --assumeno always exits 1, so the transaction is judged from its output.
|
||||
# shellcheck disable=SC2086
|
||||
out=$(dnf install --assumeno $OL_RUNTIME_PACKAGES 2>&1) || true
|
||||
if ! grep -q "^Operation aborted" <<< "$out" || grep -qE "^(Error|Problem)|No match for argument" <<< "$out"; then
|
||||
echo "$out"
|
||||
echo "FAIL: the Oracle Linux package list does not resolve" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# repo_of <package> — the repository dnf would install the package from.
|
||||
repo_of() { awk -v p="$1" "\$1 == p { print \$4; exit }" <<< "$out"; }
|
||||
for want in "qemu-kvm $OL_KVM_UTILS_REPO" "qemu-img $OL_KVM_UTILS_REPO" \
|
||||
"openvswitch3.5 $NFV_SIG_REPO_ID" "ovn25.03-central $NFV_SIG_REPO_ID"; do
|
||||
pkg=${want% *} repo=${want#* }
|
||||
got=$(repo_of "$pkg")
|
||||
if [ "$got" != "$repo" ]; then
|
||||
echo "$out"
|
||||
echo "FAIL: $pkg resolves from ${got:-nowhere}, not $repo" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
'
|
||||
|
||||
echo "OK: every package resolves on ${IMAGE}, QEMU from KVM Utilities and OVS/OVN from the NFV SIG"
|
||||
+92
-14
@@ -57,6 +57,55 @@ run() {
|
||||
"$@"
|
||||
}
|
||||
|
||||
# --- BEGIN distro.sh probes ---
|
||||
# Copied from scripts/lib/distro.sh: install-node.sh --wipe copies this script
|
||||
# alone to each host, so it cannot source the library. distro_test.sh keeps them equal.
|
||||
distro_unit_exists() {
|
||||
local dir
|
||||
for dir in /etc/systemd/system /run/systemd/system /usr/lib/systemd/system /lib/systemd/system; do
|
||||
if [ -e "${DISTRO_ROOT:-}$dir/$1" ] || [ -L "${DISTRO_ROOT:-}$dir/$1" ]; then
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
distro_ovs_unit() {
|
||||
local deb=false el=false
|
||||
distro_unit_exists openvswitch-switch.service && deb=true
|
||||
distro_unit_exists openvswitch.service && el=true
|
||||
if $deb && $el; then
|
||||
echo "distro.sh: OVS unit: found both openvswitch-switch.service and openvswitch.service; cannot tell which one owns OVS" >&2
|
||||
return 1
|
||||
fi
|
||||
if $deb; then echo openvswitch-switch.service; return 0; fi
|
||||
if $el; then echo openvswitch.service; return 0; fi
|
||||
echo "distro.sh: OVS unit: found neither openvswitch-switch.service nor openvswitch.service; is Open vSwitch installed?" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
distro_ca_trust() {
|
||||
local deb=false el=false
|
||||
command -v update-ca-certificates >/dev/null 2>&1 && deb=true
|
||||
command -v update-ca-trust >/dev/null 2>&1 && el=true
|
||||
if $deb && $el; then
|
||||
echo "distro.sh: CA trust: found both update-ca-certificates and update-ca-trust; cannot tell which trust store TLS clients read" >&2
|
||||
return 1
|
||||
fi
|
||||
if $deb; then echo "update-ca-certificates /usr/local/share/ca-certificates"; return 0; fi
|
||||
if $el; then echo "update-ca-trust /etc/pki/ca-trust/source/anchors"; return 0; fi
|
||||
echo "distro.sh: CA trust: found neither update-ca-certificates nor update-ca-trust on PATH" >&2
|
||||
return 1
|
||||
}
|
||||
# --- END distro.sh probes ---
|
||||
|
||||
# Resolved before anything stops, so a host whose OVS unit is ambiguous is
|
||||
# refused with nothing changed.
|
||||
OVS_UNIT=""
|
||||
if command -v ovs-vsctl >/dev/null 2>&1; then
|
||||
OVS_UNIT=$(distro_ovs_unit) || exit 1
|
||||
fi
|
||||
|
||||
WIPE_DIRS=("$ETC_DIR" "$LOG_DIR" "$RUN_DIR")
|
||||
$KEEP_DATA || WIPE_DIRS+=("$DATA_DIR")
|
||||
|
||||
@@ -102,6 +151,9 @@ fi
|
||||
|
||||
log "stopping services"
|
||||
run sudo systemctl stop spinifex.target 2>/dev/null || true
|
||||
# spinifex-hostnet starts with OVS, which is started below to delete bridges; it
|
||||
# would rebuild the old veth pairs. setup-ovn.sh enables it again.
|
||||
run sudo systemctl disable --now spinifex-hostnet.service 2>/dev/null || true
|
||||
run sudo systemctl reset-failed 'spinifex-*' 2>/dev/null || true
|
||||
|
||||
# Stopping the target is not the same as its services being stopped.
|
||||
@@ -134,8 +186,9 @@ fi
|
||||
# `pgrep -x` and `pkill -x` match against comm, which the kernel truncates to 15
|
||||
# characters, so `qemu-system-x86_64` is stored as `qemu-system-x86` and an -x
|
||||
# match on the full name can never fire. Match the prefix instead: it is 11
|
||||
# characters, well under the limit, and covers both x86_64 and aarch64. This is
|
||||
# the same rule qemuProcessPrefix applies in spinifex/runtime/compute/vm/orphan_scan.go.
|
||||
# characters, well under the limit, and covers both x86_64 and aarch64. EL-family
|
||||
# hosts run /usr/libexec/qemu-kvm, whose comm is exactly `qemu-kvm`. This is the
|
||||
# rule hostos.IsQEMUProcess applies in spinifex/runtime/host/hostos.
|
||||
#
|
||||
# Deliberately not -f. That matches the whole command line, where argv[0] may
|
||||
# carry a path, so an anchored pattern misses the process it was written for.
|
||||
@@ -144,7 +197,7 @@ fi
|
||||
# ones has both under the same comm and an unscoped kill would take the wrong
|
||||
# set down. The daemon runs guests as its own service user on an installed node;
|
||||
# a dev install runs them as whoever invoked spx.
|
||||
QEMU_PGREP=('^qemu-system')
|
||||
QEMU_PGREP=('^(qemu-system|qemu-kvm$)')
|
||||
for qemu_user in spinifex-daemon spinifex "${SUDO_USER:-$(id -un)}"; do
|
||||
if id -u "$qemu_user" >/dev/null 2>&1; then
|
||||
QEMU_PGREP=(-u "$qemu_user" "${QEMU_PGREP[@]}")
|
||||
@@ -178,8 +231,10 @@ fi
|
||||
# node keeping its old system-id across a reset re-registers under the old
|
||||
# chassis name and collides with the new one over the encap IP.
|
||||
log "removing OVS bridges and identity"
|
||||
if command -v ovs-vsctl >/dev/null 2>&1; then
|
||||
run sudo systemctl start openvswitch-switch 2>/dev/null || true
|
||||
# A running ovn-controller recreates br-int as soon as it is deleted.
|
||||
run sudo systemctl stop ovn-controller 2>/dev/null || true
|
||||
if [ -n "$OVS_UNIT" ]; then
|
||||
run sudo systemctl start "$OVS_UNIT" 2>/dev/null || true
|
||||
$DRY_RUN || sleep 1
|
||||
# Listed even under --dry-run: "which bridges" is the question an operator
|
||||
# actually has here, and br-wan is a Linux bridge so it is never in this set.
|
||||
@@ -188,7 +243,7 @@ if command -v ovs-vsctl >/dev/null 2>&1; then
|
||||
run sudo ovs-vsctl --if-exists del-br "$br"
|
||||
done
|
||||
run sudo ovs-vsctl --if-exists clear Open_vSwitch . external_ids 2>/dev/null || true
|
||||
run sudo systemctl stop openvswitch-switch 2>/dev/null || true
|
||||
run sudo systemctl stop "$OVS_UNIT" 2>/dev/null || true
|
||||
fi
|
||||
run sudo rm -f /etc/openvswitch/system-id.conf
|
||||
|
||||
@@ -233,14 +288,20 @@ if [ -e "$ETC_DIR/firewall/custom.nft" ]; then
|
||||
fi
|
||||
|
||||
log "removing OVN databases"
|
||||
run sudo systemctl stop ovn-central 2>/dev/null || true
|
||||
# Debian's ovn-central takes its split units down with it; EL's ovn-northd runs
|
||||
# both databases itself. A compute node has neither.
|
||||
for unit in ovn-central.service ovn-northd.service; do
|
||||
if distro_unit_exists "$unit"; then
|
||||
run sudo systemctl stop "$unit" 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
run sudo systemctl stop ovn-controller 2>/dev/null || true
|
||||
run sudo rm -f /var/lib/ovn/ovnnb_db.db /var/lib/ovn/ovnsb_db.db
|
||||
|
||||
# The RAFT membership flags live in this file, not in the databases just
|
||||
# deleted. Left behind, ovn-central recreates the DB as a cluster member and
|
||||
# blocks dialling a peer that has itself been reset.
|
||||
run sudo rm -f /etc/default/ovn-central
|
||||
# The RAFT membership flags live in these files (Debian, EL), not in the
|
||||
# databases just deleted. Left behind, OVN central recreates the DB as a
|
||||
# cluster member and blocks dialling a peer that has itself been reset.
|
||||
run sudo rm -f /etc/default/ovn-central /etc/sysconfig/ovn-northd
|
||||
|
||||
if ip link show veth-wan-br >/dev/null 2>&1; then
|
||||
log " deleting veth pair: veth-wan-br <-> veth-wan-ovs"
|
||||
@@ -259,6 +320,21 @@ if [ -e /etc/systemd/network/15-spinifex-veth-wan.netdev ] ||
|
||||
run sudo networkctl reload 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# The routed-NAT transit veth keeps 100.127.0.1/24 and the routes behind it.
|
||||
if ip link show spx-nat-host >/dev/null 2>&1; then
|
||||
log " deleting veth pair: spx-nat-host <-> spx-nat-ovs"
|
||||
run sudo ip link del spx-nat-host 2>/dev/null || true
|
||||
fi
|
||||
if [ -e /etc/systemd/network/17-spinifex-nat.netdev ] ||
|
||||
[ -e /etc/systemd/network/17-spinifex-nat.network ] ||
|
||||
[ -e /etc/systemd/network/18-spinifex-nat-ovs.network ]; then
|
||||
log " deleting NAT transit persistence units"
|
||||
run sudo rm -f /etc/systemd/network/17-spinifex-nat.netdev \
|
||||
/etc/systemd/network/17-spinifex-nat.network \
|
||||
/etc/systemd/network/18-spinifex-nat-ovs.network
|
||||
run sudo networkctl reload 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# `rm -rf` is wrong here on two counts. On an ISO-installed node several of
|
||||
# these are separate ZFS datasets — rpool/log and rpool/data/{nats,viperblock,
|
||||
# predastore,predastore-db,predastore-nodes} — and a mountpoint cannot be
|
||||
@@ -374,10 +450,12 @@ fi
|
||||
|
||||
# The next init writes a fresh CA. Leaving the old one trusted means the host
|
||||
# trusts a CA nobody holds the key for.
|
||||
if [ -f /usr/local/share/ca-certificates/spinifex-ca.crt ]; then
|
||||
CA_TRUST=$(distro_ca_trust) || { echo "ERROR: cannot tell which trust store holds the stale CA" >&2; exit 1; }
|
||||
CA_ANCHOR="${CA_TRUST#* }/spinifex-ca.crt"
|
||||
if [ -f "$CA_ANCHOR" ]; then
|
||||
log "removing the stale CA from the trust store"
|
||||
run sudo rm -f /usr/local/share/ca-certificates/spinifex-ca.crt
|
||||
run sudo update-ca-certificates
|
||||
run sudo rm -f "$CA_ANCHOR"
|
||||
run sudo "${CA_TRUST%% *}"
|
||||
fi
|
||||
|
||||
log "done — node is installed but uninitialized, ready to init or join"
|
||||
|
||||
Executable
+90
@@ -0,0 +1,90 @@
|
||||
#!/bin/sh
|
||||
# node-reset_test.sh — tests the QEMU process match node-reset.sh uses to stop
|
||||
# guests before it deletes Viperblock state, against real processes.
|
||||
#
|
||||
# Each fake guest is a copy of sleep named like a QEMU binary, so the kernel
|
||||
# gives it the same comm, including the 15-character truncation.
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
|
||||
TMP=$(mktemp -d)
|
||||
pids=""
|
||||
cleanup() {
|
||||
for pid in $pids; do kill "$pid" 2>/dev/null || true; done
|
||||
rm -rf "$TMP"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
fails=0
|
||||
pass() { echo " ok - $1"; }
|
||||
fail() { echo " FAIL - $1"; fails=$((fails + 1)); }
|
||||
|
||||
echo "-- node-reset.sh QEMU process match"
|
||||
|
||||
# pattern_in <script> — the pgrep pattern the script assigns to QEMU_PGREP.
|
||||
pattern_in() {
|
||||
sed -n "s/^QEMU_PGREP=('\(.*\)')$/\1/p" "$1"
|
||||
}
|
||||
|
||||
PATTERN=$(pattern_in "$SCRIPT_DIR/node-reset.sh")
|
||||
if [ -z "$PATTERN" ]; then
|
||||
echo " FAIL - no QEMU_PGREP=('...') line in node-reset.sh"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# reset-dev-env.sh stops guests the same way and must not drift from it.
|
||||
dev_pattern=$(pattern_in "$SCRIPT_DIR/reset-dev-env.sh")
|
||||
if [ "$dev_pattern" = "$PATTERN" ]; then
|
||||
pass "reset-dev-env.sh uses node-reset.sh's pattern"
|
||||
else
|
||||
fail "reset-dev-env.sh pattern '$dev_pattern' differs from node-reset.sh's '$PATTERN'"
|
||||
fi
|
||||
|
||||
# start <name> — runs a process whose comm is <name>, truncated to 15
|
||||
# characters. Its PID is left in $!.
|
||||
start() {
|
||||
cp "$(command -v sleep)" "$TMP/$1"
|
||||
"$TMP/$1" 300 &
|
||||
pids="$pids $!"
|
||||
}
|
||||
start qemu-system-x86_64; pid_x86=$!
|
||||
start qemu-system-aarch64; pid_arm=$!
|
||||
start qemu-kvm; pid_kvm=$!
|
||||
start qemu-img; pid_img=$!
|
||||
start qemu-kvm-helper; pid_helper=$!
|
||||
|
||||
me=$(id -un)
|
||||
found=" $(pgrep -u "$me" "$PATTERN" | tr '\n' ' ' || true) "
|
||||
|
||||
check_found() {
|
||||
case "$found" in
|
||||
*" $2 "*) pass "finds a $1 guest owned by the service user" ;;
|
||||
*) fail "misses a $1 guest (pid $2) owned by the service user" ;;
|
||||
esac
|
||||
}
|
||||
check_found qemu-system-x86_64 "$pid_x86"
|
||||
check_found qemu-system-aarch64 "$pid_arm"
|
||||
check_found qemu-kvm "$pid_kvm"
|
||||
|
||||
check_ignored() {
|
||||
case "$found" in
|
||||
*" $2 "*) fail "matches $1 (pid $2), which is not a guest" ;;
|
||||
*) pass "ignores $1" ;;
|
||||
esac
|
||||
}
|
||||
check_ignored qemu-img "$pid_img"
|
||||
check_ignored qemu-kvm-helper "$pid_helper"
|
||||
|
||||
# The owner scope: the same guests, asked for as another user, are not found.
|
||||
other=root
|
||||
[ "$me" = root ] && other=nobody
|
||||
other_found=" $(pgrep -u "$other" "$PATTERN" | tr '\n' ' ' || true) "
|
||||
case "$other_found" in
|
||||
*" $pid_kvm "* | *" $pid_x86 "*) fail "a scope of -u $other still matches guests owned by $me" ;;
|
||||
*) pass "ignores guests owned by another user" ;;
|
||||
esac
|
||||
|
||||
if [ "$fails" -gt 0 ]; then
|
||||
echo "$fails failure(s)"
|
||||
exit 1
|
||||
fi
|
||||
@@ -50,6 +50,8 @@ CONFIG_FILE="$ETC_DIR/spinifex.toml"
|
||||
# --- Script context ---
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
# shellcheck source=lib/distro.sh
|
||||
. "$SCRIPT_DIR/lib/distro.sh"
|
||||
|
||||
# Resolve invoking user's HOME. $HOME is unsafe under sudo (resolves to /root).
|
||||
INVOKING_USER="${SUDO_USER:-$(id -un)}"
|
||||
@@ -359,17 +361,29 @@ fi
|
||||
# --- Shutdown services ---
|
||||
echo "==> Stopping services"
|
||||
sudo systemctl stop spinifex.target 2>/dev/null || true
|
||||
# spinifex-hostnet starts with OVS, which is started below to delete bridges; it
|
||||
# would rebuild the old veth pairs. setup-ovn.sh enables it again.
|
||||
sudo systemctl disable --now spinifex-hostnet.service 2>/dev/null || true
|
||||
sudo systemctl reset-failed 'spinifex-*' 2>/dev/null || true
|
||||
sudo pkill -x qemu-system-x86_64 2>/dev/null || true
|
||||
sudo pkill -x qemu-system-aarch64 2>/dev/null || true
|
||||
# comm truncates to 15 characters, so match qemu-system-* by prefix; EL-family
|
||||
# hosts run qemu-kvm. Scoped to the user the daemon runs guests as, as in
|
||||
# node-reset.sh, so libvirt guests on the same host are left alone.
|
||||
QEMU_PGREP=('^(qemu-system|qemu-kvm$)')
|
||||
for qemu_user in spinifex-daemon spinifex "$INVOKING_USER"; do
|
||||
if id -u "$qemu_user" >/dev/null 2>&1; then
|
||||
QEMU_PGREP=(-u "$qemu_user" "${QEMU_PGREP[@]}")
|
||||
break
|
||||
fi
|
||||
done
|
||||
sudo pkill "${QEMU_PGREP[@]}" 2>/dev/null || true
|
||||
|
||||
# Wait for QEMU to fully exit before tearing down viperblock state.
|
||||
timeout=30
|
||||
elapsed=0
|
||||
while pgrep -x 'qemu-system-x86_64|qemu-system-aarch64' > /dev/null 2>&1; do
|
||||
while pgrep "${QEMU_PGREP[@]}" > /dev/null 2>&1; do
|
||||
if [ "$elapsed" -ge "$timeout" ]; then
|
||||
echo "❌ QEMU still running after ${timeout}s:"
|
||||
pgrep -af 'qemu-system-' || true
|
||||
pgrep -a "${QEMU_PGREP[@]}" || true
|
||||
echo " Kill them manually and re-run this script."
|
||||
exit 1
|
||||
fi
|
||||
@@ -380,21 +394,26 @@ done
|
||||
# --- Clean OVS / OVN ---
|
||||
echo "==> Cleaning OVS bridges"
|
||||
if command -v ovs-vsctl >/dev/null 2>&1; then
|
||||
sudo systemctl start openvswitch-switch 2>/dev/null || true
|
||||
OVS_UNIT=$(distro_ovs_unit) || exit 1
|
||||
sudo systemctl start "$OVS_UNIT" 2>/dev/null || true
|
||||
sleep 1
|
||||
for br in $(sudo ovs-vsctl list-br 2>/dev/null); do
|
||||
echo " Deleting bridge: $br"
|
||||
sudo ovs-vsctl --if-exists del-br "$br"
|
||||
done
|
||||
sudo ovs-vsctl --if-exists clear Open_vSwitch . external_ids 2>/dev/null || true
|
||||
sudo systemctl stop openvswitch-switch 2>/dev/null || true
|
||||
sudo systemctl stop "$OVS_UNIT" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# Delete OVN DB files outright — setup-ovn.sh will restart ovn-central with
|
||||
# fresh empty DBs. Eliminates stale SB state (chassis entries, port bindings)
|
||||
# that accumulates across resets and triggers ovn-controller commit loops.
|
||||
echo "==> Removing OVN database files"
|
||||
sudo systemctl stop ovn-central 2>/dev/null || true
|
||||
for unit in ovn-central.service ovn-northd.service; do
|
||||
if distro_unit_exists "$unit"; then
|
||||
sudo systemctl stop "$unit" 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
sudo systemctl stop ovn-controller 2>/dev/null || true
|
||||
if [ -d /var/lib/ovn ]; then
|
||||
sudo rm -f /var/lib/ovn/ovnnb_db.db /var/lib/ovn/ovnsb_db.db
|
||||
@@ -516,10 +535,16 @@ sleep 5
|
||||
echo "==> Building and installing microVM artifacts"
|
||||
cd "$PROJECT_ROOT" && make build-lb-agent install-microvm
|
||||
|
||||
# By absolute path: sudo's secure_path leaves out /usr/local/bin on some hosts,
|
||||
# Oracle Linux among them, and setup.sh installs the AWS CLI there.
|
||||
AWS_BIN=$(command -v aws || true)
|
||||
[ -z "$AWS_BIN" ] && [ -x /usr/local/bin/aws ] && AWS_BIN=/usr/local/bin/aws
|
||||
[ -n "$AWS_BIN" ] || { echo "ERROR: aws CLI not found on PATH or at /usr/local/bin/aws" >&2; exit 1; }
|
||||
|
||||
# --- Authorize SSH ingress on the default security group ---
|
||||
# Default SG denies all inbound, so smoke-test.sh's SSH probe times out.
|
||||
# Open port 22 on the default VPC's default SG before the smoke test runs.
|
||||
aws_as_user() { sudo -u "$INVOKING_USER" env HOME="$INVOKING_HOME" AWS_PROFILE=spinifex aws "$@"; }
|
||||
aws_as_user() { sudo -u "$INVOKING_USER" env HOME="$INVOKING_HOME" AWS_PROFILE=spinifex "$AWS_BIN" "$@"; }
|
||||
|
||||
echo "==> Waiting for EC2 daemon"
|
||||
DAEMON_TIMEOUT=60
|
||||
|
||||
@@ -66,7 +66,7 @@ MAX_QEMU_ATTEMPTS=60
|
||||
QEMU_CMD=""
|
||||
|
||||
while [ $QEMU_ATTEMPTS -lt $MAX_QEMU_ATTEMPTS ]; do
|
||||
QEMU_CMD=$(ps auxw | grep "$INSTANCE_ID" | grep qemu-system | grep -v grep)
|
||||
QEMU_CMD=$(ps auxw | grep "$INSTANCE_ID" | grep -E "qemu-system|qemu-kvm" | grep -v grep)
|
||||
|
||||
if [ -n "$QEMU_CMD" ]; then
|
||||
break
|
||||
|
||||
+501
-118
@@ -4,12 +4,20 @@
|
||||
#
|
||||
# This script bootstraps a compute node for OVN-based VPC networking:
|
||||
# 1. Installs OVN/OVS packages (if not present)
|
||||
# 2. Enables required services (openvswitch-switch, ovn-controller)
|
||||
# 2. Enables required services (Open vSwitch, ovn-controller)
|
||||
# 3. Creates br-int with secure fail-mode
|
||||
# 4. Configures WAN bridge for public subnet uplink (auto-detected or manual)
|
||||
# 5. Configures OVS external_ids for OVN chassis identity
|
||||
# 6. Applies sysctl tuning for overlay networking
|
||||
#
|
||||
# Host network persistence: the links this script creates are kernel-only. On
|
||||
# a host where systemd-networkd is enabled, or that already has Spinifex
|
||||
# networkd files, they persist in /etc/systemd/network. Otherwise (for example
|
||||
# a NetworkManager host) the topology goes to /etc/spinifex/hostnet.json and
|
||||
# spinifex-hostnet.service re-creates it at boot and on every OVS start. The
|
||||
# first run records which in /etc/spinifex/hostnet-backend; it never changes.
|
||||
# Where NetworkManager runs, the links Spinifex owns are made unmanaged.
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/setup-ovn.sh [options]
|
||||
#
|
||||
@@ -121,6 +129,19 @@
|
||||
|
||||
set -e
|
||||
|
||||
# Host-OS capability probes: beside this script in a checkout and in the
|
||||
# installed share dir, else where setup.sh installs them.
|
||||
DISTRO_LIB=""
|
||||
for candidate in "$(dirname "${BASH_SOURCE[0]}")/lib/distro.sh" /usr/local/share/spinifex/lib/distro.sh; do
|
||||
if [ -f "$candidate" ]; then DISTRO_LIB="$candidate"; break; fi
|
||||
done
|
||||
if [ -z "$DISTRO_LIB" ]; then
|
||||
echo "ERROR: distro.sh not found at $(dirname "${BASH_SOURCE[0]}")/lib/distro.sh or /usr/local/share/spinifex/lib/distro.sh; re-run setup.sh to install it" >&2
|
||||
exit 1
|
||||
fi
|
||||
# shellcheck source=lib/distro.sh
|
||||
. "$DISTRO_LIB"
|
||||
|
||||
# Defaults
|
||||
MANAGEMENT=false
|
||||
WAN_BRIDGE=""
|
||||
@@ -145,6 +166,10 @@ DB_CLUSTER_REMOTE_ADDR=""
|
||||
# Recreating the NB/SB DBs discards all logical network state, so it is opt-in.
|
||||
RECREATE_DB=false
|
||||
OVN_DBDIR="${OVN_DBDIR:-/var/lib/ovn}"
|
||||
# Records that --management set this node up. Nodes set up before it existed
|
||||
# carry only the legacy marker, OVN_CTL_OPTS in /etc/default/ovn-central.
|
||||
OVN_ROLE_FILE=/etc/spinifex/ovn-role
|
||||
LEGACY_OVN_ROLE_FILE=/etc/default/ovn-central
|
||||
# NODE_NAME is left empty by default. The chassis-id pin block at Step 4
|
||||
# only runs when --node-name=NAME is explicitly given. Passing nothing
|
||||
# preserves whatever system-id already lives in OVS (gold-image UUID,
|
||||
@@ -222,6 +247,152 @@ iface_master() {
|
||||
printf '%s' "$m"
|
||||
}
|
||||
|
||||
# --- Host network persistence ---
|
||||
#
|
||||
# Links this script creates are kernel-only and vanish on reboot. The first run
|
||||
# picks how they persist and records it: networkd where systemd-networkd is
|
||||
# enabled, else hostnet (spinifex-hostnet.service replays HOSTNET_STATE at boot
|
||||
# and on every ovs-vswitchd start). There is no switching; reset the node.
|
||||
HOSTNET_BACKEND_FILE=/etc/spinifex/hostnet-backend
|
||||
HOSTNET_STATE=/etc/spinifex/hostnet.json
|
||||
HOSTNET_LOCK=/run/spinifex/hostnet.lock
|
||||
NETWORKD_DIR=/etc/systemd/network
|
||||
NM_UNMANAGED_CONF=/etc/NetworkManager/conf.d/00-spinifex-unmanaged.conf
|
||||
NM_UNMANAGED_RULE=/etc/udev/rules.d/90-spinifex-nm-unmanaged.rules
|
||||
# System-instance management TAPs: "mg" plus 13 hex digits of the instance ID.
|
||||
# NetworkManager globs have no character classes, so this match goes to udev.
|
||||
MGMT_TAP_GLOB='mg[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]'
|
||||
|
||||
# Prints the recorded backend, or picks one and records it. A node with
|
||||
# Spinifex networkd files and no record was set up before the record existed.
|
||||
select_hostnet_backend() {
|
||||
local backend
|
||||
if sudo test -e "$HOSTNET_BACKEND_FILE"; then
|
||||
backend=$(sudo cat "$HOSTNET_BACKEND_FILE" | tr -d '[:space:]')
|
||||
case "$backend" in
|
||||
networkd|hostnet) printf '%s\n' "$backend"; return 0 ;;
|
||||
esac
|
||||
echo "ERROR: $HOSTNET_BACKEND_FILE says '$backend'; expected networkd or hostnet" >&2
|
||||
return 1
|
||||
fi
|
||||
if compgen -G "$NETWORKD_DIR/*-spinifex-*.network" >/dev/null || compgen -G "$NETWORKD_DIR/*-spinifex-*.netdev" >/dev/null; then
|
||||
backend=networkd
|
||||
elif systemctl is-enabled --quiet systemd-networkd.service 2>/dev/null; then
|
||||
backend=networkd
|
||||
else
|
||||
backend=hostnet
|
||||
fi
|
||||
sudo mkdir -p "$(dirname "$HOSTNET_BACKEND_FILE")"
|
||||
printf '%s\n' "$backend" | sudo tee "$HOSTNET_BACKEND_FILE" >/dev/null
|
||||
# Read by `spx admin hostnet check`, which the daemon and vpcd run unprivileged.
|
||||
sudo chmod 0644 "$HOSTNET_BACKEND_FILE"
|
||||
printf '%s\n' "$backend"
|
||||
}
|
||||
|
||||
# write_hostnet_state <true|false> — records the topology for
|
||||
# `spx admin hostnet apply`, atomically. true marks it pending: apply refuses
|
||||
# it until the run that wrote it finishes.
|
||||
write_hostnet_state() {
|
||||
local pending="$1" body
|
||||
body=$(printf '{\n "version": 1,\n "pending": %s,\n "int_bridge": "br-int"' "$pending")
|
||||
case "$WAN_BRIDGE_MODE" in
|
||||
existing)
|
||||
body="$body"$(printf ',\n "wan": {"mode": "existing", "ovs_bridge": "%s"}' "$WAN_BRIDGE") ;;
|
||||
veth)
|
||||
body="$body"$(printf ',\n "wan": {"mode": "veth", "ovs_bridge": "%s", "linux_bridge": "%s", "host_end": "veth-wan-br", "ovs_end": "veth-wan-ovs"}' "$WAN_BRIDGE" "$LINUX_BRIDGE") ;;
|
||||
direct)
|
||||
body="$body"$(printf ',\n "wan": {"mode": "direct", "ovs_bridge": "%s", "nic": "%s"}' "$WAN_BRIDGE" "$WAN_IFACE") ;;
|
||||
nat)
|
||||
body="$body"$(printf ',\n "wan": {"mode": "nat", "ovs_bridge": "%s", "host_end": "spx-nat-host", "ovs_end": "spx-nat-ovs", "host_mac": "%s", "host_cidr": "%s"}' "$WAN_BRIDGE" "$NAT_TRANSIT_HOST_MAC" "$NAT_TRANSIT_GW_CIDR") ;;
|
||||
esac
|
||||
if [ "$MGMT_BRIDGE_ENABLED" = true ]; then
|
||||
body="$body"$(printf ',\n "mgmt": {"bridge": "%s", "cidr": "%s", "nic": "%s"}' "$MGMT_BRIDGE" "$MGMT_CIDR" "$MGMT_IFACE")
|
||||
fi
|
||||
if [ -n "${ENCAP_ROUTE_DST:-}" ]; then
|
||||
body="$body"$(printf ',\n "encap_route": {"dst": "%s", "dev": "%s", "src": "%s", "metric": %s}' \
|
||||
"$ENCAP_ROUTE_DST" "$ENCAP_ROUTE_DEV" "$ENCAP_IP" "$ENCAP_ROUTE_METRIC")
|
||||
fi
|
||||
sudo mkdir -p "$(dirname "$HOSTNET_STATE")"
|
||||
printf '%s\n}\n' "$body" | sudo tee "$HOSTNET_STATE.tmp" >/dev/null
|
||||
sudo chmod 0644 "$HOSTNET_STATE.tmp"
|
||||
sudo mv -f "$HOSTNET_STATE.tmp" "$HOSTNET_STATE"
|
||||
}
|
||||
|
||||
# Holds the lock `spx admin hostnet apply` takes, from a root flock process fed
|
||||
# by a pipe, so an apply started by an OVS restart waits for this run's links.
|
||||
# Never run systemctl against OVS or spinifex-hostnet while holding it.
|
||||
hostnet_lock() {
|
||||
local reply=""
|
||||
sudo mkdir -p "$(dirname "$HOSTNET_LOCK")"
|
||||
coproc HOSTNET_LOCK_HOLDER { exec sudo flock -w 120 "$HOSTNET_LOCK" sh -c 'echo locked; read -r _'; }
|
||||
read -r reply <&"${HOSTNET_LOCK_HOLDER[0]}" || true
|
||||
if [ "$reply" != locked ]; then
|
||||
echo "ERROR: $HOSTNET_LOCK still held after 120 s; is spx admin hostnet apply stuck? (systemctl status spinifex-hostnet)" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
hostnet_unlock() {
|
||||
local fd="${HOSTNET_LOCK_HOLDER[1]}" pid="$HOSTNET_LOCK_HOLDER_PID"
|
||||
exec {fd}>&-
|
||||
wait "$pid" 2>/dev/null || true
|
||||
}
|
||||
|
||||
nm_running() {
|
||||
systemctl is-active --quiet NetworkManager.service 2>/dev/null
|
||||
}
|
||||
|
||||
# nm_release_nic <nic> — before a NIC moves into OVS, refuse the one carrying
|
||||
# the default route or this SSH session, then take it from NetworkManager. Its
|
||||
# profiles are left alone: once unmanaged, none can activate on it.
|
||||
nm_release_nic() {
|
||||
local nic="$1" ssh_ip state
|
||||
if sudo ovs-vsctl port-to-br "$nic" >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
if ip -4 route show default 2>/dev/null | awk '{for(i=1;i<NF;i++) if($i=="dev") print $(i+1)}' | grep -qx "$nic"; then
|
||||
echo "ERROR: $nic carries the default route; moving it into OVS would cut this host off" >&2
|
||||
exit 1
|
||||
fi
|
||||
ssh_ip=$(printf '%s\n' "${SSH_CONNECTION:-}" | awk '{print $3}')
|
||||
if [ -n "$ssh_ip" ] && ip -o -4 addr show dev "$nic" 2>/dev/null | awk '{print $4}' | cut -d/ -f1 | grep -qxF "$ssh_ip"; then
|
||||
echo "ERROR: this SSH session arrived on $nic ($ssh_ip); moving it into OVS would drop it" >&2
|
||||
exit 1
|
||||
fi
|
||||
state=$(nmcli -g GENERAL.STATE device show "$nic" 2>/dev/null || true)
|
||||
case "$state" in
|
||||
*"(connect"*)
|
||||
sudo nmcli device disconnect "$nic" >/dev/null
|
||||
echo " $nic: disconnected from NetworkManager"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Writes the NetworkManager config and udev rule that leave every link Spinifex
|
||||
# owns unmanaged, then has NetworkManager and udev reread them. The host uplink
|
||||
# (a Linux bridge such as br-wan, and its NIC) stays NetworkManager's.
|
||||
write_nm_unmanaged() {
|
||||
local names=(br-int "$WAN_BRIDGE" 'veth-wan-*' 'spx-nat-*' 'tap*' 'ime-*' 'imi-*' 'vhp-*' br-imds 'genev_sys_*')
|
||||
local list="" name
|
||||
if [ "$MGMT_BRIDGE_ENABLED" = true ]; then
|
||||
names+=("$MGMT_BRIDGE" "$MGMT_IFACE")
|
||||
fi
|
||||
if [ "$WAN_BRIDGE_MODE" = direct ]; then
|
||||
names+=("$WAN_IFACE")
|
||||
fi
|
||||
for name in "${names[@]}"; do
|
||||
[ -n "$name" ] && list="${list:+$list;}interface-name:$name"
|
||||
done
|
||||
sudo mkdir -p "$(dirname "$NM_UNMANAGED_CONF")" "$(dirname "$NM_UNMANAGED_RULE")"
|
||||
printf '# Written by setup-ovn.sh on every run: links Spinifex owns.\n[keyfile]\nunmanaged-devices+=%s\n' "$list" \
|
||||
| sudo tee "$NM_UNMANAGED_CONF" >/dev/null
|
||||
printf '# Written by setup-ovn.sh: system-instance management TAPs.\nSUBSYSTEM=="net", KERNEL=="%s", ENV{NM_UNMANAGED}="1"\n' "$MGMT_TAP_GLOB" \
|
||||
| sudo tee "$NM_UNMANAGED_RULE" >/dev/null
|
||||
sudo udevadm control --reload
|
||||
sudo udevadm trigger --action=change --subsystem-match=net --sysname-match='mg*'
|
||||
sudo nmcli general reload conf
|
||||
}
|
||||
|
||||
teardown() {
|
||||
echo "Tearing down the OVN host wiring built by this script"
|
||||
[ "$DRY_RUN" = true ] && echo " (dry run — nothing is changed)"
|
||||
@@ -469,6 +640,15 @@ else
|
||||
detect_wan_bridge
|
||||
fi
|
||||
|
||||
# Routed NAT transit, used by nat mode and recorded for spinifex-hostnet.
|
||||
NAT_TRANSIT_CIDR="100.127.0.0/24"
|
||||
NAT_TRANSIT_GW_CIDR="100.127.0.1/24"
|
||||
# Same on every node. OVN keeps one MAC binding for the transit nexthop
|
||||
# cluster-wide, so a random per-node MAC sends every chassis's egress to
|
||||
# whichever node seeded it; the pair is node-local, so one address everywhere
|
||||
# is unambiguous.
|
||||
NAT_TRANSIT_HOST_MAC="02:00:64:7f:00:01"
|
||||
|
||||
# Auto-detect encap IP if not specified
|
||||
if [ -z "$ENCAP_IP" ]; then
|
||||
# Prefer br-vpc IP if it exists (dedicated VPC data plane)
|
||||
@@ -575,19 +755,104 @@ ensure_clustered_db_storage() {
|
||||
# fresh SB and re-registers under whatever system-id it started with — which
|
||||
# on a renamed node is the old one, and that row then holds this node's encap
|
||||
# IP so the correctly-named chassis can never commit. Step 5 restarts it.
|
||||
sudo systemctl stop ovn-northd ovn-ovsdb-server-nb ovn-ovsdb-server-sb ovn-central ovn-controller 2>/dev/null || true
|
||||
sudo systemctl stop "${OVN_CENTRAL_UNITS[@]}" ovn-controller 2>/dev/null || true
|
||||
sudo rm -f "${standalone[@]}"
|
||||
}
|
||||
|
||||
# set_root_service_user <file> <var> — sets <var>="root:root" in <file> when
|
||||
# the file exists. Returns 0 only when it changed the file.
|
||||
set_root_service_user() {
|
||||
local file=$1 var=$2
|
||||
[ -f "$file" ] || return 1
|
||||
grep -qx "$var=\"root:root\"" "$file" && return 1
|
||||
if grep -q "^$var=" "$file"; then
|
||||
sudo sed -i "s|^$var=.*|$var=\"root:root\"|" "$file"
|
||||
else
|
||||
echo "$var=\"root:root\"" | sudo tee -a "$file" >/dev/null
|
||||
fi
|
||||
echo " $file: $var=\"root:root\""
|
||||
}
|
||||
|
||||
# write_ovn_central_opts <flags> — writes the ovn-ctl flags to the options
|
||||
# file the OVN central units read for the probed layout.
|
||||
write_ovn_central_opts() {
|
||||
echo "$OVN_CENTRAL_OPTS_VAR=\"$1\"" | sudo tee "$OVN_CENTRAL_OPTS_FILE" >/dev/null
|
||||
echo " wrote $OVN_CENTRAL_OPTS_FILE"
|
||||
}
|
||||
|
||||
record_management_role() {
|
||||
sudo mkdir -p "$(dirname "$OVN_ROLE_FILE")"
|
||||
echo management | sudo tee "$OVN_ROLE_FILE" >/dev/null
|
||||
}
|
||||
|
||||
# management_role_markers — prints each file recording that a previous run set
|
||||
# this node up with --management, one per line; returns 1 when there is none.
|
||||
management_role_markers() {
|
||||
local found=1
|
||||
if sudo grep -qx management "$OVN_ROLE_FILE" 2>/dev/null; then
|
||||
echo "$OVN_ROLE_FILE"
|
||||
found=0
|
||||
fi
|
||||
if sudo grep -q '^OVN_CTL_OPTS=' "$LEGACY_OVN_ROLE_FILE" 2>/dev/null; then
|
||||
echo "$LEGACY_OVN_ROLE_FILE"
|
||||
found=0
|
||||
fi
|
||||
return $found
|
||||
}
|
||||
|
||||
echo ""
|
||||
echo "Step 2: Enabling services..."
|
||||
|
||||
sudo systemctl enable openvswitch-switch
|
||||
sudo systemctl start openvswitch-switch
|
||||
echo " openvswitch-switch: started"
|
||||
# Unit names differ by distro: the OVS umbrella, and whether the OVN databases
|
||||
# have units of their own (Debian) or are started by ovn-northd.service (EL).
|
||||
OVS_UNIT=$(distro_ovs_unit) || exit 1
|
||||
OVN_CENTRAL_LAYOUT=""
|
||||
if [ "$MANAGEMENT" = true ] || distro_unit_exists ovn-central.service || distro_unit_exists ovn-northd.service; then
|
||||
OVN_CENTRAL_LAYOUT=$(distro_ovn_central_layout) || exit 1
|
||||
fi
|
||||
case "$OVN_CENTRAL_LAYOUT" in
|
||||
split)
|
||||
OVN_CENTRAL_UNIT=ovn-central.service
|
||||
OVN_DB_UNITS=(ovn-ovsdb-server-nb.service ovn-ovsdb-server-sb.service)
|
||||
OVN_SERVER_UNITS=("${OVN_DB_UNITS[@]}" ovn-northd.service)
|
||||
OVN_CENTRAL_UNITS=(ovn-central.service "${OVN_SERVER_UNITS[@]}")
|
||||
OVN_CENTRAL_OPTS_FILE=/etc/default/ovn-central
|
||||
OVN_CENTRAL_OPTS_VAR=OVN_CTL_OPTS
|
||||
;;
|
||||
lone-northd)
|
||||
OVN_CENTRAL_UNIT=ovn-northd.service
|
||||
OVN_DB_UNITS=(ovn-northd.service)
|
||||
OVN_SERVER_UNITS=(ovn-northd.service)
|
||||
OVN_CENTRAL_UNITS=(ovn-northd.service)
|
||||
OVN_CENTRAL_OPTS_FILE=/etc/sysconfig/ovn-northd
|
||||
OVN_CENTRAL_OPTS_VAR=OVN_NORTHD_OPTS
|
||||
;;
|
||||
esac
|
||||
|
||||
# OVS and OVN run as root, as on Debian: the service users reach the sockets
|
||||
# through the spinifex group, a model tested only against root-owned daemons.
|
||||
# EL ships non-root users and re-chowns the run dirs to them on every start.
|
||||
OVS_USER_CHANGED=false
|
||||
if set_root_service_user /etc/sysconfig/openvswitch OVS_USER_ID; then
|
||||
OVS_USER_CHANGED=true
|
||||
fi
|
||||
# Takes effect below: ovn-northd and ovn-controller are restarted either way.
|
||||
set_root_service_user /etc/sysconfig/ovn OVN_USER_ID || true
|
||||
|
||||
sudo systemctl enable "$OVS_UNIT"
|
||||
if [ "$OVS_USER_CHANGED" = true ]; then
|
||||
# start is a no-op on a running OVS, which would keep its old user.
|
||||
sudo systemctl restart "$OVS_UNIT"
|
||||
else
|
||||
sudo systemctl start "$OVS_UNIT"
|
||||
fi
|
||||
echo " ${OVS_UNIT%.service}: started"
|
||||
|
||||
if [ "$MANAGEMENT" = true ]; then
|
||||
sudo systemctl enable ovn-central
|
||||
sudo systemctl enable "$OVN_CENTRAL_UNIT"
|
||||
# Recorded before anything else changes, so a run that stops part way
|
||||
# still makes a later run without --management refuse.
|
||||
record_management_role
|
||||
|
||||
# LAN-plane NB/SB client listener, added to the loopback one set below.
|
||||
# Guard is load-bearing: --db-nb-addr defaults to 0.0.0.0, so
|
||||
@@ -600,8 +865,8 @@ if [ "$MANAGEMENT" = true ]; then
|
||||
if [ -n "$DB_CLUSTER_LOCAL_ADDR" ]; then
|
||||
ensure_clustered_db_storage
|
||||
|
||||
# Clustered NB/SB via native OVSDB RAFT. Both per-DB units source one
|
||||
# shared OVN_CTL_OPTS from /etc/default/ovn-central; each run_*_ovsdb
|
||||
# Clustered NB/SB via native OVSDB RAFT. Every OVN central unit reads
|
||||
# one shared options file (write_ovn_central_opts); each run_*_ovsdb
|
||||
# consumes only its own --db-{nb,sb}-* flags. RAFT ports default to NB
|
||||
# 6643 / SB 6644. ovn-ctl creates the cluster when no remote-addr is
|
||||
# given (and no existing .db file), joins when one is.
|
||||
@@ -635,26 +900,28 @@ if [ "$MANAGEMENT" = true ]; then
|
||||
echo " NB/SB client listen: 127.0.0.1"
|
||||
fi
|
||||
|
||||
echo "OVN_CTL_OPTS=\"$OVN_CTL_OPTS\"" | sudo tee /etc/default/ovn-central >/dev/null
|
||||
echo " wrote /etc/default/ovn-central"
|
||||
write_ovn_central_opts "$OVN_CTL_OPTS"
|
||||
|
||||
# The packaged ovn-northd.service ExecStop runs `ovn-ctl stop_northd`
|
||||
# without --ovn-manage-ovsdb=no, so restarting northd also tears down the
|
||||
# NB/SB ovsdb-server units. With the split clustered units those DBs are
|
||||
# owned by their own units, so override ExecStop to leave them alone —
|
||||
# otherwise the restart below races and kills the freshly-started DBs.
|
||||
sudo mkdir -p /etc/systemd/system/ovn-northd.service.d
|
||||
sudo tee /etc/systemd/system/ovn-northd.service.d/no-manage-ovsdb.conf >/dev/null <<'EOF'
|
||||
# EL's lone ovn-northd.service owns the DBs, so stopping them is right there.
|
||||
if [ "$OVN_CENTRAL_LAYOUT" = split ]; then
|
||||
sudo mkdir -p /etc/systemd/system/ovn-northd.service.d
|
||||
sudo tee /etc/systemd/system/ovn-northd.service.d/no-manage-ovsdb.conf >/dev/null <<'EOF'
|
||||
[Service]
|
||||
ExecStop=
|
||||
ExecStop=/usr/share/ovn/scripts/ovn-ctl stop_northd --no-monitor --ovn-manage-ovsdb=no
|
||||
EOF
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl daemon-reload
|
||||
fi
|
||||
|
||||
# The ovn-central aggregator is ExecStart=/bin/true, so restarting it
|
||||
# won't restart the children — restart the per-DB units directly to
|
||||
# pick up the new OVN_CTL_OPTS.
|
||||
sudo systemctl restart ovn-ovsdb-server-nb ovn-ovsdb-server-sb ovn-northd
|
||||
# Debian's ovn-central aggregator is ExecStart=/bin/true, so restarting
|
||||
# it won't restart the children — restart the DB and northd units
|
||||
# directly to pick up the new OVN_CTL_OPTS.
|
||||
sudo systemctl restart "${OVN_SERVER_UNITS[@]}"
|
||||
echo " ovn-central: started clustered (NB DB + SB DB + ovn-northd)"
|
||||
else
|
||||
# Always write the file, even when there is nothing to put in it. This
|
||||
@@ -662,14 +929,13 @@ EOF
|
||||
# and the RAFT flags live here rather than in the database — leaving a
|
||||
# stale file in place restarts the DB as a cluster member dialling a
|
||||
# peer that no longer exists, which looks like a hung leader election.
|
||||
echo "OVN_CTL_OPTS=\"$OVN_DB_LISTEN_OPTS\"" | sudo tee /etc/default/ovn-central >/dev/null
|
||||
echo " wrote /etc/default/ovn-central"
|
||||
sudo systemctl start ovn-central
|
||||
write_ovn_central_opts "$OVN_DB_LISTEN_OPTS"
|
||||
sudo systemctl start "$OVN_CENTRAL_UNIT"
|
||||
|
||||
# ovn-central is ExecStart=/bin/true; restarting it does not restart
|
||||
# its children, so a re-run with changed options needs the per-DB units
|
||||
# restarted directly to pick them up.
|
||||
sudo systemctl restart ovn-ovsdb-server-nb ovn-ovsdb-server-sb
|
||||
# Debian's ovn-central is ExecStart=/bin/true; restarting it does not
|
||||
# restart its children, so a re-run with changed options needs the DB
|
||||
# units restarted directly to pick them up.
|
||||
sudo systemctl restart "${OVN_DB_UNITS[@]}"
|
||||
echo " ovn-central: started (NB DB + SB DB + ovn-northd)"
|
||||
if [ -n "$LAN_ADDR" ]; then
|
||||
echo " NB/SB client listen: 127.0.0.1, $LAN_ADDR"
|
||||
@@ -731,7 +997,7 @@ EOF
|
||||
done
|
||||
if [ "$(printf '%s\n' "$NB_MEMBERS" | grep -c .)" -gt 1 ]; then
|
||||
sudo sed -i "s|\"\$| --ovn-northd-nb-db=$NB_LIST --ovn-northd-sb-db=$SB_LIST\"|" \
|
||||
/etc/default/ovn-central
|
||||
"$OVN_CENTRAL_OPTS_FILE"
|
||||
sudo systemctl restart ovn-northd
|
||||
echo " ovn-northd: NB=$NB_LIST SB=$SB_LIST (derived from RAFT membership)"
|
||||
else
|
||||
@@ -796,26 +1062,27 @@ else
|
||||
# 6641, and vpcd crash-loops on "connection refused" with no hint that this
|
||||
# script caused it. Seen on mulga-poc at 15,061 restarts.
|
||||
#
|
||||
# The signal is /etc/default/ovn-central, not the presence of a database:
|
||||
# the ovn-central *package* creates DBs on install, so every compute node
|
||||
# has them and testing for them would refuse every legitimate compute node.
|
||||
# Only the --management path below writes OVN_CTL_OPTS to that file.
|
||||
if [ -f /etc/default/ovn-central ] && grep -q '^OVN_CTL_OPTS=' /etc/default/ovn-central 2>/dev/null; then
|
||||
# The signal is a role marker, not the presence of a database: the OVN
|
||||
# central *package* creates DBs on install, so every compute node has them
|
||||
# and testing for them would refuse every legitimate compute node. Only the
|
||||
# --management path above writes a marker (management_role_markers).
|
||||
if ROLE_MARKERS=$(management_role_markers); then
|
||||
ROLE_MARKERS=${ROLE_MARKERS//$'\n'/ }
|
||||
echo "ERROR: this node was configured as a management node by a previous run"
|
||||
echo " (/etc/default/ovn-central holds OVN_CTL_OPTS), but --management was"
|
||||
echo " not given. Re-running without it stops and disables ovn-central and"
|
||||
echo " takes the control plane down."
|
||||
echo " (recorded in $ROLE_MARKERS), but --management was not given."
|
||||
echo " Re-running without it stops and disables OVN central and takes the"
|
||||
echo " control plane down."
|
||||
echo ""
|
||||
echo " If this is a management node, re-run with --management."
|
||||
echo " If it is genuinely being demoted to a compute node:"
|
||||
echo " sudo rm -f /etc/default/ovn-central"
|
||||
echo " sudo rm -f $ROLE_MARKERS"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if systemctl is-enabled ovn-central >/dev/null 2>&1 ||
|
||||
systemctl is-active ovn-central >/dev/null 2>&1; then
|
||||
sudo systemctl stop ovn-central ovn-northd ovn-ovsdb-server-nb ovn-ovsdb-server-sb 2>/dev/null || true
|
||||
sudo systemctl disable ovn-central ovn-northd ovn-ovsdb-server-nb ovn-ovsdb-server-sb 2>/dev/null || true
|
||||
if [ -n "$OVN_CENTRAL_LAYOUT" ] && { systemctl is-enabled "$OVN_CENTRAL_UNIT" >/dev/null 2>&1 ||
|
||||
systemctl is-active "$OVN_CENTRAL_UNIT" >/dev/null 2>&1; }; then
|
||||
sudo systemctl stop "${OVN_CENTRAL_UNITS[@]}" 2>/dev/null || true
|
||||
sudo systemctl disable "${OVN_CENTRAL_UNITS[@]}" 2>/dev/null || true
|
||||
echo " ovn-central: stopped and disabled (compute node runs no database)"
|
||||
fi
|
||||
for db in "$OVN_DBDIR/ovnnb_db.db" "$OVN_DBDIR/ovnsb_db.db"; do
|
||||
@@ -825,6 +1092,34 @@ else
|
||||
done
|
||||
fi
|
||||
|
||||
# --- Step 2b: Host network persistence and NetworkManager ---
|
||||
echo ""
|
||||
echo "Step 2b: Host network persistence..."
|
||||
HOSTNET_BACKEND=$(select_hostnet_backend)
|
||||
echo " backend: $HOSTNET_BACKEND (recorded in $HOSTNET_BACKEND_FILE)"
|
||||
|
||||
if nm_running; then
|
||||
if [ "$MGMT_BRIDGE_ENABLED" = true ] && [ -n "$MGMT_IFACE" ]; then
|
||||
nm_release_nic "$MGMT_IFACE"
|
||||
fi
|
||||
if [ "$WAN_BRIDGE_MODE" = direct ]; then
|
||||
nm_release_nic "$WAN_IFACE"
|
||||
fi
|
||||
write_nm_unmanaged
|
||||
echo " NetworkManager: Spinifex links unmanaged ($NM_UNMANAGED_CONF, $NM_UNMANAGED_RULE)"
|
||||
fi
|
||||
|
||||
# Held until the wiring below is complete; a run that dies before then leaves
|
||||
# the state pending, and spinifex-hostnet refuses it until this script is rerun.
|
||||
ENCAP_ROUTE_DST=""
|
||||
ENCAP_ROUTE_DEV=""
|
||||
ENCAP_ROUTE_METRIC=50
|
||||
if [ "$HOSTNET_BACKEND" = hostnet ]; then
|
||||
hostnet_lock
|
||||
write_hostnet_state true
|
||||
echo " $HOSTNET_STATE: pending until this run finishes wiring"
|
||||
fi
|
||||
|
||||
# --- Step 3: Create and configure br-int ---
|
||||
echo ""
|
||||
echo "Step 3: Configuring br-int..."
|
||||
@@ -842,7 +1137,7 @@ echo " br-int: created, fail-mode=secure, up"
|
||||
# still forwards, but ovn-controller flow programming and any tooling that
|
||||
# probes link state misbehave. Unmanaged=yes keeps OVS in sole control.
|
||||
OVS_INTERNAL_NET=/etc/systemd/network/05-spinifex-ovs-internal.network
|
||||
if [ ! -f "$OVS_INTERNAL_NET" ]; then
|
||||
if [ "$HOSTNET_BACKEND" = networkd ] && [ ! -f "$OVS_INTERNAL_NET" ]; then
|
||||
sudo tee "$OVS_INTERNAL_NET" >/dev/null <<'NETWORK'
|
||||
[Match]
|
||||
Name=br-int br-ext
|
||||
@@ -956,23 +1251,24 @@ if [ -n "$WAN_BRIDGE" ]; then
|
||||
# existing networkd config. networkd searches /etc, /run, /usr/lib;
|
||||
# check all three. Our own file is excluded so idempotent re-runs
|
||||
# still rewrite when needed.
|
||||
BR_WAN_NETDEV="/etc/systemd/network/14-spinifex-br-wan.netdev"
|
||||
VETH_NETDEV="/etc/systemd/network/15-spinifex-veth-wan.netdev"
|
||||
VETH_NETWORK="/etc/systemd/network/15-spinifex-veth-wan.network"
|
||||
VETH_OVS_NETWORK="/etc/systemd/network/16-spinifex-veth-wan-ovs.network"
|
||||
EXISTING_BR_NETDEV=$(grep -rls --include="*.netdev" -E "^\s*Name=$LINUX_BRIDGE\s*$" \
|
||||
/etc/systemd/network /run/systemd/network /usr/lib/systemd/network 2>/dev/null \
|
||||
| grep -v "^$BR_WAN_NETDEV$" || true)
|
||||
if [ -n "$EXISTING_BR_NETDEV" ]; then
|
||||
echo " skipping $BR_WAN_NETDEV — operator-managed NetDev already declares $LINUX_BRIDGE: $EXISTING_BR_NETDEV"
|
||||
else
|
||||
sudo tee "$BR_WAN_NETDEV" >/dev/null <<NETDEV
|
||||
if [ "$HOSTNET_BACKEND" = networkd ]; then
|
||||
BR_WAN_NETDEV="/etc/systemd/network/14-spinifex-br-wan.netdev"
|
||||
VETH_NETDEV="/etc/systemd/network/15-spinifex-veth-wan.netdev"
|
||||
VETH_NETWORK="/etc/systemd/network/15-spinifex-veth-wan.network"
|
||||
VETH_OVS_NETWORK="/etc/systemd/network/16-spinifex-veth-wan-ovs.network"
|
||||
EXISTING_BR_NETDEV=$(grep -rls --include="*.netdev" -E "^\s*Name=$LINUX_BRIDGE\s*$" \
|
||||
/etc/systemd/network /run/systemd/network /usr/lib/systemd/network 2>/dev/null \
|
||||
| grep -v "^$BR_WAN_NETDEV$" || true)
|
||||
if [ -n "$EXISTING_BR_NETDEV" ]; then
|
||||
echo " skipping $BR_WAN_NETDEV — operator-managed NetDev already declares $LINUX_BRIDGE: $EXISTING_BR_NETDEV"
|
||||
else
|
||||
sudo tee "$BR_WAN_NETDEV" >/dev/null <<NETDEV
|
||||
[NetDev]
|
||||
Name=$LINUX_BRIDGE
|
||||
Kind=bridge
|
||||
NETDEV
|
||||
fi
|
||||
sudo tee "$VETH_NETDEV" >/dev/null <<NETDEV
|
||||
fi
|
||||
sudo tee "$VETH_NETDEV" >/dev/null <<NETDEV
|
||||
[NetDev]
|
||||
Name=veth-wan-br
|
||||
Kind=veth
|
||||
@@ -980,7 +1276,7 @@ Kind=veth
|
||||
[Peer]
|
||||
Name=veth-wan-ovs
|
||||
NETDEV
|
||||
sudo tee "$VETH_NETWORK" >/dev/null <<NETWORK
|
||||
sudo tee "$VETH_NETWORK" >/dev/null <<NETWORK
|
||||
[Match]
|
||||
Name=veth-wan-br
|
||||
|
||||
@@ -988,12 +1284,12 @@ Name=veth-wan-br
|
||||
Bridge=$LINUX_BRIDGE
|
||||
ConfigureWithoutCarrier=yes
|
||||
NETWORK
|
||||
# Second unit admin-ups the OVS end of the pair. OVS owns the port
|
||||
# (enslaved via ovs-vsctl add-port above) but does not flip admin
|
||||
# state on external ports — that's networkd's job. Without this,
|
||||
# veth-wan-ovs stays DOWN after reboot, peer goes LOWERLAYERDOWN,
|
||||
# br-wan loses carrier.
|
||||
sudo tee "$VETH_OVS_NETWORK" >/dev/null <<NETWORK
|
||||
# Second unit admin-ups the OVS end of the pair. OVS owns the port
|
||||
# (enslaved via ovs-vsctl add-port above) but does not flip admin
|
||||
# state on external ports — that's networkd's job. Without this,
|
||||
# veth-wan-ovs stays DOWN after reboot, peer goes LOWERLAYERDOWN,
|
||||
# br-wan loses carrier.
|
||||
sudo tee "$VETH_OVS_NETWORK" >/dev/null <<NETWORK
|
||||
[Match]
|
||||
Name=veth-wan-ovs
|
||||
|
||||
@@ -1003,8 +1299,9 @@ RequiredForOnline=no
|
||||
[Network]
|
||||
ConfigureWithoutCarrier=yes
|
||||
NETWORK
|
||||
sudo networkctl reload 2>/dev/null || true
|
||||
echo " wrote $VETH_NETDEV + $VETH_NETWORK + $VETH_OVS_NETWORK (veth persists on reboot)"
|
||||
sudo networkctl reload 2>/dev/null || true
|
||||
echo " wrote $VETH_NETDEV + $VETH_NETWORK + $VETH_OVS_NETWORK (veth persists on reboot)"
|
||||
fi
|
||||
;;
|
||||
|
||||
direct)
|
||||
@@ -1038,14 +1335,6 @@ NETWORK
|
||||
# forwards and masquerades the transit /24 out whatever uplink it
|
||||
# has. OVN SNATs each VPC CIDR to its gateway LRP transit IP, so
|
||||
# the masquerade rule below is the only host-side NAT state.
|
||||
NAT_TRANSIT_CIDR="100.127.0.0/24"
|
||||
NAT_TRANSIT_GW_CIDR="100.127.0.1/24"
|
||||
# Same on every node. OVN keeps one MAC binding for the transit
|
||||
# nexthop cluster-wide, so a random per-node MAC sends every
|
||||
# chassis's egress to whichever node seeded it; the pair is
|
||||
# node-local, so one address everywhere is unambiguous.
|
||||
NAT_TRANSIT_HOST_MAC="02:00:64:7f:00:01"
|
||||
|
||||
if ! sudo ovs-vsctl br-exists "$WAN_BRIDGE" 2>/dev/null; then
|
||||
sudo ovs-vsctl --may-exist add-br "$WAN_BRIDGE"
|
||||
echo " created OVS bridge: $WAN_BRIDGE"
|
||||
@@ -1073,10 +1362,11 @@ NETWORK
|
||||
|
||||
# Persist the veth pair + transit IP across reboot (veths are
|
||||
# kernel-only; same rationale as veth mode above).
|
||||
NAT_NETDEV="/etc/systemd/network/17-spinifex-nat.netdev"
|
||||
NAT_NETWORK="/etc/systemd/network/17-spinifex-nat.network"
|
||||
NAT_OVS_NETWORK="/etc/systemd/network/18-spinifex-nat-ovs.network"
|
||||
sudo tee "$NAT_NETDEV" >/dev/null <<NETDEV
|
||||
if [ "$HOSTNET_BACKEND" = networkd ]; then
|
||||
NAT_NETDEV="/etc/systemd/network/17-spinifex-nat.netdev"
|
||||
NAT_NETWORK="/etc/systemd/network/17-spinifex-nat.network"
|
||||
NAT_OVS_NETWORK="/etc/systemd/network/18-spinifex-nat-ovs.network"
|
||||
sudo tee "$NAT_NETDEV" >/dev/null <<NETDEV
|
||||
[NetDev]
|
||||
Name=spx-nat-host
|
||||
Kind=veth
|
||||
@@ -1085,7 +1375,7 @@ MACAddress=$NAT_TRANSIT_HOST_MAC
|
||||
[Peer]
|
||||
Name=spx-nat-ovs
|
||||
NETDEV
|
||||
sudo tee "$NAT_NETWORK" >/dev/null <<NETWORK
|
||||
sudo tee "$NAT_NETWORK" >/dev/null <<NETWORK
|
||||
[Match]
|
||||
Name=spx-nat-host
|
||||
|
||||
@@ -1093,9 +1383,9 @@ Name=spx-nat-host
|
||||
Address=$NAT_TRANSIT_GW_CIDR
|
||||
ConfigureWithoutCarrier=yes
|
||||
NETWORK
|
||||
# Admin-up the OVS end after reboot (OVS owns the port but does
|
||||
# not flip admin state on external ports — same as veth mode).
|
||||
sudo tee "$NAT_OVS_NETWORK" >/dev/null <<NETWORK
|
||||
# Admin-up the OVS end after reboot (OVS owns the port but does
|
||||
# not flip admin state on external ports — same as veth mode).
|
||||
sudo tee "$NAT_OVS_NETWORK" >/dev/null <<NETWORK
|
||||
[Match]
|
||||
Name=spx-nat-ovs
|
||||
|
||||
@@ -1105,8 +1395,9 @@ RequiredForOnline=no
|
||||
[Network]
|
||||
ConfigureWithoutCarrier=yes
|
||||
NETWORK
|
||||
sudo networkctl reload 2>/dev/null || true
|
||||
echo " wrote $NAT_NETDEV + $NAT_NETWORK + $NAT_OVS_NETWORK (transit veth persists on reboot)"
|
||||
sudo networkctl reload 2>/dev/null || true
|
||||
echo " wrote $NAT_NETDEV + $NAT_NETWORK + $NAT_OVS_NETWORK (transit veth persists on reboot)"
|
||||
fi
|
||||
|
||||
# Kernel egress: masquerade the transit /24 out any uplink and
|
||||
# accept forwarded transit traffic. vpcd re-ensures these on every
|
||||
@@ -1261,8 +1552,9 @@ if [ "$MGMT_BRIDGE_ENABLED" = true ]; then
|
||||
# Persist the IP across reboots via a systemd-networkd drop-in. The
|
||||
# OVS bridge definition itself survives in ovsdb; only the L3 IP
|
||||
# needs re-applying on boot.
|
||||
MGMT_NETD_UNIT="/etc/systemd/network/10-spinifex-mgmt.network"
|
||||
sudo tee "$MGMT_NETD_UNIT" >/dev/null <<NETD
|
||||
if [ "$HOSTNET_BACKEND" = networkd ]; then
|
||||
MGMT_NETD_UNIT="/etc/systemd/network/10-spinifex-mgmt.network"
|
||||
sudo tee "$MGMT_NETD_UNIT" >/dev/null <<NETD
|
||||
[Match]
|
||||
Name=$MGMT_BRIDGE
|
||||
|
||||
@@ -1270,7 +1562,8 @@ Name=$MGMT_BRIDGE
|
||||
Address=$MGMT_CIDR
|
||||
ConfigureWithoutCarrier=yes
|
||||
NETD
|
||||
echo " wrote $MGMT_NETD_UNIT (IP persists on reboot via systemd-networkd)"
|
||||
echo " wrote $MGMT_NETD_UNIT (IP persists on reboot via systemd-networkd)"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
@@ -1384,6 +1677,9 @@ sudo tee /etc/sysctl.d/99-spinifex-vpc.conf >/dev/null <<'SYSCTL'
|
||||
net.ipv4.ip_forward=1
|
||||
net.ipv4.conf.all.rp_filter=0
|
||||
net.ipv4.conf.default.rp_filter=0
|
||||
# EL's 50-redhat.conf sets every interface strict, reapplied by udev on each new
|
||||
# netdev; a strict IMDS endpoint drops guest requests, so keep those loose here.
|
||||
net.ipv4.conf.ime-*.rp_filter=0
|
||||
SYSCTL
|
||||
sudo sysctl --system -q
|
||||
echo " ip_forward=1, rp_filter=0"
|
||||
@@ -1401,8 +1697,10 @@ DATA_IFACE=$(ip -o -4 addr show | awk -v ip="$ENCAP_IP" '$0 ~ ip"/" {print $2}')
|
||||
if [ -n "$DATA_IFACE" ]; then
|
||||
SUBNET=$(ip -o -4 route show dev "$DATA_IFACE" proto kernel scope link | awk '{print $1}' | head -1)
|
||||
if [ -n "$SUBNET" ]; then
|
||||
sudo ip route replace "$SUBNET" dev "$DATA_IFACE" src "$ENCAP_IP" metric 50
|
||||
echo " data route: $SUBNET via $DATA_IFACE src $ENCAP_IP (metric 50)"
|
||||
sudo ip route replace "$SUBNET" dev "$DATA_IFACE" src "$ENCAP_IP" metric "$ENCAP_ROUTE_METRIC"
|
||||
echo " data route: $SUBNET via $DATA_IFACE src $ENCAP_IP (metric $ENCAP_ROUTE_METRIC)"
|
||||
ENCAP_ROUTE_DST="$SUBNET"
|
||||
ENCAP_ROUTE_DEV="$DATA_IFACE"
|
||||
else
|
||||
echo " skipped: no kernel route found for $DATA_IFACE"
|
||||
fi
|
||||
@@ -1410,6 +1708,12 @@ else
|
||||
echo " skipped: could not find interface for $ENCAP_IP"
|
||||
fi
|
||||
|
||||
if [ "$HOSTNET_BACKEND" = hostnet ]; then
|
||||
write_hostnet_state false
|
||||
hostnet_unlock
|
||||
echo " $HOSTNET_STATE: recorded for spinifex-hostnet"
|
||||
fi
|
||||
|
||||
# --- Step 7: Verify Geneve kernel support ---
|
||||
echo ""
|
||||
echo "Step 7: Verifying Geneve kernel module..."
|
||||
@@ -1509,17 +1813,24 @@ echo " OVS/OVN sockets: 0660 root:spinifex (group-scoped, no world access)"
|
||||
# file is the old 0666 override, and skipping would leave that exposure in place.
|
||||
# openvswitch-ipsec is included because Step 10 restarts it after this sweep,
|
||||
# so its ctl socket would otherwise be the one file left at the shipped mode.
|
||||
for unit in openvswitch-switch:/var/run/openvswitch/db.sock \
|
||||
"ovn-controller:/var/run/ovn/*.ctl" \
|
||||
"openvswitch-ipsec:/var/run/openvswitch/ovs-monitor-ipsec.*.ctl"; do
|
||||
# ovs-vswitchd starts after ovsdb-server and restarts with it (Requires=), so
|
||||
# its hook sees both daemons' sockets and fires on a restart of either.
|
||||
PERMS_UNITS=("ovs-vswitchd:/var/run/openvswitch/ovs-vswitchd.*.ctl"
|
||||
"ovn-controller:/var/run/ovn/*.ctl"
|
||||
"openvswitch-ipsec:/var/run/openvswitch/ovs-monitor-ipsec.*.ctl")
|
||||
# EL's ovn-northd.service starts the NB and SB databases as well as northd.
|
||||
if [ "$OVN_CENTRAL_LAYOUT" = lone-northd ]; then
|
||||
PERMS_UNITS+=("ovn-northd:/var/run/ovn/ovn-northd.*.ctl")
|
||||
fi
|
||||
for unit in "${PERMS_UNITS[@]}"; do
|
||||
UNIT="${unit%%:*}"
|
||||
WAIT_GLOB="${unit#*:}"
|
||||
OVERRIDE_DIR="/etc/systemd/system/${UNIT}.service.d"
|
||||
sudo mkdir -p "$OVERRIDE_DIR"
|
||||
# `ExecStartPost=-` so a failure here cannot stop the unit. This drop-in goes
|
||||
# into another package's service, and tightening a socket mode is hardening,
|
||||
# not a liveness requirement — it must never be the reason openvswitch-switch
|
||||
# or ovn-controller will not start. Without the dash, removing the helper
|
||||
# not a liveness requirement — it must never be the reason OVS or
|
||||
# ovn-controller will not start. Without the dash, removing the helper
|
||||
# (an uninstall, a partial re-run that has not reached this step yet) takes
|
||||
# OVS down across the whole host with `Failed at step EXEC`, which reads as
|
||||
# an OVS fault and is nothing of the sort.
|
||||
@@ -1540,23 +1851,45 @@ else
|
||||
echo " sudoers rule: not found — run setup.sh first, or install manually"
|
||||
fi
|
||||
|
||||
# --- Step 9: Configure OVN log rotation ---
|
||||
# The ovn-common package provides /etc/logrotate.d/ovn-common which handles
|
||||
# rotation and vlog/reopen. We just add maxsize + rotate to cap disk usage.
|
||||
# --- Step 9: Configure OVS and OVN log rotation ---
|
||||
# The packages' logrotate files handle rotation and vlog/reopen; this caps
|
||||
# disk usage and rotates as the user the daemons now run as.
|
||||
echo ""
|
||||
echo "Step 9: Configuring OVN log rotation..."
|
||||
echo "Step 9: Configuring OVS and OVN log rotation..."
|
||||
|
||||
OVN_LOGROTATE="/etc/logrotate.d/ovn-common"
|
||||
if [ -f "$OVN_LOGROTATE" ]; then
|
||||
if ! grep -q 'maxsize' "$OVN_LOGROTATE"; then
|
||||
sudo sed -i '/^\/var\/log\/ovn\/\*\.log {/a\ rotate 5\n maxsize 100M' "$OVN_LOGROTATE"
|
||||
echo " added maxsize 100M + rotate 5 to $OVN_LOGROTATE"
|
||||
else
|
||||
echo " $OVN_LOGROTATE already has maxsize configured"
|
||||
# cap_logrotate <file> — adds maxsize 100M, and rotate 5 where the file keeps
|
||||
# no rotated logs, to its /var/log/<dir>/*.log stanza. A su line naming another
|
||||
# user becomes su root root: rotating root-owned logs as that user fails.
|
||||
cap_logrotate() {
|
||||
local file=$1 lines=' maxsize 100M' added='maxsize 100M'
|
||||
if sudo grep -qE '^[[:space:]]*su[[:space:]]' "$file" &&
|
||||
! sudo grep -qE '^[[:space:]]*su[[:space:]]+root[[:space:]]+root[[:space:]]*$' "$file"; then
|
||||
sudo sed -i -E 's/^([[:space:]]*)su[[:space:]].*/\1su root root/' "$file"
|
||||
echo " $file: su root root"
|
||||
fi
|
||||
else
|
||||
echo " WARNING: $OVN_LOGROTATE not found — install ovn-common package"
|
||||
fi
|
||||
if sudo grep -q 'maxsize' "$file"; then
|
||||
echo " $file already has maxsize configured"
|
||||
return 0
|
||||
fi
|
||||
if ! sudo grep -qE '^[[:space:]]*rotate[[:space:]]' "$file"; then
|
||||
lines=' rotate 5\n maxsize 100M'
|
||||
added='maxsize 100M + rotate 5'
|
||||
fi
|
||||
sudo sed -i "/^\/var\/log\/[a-z]*\/\*\.log {/a\\$lines" "$file"
|
||||
if ! sudo grep -q 'maxsize' "$file"; then
|
||||
echo " WARNING: $file has no /var/log/<dir>/*.log stanza; log size left uncapped"
|
||||
return 0
|
||||
fi
|
||||
echo " added $added to $file"
|
||||
}
|
||||
|
||||
for probe in distro_ovs_logrotate distro_ovn_logrotate; do
|
||||
if LOGROTATE_FILE=$($probe); then
|
||||
cap_logrotate "$LOGROTATE_FILE"
|
||||
else
|
||||
echo " WARNING: log size left uncapped (see above)"
|
||||
fi
|
||||
done
|
||||
|
||||
# Remove our old custom config if present (superseded by patching ovn-common)
|
||||
if [ -f /etc/logrotate.d/ovn-spinifex ]; then
|
||||
@@ -1570,22 +1903,72 @@ fi
|
||||
# to prevent log spam during those retries.
|
||||
echo ""
|
||||
echo "Step 10: Enabling OVN auto-start on boot..."
|
||||
sudo systemctl enable openvswitch-switch 2>/dev/null || true
|
||||
sudo systemctl enable "$OVS_UNIT" 2>/dev/null || true
|
||||
sudo systemctl enable ovn-controller 2>/dev/null || true
|
||||
# ovs-monitor-ipsec drives strongSwan from OVS DB cert pointers. The daemon's
|
||||
# enableOVNIPSec() flips ipsec_encapsulation=true at runtime and silently drops
|
||||
# tunnel traffic if this unit isn't already up — enable at provision time so
|
||||
# daemon never needs systemd-write capability (only is-active read).
|
||||
sudo systemctl enable openvswitch-ipsec.service 2>/dev/null || true
|
||||
# When openvswitch-ipsec is installed before strongswan-starter, the deb
|
||||
# post-install starts the service against a missing /usr/sbin/ipsec and
|
||||
# leaves it in 'failed' state. Clear the failure and restart unconditionally
|
||||
# now that strongswan-starter is present.
|
||||
sudo systemctl reset-failed openvswitch-ipsec.service 2>/dev/null || true
|
||||
sudo systemctl restart openvswitch-ipsec.service 2>/dev/null || true
|
||||
echo " openvswitch-switch: enabled on boot"
|
||||
echo " ovn-controller: enabled on boot"
|
||||
echo " openvswitch-ipsec: enabled on boot"
|
||||
printf ' %-21s enabled on boot\n' "${OVS_UNIT%.service}:" ovn-controller:
|
||||
if distro_has_libreswan; then
|
||||
# OVN IPsec does not run on libreswan yet, so it stays off here. Starting
|
||||
# openvswitch-ipsec would also start pluto, which listens on UDP 500/4500.
|
||||
IPSEC_UNITS=(ipsec.service)
|
||||
distro_unit_exists openvswitch-ipsec.service && IPSEC_UNITS+=(openvswitch-ipsec.service)
|
||||
if ! sudo systemctl mask --now "${IPSEC_UNITS[@]}" >/dev/null; then
|
||||
echo "ERROR: could not stop and mask ${IPSEC_UNITS[*]}; IPsec must stay off on libreswan" >&2
|
||||
exit 1
|
||||
fi
|
||||
printf ' %-21s %s\n' openvswitch-ipsec: "masked: IPsec stays off on libreswan" ipsec: "masked: IPsec stays off on libreswan"
|
||||
elif [ "$(systemctl is-enabled openvswitch-ipsec.service 2>/dev/null)" = masked ]; then
|
||||
# The daemon masks it while IPsec is off and unmasks it to turn IPsec on.
|
||||
printf ' %-21s %s\n' openvswitch-ipsec: "masked: IPsec is off, left to the daemon"
|
||||
else
|
||||
# ovs-monitor-ipsec drives strongSwan from OVS DB cert pointers. The daemon's
|
||||
# enableOVNIPSec() flips ipsec_encapsulation=true at runtime and silently drops
|
||||
# tunnel traffic if this unit isn't already up — enable at provision time so
|
||||
# daemon never needs systemd-write capability (only is-active read).
|
||||
sudo systemctl enable openvswitch-ipsec.service 2>/dev/null || true
|
||||
# When openvswitch-ipsec is installed before strongswan-starter, the deb
|
||||
# post-install starts the service against a missing /usr/sbin/ipsec and
|
||||
# leaves it in 'failed' state. Clear the failure and restart unconditionally
|
||||
# now that strongswan-starter is present.
|
||||
sudo systemctl reset-failed openvswitch-ipsec.service 2>/dev/null || true
|
||||
sudo systemctl restart openvswitch-ipsec.service 2>/dev/null || true
|
||||
printf ' %-21s enabled on boot\n' openvswitch-ipsec:
|
||||
fi
|
||||
|
||||
# Applied now, outside the hostnet lock, so the daemon and vpcd find its
|
||||
# readiness marker; on the networkd backend the apply does nothing.
|
||||
if [ -f /etc/systemd/system/spinifex-hostnet.service ]; then
|
||||
sudo systemctl enable spinifex-hostnet.service >/dev/null 2>&1
|
||||
if ! sudo systemctl restart spinifex-hostnet.service; then
|
||||
echo "ERROR: spinifex-hostnet could not apply the host network: journalctl -u spinifex-hostnet" >&2
|
||||
exit 1
|
||||
fi
|
||||
printf ' %-21s enabled on boot, applied (%s backend)\n' spinifex-hostnet: "$HOSTNET_BACKEND"
|
||||
else
|
||||
echo " spinifex-hostnet: not installed yet — run setup.sh, then re-run this script"
|
||||
fi
|
||||
|
||||
# systemd silently drops ordering against a unit that is not installed, so a
|
||||
# missing OVS or OVN unit would leave Spinifex starting unordered against OVS.
|
||||
SPINIFEX_UNITS=()
|
||||
for f in /etc/systemd/system/spinifex.target /etc/systemd/system/spinifex-*.service; do
|
||||
[ -f "$f" ] && SPINIFEX_UNITS+=("$f")
|
||||
done
|
||||
if [ ${#SPINIFEX_UNITS[@]} -eq 0 ]; then
|
||||
echo " Spinifex units: not installed yet — run setup.sh, then re-run this script"
|
||||
else
|
||||
MISSING_DEPS=$(distro_missing_ovs_deps "${SPINIFEX_UNITS[@]}") || exit 1
|
||||
DEPS_OK=true
|
||||
for dep in $MISSING_DEPS; do
|
||||
if [ "$dep" = openvswitch-ipsec.service ]; then
|
||||
echo " WARNING: $dep is not installed; IPsec cannot be turned on"
|
||||
else
|
||||
echo "ERROR: the Spinifex units order against $dep, which is not installed" >&2
|
||||
DEPS_OK=false
|
||||
fi
|
||||
done
|
||||
[ "$DEPS_OK" = true ] || exit 1
|
||||
echo " OVS/OVN units the Spinifex units name: installed"
|
||||
fi
|
||||
|
||||
# --- Step 11: Health check ---
|
||||
echo ""
|
||||
|
||||
+239
-1
@@ -131,7 +131,7 @@ esac
|
||||
|
||||
# The script and the Go constant must agree: vpcd repairs a veth made before
|
||||
# this existed, and a disagreement would have the two fighting every pass.
|
||||
script_mac=$(printf '%s\n' "$nat_mode_body" | sed -n 's/.*NAT_TRANSIT_HOST_MAC="\([^"]*\)".*/\1/p' | head -1)
|
||||
script_mac=$(sed -n 's/^NAT_TRANSIT_HOST_MAC="\([^"]*\)".*/\1/p' "$SETUP" | head -1)
|
||||
go_mac=$(sed -n 's/.*NATTransitHostMAC = "\([^"]*\)".*/\1/p' "$SCRIPT_DIR/../spinifex/domains/network/host/routed.go" | head -1)
|
||||
if [ -n "$script_mac" ] && [ "$script_mac" = "$go_mac" ]; then
|
||||
pass "the transit MAC matches host.NATTransitHostMAC"
|
||||
@@ -139,6 +139,244 @@ else
|
||||
fail "transit MAC disagrees: script=$script_mac go=$go_mac"
|
||||
fi
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# OVS and OVN service users.
|
||||
# --------------------------------------------------------------------------
|
||||
# EL runs OVS and OVN as non-root users and re-chowns their sockets on every
|
||||
# start. The function is run on its own against scratch files, through a sudo
|
||||
# that only runs its arguments.
|
||||
awk '$0 == "set_root_service_user() {" { on = 1 } on { print } on && $0 == "}" { exit }' "$SETUP" >"$TMP/user_fn.sh"
|
||||
mkdir -p "$TMP/bin"
|
||||
printf '#!/bin/sh\nexec "$@"\n' >"$TMP/bin/sudo"
|
||||
chmod +x "$TMP/bin/sudo"
|
||||
|
||||
# set_user <file> — runs the function on <file> for OVS_USER_ID; sets RC.
|
||||
set_user() {
|
||||
RC=0
|
||||
PATH="$TMP/bin:$PATH" bash -c '. "$1"; set_root_service_user "$2" OVS_USER_ID' _ "$TMP/user_fn.sh" "$1" >/dev/null || RC=$?
|
||||
}
|
||||
|
||||
set_user "$TMP/absent"
|
||||
if [ "$RC" != 0 ] && [ ! -e "$TMP/absent" ]; then pass "service user: no file, no change"; else fail "service user: absent file gave rc=$RC or was created"; fi
|
||||
|
||||
printf '# OVS_USER_ID="openvswitch:openvswitch"\nOPTIONS=""\nOVS_USER_ID="openvswitch:hugetlbfs"\n' >"$TMP/el"
|
||||
set_user "$TMP/el"
|
||||
want='# OVS_USER_ID="openvswitch:openvswitch"
|
||||
OPTIONS=""
|
||||
OVS_USER_ID="root:root"'
|
||||
if [ "$RC" = 0 ] && [ "$(cat "$TMP/el")" = "$want" ]; then pass "service user: EL default replaced in place, comments and other lines kept"; else fail "service user: EL file rc=$RC: $(cat "$TMP/el")"; fi
|
||||
|
||||
set_user "$TMP/el"
|
||||
if [ "$RC" != 0 ] && [ "$(cat "$TMP/el")" = "$want" ]; then pass "service user: already root reports no change"; else fail "service user: second run rc=$RC: $(cat "$TMP/el")"; fi
|
||||
|
||||
printf 'OPTIONS=""\n' >"$TMP/unset"
|
||||
set_user "$TMP/unset"
|
||||
if [ "$RC" = 0 ] && [ "$(cat "$TMP/unset")" = 'OPTIONS=""
|
||||
OVS_USER_ID="root:root"' ]; then pass "service user: appended when the file does not set it"; else fail "service user: unset file rc=$RC: $(cat "$TMP/unset")"; fi
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Host network persistence backend.
|
||||
# --------------------------------------------------------------------------
|
||||
# The backend is picked once and recorded; every later run reads the record.
|
||||
# The functions run on their own against scratch paths, with systemctl stubbed.
|
||||
for fn in select_hostnet_backend write_nm_unmanaged; do
|
||||
awk -v f="$fn() {" '$0 == f { on = 1 } on { print } on && $0 == "}" { exit }' "$SETUP" >>"$TMP/hostnet_fn.sh"
|
||||
done
|
||||
|
||||
# backend <networkd-enabled:yes|no> — runs select_hostnet_backend; sets RC, OUT.
|
||||
backend() {
|
||||
printf '#!/bin/sh
|
||||
[ "$1 $2" = "is-enabled --quiet" ] && [ "%s" = yes ]
|
||||
' "$1" >"$TMP/bin/systemctl"
|
||||
chmod +x "$TMP/bin/systemctl"
|
||||
RC=0
|
||||
OUT=$(PATH="$TMP/bin:$PATH" HOSTNET_BACKEND_FILE="$TMP/hn/etc/hostnet-backend" NETWORKD_DIR="$TMP/hn/network" \
|
||||
bash -c '. "$1"; select_hostnet_backend' _ "$TMP/hostnet_fn.sh" 2>&1) || RC=$?
|
||||
}
|
||||
|
||||
reset_hn() { rm -rf "$TMP/hn"; mkdir -p "$TMP/hn/network"; }
|
||||
|
||||
reset_hn
|
||||
backend no
|
||||
if [ "$RC" = 0 ] && [ "$OUT" = hostnet ] && [ "$(cat "$TMP/hn/etc/hostnet-backend")" = hostnet ]; then pass "backend: no networkd picks and records hostnet"; else fail "backend: no networkd gave rc=$RC out=$OUT"; fi
|
||||
|
||||
reset_hn
|
||||
backend yes
|
||||
if [ "$RC" = 0 ] && [ "$OUT" = networkd ] && [ "$(cat "$TMP/hn/etc/hostnet-backend")" = networkd ]; then pass "backend: networkd enabled picks networkd"; else fail "backend: networkd enabled gave rc=$RC out=$OUT"; fi
|
||||
|
||||
reset_hn
|
||||
: >"$TMP/hn/network/10-spinifex-mgmt.network"
|
||||
backend no
|
||||
if [ "$RC" = 0 ] && [ "$OUT" = networkd ]; then pass "backend: legacy Spinifex networkd files keep networkd"; else fail "backend: legacy files gave rc=$RC out=$OUT"; fi
|
||||
|
||||
reset_hn
|
||||
mkdir -p "$TMP/hn/etc"
|
||||
echo hostnet >"$TMP/hn/etc/hostnet-backend"
|
||||
backend yes
|
||||
if [ "$RC" = 0 ] && [ "$OUT" = hostnet ]; then pass "backend: the record wins over networkd being enabled"; else fail "backend: recorded hostnet gave rc=$RC out=$OUT"; fi
|
||||
|
||||
echo ifupdown >"$TMP/hn/etc/hostnet-backend"
|
||||
backend yes
|
||||
case "$RC:$OUT" in
|
||||
0:*) fail "backend: an unknown record was accepted: $OUT" ;;
|
||||
*"expected networkd or hostnet"*) pass "backend: an unknown record is refused" ;;
|
||||
*) fail "backend: an unknown record gave rc=$RC out=$OUT" ;;
|
||||
esac
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# NetworkManager unmanaged config.
|
||||
# --------------------------------------------------------------------------
|
||||
# NetworkManager globs have no character classes, so the management TAPs go to
|
||||
# a udev rule; the keyfile lists the rest. The host uplink bridge stays managed.
|
||||
printf '#!/bin/sh
|
||||
exit 0
|
||||
' >"$TMP/bin/udevadm"
|
||||
cp "$TMP/bin/udevadm" "$TMP/bin/nmcli"
|
||||
chmod +x "$TMP/bin/udevadm" "$TMP/bin/nmcli"
|
||||
glob=$(sed -n "s/^MGMT_TAP_GLOB='\([^']*\)'$/\1/p" "$SETUP")
|
||||
RC=0
|
||||
PATH="$TMP/bin:$PATH" NM_UNMANAGED_CONF="$TMP/nm/conf.d/00-spinifex-unmanaged.conf" NM_UNMANAGED_RULE="$TMP/nm/rules.d/90.rules" \
|
||||
MGMT_TAP_GLOB="$glob" WAN_BRIDGE=br-ext WAN_BRIDGE_MODE=direct WAN_IFACE=eth3 LINUX_BRIDGE=br-wan \
|
||||
MGMT_BRIDGE_ENABLED=true MGMT_BRIDGE=br-mgmt MGMT_IFACE=eth2 \
|
||||
bash -c '. "$1"; write_nm_unmanaged' _ "$TMP/hostnet_fn.sh" >/dev/null 2>&1 || RC=$?
|
||||
conf=$(cat "$TMP/nm/conf.d/00-spinifex-unmanaged.conf" 2>/dev/null || true)
|
||||
line=$(printf '%s\n' "$conf" | grep '^unmanaged-devices+=' || true)
|
||||
if [ "$RC" = 0 ] && printf '%s\n' "$conf" | grep -qx '\[keyfile\]' && [ -n "$line" ]; then pass "nm: keyfile section appends to unmanaged-devices"; else fail "nm: rc=$RC conf=$conf"; fi
|
||||
for want in br-int br-ext 'veth-wan-*' 'spx-nat-*' 'tap*' br-mgmt eth2 eth3; do
|
||||
case ";${line#unmanaged-devices+=};" in
|
||||
*";interface-name:$want;"*) pass "nm: $want unmanaged" ;;
|
||||
*) fail "nm: $want missing from $line" ;;
|
||||
esac
|
||||
done
|
||||
case "$line" in
|
||||
*br-wan*) fail "nm: the host uplink bridge br-wan was made unmanaged" ;;
|
||||
*'['*) fail "nm: keyfile uses a character class NetworkManager cannot match: $line" ;;
|
||||
*) pass "nm: host uplink left managed and no character classes" ;;
|
||||
esac
|
||||
rule=$(cat "$TMP/nm/rules.d/90.rules" 2>/dev/null || true)
|
||||
case "$rule" in
|
||||
*"KERNEL==\"$glob\", ENV{NM_UNMANAGED}=\"1\""*) pass "nm: udev rule unmanages management TAPs" ;;
|
||||
*) fail "nm: udev rule is $rule" ;;
|
||||
esac
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# OVN central options file and management role.
|
||||
# --------------------------------------------------------------------------
|
||||
# The functions run on their own against scratch paths, through the stub sudo.
|
||||
for fn in write_ovn_central_opts record_management_role management_role_markers cap_logrotate; do
|
||||
awk -v f="$fn() {" '$0 == f { on = 1 } on { print } on && $0 == "}" { exit }' "$SETUP" >>"$TMP/ovn_fn.sh"
|
||||
done
|
||||
|
||||
# ovn_fn <cmd>... — runs the functions against scratch role files; sets RC, OUT.
|
||||
ovn_fn() {
|
||||
RC=0
|
||||
OUT=$(PATH="$TMP/bin:$PATH" OVN_ROLE_FILE="$TMP/role/etc/spinifex/ovn-role" \
|
||||
LEGACY_OVN_ROLE_FILE="$TMP/role/etc/default/ovn-central" \
|
||||
bash -c '. "$1"; shift; "$@"' _ "$TMP/ovn_fn.sh" "$@" 2>&1) || RC=$?
|
||||
}
|
||||
|
||||
ovn_fn env OVN_CENTRAL_OPTS_FILE="$TMP/sysconfig-ovn-northd" OVN_CENTRAL_OPTS_VAR=OVN_NORTHD_OPTS \
|
||||
bash -c '. "$1"; write_ovn_central_opts "--db-nb-addr=10.0.0.1 --db-nb-create-insecure-remote=yes"' _ "$TMP/ovn_fn.sh"
|
||||
if [ "$RC" = 0 ] && [ "$(cat "$TMP/sysconfig-ovn-northd")" = 'OVN_NORTHD_OPTS="--db-nb-addr=10.0.0.1 --db-nb-create-insecure-remote=yes"' ]; then
|
||||
pass "opts: EL file holds OVN_NORTHD_OPTS"
|
||||
else
|
||||
fail "opts: rc=$RC $(cat "$TMP/sysconfig-ovn-northd" 2>/dev/null)"
|
||||
fi
|
||||
|
||||
rm -rf "$TMP/role"
|
||||
ovn_fn management_role_markers
|
||||
if [ "$RC" != 0 ] && [ -z "$OUT" ]; then pass "role: a fresh node has no marker"; else fail "role: fresh node rc=$RC out=$OUT"; fi
|
||||
|
||||
ovn_fn record_management_role
|
||||
ovn_fn management_role_markers
|
||||
if [ "$RC" = 0 ] && [ "$OUT" = "$TMP/role/etc/spinifex/ovn-role" ]; then pass "role: recorded marker is found"; else fail "role: recorded marker rc=$RC out=$OUT"; fi
|
||||
|
||||
mkdir -p "$TMP/role/etc/default"
|
||||
echo 'OVN_CTL_OPTS=""' >"$TMP/role/etc/default/ovn-central"
|
||||
ovn_fn management_role_markers
|
||||
if [ "$RC" = 0 ] && [ "$OUT" = "$TMP/role/etc/spinifex/ovn-role
|
||||
$TMP/role/etc/default/ovn-central" ]; then pass "role: both markers are named, so the demotion hint removes both"; else fail "role: both markers rc=$RC out=$OUT"; fi
|
||||
|
||||
rm -f "$TMP/role/etc/spinifex/ovn-role"
|
||||
ovn_fn management_role_markers
|
||||
if [ "$RC" = 0 ] && [ "$OUT" = "$TMP/role/etc/default/ovn-central" ]; then pass "role: the legacy Debian marker alone still counts"; else fail "role: legacy marker rc=$RC out=$OUT"; fi
|
||||
|
||||
# The package's own file sets only commented defaults; it is not a marker.
|
||||
printf '# OVN_CTL_OPTS="--db-nb-addr=0.0.0.0"\n' >"$TMP/role/etc/default/ovn-central"
|
||||
echo compute >"$TMP/role/etc/spinifex/ovn-role"
|
||||
ovn_fn management_role_markers
|
||||
if [ "$RC" != 0 ]; then pass "role: a package-default options file and another role are not markers"; else fail "role: non-markers gave rc=$RC out=$OUT"; fi
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# OVS and OVN log rotation.
|
||||
# --------------------------------------------------------------------------
|
||||
# Fixtures are the stanzas the Debian 13 and NFV SIG el10s packages ship.
|
||||
mkdir -p "$TMP/lr"
|
||||
cat >"$TMP/lr/ovn-common" <<'EOF'
|
||||
/var/log/ovn/*.log {
|
||||
su root root
|
||||
daily
|
||||
compress
|
||||
sharedscripts
|
||||
missingok
|
||||
}
|
||||
EOF
|
||||
cat >"$TMP/lr/openvswitch-switch" <<'EOF'
|
||||
/var/log/openvswitch/*.log {
|
||||
daily
|
||||
compress
|
||||
sharedscripts
|
||||
create 640 root adm
|
||||
missingok
|
||||
rotate 30
|
||||
}
|
||||
EOF
|
||||
cat >"$TMP/lr/ovn" <<'EOF'
|
||||
/var/log/ovn/*.log {
|
||||
su openvswitch openvswitch
|
||||
daily
|
||||
compress
|
||||
sharedscripts
|
||||
missingok
|
||||
}
|
||||
EOF
|
||||
printf '/var/log/other.log {\n daily\n}\n' >"$TMP/lr/odd"
|
||||
|
||||
# directives <file> — the stanza's directives, sorted, one per line.
|
||||
directives() { sed -n 's/^ //p' "$1" | sort; }
|
||||
|
||||
ovn_fn cap_logrotate "$TMP/lr/ovn-common"
|
||||
if [ "$RC" = 0 ] && [ "$(directives "$TMP/lr/ovn-common")" = "$(printf 'compress\ndaily\nmaxsize 100M\nmissingok\nrotate 5\nsharedscripts\nsu root root\n')" ]; then
|
||||
pass "logrotate: Debian OVN gets rotate 5 and maxsize, as before"
|
||||
else
|
||||
fail "logrotate: Debian OVN rc=$RC: $(cat "$TMP/lr/ovn-common")"
|
||||
fi
|
||||
|
||||
ovn_fn cap_logrotate "$TMP/lr/openvswitch-switch"
|
||||
if [ "$RC" = 0 ] && grep -qx ' maxsize 100M' "$TMP/lr/openvswitch-switch" &&
|
||||
[ "$(grep -c '^ *rotate ' "$TMP/lr/openvswitch-switch")" = 1 ] && grep -qx ' rotate 30' "$TMP/lr/openvswitch-switch"; then
|
||||
pass "logrotate: Debian OVS keeps its rotate 30 and gains maxsize"
|
||||
else
|
||||
fail "logrotate: Debian OVS rc=$RC: $(cat "$TMP/lr/openvswitch-switch")"
|
||||
fi
|
||||
|
||||
ovn_fn cap_logrotate "$TMP/lr/ovn"
|
||||
if [ "$RC" = 0 ] && [ "$(directives "$TMP/lr/ovn")" = "$(printf 'compress\ndaily\nmaxsize 100M\nmissingok\nrotate 5\nsharedscripts\nsu root root\n')" ]; then
|
||||
pass "logrotate: EL OVN rotates as root, with rotate 5 and maxsize"
|
||||
else
|
||||
fail "logrotate: EL OVN rc=$RC: $(cat "$TMP/lr/ovn")"
|
||||
fi
|
||||
|
||||
before=$(cat "$TMP/lr/ovn")
|
||||
ovn_fn cap_logrotate "$TMP/lr/ovn"
|
||||
if [ "$RC" = 0 ] && [ "$(cat "$TMP/lr/ovn")" = "$before" ]; then pass "logrotate: a rerun changes nothing"; else fail "logrotate: rerun changed $TMP/lr/ovn: $(cat "$TMP/lr/ovn")"; fi
|
||||
|
||||
ovn_fn cap_logrotate "$TMP/lr/odd"
|
||||
case "$RC:$OUT" in
|
||||
0:*"left uncapped"*) pass "logrotate: a file without the expected stanza warns" ;;
|
||||
*) fail "logrotate: unexpected stanza gave rc=$RC out=$OUT" ;;
|
||||
esac
|
||||
|
||||
if [ "$fails" -ne 0 ]; then
|
||||
echo " setup-ovn.sh: $fails failure(s)"
|
||||
exit 1
|
||||
|
||||
+617
-79
@@ -7,7 +7,7 @@
|
||||
# INSTALL_SPINIFEX_CHANNEL Release channel: latest (default), dev
|
||||
# INSTALL_SPINIFEX_VERSION Pin to specific version (overrides channel)
|
||||
# INSTALL_SPINIFEX_TARBALL Path to local tarball (skips download, for testing/air-gapped)
|
||||
# INSTALL_SPINIFEX_SKIP_APT Set to 1 to skip apt dependency install
|
||||
# INSTALL_SPINIFEX_SKIP_APT Set to 1 to skip the system package install (apt or dnf)
|
||||
# INSTALL_SPINIFEX_SKIP_AWS Set to 1 to skip AWS CLI install
|
||||
# INSTALL_SPINIFEX_SKIP_NEWGRP Set to 1 to skip newgrp exec at end (for callers like dev-install.sh)
|
||||
# ISO_BUILD Set to 1 when running inside a debootstrap chroot from the ISO
|
||||
@@ -15,9 +15,9 @@
|
||||
# skip systemctl daemon-reload + enable, short-circuit setup_sudo.
|
||||
# VERBOSE Set to 1 to echo "[setup] <stage>" before each top-level step.
|
||||
# SETUP_STAGES Comma-separated subset of stages to run:
|
||||
# deps, aws, users, sudoers, firewall, timesync, files,
|
||||
# directories, env, systemd, sysctl, logrotate, udev,
|
||||
# fixown, migrations
|
||||
# deps, aws, dns, users, sudoers, firewall, timesync, files,
|
||||
# directories, env, systemd, sysctl, modules, logrotate,
|
||||
# udev, fixown, migrations
|
||||
# Unset = run every stage appropriate for the current mode.
|
||||
|
||||
set -e
|
||||
@@ -39,9 +39,13 @@ FIREWALL_PEERS="${FIREWALL_DIR}/peers.nft"
|
||||
FIREWALL_LOCAL="${FIREWALL_DIR}/local.nft"
|
||||
FIREWALL_OPEN="${FIREWALL_DIR}/open-ports.nft"
|
||||
FIREWALL_CUSTOM="${FIREWALL_DIR}/custom.nft"
|
||||
FIREWALL_MODE_FILE="${FIREWALL_DIR}/mode"
|
||||
FIREWALL_MODE_FILE="${SPINIFEX_FIREWALL_MODE_FILE:-${FIREWALL_DIR}/mode}"
|
||||
FIREWALL_APPLY="/usr/local/lib/spinifex/spinifex-firewall-apply"
|
||||
|
||||
# The daemon runs with ProtectKernelModules=yes, so every module it needs must
|
||||
# already be loaded when it starts.
|
||||
SPINIFEX_KERNEL_MODULES="openvswitch geneve vhost_net tun"
|
||||
|
||||
# Whether the policy is armed, not merely installed. The ISO is an appliance on
|
||||
# hardware we define, so a default-deny policy is the product; a curl-to-bash
|
||||
# install lands on a machine that was already doing something, where arming a
|
||||
@@ -58,6 +62,48 @@ info() { echo -e "${GREEN}[INFO]${NC} $*"; }
|
||||
warn() { echo -e "${YELLOW}[WARN]${NC} $*"; }
|
||||
fatal() { echo -e "${RED}[ERROR]${NC} $*"; exit 1; }
|
||||
|
||||
# --- BEGIN distro.sh probes ---
|
||||
# Copied from scripts/lib/distro.sh, which this script installs but cannot source
|
||||
# when piped from the install endpoint. distro_test.sh keeps the copies equal.
|
||||
distro_pkg_manager() {
|
||||
local apt=false dnf=false
|
||||
command -v apt-get >/dev/null 2>&1 && apt=true
|
||||
command -v dnf >/dev/null 2>&1 && dnf=true
|
||||
if $apt && $dnf; then
|
||||
echo "distro.sh: package manager: found both apt-get and dnf; cannot tell which one manages this host" >&2
|
||||
return 1
|
||||
fi
|
||||
if $apt; then echo apt; return 0; fi
|
||||
if $dnf; then echo dnf; return 0; fi
|
||||
echo "distro.sh: package manager: found neither apt-get nor dnf on PATH" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
distro_unit_exists() {
|
||||
local dir
|
||||
for dir in /etc/systemd/system /run/systemd/system /usr/lib/systemd/system /lib/systemd/system; do
|
||||
if [ -e "${DISTRO_ROOT:-}$dir/$1" ] || [ -L "${DISTRO_ROOT:-}$dir/$1" ]; then
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
distro_chrony_conf() {
|
||||
local deb=false el=false
|
||||
[ -f "${DISTRO_ROOT:-}/etc/chrony/chrony.conf" ] && deb=true
|
||||
[ -f "${DISTRO_ROOT:-}/etc/chrony.conf" ] && el=true
|
||||
if $deb && $el; then
|
||||
echo "distro.sh: chrony config: found both /etc/chrony/chrony.conf and /etc/chrony.conf; cannot tell which one chronyd reads" >&2
|
||||
return 1
|
||||
fi
|
||||
if $deb; then echo /etc/chrony/chrony.conf; return 0; fi
|
||||
if $el; then echo /etc/chrony.conf; return 0; fi
|
||||
echo "distro.sh: chrony config: found neither /etc/chrony/chrony.conf nor /etc/chrony.conf; is chrony installed?" >&2
|
||||
return 1
|
||||
}
|
||||
# --- END distro.sh probes ---
|
||||
|
||||
stage() {
|
||||
[ "${VERBOSE:-0}" = "1" ] && echo "[setup] $*"
|
||||
return 0
|
||||
@@ -92,14 +138,25 @@ setup_sudo() {
|
||||
}
|
||||
|
||||
# --- OS detection ---
|
||||
# os_release_field <key> <file> — the value of KEY=value, with one level of quotes removed.
|
||||
os_release_field() {
|
||||
sed -n "s/^$1=//p" "$2" | head -n 1 | sed -e 's/^"\(.*\)"$/\1/' -e "s/^'\(.*\)'$/\1/"
|
||||
}
|
||||
|
||||
# The only place that keys on the OS ID: the gate and, later, the package table.
|
||||
# Everything else probes for the capability (scripts/lib/distro.sh).
|
||||
detect_os() {
|
||||
if [ ! -f /etc/os-release ]; then
|
||||
fatal "Cannot detect OS: /etc/os-release not found"
|
||||
local os_release="${SPINIFEX_OS_RELEASE:-/etc/os-release}"
|
||||
if [ ! -f "$os_release" ]; then
|
||||
fatal "Cannot detect OS: $os_release not found"
|
||||
fi
|
||||
|
||||
. /etc/os-release
|
||||
# Parsed, not sourced, so a malformed file is refused instead of executed.
|
||||
ID=$(os_release_field ID "$os_release")
|
||||
VERSION_ID=$(os_release_field VERSION_ID "$os_release")
|
||||
PRETTY_NAME=$(os_release_field PRETTY_NAME "$os_release")
|
||||
|
||||
case "$ID" in
|
||||
case "${ID:-}" in
|
||||
debian)
|
||||
if [ "${VERSION_ID%%.*}" -lt 13 ] 2>/dev/null; then
|
||||
fatal "Debian $VERSION_ID is not supported. Minimum: Debian 13"
|
||||
@@ -111,12 +168,77 @@ detect_os() {
|
||||
fatal "Ubuntu $VERSION_ID is not supported. Minimum: Ubuntu 24.04"
|
||||
fi
|
||||
;;
|
||||
ol)
|
||||
check_oracle_linux
|
||||
;;
|
||||
*)
|
||||
fatal "Unsupported OS: $ID $VERSION_ID. Spinifex requires Debian 13+ or Ubuntu 24.04+"
|
||||
fatal "Unsupported OS: ${ID:-unknown} ${VERSION_ID:-}. Spinifex requires Debian 13+, Ubuntu 24.04+ or Oracle Linux 10.2+"
|
||||
;;
|
||||
esac
|
||||
|
||||
info "Detected OS: $PRETTY_NAME"
|
||||
info "Detected OS: ${PRETTY_NAME:-$ID $VERSION_ID}"
|
||||
}
|
||||
|
||||
# Oracle Linux 10.2+ on x86_64 only, running UEK R8: the KVM Utilities QEMU
|
||||
# needs UEK, and Oracle ships kernel modules per kernel, so installing a newer
|
||||
# kernel-uek alone leaves the running one without openvswitch.ko.
|
||||
check_oracle_linux() {
|
||||
local unsupported="Oracle Linux ${VERSION_ID:-<no VERSION_ID>} is not supported. Spinifex requires Oracle Linux 10.2 or a later 10.x"
|
||||
local major="${VERSION_ID%%.*}" minor="${VERSION_ID#*.}" kernel
|
||||
minor="${minor%%.*}"
|
||||
case "${VERSION_ID:-}" in *.*) ;; *) fatal "$unsupported" ;; esac
|
||||
case "$major" in "" | *[!0-9]*) fatal "$unsupported" ;; esac
|
||||
case "$minor" in "" | *[!0-9]*) fatal "$unsupported" ;; esac
|
||||
if [ "$major" -ne 10 ] || [ "$minor" -lt 2 ]; then
|
||||
fatal "$unsupported"
|
||||
fi
|
||||
|
||||
if [ "$(uname -m)" != "x86_64" ]; then
|
||||
fatal "Oracle Linux on $(uname -m) is not supported. Spinifex supports Oracle Linux on x86_64 only"
|
||||
fi
|
||||
|
||||
kernel=$(uname -r)
|
||||
case "$kernel" in
|
||||
6.12.0-*uek*) ;;
|
||||
*) fatal "The running kernel $kernel is not UEK R8 (6.12.0-*uek). Install kernel-uek, reboot into it, then re-run this installer" ;;
|
||||
esac
|
||||
|
||||
check_x86_64_v3
|
||||
check_selinux_not_enforcing
|
||||
}
|
||||
|
||||
# Spinifex ships no SELinux policy yet. Oracle's templates boot enforcing, and
|
||||
# setenforce alone reverts on reboot, so the config file is checked as well.
|
||||
check_selinux_not_enforcing() {
|
||||
local config="${SPINIFEX_SELINUX_CONFIG:-/etc/selinux/config}" now="" boot=""
|
||||
if command -v getenforce >/dev/null 2>&1; then
|
||||
now=$(getenforce 2>/dev/null | tr '[:upper:]' '[:lower:]')
|
||||
fi
|
||||
if [ -f "$config" ]; then
|
||||
boot=$(os_release_field SELINUX "$config" | tr '[:upper:]' '[:lower:]')
|
||||
fi
|
||||
if [ "$now" = "enforcing" ]; then
|
||||
fatal "SELinux is enforcing, and Spinifex has no SELinux policy. Run 'sudo setenforce 0', set SELINUX=permissive in $config, then re-run this installer"
|
||||
fi
|
||||
if [ "$boot" = "enforcing" ]; then
|
||||
fatal "SELinux is set to enforcing at next boot (SELINUX=enforcing in $config). Set SELINUX=permissive there, then re-run this installer"
|
||||
fi
|
||||
}
|
||||
|
||||
# The NFV SIG OVS/OVN and the KVM Utilities QEMU are built for x86-64-v3, so an
|
||||
# older CPU installs cleanly and then dies on an illegal instruction.
|
||||
check_x86_64_v3() {
|
||||
local cpuinfo="${SPINIFEX_CPUINFO:-/proc/cpuinfo}" flags flag missing=""
|
||||
flags=" $(grep -m1 '^flags' "$cpuinfo" 2>/dev/null | cut -d: -f2) "
|
||||
for flag in cx16 lahf_lm popcnt sse4_1 sse4_2 ssse3 avx avx2 bmi1 bmi2 f16c fma abm movbe xsave; do
|
||||
case "$flags" in
|
||||
*" $flag "*) ;;
|
||||
*) missing="${missing:+$missing }$flag" ;;
|
||||
esac
|
||||
done
|
||||
if [ -n "$missing" ]; then
|
||||
fatal "This CPU is below x86-64-v3 (missing: $missing). Oracle Linux's OVS, OVN and QEMU builds require x86-64-v3"
|
||||
fi
|
||||
}
|
||||
|
||||
# --- Architecture detection ---
|
||||
@@ -287,7 +409,12 @@ case "$1" in
|
||||
exec systemctl start openvswitch-ipsec.service
|
||||
;;
|
||||
off)
|
||||
exec systemctl mask --now openvswitch-ipsec.service
|
||||
systemctl mask --now openvswitch-ipsec.service
|
||||
# libreswan's pluto listens on UDP 500/4500 whenever ipsec.service runs.
|
||||
# On Debian that name is strongswan-starter's alias, masked above.
|
||||
if [ -x /usr/libexec/ipsec/pluto ]; then
|
||||
systemctl mask --now ipsec.service
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
echo "state not permitted: $1" >&2
|
||||
@@ -326,6 +453,73 @@ sshd_ports() {
|
||||
printf '%s' "$_out"
|
||||
}
|
||||
|
||||
# An unset --firewall keeps the mode a previous install recorded, so a rerun
|
||||
# never disarms a node. The record is read as root: /etc/spinifex is closed to
|
||||
# anyone outside the spinifex group, who would otherwise see no record at all.
|
||||
resolve_firewall_mode() {
|
||||
local recorded
|
||||
case "${INSTALL_SPINIFEX_FIREWALL}" in
|
||||
on|off) return 0 ;;
|
||||
"") ;;
|
||||
*) fatal "--firewall must be 'on' or 'off', got: ${INSTALL_SPINIFEX_FIREWALL}" ;;
|
||||
esac
|
||||
if ${SUDO:-} test -e "$FIREWALL_MODE_FILE"; then
|
||||
recorded=$(${SUDO:-} cat "$FIREWALL_MODE_FILE") \
|
||||
|| fatal "Cannot read the recorded firewall mode in $FIREWALL_MODE_FILE. Re-run with --firewall=on or --firewall=off"
|
||||
recorded=$(printf '%s' "$recorded" | tr -d '[:space:]')
|
||||
case "$recorded" in
|
||||
on|off) INSTALL_SPINIFEX_FIREWALL="$recorded"
|
||||
info "Host firewall: keeping the recorded mode '$recorded' ($FIREWALL_MODE_FILE)"
|
||||
return 0 ;;
|
||||
*) fatal "$FIREWALL_MODE_FILE holds '${recorded}', not 'on' or 'off'. Re-run with --firewall=on or --firewall=off" ;;
|
||||
esac
|
||||
fi
|
||||
if [ "${ISO_BUILD:-0}" != "1" ] && firewalld_active; then
|
||||
INSTALL_SPINIFEX_FIREWALL="on"
|
||||
info "Host firewall: firewalld is active, so the Spinifex policy will be armed in its place"
|
||||
return 0
|
||||
fi
|
||||
# The ISO is an appliance and arms; any other host was already doing something.
|
||||
if [ "${ISO_BUILD:-0}" = "1" ]; then
|
||||
INSTALL_SPINIFEX_FIREWALL="on"
|
||||
else
|
||||
INSTALL_SPINIFEX_FIREWALL="off"
|
||||
fi
|
||||
}
|
||||
|
||||
firewalld_active() {
|
||||
systemctl is-active --quiet firewalld.service 2>/dev/null
|
||||
}
|
||||
|
||||
# Disable firewalld if present. The node firewall is spinifex-managed: with
|
||||
# firewalld running, a packet must be accepted by both its table and ours, so
|
||||
# routed NAT, elastic IPs and IMDS on ime-* ports break with nothing logged.
|
||||
# Masked, not just disabled, so a package update or D-Bus activation cannot
|
||||
# bring it back. Its rules are not saved or migrated.
|
||||
mask_firewalld() {
|
||||
local state
|
||||
[ "${ISO_BUILD:-0}" = "1" ] && return 0
|
||||
state=$(systemctl is-enabled firewalld.service 2>/dev/null) || true
|
||||
case "$state" in
|
||||
"" | not-found) return 0 ;;
|
||||
masked) if ! firewalld_active; then
|
||||
info " firewalld already masked"
|
||||
return 0
|
||||
fi ;;
|
||||
esac
|
||||
$SUDO systemctl mask --now firewalld.service > /dev/null \
|
||||
|| fatal "Cannot stop and mask firewalld. The Spinifex nftables policy cannot run beside it"
|
||||
warn "firewalld is now stopped and masked: the Spinifex nftables policy is the only supported node firewall."
|
||||
warn " firewalld's rules were not migrated. The Spinifex policy filters input only, accepts ssh"
|
||||
warn " from every IPv4 source until \$trusted_ssh_peers in $FIREWALL_CUSTOM narrows it, and opens"
|
||||
warn " 53, 443, 3000, 8443 and 9999 to any source. Restate any other restriction in that file."
|
||||
if [ "$INSTALL_SPINIFEX_FIREWALL" = "on" ]; then
|
||||
warn " Until the daemon first loads the policy, this host accepts every connection."
|
||||
else
|
||||
warn " The Spinifex policy is not armed (--firewall=off), so this host now has no firewall."
|
||||
fi
|
||||
}
|
||||
|
||||
# The policy is written here rather than shipped in the tarball so the ISO chroot
|
||||
# and the ansible path get an identical file without a packaging step. Installed
|
||||
# but NOT enabled: the peer set is empty until spinifex-daemon resolves the
|
||||
@@ -499,6 +693,10 @@ CUSTOM
|
||||
$SUDO chmod 0644 "$FIREWALL_MODE_FILE"
|
||||
info " $FIREWALL_MODE_FILE ($INSTALL_SPINIFEX_FIREWALL)"
|
||||
|
||||
# After the mode file, so a retry after an interrupted install still finds
|
||||
# 'on' once firewalld is no longer active.
|
||||
mask_firewalld
|
||||
|
||||
$SUDO tee "$FIREWALL_APPLY" > /dev/null << 'APPLY'
|
||||
#!/bin/sh
|
||||
# Applies the spinifex host firewall. Fails without changing the ruleset when
|
||||
@@ -650,21 +848,171 @@ APPLY
|
||||
fi
|
||||
}
|
||||
|
||||
# --- Host DNS ---
|
||||
# The one resolver setup supported on a host with systemd-resolved installed:
|
||||
# NetworkManager hands link DNS to resolved, and /etc/resolv.conf is its stub.
|
||||
RESOLV_CONF="${SPINIFEX_RESOLV_CONF:-/etc/resolv.conf}"
|
||||
RESOLVED_STUB=/run/systemd/resolve/stub-resolv.conf
|
||||
# node-reset.sh keeps /var/lib/spinifex-reset, so the copy survives until uninstall.
|
||||
RESOLV_CONF_SAVED="${SPINIFEX_RESOLV_CONF_SAVED:-/var/lib/spinifex-reset/resolv.conf.orig}"
|
||||
|
||||
# systemd-resolved ships resolvconf as a link to resolvectl, which only works
|
||||
# while resolved runs, so that one is not a resolvconf backend.
|
||||
real_resolvconf() {
|
||||
local path
|
||||
path=$(command -v resolvconf) || return 1
|
||||
[ "$(basename "$(readlink -f "$path")")" != resolvectl ]
|
||||
}
|
||||
|
||||
# Prints the NetworkManager setting that keeps it from handing DNS to resolved,
|
||||
# or nothing. Only explicitly set keys are uncommented in --print-config.
|
||||
nm_dns_blocker() {
|
||||
local config
|
||||
config=$($SUDO NetworkManager --print-config) || return 1
|
||||
printf '%s\n' "$config" | awk -F= '
|
||||
/^\[/ { main = ($0 == "[main]"); next }
|
||||
!main || /^#/ { next }
|
||||
$1 == "dns" && $2 != "default" && $2 != "systemd-resolved" { print; exit }
|
||||
$1 == "systemd-resolved" && ($2 == "false" || $2 == "no" || $2 == "0" || $2 == "off") { print; exit }'
|
||||
}
|
||||
|
||||
# The URL of the first enabled package repository, which the install has
|
||||
# reached unless system packages were skipped.
|
||||
first_repo_url() {
|
||||
# shellcheck disable=SC2016 # apt's own $(FIELD) template, not a shell expansion
|
||||
case "$(distro_pkg_manager 2>/dev/null)" in
|
||||
apt) apt-get indextargets --format '$(REPO_URI)' 2>/dev/null | head -n 1 ;;
|
||||
dnf) dnf config-manager --dump '*' 2>/dev/null | awk '
|
||||
function flush() { if (enabled == "1" && url != "") { print url; done = 1; exit } }
|
||||
/^\[/ { flush(); enabled = ""; url = ""; next }
|
||||
/^(baseurl|mirrorlist|metalink) = / { v = $3; sub(/,.*/, "", v); if (url == "") url = v }
|
||||
/^enabled = / { enabled = $3 }
|
||||
END { if (!done) flush() }' ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# The host part of a URL. Prints nothing for a file:// or schemeless URL.
|
||||
url_host() {
|
||||
local rest="${1#*://}"
|
||||
case "$1" in file://* | "$rest") return 0 ;; esac
|
||||
rest="${rest%%/*}"
|
||||
rest="${rest##*@}"
|
||||
case "$rest" in
|
||||
\[*) echo "${rest%%]*}]" ;;
|
||||
*) echo "${rest%%:*}" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
dns_handover_failed() {
|
||||
fatal "Host DNS through systemd-resolved does not work: $1.
|
||||
To put host DNS back as it was before this installer:
|
||||
sudo systemctl disable --now systemd-resolved
|
||||
sudo cp -a --remove-destination $RESOLV_CONF_SAVED $RESOLV_CONF
|
||||
sudo systemctl restart NetworkManager"
|
||||
}
|
||||
|
||||
# Checks resolved has upstream servers and still resolves the package
|
||||
# repository host, the one name this host is known to have resolved.
|
||||
check_resolved_dns() {
|
||||
local i url host
|
||||
for i in 1 2 3 4 5 6 7 8 9 10; do
|
||||
resolvectl dns 2>/dev/null | grep -q ':[[:space:]]*[^[:space:]]' && break
|
||||
[ "$i" = 10 ] && dns_handover_failed "'resolvectl dns' lists no DNS servers 10 seconds after NetworkManager restarted"
|
||||
sleep 1
|
||||
done
|
||||
|
||||
url=$(first_repo_url)
|
||||
host=$(url_host "$url")
|
||||
case "$host" in
|
||||
"")
|
||||
warn "No package repository with a host name to look up (${url:-none found}); not checking that names resolve"
|
||||
return 0 ;;
|
||||
\[*) ;;
|
||||
*[!0-9.]*)
|
||||
getent ahosts "$host" > /dev/null || dns_handover_failed "the package repository host $host no longer resolves"
|
||||
return 0 ;;
|
||||
esac
|
||||
warn "The package repository host $host is an IP address; not checking that names resolve"
|
||||
}
|
||||
|
||||
# Where systemd-resolved is installed but not running and nothing else manages
|
||||
# DNS, turns it on behind NetworkManager. The original resolv.conf is saved
|
||||
# once, so a rerun never replaces it with one this installer wrote.
|
||||
hand_dns_to_resolved() {
|
||||
stage "handing host DNS to systemd-resolved"
|
||||
local blocker
|
||||
[ "${ISO_BUILD:-0}" = "1" ] && return 0
|
||||
|
||||
if systemctl is-active --quiet systemd-resolved.service; then
|
||||
# A handover made by an earlier run is checked again on every run.
|
||||
if [ -e "$RESOLV_CONF_SAVED" ] || [ -L "$RESOLV_CONF_SAVED" ]; then
|
||||
check_resolved_dns
|
||||
fi
|
||||
return 0
|
||||
fi
|
||||
if ! distro_unit_exists systemd-resolved.service; then
|
||||
return 0
|
||||
fi
|
||||
if real_resolvconf; then
|
||||
info "resolvconf manages host DNS; leaving it as it is"
|
||||
return 0
|
||||
fi
|
||||
if ! systemctl is-active --quiet NetworkManager.service; then
|
||||
warn "systemd-resolved is installed but not running, and NetworkManager is not running to hand it DNS. Leaving host DNS as it is: Spinifex names will not resolve on this host until its resolver forwards them to northstar"
|
||||
return 0
|
||||
fi
|
||||
blocker=$(nm_dns_blocker) || fatal "Cannot read NetworkManager's configuration with 'NetworkManager --print-config'"
|
||||
if [ -n "$blocker" ]; then
|
||||
fatal "NetworkManager sets $blocker, so it cannot hand DNS to systemd-resolved. Remove that setting ('NetworkManager --print-config' shows the files it reads) and re-run this installer"
|
||||
fi
|
||||
|
||||
info "Handing host DNS to systemd-resolved..."
|
||||
if [ ! -e "$RESOLV_CONF_SAVED" ] && [ ! -L "$RESOLV_CONF_SAVED" ]; then
|
||||
$SUDO mkdir -p "$(dirname "$RESOLV_CONF_SAVED")"
|
||||
$SUDO cp -a "$RESOLV_CONF" "$RESOLV_CONF_SAVED" || fatal "Cannot save $RESOLV_CONF to $RESOLV_CONF_SAVED"
|
||||
info " saved $RESOLV_CONF to $RESOLV_CONF_SAVED"
|
||||
fi
|
||||
$SUDO systemctl enable --now systemd-resolved.service > /dev/null 2>&1 \
|
||||
|| dns_handover_failed "systemd-resolved did not start"
|
||||
$SUDO systemctl restart NetworkManager.service || dns_handover_failed "NetworkManager did not restart"
|
||||
$SUDO ln -sfn "$RESOLVED_STUB" "$RESOLV_CONF"
|
||||
check_resolved_dns
|
||||
info " $RESOLV_CONF -> $RESOLVED_STUB"
|
||||
}
|
||||
|
||||
# Debian's stock chrony.conf carries `makestep 1 3`: step the clock only for the
|
||||
# first three updates, then slew at ~83us/s, which is roughly 7 seconds of
|
||||
# correction per day. A node that burns those three before the network is usable
|
||||
# then crawls toward correct time for days, and SigV4 rejects its requests the
|
||||
# whole time. A drop-in rather than an edit to chrony.conf, which is a dpkg
|
||||
# conffile and would prompt on every future upgrade.
|
||||
#
|
||||
# EL's chrony.conf reads no drop-ins, so a confdir line is appended to it.
|
||||
# chronyd keeps the last makestep it reads, so drop-ins must come after the
|
||||
# stock line. DISTRO_ROOT prefixes the paths here as it does for the probes.
|
||||
install_chrony_conf() {
|
||||
stage "installing chrony drop-in"
|
||||
local root="${DISTRO_ROOT:-}" conf dir="" confdir_line=0 makestep_line
|
||||
|
||||
if [ ! -d /etc/chrony/conf.d ]; then
|
||||
warn "No /etc/chrony/conf.d — skipping chrony drop-in"
|
||||
if ! conf=$(distro_chrony_conf); then
|
||||
warn "Skipping the chrony drop-in"
|
||||
return
|
||||
fi
|
||||
read -r confdir_line dir < <(awk '$1 == "confdir" { print NR, $2; exit }' "$root$conf") || true
|
||||
if [ -z "$dir" ]; then
|
||||
dir=/etc/chrony.d
|
||||
printf '\n# Added by the Spinifex installer. Last, so a drop-in overrides the settings above.\nconfdir %s\n' "$dir" \
|
||||
| $SUDO tee -a "$root$conf" > /dev/null
|
||||
confdir_line=$(wc -l < "$root$conf")
|
||||
info " $conf: confdir $dir"
|
||||
fi
|
||||
makestep_line=$(awk '$1 == "makestep" { n = NR } END { print n + 0 }' "$root$conf")
|
||||
if [ "$makestep_line" -gt "$confdir_line" ]; then
|
||||
warn "$conf sets makestep on line $makestep_line, after its confdir line, which overrides the Spinifex drop-in. Move the confdir line to the end of $conf"
|
||||
fi
|
||||
|
||||
$SUDO tee /etc/chrony/conf.d/spinifex.conf > /dev/null << 'CHRONY'
|
||||
$SUDO mkdir -p "$root$dir"
|
||||
$SUDO tee "$root$dir/spinifex.conf" > /dev/null << 'CHRONY'
|
||||
# Managed by scripts/setup.sh — edits are overwritten.
|
||||
# Step at any offset over a second, however many times it takes. The clock can
|
||||
# jump backwards under load, which is the accepted cost: everything
|
||||
@@ -672,8 +1020,8 @@ install_chrony_conf() {
|
||||
# step fails every signed request until it converges.
|
||||
makestep 1 -1
|
||||
CHRONY
|
||||
$SUDO chmod 0644 /etc/chrony/conf.d/spinifex.conf
|
||||
info " /etc/chrony/conf.d/spinifex.conf"
|
||||
$SUDO chmod 0644 "$root$dir/spinifex.conf"
|
||||
info " $dir/spinifex.conf"
|
||||
}
|
||||
|
||||
install_sudoers() {
|
||||
@@ -733,7 +1081,7 @@ SUDOERS
|
||||
info "Scoped sudoers rules installed for spinifex-daemon (spinifex-vpcd needs none)"
|
||||
}
|
||||
|
||||
# --- Install apt dependencies ---
|
||||
# --- Install system dependencies ---
|
||||
# NOTE: Runtime deps must stay in sync with the ISO package list at
|
||||
# scripts/iso-builder/build/packages.list in the mulga repo. When you add,
|
||||
# remove, or change a runtime package here, review that file too — drift means
|
||||
@@ -750,29 +1098,43 @@ jq curl iproute2 ethtool netcat-openbsd wget unzip xz-utils file
|
||||
ovn-central ovn-host openvswitch-switch openvswitch-ipsec strongswan-charon dhcpcd-base
|
||||
chrony nftables"
|
||||
|
||||
install_apt_deps() {
|
||||
stage "installing apt dependencies"
|
||||
# The Oracle Linux equivalents, plus what a minimal OL install lacks. OVS 3.5
|
||||
# and OVN 25.03 come from the CentOS NFV SIG; 25.03 is Debian 13's OVN, which
|
||||
# the hand-written OVN NB models are written against.
|
||||
OL_RUNTIME_PACKAGES="nbdkit nbdkit-basic-filters
|
||||
qemu-kvm qemu-img edk2-ovmf seabios-bin less
|
||||
pciutils
|
||||
jq curl iproute ethtool nmap-ncat wget unzip xz file
|
||||
ovn25.03-central ovn25.03-host openvswitch3.5 openvswitch3.5-ipsec libreswan dhcpcd
|
||||
chrony nftables iptables-nft systemd-resolved"
|
||||
|
||||
# Oracle ships no OVS or OVN for OL10. The allowlist stops the SIG supplying
|
||||
# anything but OVS, OVN and the two SIG-only packages they depend on.
|
||||
NFV_SIG_REPO_ID="spinifex-nfv-sig"
|
||||
NFV_SIG_KEY_URL="https://www.centos.org/keys/RPM-GPG-KEY-CentOS-SIG-NFV"
|
||||
NFV_SIG_KEY_FILE="/etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-NFV"
|
||||
NFV_SIG_KEY_FPR="35154228174901BEFA8E69A621465E289D2A76A7"
|
||||
NFV_SIG_INCLUDEPKGS="openvswitch3.5* ovn25.03* python3-openvswitch3.5 openvswitch-selinux-extra-policy"
|
||||
|
||||
# Only the KVM Utilities QEMU has the microvm machine. Excluding QEMU from
|
||||
# AppStream stops a later AppStream build ever replacing part of that stack.
|
||||
OL_KVM_UTILS_REPO="ol10_kvm_utils"
|
||||
OL_APPSTREAM_REPO="ol10_appstream"
|
||||
OL_APPSTREAM_EXCLUDES="qemu-kvm* qemu-img*"
|
||||
# The running kernel's modules come from here, and KVM Utilities requires kernel-uek.
|
||||
OL_UEK_REPO="ol10_UEKR8"
|
||||
|
||||
install_deps() {
|
||||
stage "installing system dependencies"
|
||||
if [ "${INSTALL_SPINIFEX_SKIP_APT}" = "1" ]; then
|
||||
info "Skipping apt dependencies (INSTALL_SPINIFEX_SKIP_APT=1)"
|
||||
info "Skipping system dependencies (INSTALL_SPINIFEX_SKIP_APT=1)"
|
||||
else
|
||||
info "Installing system dependencies..."
|
||||
$SUDO apt-get update -qq
|
||||
|
||||
# Unquoted on purpose: both variables are whitespace-separated package lists.
|
||||
# shellcheck disable=SC2086
|
||||
DEBIAN_FRONTEND=noninteractive $SUDO apt-get install -y -qq \
|
||||
$QEMU_PACKAGES $APT_RUNTIME_PACKAGES \
|
||||
> /dev/null
|
||||
|
||||
info "System dependencies installed"
|
||||
|
||||
# Only on a machine that already boots from ZFS. Pulling zfsutils-linux
|
||||
# in unconditionally would drag zfs-dkms onto every dev box and rebuild
|
||||
# the module on each kernel upgrade for no benefit.
|
||||
if [ "$(findmnt -no FSTYPE /)" = "zfs" ] && ! command -v zpool >/dev/null 2>&1; then
|
||||
info "ZFS root detected, installing zfsutils-linux..."
|
||||
DEBIAN_FRONTEND=noninteractive $SUDO apt-get install -y -qq zfsutils-linux > /dev/null
|
||||
fi
|
||||
local pm
|
||||
pm=$(distro_pkg_manager) || fatal "Cannot tell which package manager this host uses"
|
||||
case "$pm" in
|
||||
apt) install_apt_deps ;;
|
||||
dnf) install_dnf_deps ;;
|
||||
esac
|
||||
fi
|
||||
|
||||
# Mask the standalone dhcpcd.service auto-enabled on Debian Trixie when
|
||||
@@ -786,6 +1148,150 @@ install_apt_deps() {
|
||||
$SUDO systemctl mask dhcpcd.service 2>/dev/null || true
|
||||
}
|
||||
|
||||
install_apt_deps() {
|
||||
info "Installing system dependencies..."
|
||||
$SUDO apt-get update -qq
|
||||
|
||||
# Unquoted on purpose: both variables are whitespace-separated package lists.
|
||||
# shellcheck disable=SC2086
|
||||
DEBIAN_FRONTEND=noninteractive $SUDO apt-get install -y -qq \
|
||||
$QEMU_PACKAGES $APT_RUNTIME_PACKAGES \
|
||||
> /dev/null
|
||||
|
||||
info "System dependencies installed"
|
||||
|
||||
# Only on a machine that already boots from ZFS. Pulling zfsutils-linux
|
||||
# in unconditionally would drag zfs-dkms onto every dev box and rebuild
|
||||
# the module on each kernel upgrade for no benefit.
|
||||
if [ "$(findmnt -no FSTYPE /)" = "zfs" ] && ! command -v zpool >/dev/null 2>&1; then
|
||||
info "ZFS root detected, installing zfsutils-linux..."
|
||||
DEBIAN_FRONTEND=noninteractive $SUDO apt-get install -y -qq zfsutils-linux > /dev/null
|
||||
fi
|
||||
}
|
||||
|
||||
# The package table is the other place keyed on the OS ID: dnf alone does not
|
||||
# say which repositories or package names a host has.
|
||||
install_dnf_deps() {
|
||||
case "${ID:-}" in
|
||||
ol) ;;
|
||||
*) fatal "No dnf package list for ${ID:-an unknown OS}. Spinifex installs with dnf on Oracle Linux only" ;;
|
||||
esac
|
||||
if [ "$(findmnt -no FSTYPE /)" = "zfs" ]; then
|
||||
fatal "A ZFS root is not supported on Oracle Linux"
|
||||
fi
|
||||
|
||||
ensure_ol_repos
|
||||
install_running_kernel_modules
|
||||
|
||||
info "Installing system dependencies..."
|
||||
# shellcheck disable=SC2086 # a whitespace-separated package list
|
||||
$SUDO dnf install -y -q $OL_RUNTIME_PACKAGES > /dev/null
|
||||
info "System dependencies installed"
|
||||
|
||||
# shellcheck disable=SC2086 # a whitespace-separated module list
|
||||
load_kernel_modules $SPINIFEX_KERNEL_MODULES
|
||||
}
|
||||
|
||||
# Enables KVM Utilities, pins QEMU to it, and adds the NFV SIG repository with
|
||||
# a key whose fingerprint must match the one recorded above.
|
||||
ensure_ol_repos() {
|
||||
info "Configuring Oracle Linux repositories..."
|
||||
local repo
|
||||
$SUDO dnf install -y -q dnf-plugins-core gnupg2 curl > /dev/null
|
||||
|
||||
for repo in "$OL_UEK_REPO" "$OL_KVM_UTILS_REPO" "$OL_APPSTREAM_REPO"; do
|
||||
dnf config-manager --dump "$repo" > /dev/null 2>&1 \
|
||||
|| fatal "Repository $repo is not defined on this host. Oracle Linux's release packages define it; reinstall oraclelinux-release-el10"
|
||||
done
|
||||
$SUDO dnf config-manager --set-enabled "$OL_UEK_REPO" "$OL_KVM_UTILS_REPO" > /dev/null
|
||||
exclude_from_appstream
|
||||
|
||||
install_nfv_sig_key
|
||||
$SUDO tee "/etc/yum.repos.d/${NFV_SIG_REPO_ID}.repo" > /dev/null <<EOF
|
||||
# Written by the Spinifex installer. Re-running it rewrites this file.
|
||||
[${NFV_SIG_REPO_ID}]
|
||||
name=CentOS NFV SIG Open vSwitch and OVN, for Spinifex
|
||||
baseurl=https://mirror.stream.centos.org/SIGs/10-stream/nfv/\$basearch/openvswitch-2/
|
||||
enabled=1
|
||||
gpgcheck=1
|
||||
gpgkey=file://${NFV_SIG_KEY_FILE}
|
||||
includepkgs=${NFV_SIG_INCLUDEPKGS}
|
||||
EOF
|
||||
|
||||
for repo in "$OL_UEK_REPO" "$OL_APPSTREAM_REPO" "$OL_KVM_UTILS_REPO" "$NFV_SIG_REPO_ID"; do
|
||||
$SUDO dnf -q makecache --repo "$repo" > /dev/null \
|
||||
|| fatal "Repository $repo is unreachable. Check this host's network access and proxy settings, then re-run this installer"
|
||||
done
|
||||
}
|
||||
|
||||
# Adds the QEMU excludes to AppStream, keeping any exclude already set there.
|
||||
exclude_from_appstream() {
|
||||
local current pattern merged="" patterns
|
||||
current=$(dnf config-manager --dump "$OL_APPSTREAM_REPO" | sed -n 's/^excludepkgs = //p' | tr ',' ' ')
|
||||
# read, not an unquoted expansion, so the patterns are never globbed.
|
||||
read -ra patterns <<< "$current $OL_APPSTREAM_EXCLUDES"
|
||||
for pattern in "${patterns[@]}"; do
|
||||
case " $merged " in
|
||||
*" $pattern "*) ;;
|
||||
*) merged="${merged:+$merged }$pattern" ;;
|
||||
esac
|
||||
done
|
||||
$SUDO dnf config-manager --save --setopt="${OL_APPSTREAM_REPO}.excludepkgs=${merged// /,}" > /dev/null
|
||||
}
|
||||
|
||||
# gpg_fingerprints <file> — the primary-key fingerprint of each key in file.
|
||||
gpg_fingerprints() {
|
||||
local home
|
||||
home=$(mktemp -d)
|
||||
GNUPGHOME="$home" gpg --batch --with-colons --show-keys "$1" 2>/dev/null \
|
||||
| awk -F: '$1 == "pub" { want = 1; next } want && $1 == "fpr" { print $10; want = 0 }'
|
||||
rm -rf "$home"
|
||||
}
|
||||
|
||||
install_nfv_sig_key() {
|
||||
if [ -f "$NFV_SIG_KEY_FILE" ] && [ "$(gpg_fingerprints "$NFV_SIG_KEY_FILE")" = "$NFV_SIG_KEY_FPR" ]; then
|
||||
$SUDO rpm --import "$NFV_SIG_KEY_FILE"
|
||||
return 0
|
||||
fi
|
||||
local tmp got
|
||||
tmp=$(mktemp)
|
||||
curl -fsSL "$NFV_SIG_KEY_URL" -o "$tmp" \
|
||||
|| { rm -f "$tmp"; fatal "Cannot download the CentOS NFV SIG key from $NFV_SIG_KEY_URL"; }
|
||||
got=$(gpg_fingerprints "$tmp")
|
||||
if [ "$got" != "$NFV_SIG_KEY_FPR" ]; then
|
||||
rm -f "$tmp"
|
||||
fatal "The key at $NFV_SIG_KEY_URL has fingerprint ${got:-<unreadable>}, not the CentOS NFV SIG's $NFV_SIG_KEY_FPR. Refusing to trust it"
|
||||
fi
|
||||
$SUDO install -D -m 0644 "$tmp" "$NFV_SIG_KEY_FILE"
|
||||
rm -f "$tmp"
|
||||
$SUDO rpm --import "$NFV_SIG_KEY_FILE"
|
||||
}
|
||||
|
||||
# Oracle ships kernel modules per kernel version. The unversioned package
|
||||
# follows the newest kernel and leaves the running one without openvswitch.ko.
|
||||
install_running_kernel_modules() {
|
||||
local pkg
|
||||
pkg="kernel-uek-modules-$(uname -r)"
|
||||
if rpm -q "$pkg" > /dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
if ! $SUDO dnf -q list --available "$pkg" > /dev/null 2>&1; then
|
||||
fatal "$pkg, the modules for the running kernel, is no longer in the repositories. Run 'sudo dnf update kernel-uek', reboot into the new kernel, then re-run this installer"
|
||||
fi
|
||||
info "Installing $pkg (it pulls in linux-firmware, about 800 MB)..."
|
||||
$SUDO dnf install -y -q "$pkg" > /dev/null
|
||||
}
|
||||
|
||||
load_kernel_modules() {
|
||||
local mod failed=""
|
||||
for mod in "$@"; do
|
||||
$SUDO modprobe "$mod" || failed="${failed:+$failed }$mod"
|
||||
done
|
||||
if [ -n "$failed" ]; then
|
||||
fatal "Cannot load kernel modules for the running kernel $(uname -r): $failed"
|
||||
fi
|
||||
}
|
||||
|
||||
# --- Install AWS CLI ---
|
||||
install_aws_cli() {
|
||||
stage "installing AWS CLI v2"
|
||||
@@ -794,8 +1300,12 @@ install_aws_cli() {
|
||||
return
|
||||
fi
|
||||
|
||||
if command -v aws >/dev/null 2>&1; then
|
||||
info "AWS CLI already installed: $(aws --version 2>&1 | head -1)"
|
||||
# The bundled installer links /usr/local/bin/aws, which OL's sudo secure_path
|
||||
# leaves out, so a PATH lookup alone misses it and reinstalls on every run.
|
||||
local aws_bin
|
||||
aws_bin=$(command -v aws 2>/dev/null || echo /usr/local/bin/aws)
|
||||
if [ -x "$aws_bin" ]; then
|
||||
info "AWS CLI already installed: $("$aws_bin" --version 2>&1 | head -1)"
|
||||
return
|
||||
fi
|
||||
|
||||
@@ -806,7 +1316,7 @@ install_aws_cli() {
|
||||
$SUDO "$AWS_TMPDIR/aws/install" --update > /dev/null
|
||||
rm -rf "$AWS_TMPDIR"
|
||||
|
||||
info "AWS CLI installed: $(aws --version 2>&1 | head -1)"
|
||||
info "AWS CLI installed: $(/usr/local/bin/aws --version 2>&1 | head -1)"
|
||||
}
|
||||
|
||||
# --- Download tarball ---
|
||||
@@ -861,6 +1371,34 @@ download_spinifex() {
|
||||
EXTRACT_DIR="$SPINIFEX_TMPDIR"
|
||||
}
|
||||
|
||||
# OL's sudo secure_path leaves out /usr/local/bin, so `sudo spx` would find
|
||||
# nothing there. A link in /usr/bin changes sudo for nothing else on the host.
|
||||
link_spx_for_sudo() {
|
||||
local sudo_path
|
||||
[ "${ISO_BUILD:-0}" = "1" ] && return 0
|
||||
command -v sudo > /dev/null 2>&1 || return 0
|
||||
sudo_path=$(sudo -n /usr/bin/env 2> /dev/null | sed -n 's/^PATH=//p') || true
|
||||
if [ -z "$sudo_path" ]; then
|
||||
warn "Cannot read the PATH sudo gives commands. If 'sudo spx' is not found, run: sudo ln -s /usr/local/bin/spx /usr/bin/spx"
|
||||
return 0
|
||||
fi
|
||||
case ":$sudo_path:" in *:/usr/local/bin:*) return 0 ;; esac
|
||||
if [ -e /usr/bin/spx ] || [ -L /usr/bin/spx ]; then
|
||||
[ "$(readlink /usr/bin/spx)" = /usr/local/bin/spx ] && return 0
|
||||
fatal "/usr/bin/spx exists and is not a link to /usr/local/bin/spx. sudo's PATH ($sudo_path) leaves out /usr/local/bin, so 'sudo spx' would run that file. Remove it and re-run this installer"
|
||||
fi
|
||||
$SUDO ln -s /usr/local/bin/spx /usr/bin/spx
|
||||
info " /usr/bin/spx -> /usr/local/bin/spx (sudo's PATH leaves out /usr/local/bin)"
|
||||
}
|
||||
|
||||
# Sets NBDKIT_PLUGINDIR from nbdkit itself; no guessed path is safe, since the
|
||||
# daemon loads the plugin from wherever systemd.env says it is.
|
||||
resolve_nbdkit_plugindir() {
|
||||
NBDKIT_PLUGINDIR=$(nbdkit --dump-config 2> /dev/null | sed -n 's/^plugindir=//p') || true
|
||||
[ -n "$NBDKIT_PLUGINDIR" ] \
|
||||
|| fatal "Cannot find nbdkit's plugin directory: 'nbdkit --dump-config' printed no plugindir. Check that nbdkit is installed and runs, then re-run this installer"
|
||||
}
|
||||
|
||||
# --- Place files ---
|
||||
install_files() {
|
||||
stage "installing binaries and scripts"
|
||||
@@ -870,19 +1408,13 @@ install_files() {
|
||||
$SUDO install -m 0755 "$EXTRACT_DIR/spx" /usr/local/bin/spx
|
||||
info " /usr/local/bin/spx"
|
||||
|
||||
link_spx_for_sudo
|
||||
|
||||
# nbdkit plugin
|
||||
PLUGINDIR=$(nbdkit --dump-config 2>/dev/null | grep ^plugindir= | cut -d= -f2)
|
||||
if [ -z "$PLUGINDIR" ]; then
|
||||
warn "Could not detect nbdkit plugin directory, using default"
|
||||
if [ "$ARCH" = "arm64" ]; then
|
||||
PLUGINDIR="/usr/lib/aarch64-linux-gnu/nbdkit/plugins"
|
||||
else
|
||||
PLUGINDIR="/usr/lib/x86_64-linux-gnu/nbdkit/plugins"
|
||||
fi
|
||||
fi
|
||||
$SUDO mkdir -p "$PLUGINDIR"
|
||||
$SUDO install -m 0755 "$EXTRACT_DIR/nbdkit-viperblock-plugin.so" "$PLUGINDIR/nbdkit-viperblock-plugin.so"
|
||||
info " $PLUGINDIR/nbdkit-viperblock-plugin.so"
|
||||
resolve_nbdkit_plugindir
|
||||
$SUDO mkdir -p "$NBDKIT_PLUGINDIR"
|
||||
$SUDO install -m 0755 "$EXTRACT_DIR/nbdkit-viperblock-plugin.so" "$NBDKIT_PLUGINDIR/nbdkit-viperblock-plugin.so"
|
||||
info " $NBDKIT_PLUGINDIR/nbdkit-viperblock-plugin.so"
|
||||
|
||||
# Setup scripts
|
||||
$SUDO mkdir -p /usr/local/share/spinifex
|
||||
@@ -891,6 +1423,13 @@ install_files() {
|
||||
info " /usr/local/share/spinifex/setup-ovn.sh"
|
||||
fi
|
||||
|
||||
# Sourced, not run, by setup-ovn.sh and uninstall-spx.sh wherever they are
|
||||
# installed, including the ISO's /usr/local/bin/setup-ovn.sh.
|
||||
if [ -f "$EXTRACT_DIR/lib/distro.sh" ]; then
|
||||
$SUDO install -D -m 0644 "$EXTRACT_DIR/lib/distro.sh" /usr/local/share/spinifex/lib/distro.sh
|
||||
info " /usr/local/share/spinifex/lib/distro.sh"
|
||||
fi
|
||||
|
||||
# Install setup.sh itself so firstboot and future re-runs can find it at a
|
||||
# stable path on both ISO and curl|bash installs.
|
||||
if [ -f "$EXTRACT_DIR/setup.sh" ]; then
|
||||
@@ -1062,23 +1601,13 @@ TMPEOF
|
||||
install_systemd_env() {
|
||||
stage "generating /etc/spinifex/systemd.env"
|
||||
|
||||
# nbdkit plugin path is arch-dependent (`nbdkit --dump-config` returns
|
||||
# /usr/lib/{x86_64,aarch64}-linux-gnu/nbdkit/plugins). Resolve it here so
|
||||
# systemd.env always matches wherever install_files placed the .so.
|
||||
local plugindir
|
||||
plugindir=$(nbdkit --dump-config 2>/dev/null | grep ^plugindir= | cut -d= -f2)
|
||||
if [ -z "$plugindir" ]; then
|
||||
if [ "${ARCH:-}" = "arm64" ]; then
|
||||
plugindir="/usr/lib/aarch64-linux-gnu/nbdkit/plugins"
|
||||
else
|
||||
plugindir="/usr/lib/x86_64-linux-gnu/nbdkit/plugins"
|
||||
fi
|
||||
fi
|
||||
# The same directory install_files placed the plugin in.
|
||||
resolve_nbdkit_plugindir
|
||||
|
||||
$SUDO mkdir -p /etc/spinifex
|
||||
$SUDO tee /etc/spinifex/systemd.env > /dev/null << EOF
|
||||
# Generated by setup.sh — install-specific environment variables
|
||||
SPINIFEX_VIPERBLOCK_PLUGIN_PATH=${plugindir}/nbdkit-viperblock-plugin.so
|
||||
SPINIFEX_VIPERBLOCK_PLUGIN_PATH=${NBDKIT_PLUGINDIR}/nbdkit-viperblock-plugin.so
|
||||
EOF
|
||||
$SUDO chown "spinifex-viperblock:${SPINIFEX_GROUP:-spinifex}" /etc/spinifex/systemd.env
|
||||
$SUDO chmod 0640 /etc/spinifex/systemd.env
|
||||
@@ -1242,6 +1771,23 @@ EOF
|
||||
info "Systemd units installed and enabled (per-service users)"
|
||||
}
|
||||
|
||||
# --- Install kernel module list ---
|
||||
# The file covers later boots; modprobe covers this one.
|
||||
install_kernel_modules() {
|
||||
stage "installing kernel module list"
|
||||
$SUDO install -d /etc/modules-load.d
|
||||
# shellcheck disable=SC2086 # one module per line
|
||||
printf '# Managed by spinifex setup.sh — do not edit.\n%s\n' $SPINIFEX_KERNEL_MODULES \
|
||||
| $SUDO tee /etc/modules-load.d/spinifex.conf > /dev/null
|
||||
$SUDO chmod 0644 /etc/modules-load.d/spinifex.conf
|
||||
# A chroot would load them into the build host's kernel.
|
||||
if [ "${ISO_BUILD:-0}" != "1" ]; then
|
||||
# shellcheck disable=SC2086 # a whitespace-separated module list
|
||||
load_kernel_modules $SPINIFEX_KERNEL_MODULES
|
||||
fi
|
||||
info "Kernel modules: $SPINIFEX_KERNEL_MODULES (/etc/modules-load.d/spinifex.conf)"
|
||||
}
|
||||
|
||||
# --- Install kernel tunables ---
|
||||
# Predastore carries blob, meta and raft over QUIC, and quic-go asks for a 7 MiB
|
||||
# UDP receive buffer. Left at the kernel default of 208 KiB it silently drops
|
||||
@@ -1418,22 +1964,12 @@ main() {
|
||||
*) fatal "--channel must be 'latest' or 'dev', got: ${INSTALL_SPINIFEX_CHANNEL} (use --version for a specific tag)" ;;
|
||||
esac
|
||||
|
||||
case "${INSTALL_SPINIFEX_FIREWALL}" in
|
||||
on|off) ;;
|
||||
# Unset resolves by install path: the ISO arms, everything else does not.
|
||||
"") if [ "${ISO_BUILD:-0}" = "1" ]; then
|
||||
INSTALL_SPINIFEX_FIREWALL="on"
|
||||
else
|
||||
INSTALL_SPINIFEX_FIREWALL="off"
|
||||
fi ;;
|
||||
*) fatal "--firewall must be 'on' or 'off', got: ${INSTALL_SPINIFEX_FIREWALL}" ;;
|
||||
esac
|
||||
|
||||
info "Spinifex installer"
|
||||
echo ""
|
||||
|
||||
# Always needed: sudo resolution, OS/arch detection.
|
||||
setup_sudo
|
||||
resolve_firewall_mode
|
||||
detect_os
|
||||
detect_arch
|
||||
|
||||
@@ -1451,8 +1987,9 @@ main() {
|
||||
download_spinifex
|
||||
fi
|
||||
|
||||
stage_enabled deps && install_apt_deps
|
||||
stage_enabled deps && install_deps
|
||||
stage_enabled aws && install_aws_cli
|
||||
stage_enabled dns && hand_dns_to_resolved
|
||||
stage_enabled users && create_service_users
|
||||
stage_enabled sudoers && install_sudoers
|
||||
stage_enabled firewall && install_firewall
|
||||
@@ -1463,6 +2000,7 @@ main() {
|
||||
stage_enabled fixown && fix_file_ownership
|
||||
stage_enabled systemd && install_systemd
|
||||
stage_enabled sysctl && install_sysctl
|
||||
stage_enabled modules && install_kernel_modules
|
||||
stage_enabled logrotate && install_logrotate
|
||||
stage_enabled udev && install_udev
|
||||
stage_enabled swap && setup_swap
|
||||
|
||||
Executable
+206
@@ -0,0 +1,206 @@
|
||||
#!/bin/sh
|
||||
# setup_firewall_test.sh — tests for how setup.sh picks the firewall mode and
|
||||
# masks firewalld.
|
||||
#
|
||||
# setup.sh is sourced with INSTALL_SPINIFEX_LIB_ONLY=1 so main never runs. A stub
|
||||
# systemctl keeps firewalld's state in fixture files, and the mode file is a
|
||||
# fixture path.
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
|
||||
SETUP="$SCRIPT_DIR/setup.sh"
|
||||
TMP=$(mktemp -d)
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
|
||||
fails=0
|
||||
pass() { echo " ok - $1"; }
|
||||
fail() { echo " FAIL - $1"; fails=$((fails + 1)); }
|
||||
|
||||
echo "-- setup.sh firewall mode and firewalld"
|
||||
|
||||
mkdir -p "$TMP/bin"
|
||||
cat > "$TMP/bin/systemctl" <<EOF
|
||||
#!/bin/sh
|
||||
state="$TMP/state"
|
||||
[ "\$1" = "--quiet" ] && shift
|
||||
case "\$1 \$2" in
|
||||
"is-active --quiet") [ -e "\$state/active" ] ;;
|
||||
"is-enabled firewalld.service") [ -e "\$state/enabled" ] || { echo not-found; exit 4; }; cat "\$state/enabled" ;;
|
||||
"mask --now")
|
||||
echo "mask" >> "\$state/calls"
|
||||
[ -e "\$state/fail-mask" ] && exit 1
|
||||
echo masked > "\$state/enabled"; rm -f "\$state/active" ;;
|
||||
*) echo "unexpected systemctl \$*" >&2; exit 2 ;;
|
||||
esac
|
||||
EOF
|
||||
chmod +x "$TMP/bin/systemctl"
|
||||
|
||||
# host <firewalld: none|inactive|active|masked> [recorded mode] — sets up the
|
||||
# fixture host. The mode file is absent when no mode is given.
|
||||
host() {
|
||||
rm -rf "$TMP/state" "$TMP/mode"
|
||||
mkdir -p "$TMP/state"
|
||||
case "$1" in
|
||||
inactive) echo disabled > "$TMP/state/enabled" ;;
|
||||
active) echo enabled > "$TMP/state/enabled"; : > "$TMP/state/active" ;;
|
||||
masked) echo masked > "$TMP/state/enabled" ;;
|
||||
esac
|
||||
if [ -n "${2:-}" ]; then printf '%s\n' "$2" > "$TMP/mode"; fi
|
||||
}
|
||||
|
||||
# run <function...> [--firewall value via FW] — sets RC and OUT, and MODE to
|
||||
# the mode setup.sh resolved.
|
||||
run() {
|
||||
RC=0
|
||||
OUT=$(PATH="$TMP/bin:$PATH" SPINIFEX_FIREWALL_MODE_FILE="$TMP/mode" INSTALL_SPINIFEX_FIREWALL="${FW:-}" \
|
||||
INSTALL_SPINIFEX_LIB_ONLY=1 bash -c '. "$1"; shift; for f in "$@"; do "$f"; done; echo "MODE=$INSTALL_SPINIFEX_FIREWALL"' \
|
||||
_ "$SETUP" "$@" 2>&1) || RC=$?
|
||||
MODE=$(printf '%s\n' "$OUT" | sed -n 's/^MODE=//p')
|
||||
unset FW
|
||||
}
|
||||
|
||||
resolves() { # resolves <desc> <mode>
|
||||
if [ "$RC" = 0 ] && [ "$MODE" = "$2" ]; then pass "$1 resolves to $2"; else fail "$1: rc=$RC mode='$MODE': $OUT"; fi
|
||||
}
|
||||
|
||||
host none
|
||||
run resolve_firewall_mode
|
||||
resolves "no firewalld, no recorded mode" off
|
||||
|
||||
host inactive
|
||||
run resolve_firewall_mode
|
||||
resolves "firewalld installed but inactive" off
|
||||
|
||||
host active
|
||||
run resolve_firewall_mode
|
||||
resolves "firewalld active" on
|
||||
|
||||
host masked on
|
||||
run resolve_firewall_mode
|
||||
resolves "a rerun after an install with --firewall=on" on
|
||||
|
||||
host active on
|
||||
run resolve_firewall_mode
|
||||
resolves "a retry after the mode file was written but before firewalld was masked" on
|
||||
|
||||
host active off
|
||||
run resolve_firewall_mode
|
||||
resolves "a recorded 'off' with firewalld active" off
|
||||
|
||||
host masked on
|
||||
FW=off run resolve_firewall_mode
|
||||
resolves "an explicit --firewall=off over a recorded 'on'" off
|
||||
|
||||
host none on
|
||||
FW=off run resolve_firewall_mode
|
||||
resolves "an explicit --firewall=off" off
|
||||
|
||||
host none
|
||||
FW=maybe run resolve_firewall_mode
|
||||
case "$RC:$OUT" in
|
||||
0:*) fail "accepts --firewall=maybe" ;;
|
||||
*"--firewall must be"*) pass "rejects --firewall=maybe" ;;
|
||||
*) fail "rejects --firewall=maybe without saying why: $OUT" ;;
|
||||
esac
|
||||
|
||||
host none garbage
|
||||
run resolve_firewall_mode
|
||||
case "$RC:$OUT" in
|
||||
0:*) fail "accepts a mode file holding 'garbage'" ;;
|
||||
*"holds 'garbage'"*) pass "rejects a mode file holding 'garbage', naming it" ;;
|
||||
*) fail "rejects a bad mode file without naming its content: $OUT" ;;
|
||||
esac
|
||||
|
||||
# The record sits under /etc/spinifex, which only root and the spinifex group
|
||||
# can enter, so an installer run by any other sudoer must read it through sudo.
|
||||
cat > "$TMP/bin/sudo" <<EOF
|
||||
#!/bin/sh
|
||||
echo "\$*" >> "$TMP/sudo-calls"
|
||||
exec "\$@"
|
||||
EOF
|
||||
chmod +x "$TMP/bin/sudo"
|
||||
host masked on
|
||||
rm -f "$TMP/sudo-calls"
|
||||
RC=0
|
||||
OUT=$(PATH="$TMP/bin:$PATH" SUDO=sudo SPINIFEX_FIREWALL_MODE_FILE="$TMP/mode" INSTALL_SPINIFEX_FIREWALL="" \
|
||||
INSTALL_SPINIFEX_LIB_ONLY=1 bash -c '. "$1"; resolve_firewall_mode; echo "MODE=$INSTALL_SPINIFEX_FIREWALL"' \
|
||||
_ "$SETUP" 2>&1) || RC=$?
|
||||
MODE=$(printf '%s\n' "$OUT" | sed -n 's/^MODE=//p')
|
||||
if [ "$RC" = 0 ] && [ "$MODE" = on ] && grep -qx "cat $TMP/mode" "$TMP/sudo-calls" 2>/dev/null; then
|
||||
pass "a non-root rerun reads the recorded mode through sudo"
|
||||
else
|
||||
fail "a non-root rerun did not read the recorded mode through sudo: rc=$RC mode='$MODE' calls='$(cat "$TMP/sudo-calls" 2>/dev/null)': $OUT"
|
||||
fi
|
||||
|
||||
# A record that exists but cannot be read must stop the install, never fall
|
||||
# through to the firewalld check, which reads a masked firewalld as "off".
|
||||
if [ "$(id -u)" != 0 ]; then
|
||||
host masked on
|
||||
chmod 000 "$TMP/mode"
|
||||
run resolve_firewall_mode
|
||||
chmod 644 "$TMP/mode"
|
||||
case "$RC:$OUT" in
|
||||
0:*) fail "an unreadable mode file resolved to '$MODE'" ;;
|
||||
*"Cannot read the recorded firewall mode"*) pass "an unreadable mode file stops the install" ;;
|
||||
*) fail "an unreadable mode file stopped without saying why: $OUT" ;;
|
||||
esac
|
||||
fi
|
||||
|
||||
masks() { # masks <desc> <expect: masked|untouched>
|
||||
calls=$(cat "$TMP/state/calls" 2>/dev/null || true)
|
||||
if [ "$RC" != 0 ]; then
|
||||
fail "$1: rc=$RC: $OUT"
|
||||
elif [ "$2" = masked ] && [ -n "$calls" ] && [ "$(cat "$TMP/state/enabled")" = masked ] && [ ! -e "$TMP/state/active" ]; then
|
||||
pass "$1: masked and stopped"
|
||||
elif [ "$2" = untouched ] && [ -z "$calls" ]; then
|
||||
pass "$1: left alone"
|
||||
else
|
||||
fail "$1: expected $2, calls='$calls': $OUT"
|
||||
fi
|
||||
}
|
||||
|
||||
host active
|
||||
run resolve_firewall_mode mask_firewalld
|
||||
masks "firewalld active" masked
|
||||
case "$OUT" in
|
||||
*"Restate any other restriction"*) pass "masking firewalld says its rules were not migrated" ;;
|
||||
*) fail "masking firewalld printed no notice: $OUT" ;;
|
||||
esac
|
||||
|
||||
host inactive
|
||||
run mask_firewalld
|
||||
masks "firewalld installed but inactive" masked
|
||||
|
||||
host none
|
||||
run mask_firewalld
|
||||
masks "no firewalld" untouched
|
||||
|
||||
host masked
|
||||
run mask_firewalld
|
||||
masks "firewalld already masked" untouched
|
||||
|
||||
host active
|
||||
echo masked > "$TMP/state/enabled"
|
||||
run mask_firewalld
|
||||
masks "firewalld masked but still running" masked
|
||||
|
||||
host active
|
||||
: > "$TMP/state/fail-mask"
|
||||
run mask_firewalld
|
||||
case "$RC:$OUT" in
|
||||
0:*) fail "succeeds when firewalld cannot be masked" ;;
|
||||
*"Cannot stop and mask firewalld"*) pass "fails by name when firewalld cannot be masked" ;;
|
||||
*) fail "fails without naming firewalld: $OUT" ;;
|
||||
esac
|
||||
|
||||
host active
|
||||
FW=off run mask_firewalld
|
||||
case "$OUT" in
|
||||
*"this host now has no firewall"*) pass "masking firewalld under --firewall=off says the host has no firewall" ;;
|
||||
*) fail "masking under --firewall=off did not say the host has no firewall: $OUT" ;;
|
||||
esac
|
||||
|
||||
if [ "$fails" -gt 0 ]; then
|
||||
echo "$fails failure(s)"
|
||||
exit 1
|
||||
fi
|
||||
Executable
+272
@@ -0,0 +1,272 @@
|
||||
#!/bin/sh
|
||||
# setup_host_test.sh — tests for setup.sh's DNS handover to systemd-resolved,
|
||||
# its chrony drop-in and its nbdkit plugin directory.
|
||||
#
|
||||
# setup.sh is sourced with INSTALL_SPINIFEX_LIB_ONLY=1 so main never runs. PATH
|
||||
# holds stub commands and links to a few real tools only, so no host resolver,
|
||||
# package manager or service is ever touched.
|
||||
# shellcheck disable=SC2016 # stub bodies are single-quoted so they expand when the stub runs
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
|
||||
SETUP="$SCRIPT_DIR/setup.sh"
|
||||
TMP=$(mktemp -d)
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
|
||||
fails=0
|
||||
pass() { echo " ok - $1"; }
|
||||
fail() { echo " FAIL - $1"; fails=$((fails + 1)); }
|
||||
|
||||
echo "-- setup.sh host DNS, chrony and nbdkit"
|
||||
|
||||
mkdir -p "$TMP/sys"
|
||||
for tool in bash awk sed grep readlink basename dirname cp ln mkdir cat head tail wc tee rm chmod; do
|
||||
ln -s "$(command -v "$tool")" "$TMP/sys/$tool"
|
||||
done
|
||||
|
||||
# stub <name> <body> — a stub command; $S is the fixture's state directory.
|
||||
stub() {
|
||||
printf '#!/bin/sh\nS="%s"\n%s\n' "$TMP/state" "$2" > "$TMP/bin/$1"
|
||||
chmod +x "$TMP/bin/$1"
|
||||
}
|
||||
|
||||
# host — a fresh fixture: resolved installed and inactive, resolvconf the
|
||||
# resolvectl link, NetworkManager running with no DNS settings, a dnf host
|
||||
# whose first enabled repository is yum.oracle.com.
|
||||
host() {
|
||||
rm -rf "${TMP:?}/state" "${TMP:?}/root" "${TMP:?}/bin"
|
||||
mkdir -p "$TMP/state/active" "$TMP/root/etc" "$TMP/root/usr/lib/systemd/system" "$TMP/bin"
|
||||
: > "$TMP/root/usr/lib/systemd/system/systemd-resolved.service"
|
||||
: > "$TMP/state/active/NetworkManager.service"
|
||||
printf 'nameserver 192.168.1.1\nsearch example.lan\n' > "$TMP/root/etc/resolv.conf"
|
||||
printf '[main]\n# dns=default\nconfigure-and-quit=no\n' > "$TMP/state/nm-config"
|
||||
printf 'Global:\nLink 2 (br-wan): 192.168.1.1\n' > "$TMP/state/resolvectl-dns"
|
||||
printf '[ol10_UEKR8]\nbaseurl = https://mirror.example/uek/\nenabled = 0\n[ol10_baseos_latest]\nbaseurl = https://yum.oracle.com/repo/OracleLinux/OL10/baseos/latest/x86_64/\nenabled = 1\n' > "$TMP/state/dnf-dump"
|
||||
stub systemctl '[ "$1" = is-active ] && { [ -e "$S/active/$3" ]; exit; }
|
||||
echo "$*" >> "$S/calls"
|
||||
[ "$1 $2" = "enable --now" ] && : > "$S/active/$3"
|
||||
exit 0'
|
||||
stub NetworkManager 'cat "$S/nm-config"'
|
||||
stub resolvectl '[ "$1" = dns ] && cat "$S/resolvectl-dns"'
|
||||
stub resolvectl-real ':'
|
||||
ln -s resolvectl "$TMP/bin/resolvconf"
|
||||
stub getent '[ ! -e "$S/getent-fail" ] && echo "$2" >> "$S/looked-up"'
|
||||
stub dnf '[ "$1 $2" = "config-manager --dump" ] && cat "$S/dnf-dump"'
|
||||
stub sleep ':'
|
||||
}
|
||||
|
||||
# run <function...> — runs setup.sh functions against the fixture. Sets RC and OUT.
|
||||
run() {
|
||||
RC=0
|
||||
OUT=$(PATH="$TMP/bin:$TMP/sys" DISTRO_ROOT="$TMP/root" \
|
||||
SPINIFEX_RESOLV_CONF="$TMP/root/etc/resolv.conf" SPINIFEX_RESOLV_CONF_SAVED="$TMP/root/saved/resolv.conf.orig" \
|
||||
INSTALL_SPINIFEX_LIB_ONLY=1 bash -c '. "$1"; shift; for f in "$@"; do $f; done' _ "$SETUP" "$@" 2>&1) || RC=$?
|
||||
}
|
||||
|
||||
calls() { cat "$TMP/state/calls" 2>/dev/null || true; }
|
||||
saved() { cat "$TMP/root/saved/resolv.conf.orig" 2>/dev/null || echo "<none>"; }
|
||||
expect_untouched() {
|
||||
if [ "$RC" = 0 ] && [ -z "$(calls)" ] && [ ! -L "$TMP/root/etc/resolv.conf" ] && [ "$(saved)" = "<none>" ]; then
|
||||
pass "$1"
|
||||
else
|
||||
fail "$1: rc=$RC calls='$(calls)' saved='$(saved)': $OUT"
|
||||
fi
|
||||
}
|
||||
expect_fatal() { # expect_fatal <desc> <text>...
|
||||
desc=$1; shift
|
||||
if [ "$RC" = 0 ]; then fail "$desc (succeeded): $OUT"; return; fi
|
||||
for want in "$@"; do
|
||||
case "$OUT" in *"$want"*) ;; *) fail "$desc (output lacks '$want'): $OUT"; return ;; esac
|
||||
done
|
||||
pass "$desc"
|
||||
}
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# DNS handover.
|
||||
# --------------------------------------------------------------------------
|
||||
host
|
||||
run hand_dns_to_resolved
|
||||
if [ "$RC" = 0 ] && [ "$(readlink "$TMP/root/etc/resolv.conf")" = /run/systemd/resolve/stub-resolv.conf ] \
|
||||
&& [ "$(saved)" = "$(printf 'nameserver 192.168.1.1\nsearch example.lan')" ] \
|
||||
&& [ "$(calls)" = "$(printf 'enable --now systemd-resolved.service\nrestart NetworkManager.service')" ] \
|
||||
&& [ "$(cat "$TMP/state/looked-up")" = yum.oracle.com ]; then
|
||||
pass "hands DNS to resolved: saves resolv.conf, starts resolved, restarts NM, links the stub, looks up the repo host"
|
||||
else
|
||||
fail "handover: rc=$RC calls='$(calls)' saved='$(saved)' link='$(readlink "$TMP/root/etc/resolv.conf")': $OUT"
|
||||
fi
|
||||
|
||||
: > "$TMP/state/calls"
|
||||
rm -f "$TMP/state/looked-up"
|
||||
run hand_dns_to_resolved
|
||||
if [ "$RC" = 0 ] && [ -z "$(calls)" ] && [ "$(cat "$TMP/state/looked-up")" = yum.oracle.com ]; then
|
||||
pass "a rerun after a handover checks DNS again and changes nothing"
|
||||
else
|
||||
fail "rerun after handover: rc=$RC calls='$(calls)': $OUT"
|
||||
fi
|
||||
|
||||
host
|
||||
mkdir -p "$TMP/root/saved"
|
||||
echo "the original" > "$TMP/root/saved/resolv.conf.orig"
|
||||
run hand_dns_to_resolved
|
||||
if [ "$RC" = 0 ] && [ "$(saved)" = "the original" ]; then
|
||||
pass "a rerun never overwrites the saved resolv.conf"
|
||||
else
|
||||
fail "saved copy overwritten: rc=$RC saved='$(saved)': $OUT"
|
||||
fi
|
||||
|
||||
host
|
||||
rm "$TMP/root/usr/lib/systemd/system/systemd-resolved.service"
|
||||
run hand_dns_to_resolved
|
||||
expect_untouched "resolved not installed: DNS is left alone"
|
||||
|
||||
host
|
||||
: > "$TMP/state/active/systemd-resolved.service"
|
||||
run hand_dns_to_resolved
|
||||
expect_untouched "resolved already active, with no earlier handover: DNS is left alone"
|
||||
|
||||
host
|
||||
rm "$TMP/bin/resolvconf"
|
||||
stub resolvconf ':'
|
||||
run hand_dns_to_resolved
|
||||
expect_untouched "a real resolvconf: DNS is left alone"
|
||||
|
||||
host
|
||||
rm "$TMP/state/active/NetworkManager.service"
|
||||
run hand_dns_to_resolved
|
||||
case "$OUT" in
|
||||
*"NetworkManager is not running"*) expect_untouched "NetworkManager not running: DNS is left alone, with a warning" ;;
|
||||
*) fail "NetworkManager not running: no warning: $OUT" ;;
|
||||
esac
|
||||
|
||||
for setting in dns=none dns=dnsmasq systemd-resolved=false; do
|
||||
host
|
||||
printf '[main]\n# dns=default\n%s\n[logging]\ndns=none\n' "$setting" > "$TMP/state/nm-config"
|
||||
run hand_dns_to_resolved
|
||||
expect_fatal "NetworkManager $setting stops the install, naming it" "NetworkManager sets $setting"
|
||||
[ -z "$(calls)" ] || fail "NetworkManager $setting: acted before refusing: $(calls)"
|
||||
done
|
||||
|
||||
host
|
||||
printf '[main]\ndns=systemd-resolved\n[logging]\ndns=none\n' > "$TMP/state/nm-config"
|
||||
run hand_dns_to_resolved
|
||||
[ "$RC" = 0 ] && pass "dns=systemd-resolved, and dns= outside [main], are accepted" || fail "dns=systemd-resolved refused: $OUT"
|
||||
|
||||
host
|
||||
printf 'Global:\nLink 2 (br-wan):\n' > "$TMP/state/resolvectl-dns"
|
||||
run hand_dns_to_resolved
|
||||
expect_fatal "no upstream servers stops the install, naming the check and printing the restore" \
|
||||
"'resolvectl dns' lists no DNS servers" "systemctl disable --now systemd-resolved" "cp -a --remove-destination $TMP/root/saved/resolv.conf.orig"
|
||||
|
||||
host
|
||||
: > "$TMP/state/getent-fail"
|
||||
run hand_dns_to_resolved
|
||||
expect_fatal "a repository host that no longer resolves stops the install, naming it" "yum.oracle.com no longer resolves" "systemctl restart NetworkManager"
|
||||
|
||||
host
|
||||
printf '[local]\nbaseurl = file:///srv/repo/\nenabled = 1\n' > "$TMP/state/dnf-dump"
|
||||
run hand_dns_to_resolved
|
||||
case "$RC:$OUT" in
|
||||
0:*"not checking that names resolve"*) pass "a file:// repository skips the name check with a warning" ;;
|
||||
*) fail "file:// repository: rc=$RC: $OUT" ;;
|
||||
esac
|
||||
|
||||
host
|
||||
printf '[local]\nbaseurl = http://10.0.0.5/repo/\nenabled = 1\n' > "$TMP/state/dnf-dump"
|
||||
run hand_dns_to_resolved
|
||||
case "$RC:$OUT" in
|
||||
0:*"10.0.0.5 is an IP address"*) pass "a repository at an IP address skips the name check with a warning" ;;
|
||||
*) fail "IP repository: rc=$RC: $OUT" ;;
|
||||
esac
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Repository URLs and their hosts.
|
||||
# --------------------------------------------------------------------------
|
||||
host
|
||||
printf '[a]\nbaseurl = \nenabled = 1\nmirrorlist = https://mirrors.example/list\n[b]\nbaseurl = https://b.example/\nenabled = 1\n' > "$TMP/state/dnf-dump"
|
||||
run first_repo_url
|
||||
[ "$OUT" = https://mirrors.example/list ] && pass "dnf: a mirrorlist stands in for an empty baseurl" || fail "dnf mirrorlist: '$OUT'"
|
||||
printf '[a]\nbaseurl = https://a1.example/, https://a2.example/\nenabled = 1\n' > "$TMP/state/dnf-dump"
|
||||
run first_repo_url
|
||||
[ "$OUT" = https://a1.example/ ] && pass "dnf: the first of several baseurls" || fail "dnf baseurl list: '$OUT'"
|
||||
|
||||
host
|
||||
rm "$TMP/bin/dnf"
|
||||
stub apt-get '[ "$1" = indextargets ] && printf "http://deb.debian.org/debian/\nhttp://deb.debian.org/debian-security/\n"'
|
||||
run first_repo_url
|
||||
[ "$OUT" = http://deb.debian.org/debian/ ] && pass "apt: the first repository URI" || fail "apt: '$OUT'"
|
||||
|
||||
for case in "https://yum.oracle.com/repo/=yum.oracle.com" "http://user@mirror.lan:8080/x=mirror.lan" \
|
||||
"http://[2001:db8::1]:80/x=[2001:db8::1]" "file:///srv/repo=" "cdrom:[Debian]/=" "=" ; do
|
||||
url=${case%=*}
|
||||
want=${case##*=}
|
||||
run "url_host $url"
|
||||
[ "$OUT" = "$want" ] && pass "url_host '$url' is '$want'" || fail "url_host '$url' = '$OUT', want '$want'"
|
||||
done
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# chrony drop-in.
|
||||
# --------------------------------------------------------------------------
|
||||
host
|
||||
mkdir -p "$TMP/root/etc/chrony"
|
||||
printf 'pool 2.debian.pool.ntp.org iburst\nmakestep 1 3\nconfdir /etc/chrony/conf.d\n' > "$TMP/root/etc/chrony/chrony.conf"
|
||||
before=$(cat "$TMP/root/etc/chrony/chrony.conf")
|
||||
run install_chrony_conf
|
||||
if [ "$RC" = 0 ] && grep -qx 'makestep 1 -1' "$TMP/root/etc/chrony/conf.d/spinifex.conf" 2>/dev/null \
|
||||
&& [ "$(cat "$TMP/root/etc/chrony/chrony.conf")" = "$before" ]; then
|
||||
case "$OUT" in
|
||||
*WARN*) fail "Debian chrony layout warned: $OUT" ;;
|
||||
*) pass "Debian: the drop-in goes in the confdir, chrony.conf is not edited" ;;
|
||||
esac
|
||||
else
|
||||
fail "Debian chrony: rc=$RC: $OUT"
|
||||
fi
|
||||
|
||||
host
|
||||
printf 'pool 2.pool.ntp.org iburst\nmakestep 1.0 3\nrtcsync' > "$TMP/root/etc/chrony.conf"
|
||||
run install_chrony_conf
|
||||
run install_chrony_conf
|
||||
if [ "$RC" = 0 ] && grep -qx 'makestep 1 -1' "$TMP/root/etc/chrony.d/spinifex.conf" 2>/dev/null \
|
||||
&& [ "$(grep -c '^confdir' "$TMP/root/etc/chrony.conf")" = 1 ] \
|
||||
&& [ "$(tail -n 1 "$TMP/root/etc/chrony.conf")" = "confdir /etc/chrony.d" ] \
|
||||
&& grep -qx rtcsync "$TMP/root/etc/chrony.conf"; then
|
||||
pass "EL: one confdir line is appended after the stock makestep, and the drop-in goes there"
|
||||
else
|
||||
fail "EL chrony: rc=$RC: $OUT: $(cat "$TMP/root/etc/chrony.conf")"
|
||||
fi
|
||||
|
||||
host
|
||||
printf 'confdir /etc/chrony.d\nmakestep 1.0 3\n' > "$TMP/root/etc/chrony.conf"
|
||||
run install_chrony_conf
|
||||
case "$RC:$OUT" in
|
||||
0:*"makestep on line 2, after its confdir line"*) pass "a makestep after the confdir line warns that it overrides the drop-in" ;;
|
||||
*) fail "late makestep: rc=$RC: $OUT" ;;
|
||||
esac
|
||||
|
||||
host
|
||||
run install_chrony_conf
|
||||
case "$RC:$OUT" in
|
||||
0:*"Skipping the chrony drop-in"*) pass "no chrony config: the drop-in is skipped with a warning" ;;
|
||||
*) fail "no chrony: rc=$RC: $OUT" ;;
|
||||
esac
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# nbdkit plugin directory.
|
||||
# --------------------------------------------------------------------------
|
||||
host
|
||||
stub nbdkit '[ "$1" = --dump-config ] && printf "binary=/usr/sbin/nbdkit\nplugindir=/usr/lib64/nbdkit/plugins\n"'
|
||||
OUT=$(PATH="$TMP/bin:$TMP/sys" INSTALL_SPINIFEX_LIB_ONLY=1 bash -c '. "$1"; resolve_nbdkit_plugindir; echo "DIR=$NBDKIT_PLUGINDIR"' _ "$SETUP" 2>&1) && RC=0 || RC=$?
|
||||
[ "$OUT" = DIR=/usr/lib64/nbdkit/plugins ] && pass "the plugin directory comes from nbdkit --dump-config" || fail "nbdkit plugindir: rc=$RC: $OUT"
|
||||
|
||||
host
|
||||
stub nbdkit 'exit 1'
|
||||
run resolve_nbdkit_plugindir
|
||||
expect_fatal "a broken nbdkit stops the install, naming it" "nbdkit --dump-config"
|
||||
|
||||
host
|
||||
run resolve_nbdkit_plugindir
|
||||
expect_fatal "no nbdkit stops the install, naming it" "nbdkit --dump-config"
|
||||
|
||||
if [ "$fails" -gt 0 ]; then
|
||||
echo "$fails failure(s)"
|
||||
exit 1
|
||||
fi
|
||||
Executable
+150
@@ -0,0 +1,150 @@
|
||||
#!/bin/sh
|
||||
# setup_repos_test.sh — tests for the Oracle Linux repository trust setup in
|
||||
# setup.sh: the NFV SIG key fingerprint check and the AppStream QEMU exclude.
|
||||
#
|
||||
# setup.sh is sourced with INSTALL_SPINIFEX_LIB_ONLY=1 so main never runs. Stub
|
||||
# gpg, curl, rpm and dnf read and log fixture files, and the key path is a
|
||||
# fixture path.
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
|
||||
SETUP="$SCRIPT_DIR/setup.sh"
|
||||
TMP=$(mktemp -d)
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
|
||||
fails=0
|
||||
pass() { echo " ok - $1"; }
|
||||
fail() { echo " FAIL - $1"; fails=$((fails + 1)); }
|
||||
|
||||
echo "-- setup.sh NFV SIG key and AppStream exclude"
|
||||
|
||||
GOOD=35154228174901BEFA8E69A621465E289D2A76A7
|
||||
BAD=0000000000000000000000000000000000000BAD
|
||||
|
||||
mkdir -p "$TMP/bin"
|
||||
# A fixture key file holds the --with-colons listing gpg would print for it.
|
||||
cat > "$TMP/bin/gpg" <<'EOF'
|
||||
#!/bin/sh
|
||||
for last; do :; done
|
||||
cat "$last"
|
||||
EOF
|
||||
cat > "$TMP/bin/curl" <<EOF
|
||||
#!/bin/sh
|
||||
echo "curl" >> "$TMP/calls"
|
||||
while [ \$# -gt 0 ]; do
|
||||
case "\$1" in -o) cp "$TMP/served" "\$2"; exit 0 ;; esac
|
||||
shift
|
||||
done
|
||||
exit 1
|
||||
EOF
|
||||
cat > "$TMP/bin/rpm" <<EOF
|
||||
#!/bin/sh
|
||||
echo "rpm \$*" >> "$TMP/calls"
|
||||
EOF
|
||||
cat > "$TMP/bin/dnf" <<EOF
|
||||
#!/bin/sh
|
||||
case "\$*" in
|
||||
"config-manager --dump "*) cat "$TMP/appstream" ;;
|
||||
"config-manager --save "*) echo "\$3" >> "$TMP/calls" ;;
|
||||
*) echo "unexpected dnf \$*" >&2; exit 2 ;;
|
||||
esac
|
||||
EOF
|
||||
chmod +x "$TMP/bin/"*
|
||||
|
||||
key() { # key <file> <pub-fpr> [sub-fpr] [second-pub-fpr]
|
||||
{
|
||||
printf 'pub:-:4096:1:AAAA:1:::-:::scESC::::::23::0:\nfpr:::::::::%s:\n' "$2"
|
||||
if [ -n "${3:-}" ]; then printf 'sub:-:4096:1:BBBB:1::::::e::::::23:\nfpr:::::::::%s:\n' "$3"; fi
|
||||
if [ -n "${4:-}" ]; then printf 'pub:-:4096:1:CCCC:1:::-:::scESC::::::23::0:\nfpr:::::::::%s:\n' "$4"; fi
|
||||
} > "$1"
|
||||
}
|
||||
|
||||
# run <function> — sets RC, OUT and CALLS, from $TMP as the working directory.
|
||||
run() {
|
||||
rm -f "$TMP/calls"
|
||||
RC=0
|
||||
OUT=$(cd "$TMP/cwd" && PATH="$TMP/bin:$PATH" INSTALL_SPINIFEX_LIB_ONLY=1 \
|
||||
bash -c '. "$1"; SUDO=""; NFV_SIG_KEY_FILE="$2/installed.key"; "$3"' _ "$SETUP" "$TMP" "$1" 2>&1) || RC=$?
|
||||
CALLS=$(cat "$TMP/calls" 2>/dev/null || true)
|
||||
}
|
||||
mkdir -p "$TMP/cwd"
|
||||
|
||||
imports_once() { [ "$(printf '%s\n' "$CALLS" | grep -c '^rpm --import')" = 1 ]; }
|
||||
|
||||
rm -f "$TMP/installed.key"
|
||||
key "$TMP/served" "$GOOD"
|
||||
run install_nfv_sig_key
|
||||
if [ "$RC" = 0 ] && imports_once && cmp -s "$TMP/served" "$TMP/installed.key"; then
|
||||
pass "the SIG's key is installed and imported once"
|
||||
else
|
||||
fail "the SIG's key was not installed and imported once: rc=$RC calls='$CALLS': $OUT"
|
||||
fi
|
||||
|
||||
refused() { # refused <desc>
|
||||
case "$RC:$OUT" in
|
||||
0:*) fail "$1 was accepted" ;;
|
||||
*"Refusing to trust it"*)
|
||||
if printf '%s\n' "$CALLS" | grep -q '^rpm' || [ -e "$TMP/installed.key" ]; then
|
||||
fail "$1 was refused, but only after it was installed or imported: $CALLS"
|
||||
else
|
||||
pass "$1 is refused before anything is trusted"
|
||||
fi ;;
|
||||
*) fail "$1 stopped without the refusal: $OUT" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
rm -f "$TMP/installed.key"
|
||||
key "$TMP/served" "$BAD"
|
||||
run install_nfv_sig_key
|
||||
refused "a key with another fingerprint"
|
||||
case "$OUT" in
|
||||
*"$BAD"*"$GOOD"*) pass "the refusal names both fingerprints" ;;
|
||||
*) fail "the refusal does not name both fingerprints: $OUT" ;;
|
||||
esac
|
||||
|
||||
rm -f "$TMP/installed.key"
|
||||
key "$TMP/served" "$GOOD" "" "$BAD"
|
||||
run install_nfv_sig_key
|
||||
refused "a file holding the SIG's key and a second key"
|
||||
|
||||
rm -f "$TMP/installed.key"
|
||||
key "$TMP/served" "$BAD" "$GOOD"
|
||||
run install_nfv_sig_key
|
||||
refused "a key whose subkey, not primary key, has the SIG's fingerprint"
|
||||
|
||||
key "$TMP/installed.key" "$BAD"
|
||||
key "$TMP/served" "$GOOD"
|
||||
run install_nfv_sig_key
|
||||
if [ "$RC" = 0 ] && printf '%s\n' "$CALLS" | grep -q '^curl' && imports_once && cmp -s "$TMP/served" "$TMP/installed.key"; then
|
||||
pass "a cached key with another fingerprint is downloaded again and replaced"
|
||||
else
|
||||
fail "a cached key with another fingerprint was not replaced: rc=$RC calls='$CALLS': $OUT"
|
||||
fi
|
||||
|
||||
key "$TMP/installed.key" "$GOOD"
|
||||
run install_nfv_sig_key
|
||||
if [ "$RC" = 0 ] && ! printf '%s\n' "$CALLS" | grep -q '^curl' && imports_once; then
|
||||
pass "a cached SIG key is imported without a download"
|
||||
else
|
||||
fail "a cached SIG key was not reused: rc=$RC calls='$CALLS': $OUT"
|
||||
fi
|
||||
|
||||
# excludes <desc> <dump excludepkgs line, or empty> <expected setopt>
|
||||
excludes() {
|
||||
if [ -n "$2" ]; then printf '[ol10_appstream]\nexcludepkgs = %s\nenabled = 1\n' "$2" > "$TMP/appstream"
|
||||
else printf '[ol10_appstream]\nenabled = 1\n' > "$TMP/appstream"; fi
|
||||
run exclude_from_appstream
|
||||
if [ "$RC" = 0 ] && [ "$CALLS" = "--setopt=ol10_appstream.excludepkgs=$3" ]; then
|
||||
pass "$1"
|
||||
else
|
||||
fail "$1: rc=$RC calls='$CALLS': $OUT"
|
||||
fi
|
||||
}
|
||||
|
||||
excludes "no exclude set gains the QEMU patterns" "" "qemu-kvm*,qemu-img*"
|
||||
excludes "an operator's excludes are kept" "foo*, bar" "foo*,bar,qemu-kvm*,qemu-img*"
|
||||
excludes "a rerun adds no duplicates" "foo*, bar, qemu-kvm*, qemu-img*" "foo*,bar,qemu-kvm*,qemu-img*"
|
||||
: > "$TMP/cwd/qemu-kvm-x"
|
||||
excludes "the patterns are never expanded against the working directory" "" "qemu-kvm*,qemu-img*"
|
||||
|
||||
[ "$fails" = 0 ] || { echo "$fails failure(s)"; exit 1; }
|
||||
@@ -0,0 +1,172 @@
|
||||
#!/bin/sh
|
||||
# setup_test.sh — tests for setup.sh's supported-OS gate.
|
||||
#
|
||||
# setup.sh is sourced with INSTALL_SPINIFEX_LIB_ONLY=1 so main never runs, and
|
||||
# detect_os reads fixture os-release, cpuinfo and SELinux config files, and stub
|
||||
# uname and getenforce commands.
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
|
||||
SETUP="$SCRIPT_DIR/setup.sh"
|
||||
TMP=$(mktemp -d)
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
|
||||
fails=0
|
||||
pass() { echo " ok - $1"; }
|
||||
fail() { echo " FAIL - $1"; fails=$((fails + 1)); }
|
||||
|
||||
echo "-- setup.sh OS gate"
|
||||
|
||||
V3_FLAGS="fpu cx16 lahf_lm popcnt sse4_1 sse4_2 ssse3 avx avx2 bmi1 bmi2 f16c fma abm movbe xsave"
|
||||
V2_FLAGS="fpu cx16 lahf_lm popcnt sse4_1 sse4_2 ssse3"
|
||||
UEK_R8="6.12.0-207.111.5.4.el10uek.x86_64"
|
||||
|
||||
# gate <os-release body> [machine] [kernel] [cpu flags] — runs detect_os and
|
||||
# sets RC and OUT (stdout and stderr together). SELINUX_NOW is what getenforce
|
||||
# prints; SELINUX_BOOT is the config file's SELINUX=, or "none" for no file.
|
||||
gate() {
|
||||
printf '%s\n' "$1" > "$TMP/os-release"
|
||||
printf 'processor\t: 0\nflags\t\t: %s\n' "${4:-$V3_FLAGS}" > "$TMP/cpuinfo"
|
||||
rm -f "$TMP/selinux-config"
|
||||
if [ "${SELINUX_BOOT:-permissive}" != none ]; then
|
||||
printf '# fixture\nSELINUX=%s\nSELINUXTYPE=targeted\n' "${SELINUX_BOOT:-permissive}" > "$TMP/selinux-config"
|
||||
fi
|
||||
mkdir -p "$TMP/bin"
|
||||
printf '#!/bin/sh\necho %s\n' "${SELINUX_NOW:-Permissive}" > "$TMP/bin/getenforce"
|
||||
chmod +x "$TMP/bin/getenforce"
|
||||
cat > "$TMP/bin/uname" <<EOF
|
||||
#!/bin/sh
|
||||
case "\$1" in
|
||||
-m) echo "${2:-x86_64}" ;;
|
||||
-r) echo "${3:-$UEK_R8}" ;;
|
||||
*) exit 1 ;;
|
||||
esac
|
||||
EOF
|
||||
chmod +x "$TMP/bin/uname"
|
||||
RC=0
|
||||
OUT=$(PATH="$TMP/bin:$PATH" SPINIFEX_OS_RELEASE="$TMP/os-release" SPINIFEX_CPUINFO="$TMP/cpuinfo" \
|
||||
SPINIFEX_SELINUX_CONFIG="$TMP/selinux-config" INSTALL_SPINIFEX_LIB_ONLY=1 bash -c '. "$1"; detect_os' _ "$SETUP" 2>&1) || RC=$?
|
||||
# Some shells keep a prefix assignment to a function call after it returns.
|
||||
unset SELINUX_NOW SELINUX_BOOT
|
||||
}
|
||||
accepts() {
|
||||
if [ "$RC" = 0 ]; then pass "accepts $1"; else fail "rejects $1: $OUT"; fi
|
||||
}
|
||||
# rejects <desc> <word> — the refusal must say why, not just exit.
|
||||
rejects() {
|
||||
if [ "$RC" = 0 ]; then
|
||||
fail "accepts $1"
|
||||
else
|
||||
case "$OUT" in
|
||||
*"$2"*) pass "rejects $1" ;;
|
||||
*) fail "rejects $1 without naming '$2': $OUT" ;;
|
||||
esac
|
||||
fi
|
||||
}
|
||||
|
||||
OL='ID="ol"
|
||||
NAME="Oracle Linux Server"
|
||||
PRETTY_NAME="Oracle Linux Server 10.2"'
|
||||
|
||||
gate 'ID=debian
|
||||
VERSION_ID="13"
|
||||
PRETTY_NAME="Debian GNU/Linux 13 (trixie)"'
|
||||
accepts "Debian 13"
|
||||
|
||||
# Debian testing and sid carry no VERSION_ID. The gate has always let them
|
||||
# through, and Debian behaviour is not this gate's to change.
|
||||
gate 'ID=debian
|
||||
PRETTY_NAME="Debian GNU/Linux forky/sid"'
|
||||
accepts "Debian testing with no VERSION_ID"
|
||||
|
||||
gate 'ID=debian
|
||||
VERSION_ID="12"'
|
||||
rejects "Debian 12" "Minimum: Debian 13"
|
||||
|
||||
gate 'ID=ubuntu
|
||||
VERSION_ID="26.04"
|
||||
PRETTY_NAME="Ubuntu 26.04 LTS"'
|
||||
accepts "Ubuntu 26.04"
|
||||
|
||||
gate 'ID=ubuntu
|
||||
VERSION_ID="22.04"'
|
||||
rejects "Ubuntu 22.04" "Minimum: Ubuntu 24.04"
|
||||
|
||||
gate "$OL
|
||||
VERSION_ID=\"10.2\""
|
||||
accepts "Oracle Linux 10.2 on UEK R8, x86_64, x86-64-v3"
|
||||
|
||||
gate "$OL
|
||||
VERSION_ID=\"10.4\""
|
||||
accepts "Oracle Linux 10.4"
|
||||
|
||||
gate "$OL
|
||||
VERSION_ID=\"10.1\""
|
||||
rejects "Oracle Linux 10.1" "Oracle Linux 10.2 or a later 10.x"
|
||||
|
||||
gate "$OL
|
||||
VERSION_ID=\"10\""
|
||||
rejects "Oracle Linux with no minor version" "Oracle Linux 10.2 or a later 10.x"
|
||||
|
||||
gate "$OL
|
||||
VERSION_ID=\"11.0\""
|
||||
rejects "Oracle Linux 11" "Oracle Linux 10.2 or a later 10.x"
|
||||
|
||||
gate "$OL
|
||||
VERSION_ID=\"9.6\"" x86_64 "5.15.0-310.184.5.2.el9uek.x86_64"
|
||||
rejects "Oracle Linux 9" "Oracle Linux 10.2 or a later 10.x"
|
||||
|
||||
gate "$OL" x86_64
|
||||
rejects "Oracle Linux with no VERSION_ID" "Oracle Linux 10.2 or a later 10.x"
|
||||
|
||||
gate "$OL
|
||||
VERSION_ID=\"10.2\"" aarch64 "6.12.0-207.111.5.4.el10uek.aarch64"
|
||||
rejects "Oracle Linux on aarch64" "x86_64 only"
|
||||
|
||||
gate "$OL
|
||||
VERSION_ID=\"10.2\"" x86_64 "6.12.0-55.9.1.el10_0.x86_64"
|
||||
rejects "Oracle Linux on the Red Hat compatible kernel" "is not UEK R8"
|
||||
|
||||
gate "$OL
|
||||
VERSION_ID=\"10.2\"" x86_64 "5.15.0-310.184.5.2.el9uek.x86_64"
|
||||
rejects "Oracle Linux on a UEK older than R8" "is not UEK R8"
|
||||
|
||||
gate "$OL
|
||||
VERSION_ID=\"10.2\"" x86_64 "$UEK_R8" "$V2_FLAGS"
|
||||
rejects "Oracle Linux on an x86-64-v2 CPU" "missing: avx avx2 bmi1 bmi2 f16c fma abm movbe xsave"
|
||||
|
||||
OL102="$OL
|
||||
VERSION_ID=\"10.2\""
|
||||
|
||||
SELINUX_NOW=Enforcing gate "$OL102"
|
||||
rejects "Oracle Linux with SELinux enforcing" "SELinux is enforcing"
|
||||
|
||||
SELINUX_BOOT=enforcing gate "$OL102"
|
||||
rejects "Oracle Linux with SELinux permissive now but enforcing at next boot" "enforcing at next boot"
|
||||
|
||||
SELINUX_BOOT='"enforcing"' gate "$OL102"
|
||||
rejects "Oracle Linux with a quoted SELINUX=\"enforcing\"" "enforcing at next boot"
|
||||
|
||||
SELINUX_NOW=Disabled SELINUX_BOOT=disabled gate "$OL102"
|
||||
accepts "Oracle Linux with SELinux disabled"
|
||||
|
||||
SELINUX_NOW=Disabled SELINUX_BOOT=none gate "$OL102"
|
||||
accepts "Oracle Linux with no SELinux config file"
|
||||
|
||||
# Spinifex has no SELinux policy on any OS, but refusing enforcing on Debian
|
||||
# would change Debian behaviour, so only the Oracle Linux gate checks it.
|
||||
SELINUX_NOW=Enforcing SELINUX_BOOT=enforcing gate 'ID=debian
|
||||
VERSION_ID="13"'
|
||||
accepts "Debian 13 regardless of SELinux"
|
||||
|
||||
gate 'ID="rocky"
|
||||
VERSION_ID="10.0"'
|
||||
rejects "Rocky Linux 10" "Unsupported OS: rocky"
|
||||
|
||||
gate 'this is not an os-release file'
|
||||
rejects "a malformed os-release" "Unsupported OS"
|
||||
|
||||
if [ "$fails" -gt 0 ]; then
|
||||
echo "$fails failure(s)"
|
||||
exit 1
|
||||
fi
|
||||
@@ -60,7 +60,12 @@ export AWS_PROFILE=spinifex
|
||||
# + CA bundle) is used when the script is called via sudo.
|
||||
INVOKING_USER="${SUDO_USER:-$(id -un)}"
|
||||
INVOKING_HOME=$(getent passwd "$INVOKING_USER" | cut -d: -f6)
|
||||
aws_as_user() { sudo -u "$INVOKING_USER" env HOME="$INVOKING_HOME" AWS_PROFILE=spinifex aws "$@"; }
|
||||
# By absolute path: sudo's secure_path leaves out /usr/local/bin on some hosts,
|
||||
# Oracle Linux among them, and setup.sh installs the AWS CLI there.
|
||||
AWS_BIN=$(command -v aws || true)
|
||||
[ -z "$AWS_BIN" ] && [ -x /usr/local/bin/aws ] && AWS_BIN=/usr/local/bin/aws
|
||||
[ -n "$AWS_BIN" ] || { echo "❌ aws CLI not found on PATH or at /usr/local/bin/aws" >&2; exit 1; }
|
||||
aws_as_user() { sudo -u "$INVOKING_USER" env HOME="$INVOKING_HOME" AWS_PROFILE=spinifex "$AWS_BIN" "$@"; }
|
||||
|
||||
# --- Wait for EC2 daemon to subscribe to NATS ---
|
||||
# Port 3000 (UI) becomes ready before the daemon finishes initialising.
|
||||
|
||||
+168
-25
@@ -12,9 +12,10 @@
|
||||
# Options:
|
||||
# --purge-data Also remove /var/lib/spinifex — volumes, S3 objects, JetStream.
|
||||
# Off by default: uninstalling software should not destroy data.
|
||||
# --purge-deps Also apt-remove the packages setup.sh installed. Names them and
|
||||
# confirms first, even under --yes. Off by default because a
|
||||
# curl|bash install lands on hosts that were already using them.
|
||||
# --purge-deps Also remove the packages setup.sh installed, with apt. Refused
|
||||
# on dnf hosts, where it cannot tell those from the host's own.
|
||||
# Names them and confirms first, even under --yes. Off by default
|
||||
# because a curl|bash install lands on hosts already using them.
|
||||
# --yes Skip the confirmation prompt. Does NOT cover --purge-data,
|
||||
# --purge-deps, or the multi-node refusal.
|
||||
# --dry-run Print the whole plan, touch nothing, exit 0.
|
||||
@@ -30,14 +31,22 @@
|
||||
# /usr/share/spinifex, every spinifex systemd unit and slice, the eight
|
||||
# spinifex-* service users, the sudoers grant, the sysctl/logrotate/chrony/udev
|
||||
# drop-ins, the host firewall policy and its loaded nft table, the OVS bridges
|
||||
# and OVN databases, and the Spinifex CA from the host trust store.
|
||||
# and OVN databases, the Spinifex resolved drop-in, and the Spinifex CA from the
|
||||
# host trust store. Where setup.sh turned on systemd-resolved, it restores
|
||||
# /etc/resolv.conf and turns resolved off.
|
||||
#
|
||||
# WHAT THIS LEAVES ALONE
|
||||
#
|
||||
# /var/lib/spinifex unless --purge-data. Every apt package unless --purge-deps.
|
||||
# /var/lib/spinifex unless --purge-data. Every package unless --purge-deps.
|
||||
# /swapfile, which setup.sh may have created but cannot be told apart from one
|
||||
# the host already had. Anything the operator configured around us: netplan,
|
||||
# /etc/iptables/rules.v4, hand-written bridges.
|
||||
# /etc/iptables/rules.v4, hand-written bridges. firewalld, which setup.sh
|
||||
# masked, is unmasked but not started. Host settings setup.sh changed and other
|
||||
# software may now rely on: /etc/modules-load.d/spinifex.conf, the confdir line
|
||||
# appended to an EL /etc/chrony.conf, the dnf repositories and AppStream QEMU
|
||||
# exclude, OVS_USER_ID and OVN_USER_ID, the OVS and OVN logrotate edits, masked
|
||||
# IPsec units, the SELinux mode, and the GPU passthrough config spx admin gpu
|
||||
# wrote. The run ends by listing each of these it finds.
|
||||
set -euo pipefail
|
||||
|
||||
ETC_DIR="${SPX_ETC_DIR:-/etc/spinifex}"
|
||||
@@ -73,6 +82,32 @@ while [[ $# -gt 0 ]]; do
|
||||
shift
|
||||
done
|
||||
|
||||
# Host-OS capability probes: beside this script in a checkout and in the
|
||||
# installed share dir, else where setup.sh installs them.
|
||||
DISTRO_LIB=""
|
||||
for candidate in "$(dirname "${BASH_SOURCE[0]}")/lib/distro.sh" /usr/local/share/spinifex/lib/distro.sh; do
|
||||
if [ -f "$candidate" ]; then DISTRO_LIB="$candidate"; break; fi
|
||||
done
|
||||
if [ -z "$DISTRO_LIB" ]; then
|
||||
echo "ERROR: distro.sh not found at $(dirname "${BASH_SOURCE[0]}")/lib/distro.sh or /usr/local/share/spinifex/lib/distro.sh; re-run setup.sh to install it" >&2
|
||||
exit 1
|
||||
fi
|
||||
# shellcheck source=lib/distro.sh
|
||||
. "$DISTRO_LIB"
|
||||
|
||||
# Resolved before anything is removed, so --purge-deps on a host with no package
|
||||
# manager, or two, stops with nothing changed.
|
||||
PKG_MANAGER=""
|
||||
if $PURGE_DEPS; then
|
||||
PKG_MANAGER=$(distro_pkg_manager) || exit 1
|
||||
if [ "$PKG_MANAGER" = dnf ]; then
|
||||
echo "ERROR: --purge-deps is not supported on dnf hosts: its package list cannot tell what setup.sh installed from what the host already had, and dnf also removes every package that requires them. Re-run without --purge-deps." >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
# A failed OVS unit is restarted after its drop-in goes; absent when OVS is.
|
||||
OVS_UNIT=$(distro_ovs_unit 2>/dev/null) || OVS_UNIT=""
|
||||
|
||||
log() { echo "[uninstall-spx] $*"; }
|
||||
run() {
|
||||
if $DRY_RUN; then
|
||||
@@ -147,7 +182,7 @@ if [ -d "$DATA_DIR" ]; then
|
||||
log " keeping $DATA_DIR ($instances instance(s), $volumes volume(s), ${size:-unknown}) — pass --purge-data to remove it"
|
||||
fi
|
||||
fi
|
||||
$PURGE_DEPS && log " --purge-deps: apt packages will be removed (confirmed separately below)"
|
||||
$PURGE_DEPS && log " --purge-deps: $PKG_MANAGER packages will be removed (confirmed separately below)"
|
||||
|
||||
if ! $ASSUME_YES && ! $DRY_RUN; then
|
||||
read -r -p "Remove the Spinifex install from this node? Type 'uninstall' to continue: " reply
|
||||
@@ -208,7 +243,7 @@ fi
|
||||
#
|
||||
# It handles the guests, the OVS bridges, the OVN databases and the JetStream
|
||||
# store, and it carries correctness that is invisible in its code: QEMU matched
|
||||
# as ^qemu-system because comm truncates at 15 characters, a wait for QEMU to
|
||||
# by a comm prefix because comm truncates at 15 characters, a wait for QEMU to
|
||||
# exit rather than for the signal, and the firewall-mode stash that stops an
|
||||
# interrupted run leaving the node unarmed. A second copy of that would drift,
|
||||
# and the failure mode of the drift is data loss.
|
||||
@@ -265,6 +300,16 @@ if command -v iptables-save >/dev/null 2>&1 && ! $DRY_RUN; then
|
||||
elif $DRY_RUN; then
|
||||
echo " would remove iptables rules commented spinifex-nat-egress / spinifex-eip-ingress / spinifex-imds / spinifex-imds-remap"
|
||||
fi
|
||||
# setup.sh masks firewalld. Unmasked but left disabled: its rules were never
|
||||
# migrated, so starting it is the operator's decision. A mask keeps the enable
|
||||
# links, so without the disable it would start again at the next boot.
|
||||
FIREWALLD_UNMASKED=false
|
||||
if [ "$(systemctl is-enabled firewalld.service 2>/dev/null)" = masked ]; then
|
||||
run sudo systemctl unmask firewalld.service
|
||||
run sudo systemctl disable firewalld.service 2>/dev/null || true
|
||||
FIREWALLD_UNMASKED=true
|
||||
$DRY_RUN || log " firewalld unmasked, left disabled"
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 6. systemd units.
|
||||
@@ -294,12 +339,14 @@ rm_path /etc/systemd/system/spinifex-*.service \
|
||||
# exactly that way on the first dev-oci round trip.
|
||||
log "removing drop-ins written into other units"
|
||||
rm_path /etc/systemd/system/openvswitch-switch.service.d/spinifex-perms.conf \
|
||||
/etc/systemd/system/ovs-vswitchd.service.d/spinifex-perms.conf \
|
||||
/etc/systemd/system/ovn-controller.service.d/spinifex-perms.conf \
|
||||
/etc/systemd/system/ovn-controller.service.d/log-level.conf \
|
||||
/etc/systemd/system/openvswitch-ipsec.service.d/spinifex-perms.conf \
|
||||
/etc/systemd/system/ovn-northd.service.d/spinifex-perms.conf \
|
||||
/etc/systemd/system/ovn-northd.service.d/no-manage-ovsdb.conf
|
||||
# Only if we emptied them — another package may own the same override dir.
|
||||
for d in openvswitch-switch ovn-controller openvswitch-ipsec ovn-northd; do
|
||||
for d in openvswitch-switch ovs-vswitchd ovn-controller openvswitch-ipsec ovn-northd; do
|
||||
dir="/etc/systemd/system/${d}.service.d"
|
||||
if sudo test -d "$dir" && [ -z "$(sudo ls -A "$dir" 2>/dev/null)" ]; then
|
||||
rm_path "$dir"
|
||||
@@ -313,8 +360,11 @@ run sudo systemctl reset-failed 2>/dev/null || true
|
||||
# attempt. Restarting them here is not housekeeping: openvswitch-switch failing
|
||||
# is how the operator discovers the drop-in problem, and leaving it failed after
|
||||
# an uninstall that claims success is the same silent failure one step later.
|
||||
# Each unit that carried one of the drop-ins above, under this host's OVS name.
|
||||
RESTART_UNITS=(ovs-vswitchd ovn-controller ovn-northd)
|
||||
[ -n "$OVS_UNIT" ] && RESTART_UNITS=("$OVS_UNIT" "${RESTART_UNITS[@]}")
|
||||
if ! $DRY_RUN; then
|
||||
for u in openvswitch-switch ovn-controller; do
|
||||
for u in "${RESTART_UNITS[@]}"; do
|
||||
if systemctl is-failed --quiet "$u" 2>/dev/null; then
|
||||
log " restarting $u, which our drop-in had failed"
|
||||
sudo systemctl reset-failed "$u" 2>/dev/null || true
|
||||
@@ -367,6 +417,10 @@ rm_path /usr/local/bin/spx \
|
||||
/usr/local/share/spinifex \
|
||||
/usr/local/lib/spinifex \
|
||||
/usr/share/spinifex
|
||||
# setup.sh adds this link where sudo's PATH leaves out /usr/local/bin.
|
||||
if [ "$(readlink /usr/bin/spx 2>/dev/null)" = /usr/local/bin/spx ]; then
|
||||
rm_path /usr/bin/spx
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 9. Drop-ins. Globbed, not listed: 99-spinifex-vpc.conf is written at runtime
|
||||
@@ -378,9 +432,15 @@ log "removing configuration drop-ins"
|
||||
rm_path /etc/sysctl.d/*spinifex* \
|
||||
/etc/sudoers.d/spinifex-network \
|
||||
/etc/chrony/conf.d/spinifex.conf \
|
||||
/etc/chrony.d/spinifex.conf \
|
||||
/etc/logrotate.d/spinifex \
|
||||
/etc/udev/rules.d/*spinifex*
|
||||
/etc/tmpfiles.d/spinifex.conf \
|
||||
/etc/udev/rules.d/*spinifex* \
|
||||
/etc/NetworkManager/conf.d/*spinifex*
|
||||
run sudo sysctl --system >/dev/null 2>&1 || true
|
||||
if systemctl is-active --quiet NetworkManager.service 2>/dev/null; then
|
||||
run sudo nmcli general reload conf 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 10. Spinifex directories.
|
||||
@@ -431,14 +491,42 @@ done
|
||||
# 12. The CA in the host trust store. Leaving it means the host trusts a CA
|
||||
# whose key no longer exists anywhere.
|
||||
# ---------------------------------------------------------------------------
|
||||
if [ -f /usr/local/share/ca-certificates/spinifex-ca.crt ]; then
|
||||
CA_TRUST=$(distro_ca_trust) || { echo "ERROR: cannot tell which trust store to remove the Spinifex CA from" >&2; exit 1; }
|
||||
CA_ANCHOR="${CA_TRUST#* }/spinifex-ca.crt"
|
||||
if [ -f "$CA_ANCHOR" ]; then
|
||||
log "removing the Spinifex CA from the host trust store"
|
||||
rm_path /usr/local/share/ca-certificates/spinifex-ca.crt
|
||||
run sudo update-ca-certificates >/dev/null 2>&1 || true
|
||||
rm_path "$CA_ANCHOR"
|
||||
run sudo "${CA_TRUST%% *}"
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 13. Swap is reported, never removed.
|
||||
# 13. Host DNS. setup.sh saved /etc/resolv.conf only where it turned on
|
||||
# systemd-resolved, so the saved copy is what says to turn it off again.
|
||||
# ---------------------------------------------------------------------------
|
||||
RESOLV_CONF_SAVED=/var/lib/spinifex-reset/resolv.conf.orig
|
||||
# The daemon's drop-in routes the cluster zones to a northstar that is gone now.
|
||||
RESOLVED_DROPIN=/etc/systemd/resolved.conf.d/spinifex-dns.conf
|
||||
if exists "$RESOLVED_DROPIN"; then
|
||||
log "removing the systemd-resolved drop-in for the cluster zones"
|
||||
rm_path "$RESOLVED_DROPIN"
|
||||
if systemctl is-active --quiet systemd-resolved.service 2>/dev/null; then
|
||||
run sudo systemctl restart systemd-resolved.service
|
||||
fi
|
||||
fi
|
||||
if exists "$RESOLV_CONF_SAVED"; then
|
||||
log "restoring /etc/resolv.conf and turning systemd-resolved off, as before the install"
|
||||
run sudo systemctl disable --now systemd-resolved.service
|
||||
run sudo cp -a --remove-destination "$RESOLV_CONF_SAVED" /etc/resolv.conf
|
||||
if systemctl is-active --quiet NetworkManager.service 2>/dev/null; then
|
||||
run sudo systemctl restart NetworkManager.service
|
||||
fi
|
||||
rm_path "$RESOLV_CONF_SAVED"
|
||||
fi
|
||||
# node-reset keeps its firewall-mode stash here too, so remove it only once empty.
|
||||
run sudo rmdir --ignore-fail-on-non-empty "$(dirname "$RESOLV_CONF_SAVED")" 2>/dev/null || true
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 14. Swap is reported, never removed.
|
||||
#
|
||||
# setup_swap creates an 8G /swapfile with the fstab line `/swapfile none swap sw
|
||||
# 0 0` — which is byte-identical to the line a host would already have if it
|
||||
@@ -455,26 +543,32 @@ if [ -f /swapfile ] && grep -q '^/swapfile ' /etc/fstab 2>/dev/null; then
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 14. apt packages, only when asked, and only after naming them.
|
||||
# 15. Packages, only when asked, and only after naming them.
|
||||
#
|
||||
# This is the one irreversible thing in the script. On a dedicated node these
|
||||
# are ours; on the host a curl|bash install actually lands on they are someone
|
||||
# else's, and openvswitch/libvirt/qemu/chrony going away underneath a running
|
||||
# workload is a far worse outcome than a few megabytes left behind. Confirmed
|
||||
# separately even under --yes, and never autoremove, which is unbounded.
|
||||
#
|
||||
# apt only: dnf hosts are refused up front.
|
||||
# ---------------------------------------------------------------------------
|
||||
if $PURGE_DEPS; then
|
||||
PACKAGES="nbdkit qemu-utils gdisk ovmf less libvirt-daemon-system libvirt-clients
|
||||
pciutils jq ethtool netcat-openbsd unzip xz-utils
|
||||
ovn-central ovn-host openvswitch-switch openvswitch-ipsec strongswan-charon
|
||||
chrony nftables"
|
||||
KEPT="curl, wget, file, iproute2, dhcpcd-base, qemu-system-*"
|
||||
REMOVE_CMD="apt-get remove -y"
|
||||
log "--purge-deps: these packages would be removed:"
|
||||
# shellcheck disable=SC2086
|
||||
for p in $PACKAGES; do echo " $p"; done
|
||||
log " NOT removed: curl, wget, file, iproute2, dhcpcd-base, qemu-system-* — too"
|
||||
log " NOT removed: $KEPT — too"
|
||||
log " widely depended on to remove from a host we do not own."
|
||||
if $DRY_RUN; then
|
||||
echo " would run: apt-get remove -y <the above>"
|
||||
if [ -z "$PACKAGES" ]; then
|
||||
log " none of them is installed"
|
||||
elif $DRY_RUN; then
|
||||
echo " would run: $REMOVE_CMD <the above>"
|
||||
else
|
||||
read -r -p "Remove these packages? Type 'remove' to continue: " reply
|
||||
if [ "$reply" = "remove" ]; then
|
||||
@@ -488,7 +582,7 @@ chrony nftables"
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 15. Verify. An uninstaller that reports success over a surviving unit file is
|
||||
# 16. Verify. An uninstaller that reports success over a surviving unit file is
|
||||
# worse than one that refuses, because the next install silently inherits it.
|
||||
# ---------------------------------------------------------------------------
|
||||
if ! $DRY_RUN; then
|
||||
@@ -496,21 +590,26 @@ if ! $DRY_RUN; then
|
||||
LEFT=()
|
||||
for p in /usr/local/bin/spx /usr/local/share/spinifex /usr/local/lib/spinifex \
|
||||
/usr/share/spinifex "$ETC_DIR" "$LOG_DIR" \
|
||||
/etc/sudoers.d/spinifex-network /etc/logrotate.d/spinifex \
|
||||
/etc/chrony/conf.d/spinifex.conf; do
|
||||
/etc/sudoers.d/spinifex-network /etc/logrotate.d/spinifex /etc/tmpfiles.d/spinifex.conf \
|
||||
/etc/chrony/conf.d/spinifex.conf /etc/chrony.d/spinifex.conf "$RESOLV_CONF_SAVED" \
|
||||
"$RESOLVED_DROPIN"; do
|
||||
exists "$p" && LEFT+=("$p")
|
||||
done
|
||||
[ "$(readlink /usr/bin/spx 2>/dev/null)" = /usr/local/bin/spx ] && LEFT+=(/usr/bin/spx)
|
||||
for p in /etc/systemd/system/spinifex-*.service /etc/systemd/system/spinifex*.slice \
|
||||
/etc/systemd/system/spinifex.target /etc/sysctl.d/*spinifex* \
|
||||
/etc/systemd/network/1[5-8]-spinifex-* \
|
||||
/etc/NetworkManager/conf.d/*spinifex* \
|
||||
/etc/systemd/system/*.service.d/spinifex-perms.conf \
|
||||
/etc/systemd/system/ovn-northd.service.d/no-manage-ovsdb.conf \
|
||||
/etc/systemd/system/ovn-controller.service.d/log-level.conf; do
|
||||
exists "$p" && LEFT+=("$p")
|
||||
done
|
||||
$FIREWALLD_UNMASKED && systemctl is-enabled --quiet firewalld.service 2>/dev/null &&
|
||||
LEFT+=("firewalld.service is enabled")
|
||||
# A unit we broke and could not restart is a failed uninstall, not a wart.
|
||||
for u in openvswitch-switch ovn-controller; do
|
||||
systemctl is-failed --quiet "$u" 2>/dev/null && LEFT+=("$u.service is failed")
|
||||
for u in "${RESTART_UNITS[@]}"; do
|
||||
systemctl is-failed --quiet "$u" 2>/dev/null && LEFT+=("${u%.service}.service is failed")
|
||||
done
|
||||
for svc in nats gw daemon storage northstar viperblock vpcd ui; do
|
||||
id -u "spinifex-${svc}" >/dev/null 2>&1 && LEFT+=("user spinifex-${svc}")
|
||||
@@ -527,7 +626,51 @@ if ! $DRY_RUN; then
|
||||
fi
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 17. Host settings setup.sh changed and this leaves, each listed only where it
|
||||
# is found, so the operator can see what to undo by hand.
|
||||
# ---------------------------------------------------------------------------
|
||||
KEPT_SETTINGS=()
|
||||
kept() { KEPT_SETTINGS+=("$1"); }
|
||||
$FIREWALLD_UNMASKED && kept "firewalld: unmasked but disabled, with the rules it had before Spinifex"
|
||||
exists /etc/modules-load.d/spinifex.conf && kept "/etc/modules-load.d/spinifex.conf: openvswitch geneve vhost_net tun load at boot"
|
||||
CHRONY_CONF=$(distro_chrony_conf 2>/dev/null) || CHRONY_CONF=""
|
||||
if [ -n "$CHRONY_CONF" ] && grep -q '^# Added by the Spinifex installer' "$CHRONY_CONF" 2>/dev/null; then
|
||||
kept "$CHRONY_CONF: the confdir line the installer appended"
|
||||
fi
|
||||
if command -v dnf >/dev/null 2>&1; then
|
||||
exists /etc/yum.repos.d/spinifex-nfv-sig.repo &&
|
||||
kept "dnf: the CentOS NFV SIG repository (/etc/yum.repos.d/spinifex-nfv-sig.repo) and its key in /etc/pki/rpm-gpg"
|
||||
for repo in ol10_kvm_utils ol10_UEKR8; do
|
||||
dnf config-manager --dump "$repo" 2>/dev/null | grep -qx 'enabled = 1' && kept "dnf: $repo enabled"
|
||||
done
|
||||
dnf config-manager --dump ol10_appstream 2>/dev/null | grep -q '^excludepkgs = .*qemu-kvm' &&
|
||||
kept "dnf: ol10_appstream excludes qemu-kvm* and qemu-img*, so QEMU stays on KVM Utilities"
|
||||
fi
|
||||
for setting in /etc/sysconfig/openvswitch:OVS_USER_ID /etc/sysconfig/ovn:OVN_USER_ID; do
|
||||
grep -q "^${setting#*:}=\"root:root\"" "${setting%%:*}" 2>/dev/null &&
|
||||
kept "${setting%%:*}: ${setting#*:}=\"root:root\""
|
||||
done
|
||||
for lr in "$(distro_ovs_logrotate 2>/dev/null)" "$(distro_ovn_logrotate 2>/dev/null)"; do
|
||||
[ -n "$lr" ] && grep -q 'maxsize 100M' "$lr" 2>/dev/null && kept "$lr: the rotate, maxsize and su edits setup-ovn.sh made"
|
||||
done
|
||||
IPSEC_UNITS=(openvswitch-ipsec.service strongswan-starter.service)
|
||||
distro_has_libreswan && IPSEC_UNITS+=(ipsec.service)
|
||||
for u in "${IPSEC_UNITS[@]}"; do
|
||||
[ "$(systemctl is-enabled "$u" 2>/dev/null)" = masked ] && kept "$u: masked, as IPsec was off"
|
||||
done
|
||||
command -v getenforce >/dev/null 2>&1 && kept "SELinux: $(getenforce 2>/dev/null); setup.sh refuses enforcing, so it was set for the install"
|
||||
GPU_FILES=""
|
||||
for gpu in /etc/modprobe.d/vfio-pci.conf /etc/dracut.conf.d/spinifex-vfio.conf; do
|
||||
exists "$gpu" && GPU_FILES="${GPU_FILES:+$GPU_FILES, }$gpu"
|
||||
done
|
||||
[ -n "$GPU_FILES" ] && kept "GPU passthrough: $GPU_FILES and the IOMMU kernel arguments, written by spx admin gpu setup"
|
||||
if [ "${#KEPT_SETTINGS[@]}" -gt 0 ]; then
|
||||
log "left in place, as other software on the host may rely on them:"
|
||||
for k in "${KEPT_SETTINGS[@]}"; do log " - $k"; done
|
||||
fi
|
||||
|
||||
log "done — Spinifex is removed from this node"
|
||||
$PURGE_DATA || { [ -d "$DATA_DIR" ] && log " $DATA_DIR was kept (--purge-data removes it)"; }
|
||||
$PURGE_DEPS || log " apt dependencies were kept (--purge-deps removes them)"
|
||||
$PURGE_DEPS || log " package dependencies were kept (--purge-deps removes them)"
|
||||
exit 0
|
||||
|
||||
@@ -56,6 +56,17 @@ grep -q "unknown option" "$TMP/err" \
|
||||
&& pass "unknown option explains itself on stderr" \
|
||||
|| fail "unknown option wrote nothing useful to stderr"
|
||||
|
||||
# --purge-deps on a dnf host stops before anything is touched. PATH holds only
|
||||
# the tools the script needs up to that check, with dnf and no apt-get.
|
||||
mkdir -p "$TMP/dnfbin"
|
||||
for tool in bash dirname; do ln -s "$(command -v "$tool")" "$TMP/dnfbin/$tool"; done
|
||||
printf '#!/bin/sh\nexit 0\n' >"$TMP/dnfbin/dnf"
|
||||
chmod +x "$TMP/dnfbin/dnf"
|
||||
PATH="$TMP/dnfbin" "$TMP/dnfbin/bash" "$UNINSTALL" --purge-deps --yes >"$TMP/out" 2>"$TMP/err" && rc=0 || rc=$?
|
||||
[ "$rc" = 1 ] && grep -q "not supported on dnf hosts" "$TMP/err" \
|
||||
&& pass "--purge-deps is refused on a dnf host, saying why" \
|
||||
|| fail "--purge-deps on a dnf host: rc=$rc: $(cat "$TMP/err")"
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# The multi-node refusal.
|
||||
#
|
||||
|
||||
@@ -29,7 +29,12 @@ SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
|
||||
INVOKING_USER="${SUDO_USER:-$(id -un)}"
|
||||
INVOKING_HOME=$(getent passwd "$INVOKING_USER" | cut -d: -f6)
|
||||
aws_as_user() { sudo -u "$INVOKING_USER" env HOME="$INVOKING_HOME" AWS_PROFILE=spinifex aws "$@"; }
|
||||
# By absolute path: sudo's secure_path leaves out /usr/local/bin on some hosts,
|
||||
# Oracle Linux among them, and setup.sh installs the AWS CLI there.
|
||||
AWS_BIN=$(command -v aws || true)
|
||||
[ -z "$AWS_BIN" ] && [ -x /usr/local/bin/aws ] && AWS_BIN=/usr/local/bin/aws
|
||||
[ -n "$AWS_BIN" ] || { echo "ERROR: aws CLI not found on PATH or at /usr/local/bin/aws" >&2; exit 1; }
|
||||
aws_as_user() { sudo -u "$INVOKING_USER" env HOME="$INVOKING_HOME" AWS_PROFILE=spinifex "$AWS_BIN" "$@"; }
|
||||
|
||||
FAILURES=0
|
||||
pass() { printf ' \033[32m✓\033[0m %s\n' "$*"; }
|
||||
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ecs"
|
||||
ecsdomain "github.com/mulgadc/spinifex/spinifex/domains/ecs"
|
||||
handlers_ecs "github.com/mulgadc/spinifex/spinifex/handlers/ecs"
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
@@ -18,7 +18,7 @@ import (
|
||||
// leaves an INACTIVE record that still lists, so a stage waiting for services
|
||||
// to disappear would never drain.
|
||||
func ECSReapers(nc *nats.Conn) []Reaper {
|
||||
return []Reaper{&ecsClusterReaper{svc: ecsdomain.NewNATSECSService(nc)}}
|
||||
return []Reaper{&ecsClusterReaper{svc: handlers_ecs.NewNATSECSService(nc)}}
|
||||
}
|
||||
|
||||
// ecsClusterReaper removes the account's ECS clusters.
|
||||
@@ -27,7 +27,7 @@ func ECSReapers(nc *nats.Conn) []Reaper {
|
||||
// so the instance reaper takes them — but only after this one has stopped the
|
||||
// tasks riding on them, which is why this is registered first.
|
||||
type ecsClusterReaper struct {
|
||||
svc ecsdomain.ECSService
|
||||
svc handlers_ecs.ECSService
|
||||
}
|
||||
|
||||
func (r *ecsClusterReaper) Kind() string { return "ecs-cluster" }
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ecs"
|
||||
"github.com/mulgadc/spinifex/internal/testkit"
|
||||
ecsdomain "github.com/mulgadc/spinifex/spinifex/domains/ecs"
|
||||
handlers_ecs "github.com/mulgadc/spinifex/spinifex/handlers/ecs"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -94,7 +94,7 @@ func (c *fakeECS) request(t *testing.T, subject string, into any) {
|
||||
}
|
||||
|
||||
func newECSReaper(nc *nats.Conn) *ecsClusterReaper {
|
||||
return &ecsClusterReaper{svc: ecsdomain.NewNATSECSService(nc)}
|
||||
return &ecsClusterReaper{svc: handlers_ecs.NewNATSECSService(nc)}
|
||||
}
|
||||
|
||||
// The listing is keyed by cluster name, which is the identity ECS takes back
|
||||
|
||||
@@ -5,7 +5,7 @@ import (
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/elbv2"
|
||||
elbv2domain "github.com/mulgadc/spinifex/spinifex/domains/elbv2"
|
||||
handlers_elbv2 "github.com/mulgadc/spinifex/spinifex/handlers/elbv2"
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
@@ -15,7 +15,7 @@ import (
|
||||
// groups, so it has to go before either can be deleted. Target groups follow
|
||||
// their balancers because a group still referenced by a listener refuses.
|
||||
func ELBv2Reapers(nc *nats.Conn) []Reaper {
|
||||
svc := elbv2domain.NewNATSELBv2Service(nc)
|
||||
svc := handlers_elbv2.NewNATSELBv2Service(nc)
|
||||
return []Reaper{
|
||||
&loadBalancerReaper{svc: svc},
|
||||
&targetGroupReaper{svc: svc},
|
||||
@@ -23,7 +23,7 @@ func ELBv2Reapers(nc *nats.Conn) []Reaper {
|
||||
}
|
||||
|
||||
type loadBalancerReaper struct {
|
||||
svc elbv2domain.ELBv2Service
|
||||
svc handlers_elbv2.ELBv2Service
|
||||
}
|
||||
|
||||
func (r *loadBalancerReaper) Kind() string { return "load-balancer" }
|
||||
@@ -58,7 +58,7 @@ func (r *loadBalancerReaper) Delete(ctx context.Context, accountID string, resou
|
||||
}
|
||||
|
||||
type targetGroupReaper struct {
|
||||
svc elbv2domain.ELBv2Service
|
||||
svc handlers_elbv2.ELBv2Service
|
||||
}
|
||||
|
||||
func (r *targetGroupReaper) Kind() string { return "target-group" }
|
||||
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/elbv2"
|
||||
elbv2domain "github.com/mulgadc/spinifex/spinifex/domains/elbv2"
|
||||
handlers_elbv2 "github.com/mulgadc/spinifex/spinifex/handlers/elbv2"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -18,7 +18,7 @@ import (
|
||||
// The embedded interface is nil, so a reaper reaching for anything beyond the
|
||||
// four methods below panics rather than passing.
|
||||
type fakeELBv2 struct {
|
||||
elbv2domain.ELBv2Service
|
||||
handlers_elbv2.ELBv2Service
|
||||
|
||||
loadBalancers []*elbv2.LoadBalancer
|
||||
targetGroups []*elbv2.TargetGroup
|
||||
|
||||
@@ -12,7 +12,7 @@ import (
|
||||
ecsv1 "github.com/mulgadc/spinifex/contracts/ecs/v1"
|
||||
"github.com/mulgadc/spinifex/internal/ecsgw"
|
||||
"github.com/mulgadc/spinifex/spinifex/agents/ecs/credentials"
|
||||
ecsdomain "github.com/mulgadc/spinifex/spinifex/domains/ecs"
|
||||
handlers_ecs "github.com/mulgadc/spinifex/spinifex/handlers/ecs"
|
||||
)
|
||||
|
||||
// controlPlane is the agent's channel to the AWS gateway. It replaces the direct
|
||||
@@ -31,7 +31,7 @@ type controlPlane interface {
|
||||
PollAssignments(cluster, instance string, ackAssigns, ackStops []string) ([]ecsv1.Assign, []ecsv1.StopDirective, error)
|
||||
// ReportTaskGPU reports the local ledger's pinned device UUIDs for a task's
|
||||
// containers, so the control plane can populate DescribeTasks gpuIds.
|
||||
ReportTaskGPU(cluster, task string, containers []ecsdomain.ContainerGPUReport) error
|
||||
ReportTaskGPU(cluster, task string, containers []handlers_ecs.ContainerGPUReport) error
|
||||
}
|
||||
|
||||
// gatewayControlPlane implements controlPlane over the SigV4 ECS gateway client.
|
||||
@@ -125,7 +125,7 @@ func (g *gatewayControlPlane) SubmitTaskState(st ecsv1.TaskState) error {
|
||||
}
|
||||
|
||||
func (g *gatewayControlPlane) PollAssignments(cluster, instance string, ackAssigns, ackStops []string) ([]ecsv1.Assign, []ecsv1.StopDirective, error) {
|
||||
in := &ecsdomain.PollAssignmentsInput{
|
||||
in := &handlers_ecs.PollAssignmentsInput{
|
||||
Cluster: cluster,
|
||||
ContainerInstance: instance,
|
||||
AckTaskIDs: ackAssigns,
|
||||
@@ -139,15 +139,15 @@ func (g *gatewayControlPlane) PollAssignments(cluster, instance string, ackAssig
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("poll: %w", err)
|
||||
}
|
||||
var out ecsdomain.PollAssignmentsOutput
|
||||
var out handlers_ecs.PollAssignmentsOutput
|
||||
if err := json.Unmarshal(resp, &out); err != nil {
|
||||
return nil, nil, fmt.Errorf("decode poll: %w", err)
|
||||
}
|
||||
return out.Assignments, out.Stops, nil
|
||||
}
|
||||
|
||||
func (g *gatewayControlPlane) ReportTaskGPU(cluster, task string, containers []ecsdomain.ContainerGPUReport) error {
|
||||
in := &ecsdomain.ReportTaskGPUInput{Cluster: cluster, Task: task, Containers: containers}
|
||||
func (g *gatewayControlPlane) ReportTaskGPU(cluster, task string, containers []handlers_ecs.ContainerGPUReport) error {
|
||||
in := &handlers_ecs.ReportTaskGPUInput{Cluster: cluster, Task: task, Containers: containers}
|
||||
body, err := json.Marshal(in)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal report-task-gpu: %w", err)
|
||||
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
|
||||
"github.com/aws/aws-sdk-go/service/ecs"
|
||||
"github.com/mulgadc/spinifex/spinifex/agents/ecs/credentials"
|
||||
ecsdomain "github.com/mulgadc/spinifex/spinifex/domains/ecs"
|
||||
handlers_ecs "github.com/mulgadc/spinifex/spinifex/handlers/ecs"
|
||||
)
|
||||
|
||||
// fixedCredsProvider is a static credentials.CredentialsProvider for tests
|
||||
@@ -101,7 +101,7 @@ func TestGatewayControlPlane_Register_NoGPUOmitsResource(t *testing.T) {
|
||||
}
|
||||
|
||||
// ReportTaskGPU posts the per-container device UUIDs under the ReportTaskGPU
|
||||
// action, decodable as the domains/ecs internal shape.
|
||||
// action, decodable as the handlers_ecs internal shape.
|
||||
func TestGatewayControlPlane_ReportTaskGPU(t *testing.T) {
|
||||
var gotTarget string
|
||||
var body []byte
|
||||
@@ -113,7 +113,7 @@ func TestGatewayControlPlane_ReportTaskGPU(t *testing.T) {
|
||||
defer srv.Close()
|
||||
|
||||
cp := newTestGatewayControlPlane(t, srv)
|
||||
err := cp.ReportTaskGPU("web", "t-1", []ecsdomain.ContainerGPUReport{
|
||||
err := cp.ReportTaskGPU("web", "t-1", []handlers_ecs.ContainerGPUReport{
|
||||
{Name: "trainer", GPUIDs: []string{"GPU-aaa"}},
|
||||
})
|
||||
if err != nil {
|
||||
@@ -123,7 +123,7 @@ func TestGatewayControlPlane_ReportTaskGPU(t *testing.T) {
|
||||
t.Fatal("no X-Amz-Target header sent")
|
||||
}
|
||||
|
||||
var in ecsdomain.ReportTaskGPUInput
|
||||
var in handlers_ecs.ReportTaskGPUInput
|
||||
if err := json.Unmarshal(body, &in); err != nil {
|
||||
t.Fatalf("decode body: %v", err)
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ecs"
|
||||
ecsv1 "github.com/mulgadc/spinifex/contracts/ecs/v1"
|
||||
ecsdomain "github.com/mulgadc/spinifex/spinifex/domains/ecs"
|
||||
handlers_ecs "github.com/mulgadc/spinifex/spinifex/handlers/ecs"
|
||||
)
|
||||
|
||||
// identity is the container instance's stable identity, assembled at boot from
|
||||
@@ -34,8 +34,8 @@ func instanceAttributes(id identity) []*ecs.Attribute {
|
||||
}
|
||||
attrs = append(attrs, &ecs.Attribute{Name: aws.String(name), Value: aws.String(value)})
|
||||
}
|
||||
add(ecsdomain.AttrInstanceType, id.InstanceType)
|
||||
add(ecsdomain.AttrAvailabilityZone, id.AZ)
|
||||
add(handlers_ecs.AttrInstanceType, id.InstanceType)
|
||||
add(handlers_ecs.AttrAvailabilityZone, id.AZ)
|
||||
return attrs
|
||||
}
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
"time"
|
||||
|
||||
ecsv1 "github.com/mulgadc/spinifex/contracts/ecs/v1"
|
||||
ecsdomain "github.com/mulgadc/spinifex/spinifex/domains/ecs"
|
||||
handlers_ecs "github.com/mulgadc/spinifex/spinifex/handlers/ecs"
|
||||
)
|
||||
|
||||
// fakeCP is a test double for the gateway control-plane. It records registers,
|
||||
@@ -37,7 +37,7 @@ type fakeCP struct {
|
||||
type gpuReport struct {
|
||||
cluster string
|
||||
task string
|
||||
containers []ecsdomain.ContainerGPUReport
|
||||
containers []handlers_ecs.ContainerGPUReport
|
||||
}
|
||||
|
||||
var _ controlPlane = (*fakeCP)(nil)
|
||||
@@ -93,7 +93,7 @@ func (f *fakeCP) PollAssignments(_, _ string, ackAssigns, ackStops []string) ([]
|
||||
return out, stops, nil
|
||||
}
|
||||
|
||||
func (f *fakeCP) ReportTaskGPU(cluster, task string, containers []ecsdomain.ContainerGPUReport) error {
|
||||
func (f *fakeCP) ReportTaskGPU(cluster, task string, containers []handlers_ecs.ContainerGPUReport) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.gpuReports = append(f.gpuReports, gpuReport{cluster: cluster, task: task, containers: containers})
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
|
||||
ecsv1 "github.com/mulgadc/spinifex/contracts/ecs/v1"
|
||||
ctrruntime "github.com/mulgadc/spinifex/spinifex/agents/ecs/runtime"
|
||||
ecsdomain "github.com/mulgadc/spinifex/spinifex/domains/ecs"
|
||||
handlers_ecs "github.com/mulgadc/spinifex/spinifex/handlers/ecs"
|
||||
)
|
||||
|
||||
// defaultStopTimeout is applied when a container carries no explicit
|
||||
@@ -413,10 +413,10 @@ func (a *Agent) reportTaskState(as *ecsv1.Assign, status, reason string, contain
|
||||
// container in the report that has them, so DescribeTasks can surface gpuIds.
|
||||
// A no-op when nothing was pinned (the common non-GPU-task case).
|
||||
func (a *Agent) reportTaskGPUs(taskID string, containers []ecsv1.ContainerStatus) {
|
||||
reports := make([]ecsdomain.ContainerGPUReport, 0, len(containers))
|
||||
reports := make([]handlers_ecs.ContainerGPUReport, 0, len(containers))
|
||||
for _, c := range containers {
|
||||
if len(c.GPUIDs) > 0 {
|
||||
reports = append(reports, ecsdomain.ContainerGPUReport{Name: c.Name, GPUIDs: c.GPUIDs})
|
||||
reports = append(reports, handlers_ecs.ContainerGPUReport{Name: c.Name, GPUIDs: c.GPUIDs})
|
||||
}
|
||||
}
|
||||
if len(reports) == 0 {
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
package preflight
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
)
|
||||
|
||||
const resolvConf = "/etc/resolv.conf"
|
||||
|
||||
// resolvedStubs are the files that send every lookup through systemd-resolved:
|
||||
// its runtime stub and the static copy it ships.
|
||||
var resolvedStubs = []string{
|
||||
"/run/systemd/resolve/stub-resolv.conf",
|
||||
"/usr/lib/systemd/resolv.conf",
|
||||
"/lib/systemd/resolv.conf",
|
||||
}
|
||||
|
||||
// checkHostDNS warns when systemd-resolved runs but /etc/resolv.conf bypasses
|
||||
// it. The Spinifex zones are routed to northstar inside resolved, so lookups
|
||||
// that skip it never see them.
|
||||
func checkHostDNS(root string) Result {
|
||||
const unit = "systemd-resolved.service"
|
||||
if !unitActive(unit) {
|
||||
return Result{Path: resolvConf, Kind: "host-dns", Status: OK}
|
||||
}
|
||||
target, err := os.Readlink(filepath.Join(root, resolvConf))
|
||||
if err == nil {
|
||||
if !filepath.IsAbs(target) {
|
||||
target = filepath.Join(filepath.Dir(resolvConf), target)
|
||||
}
|
||||
if slices.Contains(resolvedStubs, filepath.Clean(target)) {
|
||||
return Result{Path: resolvConf, Kind: "host-dns", Status: OK}
|
||||
}
|
||||
}
|
||||
what := "is not a link"
|
||||
if err == nil {
|
||||
what = fmt.Sprintf("links to %s", target)
|
||||
}
|
||||
return Result{Path: resolvConf, Kind: "host-dns", Status: Warning,
|
||||
Detail: fmt.Sprintf("%s is active but %s %s to its stub, so Spinifex names may not resolve on this host; run 'ln -sf %s %s'",
|
||||
unit, resolvConf, what, resolvedStubs[0], resolvConf)}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package preflight
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCheckHostAt_HostDNS(t *testing.T) {
|
||||
for name, tc := range map[string]struct {
|
||||
resolved bool
|
||||
link string // target of /etc/resolv.conf; "" writes a regular file
|
||||
want Status
|
||||
detail string
|
||||
}{
|
||||
"resolved inactive, plain file": {want: OK},
|
||||
"resolved on the runtime stub": {resolved: true, link: "/run/systemd/resolve/stub-resolv.conf", want: OK},
|
||||
"resolved on a relative stub link": {resolved: true, link: "../run/systemd/resolve/stub-resolv.conf", want: OK},
|
||||
"resolved on the static stub": {resolved: true, link: "/usr/lib/systemd/resolv.conf", want: OK},
|
||||
"resolved behind a plain file": {resolved: true, want: Warning, detail: "is not a link"},
|
||||
"resolved behind its uplink file": {resolved: true, link: "/run/systemd/resolve/resolv.conf", want: Warning, detail: "links to /run/systemd/resolve/resolv.conf"},
|
||||
"resolved behind NetworkManager file": {resolved: true, link: "/run/NetworkManager/resolv.conf", want: Warning, detail: "links to"},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
orig := unitActive
|
||||
t.Cleanup(func() { unitActive = orig })
|
||||
unitActive = func(unit string) bool { return tc.resolved && unit == "systemd-resolved.service" }
|
||||
root := t.TempDir()
|
||||
path := filepath.Join(root, resolvConf)
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var err error
|
||||
if tc.link == "" {
|
||||
err = os.WriteFile(path, []byte("nameserver 192.0.2.1\n"), 0o644)
|
||||
} else {
|
||||
err = os.Symlink(tc.link, path)
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := findResult(t, CheckHostAt(root), resolvConf)
|
||||
if r.Status != tc.want {
|
||||
t.Fatalf("status = %s %q, want %s", r.Status, r.Detail, tc.want)
|
||||
}
|
||||
if !strings.Contains(r.Detail, tc.detail) {
|
||||
t.Errorf("detail %q does not contain %q", r.Detail, tc.detail)
|
||||
}
|
||||
if r.Status == Warning && HasProblem([]Result{r}) {
|
||||
t.Error("a Warning fails preflight; it must only be reported")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package preflight
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ufwConfig records whether ufw is enabled. ufw.service is a oneshot that stays
|
||||
// active after `ufw disable`, so the unit state alone does not say ufw is on.
|
||||
const ufwConfig = "/etc/ufw/ufw.conf"
|
||||
|
||||
// unitActive is a variable so tests never query a real systemd.
|
||||
var unitActive = func(unit string) bool {
|
||||
return exec.Command("systemctl", "is-active", "--quiet", unit).Run() == nil
|
||||
}
|
||||
|
||||
// checkHostFirewalls fails while firewalld or ufw is active. A forwarded packet
|
||||
// must then be accepted by their ruleset as well as Spinifex's, so routed NAT,
|
||||
// EIPs and IMDS break with nothing logged.
|
||||
func checkHostFirewalls(root string) []Result {
|
||||
return []Result{checkFirewalld(), checkUFW(root)}
|
||||
}
|
||||
|
||||
func checkFirewalld() Result {
|
||||
const unit = "firewalld.service"
|
||||
if unitActive(unit) {
|
||||
return Result{Path: unit, Kind: "host-firewall", Status: Failed,
|
||||
Detail: "firewalld is active beside the Spinifex nftables policy; run 'systemctl mask --now firewalld'"}
|
||||
}
|
||||
return Result{Path: unit, Kind: "host-firewall", Status: OK}
|
||||
}
|
||||
|
||||
func checkUFW(root string) Result {
|
||||
const unit = "ufw.service"
|
||||
if unitActive(unit) && ufwEnabled(root) {
|
||||
return Result{Path: unit, Kind: "host-firewall", Status: Failed,
|
||||
Detail: "ufw is enabled beside the Spinifex nftables policy; run 'ufw disable'"}
|
||||
}
|
||||
return Result{Path: unit, Kind: "host-firewall", Status: OK}
|
||||
}
|
||||
|
||||
// ufwEnabled reads ENABLED= from ufw.conf, as ufw-init does. A missing file
|
||||
// means ufw is not installed.
|
||||
func ufwEnabled(root string) bool {
|
||||
data, err := os.ReadFile(filepath.Join(root, ufwConfig))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
sc := bufio.NewScanner(bytes.NewReader(data))
|
||||
for sc.Scan() {
|
||||
key, value, ok := strings.Cut(strings.TrimSpace(sc.Text()), "=")
|
||||
if ok && key == "ENABLED" {
|
||||
value = strings.Trim(value, `"'`)
|
||||
return value == "yes" || value == "YES" // the two spellings ufw-init accepts
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package preflight
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCheckHostAt_HostFirewalls(t *testing.T) {
|
||||
for name, tc := range map[string]struct {
|
||||
active []string
|
||||
ufwConf string
|
||||
firewalld Status
|
||||
ufw Status
|
||||
detail string
|
||||
}{
|
||||
"neither installed": {firewalld: OK, ufw: OK},
|
||||
"firewalld active": {active: []string{"firewalld.service"}, firewalld: Failed, ufw: OK, detail: "systemctl mask --now firewalld"},
|
||||
"ufw unit active after ufw disable": {active: []string{"ufw.service"}, ufwConf: "ENABLED=no\n", firewalld: OK, ufw: OK},
|
||||
"ufw enabled": {active: []string{"ufw.service"}, ufwConf: "# /etc/ufw/ufw.conf\nENABLED=yes\nLOGLEVEL=low\n", firewalld: OK, ufw: Failed, detail: "ufw disable"},
|
||||
"ufw enabled, upper case": {active: []string{"ufw.service"}, ufwConf: "ENABLED=YES\n", firewalld: OK, ufw: Failed, detail: "ufw disable"},
|
||||
"ufw enabled but its unit stopped": {ufwConf: "ENABLED=yes\n", firewalld: OK, ufw: OK},
|
||||
"both active": {active: []string{"firewalld.service", "ufw.service"}, ufwConf: "ENABLED=yes\n", firewalld: Failed, ufw: Failed},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
orig := unitActive
|
||||
t.Cleanup(func() { unitActive = orig })
|
||||
unitActive = func(unit string) bool { return slices.Contains(tc.active, unit) }
|
||||
root := t.TempDir()
|
||||
if tc.ufwConf != "" {
|
||||
path := filepath.Join(root, ufwConfig)
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(tc.ufwConf), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
results := CheckHostAt(root)
|
||||
for unit, want := range map[string]Status{"firewalld.service": tc.firewalld, "ufw.service": tc.ufw} {
|
||||
r := findResult(t, results, unit)
|
||||
if r.Status != want {
|
||||
t.Errorf("%s = %s %q, want %s", unit, r.Status, r.Detail, want)
|
||||
}
|
||||
if r.Status == Failed && !strings.Contains(r.Detail, tc.detail) {
|
||||
t.Errorf("%s detail %q does not name the fix %q", unit, r.Detail, tc.detail)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package preflight
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/host/hostnet"
|
||||
)
|
||||
|
||||
// hostnetUnit is the unit whose failure this check surfaces.
|
||||
const hostnetUnit = "spinifex-hostnet.service"
|
||||
|
||||
// hostnetOVNRemote is a variable so tests never query a real OVS.
|
||||
var hostnetOVNRemote = hostnet.DefaultEnv().OVNRemote
|
||||
|
||||
// checkHostnet runs the check the daemon and vpcd run before they start, so a
|
||||
// failed host network apply shows here, not only as a refused service start.
|
||||
func checkHostnet(root string) Result {
|
||||
env := hostnet.DefaultEnv()
|
||||
env.Paths = hostnet.PathsAt(root)
|
||||
env.OVNRemote = hostnetOVNRemote
|
||||
if err := hostnet.Check(context.Background(), env); err != nil {
|
||||
return Result{Path: hostnetUnit, Kind: "hostnet", Status: Failed, Detail: err.Error()}
|
||||
}
|
||||
return Result{Path: hostnetUnit, Kind: "hostnet", Status: OK}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package preflight
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCheckHostAt_Hostnet(t *testing.T) {
|
||||
for name, tc := range map[string]struct {
|
||||
backend string
|
||||
state bool
|
||||
want Status
|
||||
detail string
|
||||
}{
|
||||
"no record, nothing set up": {want: OK},
|
||||
"networkd backend": {backend: "networkd", want: OK},
|
||||
"hostnet backend, no state": {backend: "hostnet", want: Failed, detail: "hostnet.json"},
|
||||
"hostnet backend, no marker": {backend: "hostnet", state: true, want: Failed, detail: "host network not applied"},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
etc := filepath.Join(root, "etc/spinifex")
|
||||
if err := os.MkdirAll(etc, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if tc.backend != "" {
|
||||
if err := os.WriteFile(filepath.Join(etc, "hostnet-backend"), []byte(tc.backend+"\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if tc.state {
|
||||
state := `{"version": 1, "pending": false, "int_bridge": "br-int"}`
|
||||
if err := os.WriteFile(filepath.Join(etc, "hostnet.json"), []byte(state), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
r := findResult(t, CheckHostAt(root), hostnetUnit)
|
||||
if r.Status != tc.want || !strings.Contains(r.Detail, tc.detail) {
|
||||
t.Errorf("hostnet result = %s %q, want %s containing %q", r.Status, r.Detail, tc.want, tc.detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package preflight
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/host/hostos"
|
||||
)
|
||||
|
||||
// microvmMachine is the QEMU machine type load balancer system instances boot on.
|
||||
const microvmMachine = "microvm"
|
||||
|
||||
// microvmChecked reports whether this host runs system instances. The microvm
|
||||
// machine type exists only in x86 QEMU.
|
||||
var microvmChecked = runtime.GOARCH == "amd64"
|
||||
|
||||
// resolveQEMU and machineHelp are variables so tests never run a real QEMU.
|
||||
var (
|
||||
resolveQEMU = func() (string, error) { return hostos.QEMUBinary("x86_64") }
|
||||
machineHelp = func(binary string) ([]byte, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
return exec.CommandContext(ctx, binary, "-machine", "help").Output()
|
||||
}
|
||||
)
|
||||
|
||||
// checkMicroVM checks the QEMU binary the daemon will run, and the firmware it
|
||||
// passes, can boot a microvm. A QEMU build without microvm, such as Oracle
|
||||
// Linux's AppStream qemu-kvm, otherwise surfaces only at the first ALB launch.
|
||||
func checkMicroVM(root string) []Result {
|
||||
if !microvmChecked {
|
||||
return nil
|
||||
}
|
||||
return []Result{checkQEMUMicroVM(), checkMicroVMFirmware(root)}
|
||||
}
|
||||
|
||||
func checkQEMUMicroVM() Result {
|
||||
binary, err := resolveQEMU()
|
||||
if err != nil {
|
||||
return Result{Path: "qemu", Kind: "qemu", Status: Missing, Detail: err.Error()}
|
||||
}
|
||||
out, err := machineHelp(binary)
|
||||
if err != nil {
|
||||
return Result{Path: binary, Kind: "qemu", Status: Missing, Detail: fmt.Sprintf("%s -machine help: %v", binary, err)}
|
||||
}
|
||||
if !listsMachine(out, microvmMachine) {
|
||||
return Result{
|
||||
Path: binary,
|
||||
Kind: "qemu",
|
||||
Status: Unsupported,
|
||||
Detail: fmt.Sprintf("no %s machine type, so load balancers cannot boot; install a QEMU build that has it (on Oracle Linux, qemu-kvm from ol10_kvm_utils)", microvmMachine),
|
||||
}
|
||||
}
|
||||
return Result{Path: binary, Kind: "qemu", Status: OK}
|
||||
}
|
||||
|
||||
// listsMachine reports whether `-machine help` output has a row for name. Rows
|
||||
// are "<name> <description>", after a "Supported machines are:" header.
|
||||
func listsMachine(help []byte, name string) bool {
|
||||
sc := bufio.NewScanner(bytes.NewReader(help))
|
||||
for sc.Scan() {
|
||||
if fields := strings.Fields(sc.Text()); len(fields) > 0 && fields[0] == name {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func checkMicroVMFirmware(root string) Result {
|
||||
path := filepath.Join(vm.MicroVMDir, vm.MicroVMFirmware)
|
||||
if _, err := os.Stat(filepath.Join(root, path)); err != nil {
|
||||
return Result{Path: path, Kind: "firmware", Status: Missing, Detail: err.Error()}
|
||||
}
|
||||
return Result{Path: path, Kind: "firmware", Status: OK}
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
package preflight
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/host/hostos"
|
||||
)
|
||||
|
||||
// Debian's qemu-system-x86_64 -machine help, trimmed.
|
||||
const helpWithMicroVM = `Supported machines are:
|
||||
microvm microvm (i386)
|
||||
pc Standard PC (i440FX + PIIX, 1996) (alias of pc-i440fx-10.0)
|
||||
q35 Standard PC (Q35 + ICH9, 2009) (alias of pc-q35-10.0)
|
||||
`
|
||||
|
||||
// Oracle Linux 10's AppStream qemu-kvm 10.1.0 -machine help, trimmed. It has
|
||||
// no microvm row.
|
||||
const helpWithoutMicroVM = `Supported machines are:
|
||||
pc RHEL 10.0.0 PC (i440FX + PIIX, 1996) (alias of pc-i440fx-rhel10.0.0)
|
||||
pc-i440fx-rhel10.0.0 RHEL 10.0.0 PC (i440FX + PIIX, 1996) (default) (deprecated)
|
||||
pc-q35-rhel9.8.0 RHEL-9.8.0 PC (Q35 + ICH9, 2009) (deprecated)
|
||||
`
|
||||
|
||||
var microvmFirmwarePath = filepath.Join(vm.MicroVMDir, vm.MicroVMFirmware)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
microvmChecked = true
|
||||
resolveQEMU = func() (string, error) { return hostos.QEMUKVMPath, nil }
|
||||
machineHelp = func(string) ([]byte, error) { return []byte(helpWithMicroVM), nil }
|
||||
hostnetOVNRemote = func(context.Context) (string, error) { return "", nil }
|
||||
unitActive = func(string) bool { return false }
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
func stubQEMU(t *testing.T, binary string, resolveErr error, help string, helpErr error) {
|
||||
t.Helper()
|
||||
origResolve, origHelp := resolveQEMU, machineHelp
|
||||
t.Cleanup(func() { resolveQEMU, machineHelp = origResolve, origHelp })
|
||||
var asked string
|
||||
resolveQEMU = func() (string, error) { return binary, resolveErr }
|
||||
machineHelp = func(b string) ([]byte, error) {
|
||||
asked = b
|
||||
return []byte(help), helpErr
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if resolveErr == nil && asked != binary {
|
||||
t.Errorf("machine help ran %q, want the resolved binary %q", asked, binary)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func qemuResult(t *testing.T, results []Result) Result {
|
||||
t.Helper()
|
||||
for _, r := range results {
|
||||
if r.Kind == "qemu" {
|
||||
return r
|
||||
}
|
||||
}
|
||||
t.Fatalf("no qemu result in %+v", results)
|
||||
return Result{}
|
||||
}
|
||||
|
||||
func TestCheckHostAt_QEMUMicroVM(t *testing.T) {
|
||||
t.Run("resolved binary lists microvm", func(t *testing.T) {
|
||||
stubQEMU(t, hostos.QEMUKVMPath, nil, helpWithMicroVM, nil)
|
||||
r := qemuResult(t, CheckHostAt(t.TempDir()))
|
||||
if r.Status != OK || r.Path != hostos.QEMUKVMPath {
|
||||
t.Errorf("got %+v, want OK for %s", r, hostos.QEMUKVMPath)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("resolved binary has no microvm", func(t *testing.T) {
|
||||
stubQEMU(t, hostos.QEMUKVMPath, nil, helpWithoutMicroVM, nil)
|
||||
r := qemuResult(t, CheckHostAt(t.TempDir()))
|
||||
if r.Status != Unsupported || !strings.Contains(r.Detail, "ol10_kvm_utils") {
|
||||
t.Errorf("got %+v, want Unsupported naming ol10_kvm_utils", r)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("no QEMU resolves", func(t *testing.T) {
|
||||
noQEMU := errors.New("no QEMU for x86_64 guests: found neither /usr/libexec/qemu-kvm (package qemu-kvm) nor qemu-system-x86_64 (package qemu-system-x86) on PATH")
|
||||
stubQEMU(t, "", noQEMU, "", nil)
|
||||
r := qemuResult(t, CheckHostAt(t.TempDir()))
|
||||
if r.Status != Missing || r.Detail != noQEMU.Error() {
|
||||
t.Errorf("got %+v, want Missing carrying the resolver's error", r)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("machine help fails", func(t *testing.T) {
|
||||
stubQEMU(t, "/usr/bin/qemu-system-x86_64", nil, "", errors.New("exit status 1"))
|
||||
r := qemuResult(t, CheckHostAt(t.TempDir()))
|
||||
if r.Status != Missing || !strings.Contains(r.Detail, "exit status 1") {
|
||||
t.Errorf("got %+v, want Missing with the exec error", r)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestCheckHostAt_MicroVMFirmware(t *testing.T) {
|
||||
t.Run("absent", func(t *testing.T) {
|
||||
r := findResult(t, CheckHostAt(t.TempDir()), microvmFirmwarePath)
|
||||
if r.Status != Missing {
|
||||
t.Errorf("status = %v, want Missing", r.Status)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("present", func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
writeHelper(t, root, microvmFirmwarePath, []byte("qboot"))
|
||||
r := findResult(t, CheckHostAt(root), microvmFirmwarePath)
|
||||
if r.Status != OK {
|
||||
t.Errorf("status = %v, want OK (detail: %s)", r.Status, r.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestCheckHostAt_NoMicroVMChecksOffX86(t *testing.T) {
|
||||
orig := microvmChecked
|
||||
microvmChecked = false
|
||||
t.Cleanup(func() { microvmChecked = orig })
|
||||
|
||||
for _, r := range CheckHostAt(t.TempDir()) {
|
||||
if r.Kind == "qemu" || r.Kind == "firmware" {
|
||||
t.Errorf("unexpected %s result on a host without system instances: %+v", r.Kind, r)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -29,6 +29,13 @@ const (
|
||||
Stale
|
||||
// Ungranted means the sudoers grant for a helper is absent or missing its NOPASSWD line.
|
||||
Ungranted
|
||||
// Unsupported means the asset is present but lacks a capability this build needs.
|
||||
Unsupported
|
||||
// Failed means a host check that gates the services did not pass.
|
||||
Failed
|
||||
// Warning means a host setting degrades a feature without gating the
|
||||
// services. It is reported but does not fail preflight.
|
||||
Warning
|
||||
)
|
||||
|
||||
// String returns the human-readable name of the status, used in table output and log lines.
|
||||
@@ -42,6 +49,12 @@ func (s Status) String() string {
|
||||
return "Stale"
|
||||
case Ungranted:
|
||||
return "Ungranted"
|
||||
case Unsupported:
|
||||
return "Unsupported"
|
||||
case Failed:
|
||||
return "Failed"
|
||||
case Warning:
|
||||
return "Warning"
|
||||
default:
|
||||
return "Unknown"
|
||||
}
|
||||
@@ -50,7 +63,7 @@ func (s Status) String() string {
|
||||
// Result is the outcome of checking one managed asset against this build's manifest.
|
||||
type Result struct {
|
||||
Path string
|
||||
Kind string // "helper" | "sudoers-grant" | "service"
|
||||
Kind string // "helper" | "sudoers-grant" | "service" | "qemu" | "firmware" | "hostnet" | "host-firewall" | "host-dns"
|
||||
Status Status
|
||||
Detail string
|
||||
}
|
||||
@@ -82,13 +95,17 @@ func CheckHostAt(root string) []Result {
|
||||
}
|
||||
results = append(results, checkSudoersGrant(root))
|
||||
results = append(results, checkRunningBinaries(root)...)
|
||||
results = append(results, checkMicroVM(root)...)
|
||||
results = append(results, checkHostnet(root))
|
||||
results = append(results, checkHostFirewalls(root)...)
|
||||
results = append(results, checkHostDNS(root))
|
||||
return results
|
||||
}
|
||||
|
||||
// HasProblem reports whether any result is not OK.
|
||||
// HasProblem reports whether any result is neither OK nor a Warning.
|
||||
func HasProblem(results []Result) bool {
|
||||
return slices.ContainsFunc(results, func(r Result) bool {
|
||||
return r.Status != OK
|
||||
return r.Status != OK && r.Status != Warning
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -105,7 +105,7 @@ func TestHasProblem(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestStatusString(t *testing.T) {
|
||||
cases := map[Status]string{OK: "OK", Missing: "Missing", Stale: "Stale", Ungranted: "Ungranted"}
|
||||
cases := map[Status]string{OK: "OK", Missing: "Missing", Stale: "Stale", Ungranted: "Ungranted", Unsupported: "Unsupported", Failed: "Failed"}
|
||||
for status, want := range cases {
|
||||
if got := status.String(); got != want {
|
||||
t.Errorf("Status(%d).String() = %q, want %q", status, got, want)
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/network/systemvpc"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/ochre"
|
||||
rdsinstance "github.com/mulgadc/spinifex/spinifex/domains/rds/instance"
|
||||
gateway_bedrock "github.com/mulgadc/spinifex/spinifex/gateway/bedrock"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/compute/gpu"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
@@ -17,11 +18,11 @@ import (
|
||||
// of.
|
||||
func (d *Daemon) buildBedrockLaunchDeps() ochre.LaunchDeps {
|
||||
js, err := jetstream.New(d.natsConn)
|
||||
var weights = ochre.NoopWeightsResolver
|
||||
var weights = gateway_bedrock.NoopWeightsResolver
|
||||
if err != nil {
|
||||
slog.Warn("Bedrock: jetstream init failed; serving VMs cannot resolve staged weights", "err", err)
|
||||
} else {
|
||||
weights = ochre.NewWeightsStore(js)
|
||||
weights = gateway_bedrock.NewWeightsStore(js)
|
||||
}
|
||||
|
||||
return ochre.LaunchDeps{
|
||||
+1
-1
@@ -8,8 +8,8 @@ import (
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/mulgadc/spinifex/spinifex/daemon"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/daemon"
|
||||
)
|
||||
|
||||
// benchVM approximates a real instance record. The env19 terminated bucket
|
||||
@@ -63,9 +63,7 @@ import (
|
||||
ec2volume "github.com/mulgadc/spinifex/spinifex/domains/ec2/volume"
|
||||
ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/ecr"
|
||||
ecsdomain "github.com/mulgadc/spinifex/spinifex/domains/ecs"
|
||||
ekscluster "github.com/mulgadc/spinifex/spinifex/domains/eks/cluster"
|
||||
elbv2domain "github.com/mulgadc/spinifex/spinifex/domains/elbv2"
|
||||
iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/network/external"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/network/external/dhcp"
|
||||
@@ -80,10 +78,13 @@ import (
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/clustersize"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
|
||||
handlers_ecs "github.com/mulgadc/spinifex/spinifex/handlers/ecs"
|
||||
handlers_elbv2 "github.com/mulgadc/spinifex/spinifex/handlers/elbv2"
|
||||
"github.com/mulgadc/spinifex/spinifex/providers/ebs"
|
||||
"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/compute/gpu"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/host/hostos"
|
||||
hostprocess "github.com/mulgadc/spinifex/spinifex/runtime/host/process"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
@@ -165,10 +166,10 @@ type Daemon struct {
|
||||
spotInstanceService *ec2spotinstance.SpotInstanceServiceImpl
|
||||
vpcService *ec2vpc.VPCServiceImpl
|
||||
eipService ec2eip.EIPService
|
||||
elbv2Service *elbv2domain.ELBv2ServiceImpl
|
||||
elbv2Service *handlers_elbv2.ELBv2ServiceImpl
|
||||
eksService *ekscluster.EKSServiceImpl
|
||||
ecsService *ecsdomain.Service
|
||||
ecsScheduler *ecsdomain.Scheduler
|
||||
ecsService *handlers_ecs.Service
|
||||
ecsScheduler *handlers_ecs.Scheduler
|
||||
rdsService *rdsinstance.Service
|
||||
rdsReconciler *rdsinstance.Reconciler
|
||||
bedrockService *ochre.Service
|
||||
@@ -1395,11 +1396,17 @@ func (d *Daemon) startLocal() error {
|
||||
slog.Warn("Failed to set daemon OOM score", "err", err)
|
||||
}
|
||||
|
||||
// Warn on host assets this build expects but that are missing, stale, or
|
||||
// ungranted, so drift surfaces as a named log line rather than a later
|
||||
// cryptic failure. Never fatal — unrelated asset problems must not ground a node.
|
||||
// Log host assets that are missing, stale or ungranted, and host faults whose
|
||||
// detail names the fix, so drift surfaces as a named line rather than a later
|
||||
// cryptic failure. Never fatal: an unrelated asset problem must not ground a node.
|
||||
for _, r := range preflight.CheckHost() {
|
||||
if r.Status != preflight.OK {
|
||||
switch r.Status {
|
||||
case preflight.OK:
|
||||
case preflight.Failed:
|
||||
slog.Error("Host preflight check failed", "path", r.Path, "kind", r.Kind, "detail", r.Detail)
|
||||
case preflight.Warning:
|
||||
slog.Warn("Host preflight warning", "path", r.Path, "kind", r.Kind, "detail", r.Detail)
|
||||
default:
|
||||
slog.Warn("Host asset preflight problem", "path", r.Path, "kind", r.Kind,
|
||||
"status", r.Status.String(), "detail", r.Detail,
|
||||
"remediation", "run scripts/update-nodes.sh or make reinstall")
|
||||
@@ -1809,7 +1816,7 @@ func (d *Daemon) startCluster() error {
|
||||
// background because the first attempt routinely loses the race with
|
||||
// ovn-central accepting connections, and a node that gives up there stays
|
||||
// unconfigured while a peer may already require encryption cluster-wide.
|
||||
go host.MaintainIPSec(d.ctx, d.configPath, d.clusterConfig, d.ipsecBarrier())
|
||||
go host.MaintainIPSec(d.ctx, d.configPath, d.clusterConfig, d.ipsecBarrier(), hostos.HasLibreswan())
|
||||
|
||||
// Keep the host firewall's peer sets in step with cluster membership. Every
|
||||
// formation path reaches this, which none of the installers do. Runs in the
|
||||
@@ -2047,12 +2054,12 @@ func (d *Daemon) startCluster() error {
|
||||
// the identical master key: unlike EKS/ECS's IAM dependency, there is no
|
||||
// safe degraded mode for certificate private keys, so a missing key must
|
||||
// fail daemon startup rather than leave HTTPS listeners uncreatable.
|
||||
d.elbv2Service, err = initServiceWithRetry("ELBv2 service", func() (*elbv2domain.ELBv2ServiceImpl, error) {
|
||||
d.elbv2Service, err = initServiceWithRetry("ELBv2 service", func() (*handlers_elbv2.ELBv2ServiceImpl, error) {
|
||||
masterKey, mkErr := masterkey.ReadShared(filepath.Join(filepath.Dir(d.configPath), "master.key"))
|
||||
if mkErr != nil {
|
||||
return nil, fmt.Errorf("load ELBv2 master key: %w", mkErr)
|
||||
}
|
||||
return elbv2domain.NewELBv2ServiceImplWithNATS(d.config, d.natsConn, masterKey)
|
||||
return handlers_elbv2.NewELBv2ServiceImplWithNATS(d.config, d.natsConn, masterKey)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to initialize ELBv2 service: %w", err)
|
||||
@@ -2072,7 +2079,7 @@ func (d *Daemon) startCluster() error {
|
||||
}
|
||||
|
||||
// Route system VM launches through NATS so they fan out across the cluster.
|
||||
d.elbv2Service.InstanceLauncher = elbv2domain.NewNATSSystemInstanceLauncher(d.natsConn, 0)
|
||||
d.elbv2Service.InstanceLauncher = handlers_elbv2.NewNATSSystemInstanceLauncher(d.natsConn, 0)
|
||||
|
||||
// Provide a lazily-built KV-backed IAM service so an LB VM gets a system
|
||||
// instance profile and authenticates with IMDS instance-role creds. The
|
||||
@@ -2106,13 +2113,13 @@ func (d *Daemon) startCluster() error {
|
||||
// scheduler goroutine that owns the Layer-2 bus subscriptions and heartbeat
|
||||
// reaper. The scheduler is disabled (handlers still serve) when JetStream is
|
||||
// unavailable.
|
||||
d.ecsService = ecsdomain.NewService(d.natsConn, d.config.Region, d.clusterConfig.AWS.ServicesDomain).WithDeps(d.buildECSServiceDeps())
|
||||
d.ecsService = handlers_ecs.NewService(d.natsConn, d.config.Region, d.clusterConfig.AWS.ServicesDomain).WithDeps(d.buildECSServiceDeps())
|
||||
if js, jsErr := jetstream.New(d.natsConn); jsErr != nil {
|
||||
slog.Warn("ECS scheduler disabled: JetStream unavailable", "err", jsErr)
|
||||
} else if _, lbErr := ecsdomain.InitLeaderBucket(d.ctx, js); lbErr != nil {
|
||||
} else if _, lbErr := handlers_ecs.InitLeaderBucket(d.ctx, js); lbErr != nil {
|
||||
slog.Warn("ECS scheduler disabled: leader bucket init failed", "err", lbErr)
|
||||
} else {
|
||||
d.ecsScheduler = ecsdomain.NewScheduler(d.natsConn, d.ecsService, d.node)
|
||||
d.ecsScheduler = handlers_ecs.NewScheduler(d.natsConn, d.ecsService, d.node)
|
||||
d.shutdownWg.Go(func() {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
@@ -31,7 +31,7 @@ import (
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
const daemonTracerName = "github.com/mulgadc/spinifex/spinifex/runtime/daemon"
|
||||
const daemonTracerName = "github.com/mulgadc/spinifex/spinifex/daemon"
|
||||
|
||||
// startOpSpan opens a child span for a named instance operation under ctx.
|
||||
func startOpSpan(ctx context.Context, op, instanceID string) (context.Context, trace.Span) {
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user