mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-23 20:57:02 +00:00
Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 98b864249c | |||
| aebc2f0efb | |||
| 0c488d9470 | |||
| 9a3ab0468d | |||
| 520478dc37 |
@@ -2,7 +2,8 @@ name: Apt Packages
|
||||
|
||||
# Path-filtered like image-scripts.yml: this resolves setup.sh's runtime package
|
||||
# list against each supported suite's real indices, so a renamed, dropped, or
|
||||
# ambiguously virtual package fails here rather than on a user's machine.
|
||||
# ambiguously virtual package fails here rather than on a user's machine. The
|
||||
# dnf job does the same for Oracle Linux, through setup.sh's own repo setup.
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
@@ -11,6 +12,7 @@ on:
|
||||
paths:
|
||||
- scripts/setup.sh
|
||||
- scripts/lint-apt-packages.sh
|
||||
- scripts/lint-dnf-packages.sh
|
||||
- .github/workflows/apt-packages.yml
|
||||
pull_request:
|
||||
branches:
|
||||
@@ -19,6 +21,7 @@ on:
|
||||
paths:
|
||||
- scripts/setup.sh
|
||||
- scripts/lint-apt-packages.sh
|
||||
- scripts/lint-dnf-packages.sh
|
||||
- .github/workflows/apt-packages.yml
|
||||
workflow_dispatch:
|
||||
|
||||
@@ -44,3 +47,14 @@ jobs:
|
||||
|
||||
- name: Resolve runtime package list
|
||||
run: ./scripts/lint-apt-packages.sh ${{ matrix.image }}
|
||||
|
||||
dnf_packages:
|
||||
name: Resolve dnf list (oraclelinux:10)
|
||||
runs-on: ubuntu-26.04
|
||||
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v7.0.1
|
||||
|
||||
- name: Resolve runtime package list
|
||||
run: ./scripts/lint-dnf-packages.sh oraclelinux:10
|
||||
|
||||
@@ -102,7 +102,7 @@ jobs:
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
|
||||
- name: Build microVM artifacts (lb-agent + kernel + initramfs)
|
||||
- name: Build microVM artifacts (lb-agent + kernel + initramfs + qboot)
|
||||
if: steps.version.outputs.exists != 'true'
|
||||
run: cd spinifex && make build-microvm-image
|
||||
|
||||
@@ -144,6 +144,7 @@ jobs:
|
||||
--notes-file "$RUNNER_TEMP/notes.md" \
|
||||
"dist/spinifex-$TAG-linux-amd64.tar.gz" \
|
||||
"dist/spinifex-$TAG-linux-amd64.tar.gz.sha256" \
|
||||
dist/qboot-*.tar.gz \
|
||||
"scripts/setup.sh"
|
||||
|
||||
# The assertion that matters. A prerelease must never become the latest
|
||||
|
||||
@@ -57,7 +57,7 @@ jobs:
|
||||
go-version-file: spinifex/go.mod
|
||||
cache: false
|
||||
|
||||
- name: Build microVM artifacts (lb-agent + kernel + initramfs)
|
||||
- name: Build microVM artifacts (lb-agent + kernel + initramfs + qboot)
|
||||
run: cd spinifex && make build-microvm-image
|
||||
|
||||
- name: Build amd64 Tarball
|
||||
@@ -72,6 +72,7 @@ jobs:
|
||||
path: |
|
||||
spinifex/dist/spinifex-${{ github.ref_name }}-linux-amd64.tar.gz
|
||||
spinifex/dist/spinifex-${{ github.ref_name }}-linux-amd64.tar.gz.sha256
|
||||
spinifex/dist/qboot-*.tar.gz
|
||||
spinifex/scripts/setup.sh
|
||||
|
||||
distro-arm64:
|
||||
|
||||
@@ -19,6 +19,9 @@ bin/
|
||||
dist/
|
||||
build/microvm/vmlinuz
|
||||
build/microvm/initramfs.cpio.gz
|
||||
build/microvm/qboot.rom
|
||||
build/microvm/qboot.LICENSE
|
||||
build/qboot/
|
||||
!spinifex/runtime/roles/spinifexui/frontend/dist/
|
||||
tmp/
|
||||
*.test
|
||||
|
||||
@@ -140,12 +140,23 @@ $(MICROVM_ARTIFACTS) &: $(MICROVM_INPUTS)
|
||||
bin/lb-agent:
|
||||
$(MAKE) build-lb-agent
|
||||
|
||||
build-microvm-image: $(MICROVM_ARTIFACTS) ## Build microVM kernel + initramfs (incremental — skips when up to date)
|
||||
# qboot is the microvm firmware. The source tarball ships with each release
|
||||
# because qboot is GPL-2.0.
|
||||
QBOOT_COMMIT := $(shell sed -n 's/^QBOOT_COMMIT="\(.*\)"$$/\1/p' scripts/build-qboot.sh)
|
||||
QBOOT_SOURCE := build/qboot/qboot-$(QBOOT_COMMIT).tar.gz
|
||||
QBOOT_ARTIFACTS := $(MICROVM_OUT_DIR)/qboot.rom $(MICROVM_OUT_DIR)/qboot.LICENSE $(QBOOT_SOURCE)
|
||||
|
||||
install-microvm: $(MICROVM_ARTIFACTS) ## Install microVM artifacts to /usr/share/spinifex/microvm/
|
||||
$(QBOOT_ARTIFACTS) &: scripts/build-qboot.sh
|
||||
./scripts/build-qboot.sh
|
||||
|
||||
build-microvm-image: $(MICROVM_ARTIFACTS) $(QBOOT_ARTIFACTS) ## Build microVM kernel, initramfs and qboot firmware (incremental — skips when up to date)
|
||||
|
||||
install-microvm: $(MICROVM_ARTIFACTS) $(QBOOT_ARTIFACTS) ## Install microVM artifacts to /usr/share/spinifex/microvm/
|
||||
sudo install -d /usr/share/spinifex/microvm
|
||||
sudo install -m 0644 $(MICROVM_OUT_DIR)/vmlinuz /usr/share/spinifex/microvm/vmlinuz
|
||||
sudo install -m 0644 $(MICROVM_OUT_DIR)/initramfs.cpio.gz /usr/share/spinifex/microvm/initramfs.cpio.gz
|
||||
sudo install -m 0644 $(MICROVM_OUT_DIR)/qboot.rom /usr/share/spinifex/microvm/qboot.rom
|
||||
sudo install -m 0644 $(MICROVM_OUT_DIR)/qboot.LICENSE /usr/share/spinifex/microvm/qboot.LICENSE
|
||||
|
||||
# the pre-commit gate: manifest checks, lint, vuln, and the unit and e2e-harness tiers.
|
||||
# integration and race tests skipped to keep quick, they run in CI.
|
||||
@@ -167,7 +178,7 @@ preflight:
|
||||
# reason: between them they install the software, wire OVN, erase the state and
|
||||
# remove the install, so a shellcheck warning in one is an outage or a wrong
|
||||
# deletion, not a style note.
|
||||
NODE_SCRIPTS := scripts/setup.sh scripts/setup-ovn.sh scripts/install-node.sh scripts/smoke-test.sh scripts/node-reset.sh scripts/uninstall-spx.sh
|
||||
NODE_SCRIPTS := scripts/setup.sh scripts/setup-ovn.sh scripts/install-node.sh scripts/smoke-test.sh scripts/node-reset.sh scripts/uninstall-spx.sh scripts/lib/distro.sh
|
||||
|
||||
test-build-scripts:
|
||||
@echo -e "\n....Running build/scripts/**/*_test.sh...."
|
||||
@@ -175,8 +186,8 @@ test-build-scripts:
|
||||
echo "-- $$t"; \
|
||||
sh "$$t" || exit 1; \
|
||||
done
|
||||
@echo -e "\n....Running scripts/*_test.sh...."
|
||||
@for t in $$(find scripts -maxdepth 1 -name '*_test.sh' | sort); do \
|
||||
@echo -e "\n....Running scripts/*_test.sh and scripts/lib/*_test.sh...."
|
||||
@for t in $$( (find scripts -maxdepth 1 -name '*_test.sh'; find scripts/lib -name '*_test.sh') | sort); do \
|
||||
echo "-- $$t"; \
|
||||
sh "$$t" || exit 1; \
|
||||
done
|
||||
@@ -333,14 +344,20 @@ fuzz:
|
||||
# Microvm artifacts are reinstalled when they already exist on disk — the rule's
|
||||
# input timestamps drive a rebuild only if anything actually changed. On a fresh
|
||||
# checkout (no build/microvm/vmlinuz yet) the install step is skipped; run
|
||||
# `make install-microvm` explicitly the first time.
|
||||
# `make install-microvm` explicitly the first time. They go in before the
|
||||
# preflight, which checks the microvm firmware.
|
||||
deploy: build
|
||||
sudo install -m 755 bin/spx /usr/local/bin/spx
|
||||
@if [ -f $(MICROVM_OUT_DIR)/vmlinuz ]; then \
|
||||
$(MAKE) install-microvm; \
|
||||
else \
|
||||
echo "[deploy] microvm artifacts absent — run 'make install-microvm' for first-time setup"; \
|
||||
fi
|
||||
@if [ "$${SKIP_ASSET_PREFLIGHT:-}" != "1" ]; then \
|
||||
if ! sudo /usr/local/bin/spx admin preflight; then \
|
||||
echo ""; \
|
||||
echo "[deploy] host-asset preflight failed: this node's helper scripts or sudoers"; \
|
||||
echo " grants don't match the binary just installed. Remediate with:"; \
|
||||
echo "[deploy] host-asset preflight failed: this node's helper scripts, sudoers"; \
|
||||
echo " grants, QEMU or microvm firmware don't match the binary just installed. Remediate with:"; \
|
||||
echo " scripts/update-nodes.sh (from the mulga root)"; \
|
||||
echo " or:"; \
|
||||
echo " make reinstall"; \
|
||||
@@ -350,11 +367,6 @@ deploy: build
|
||||
else \
|
||||
echo "[deploy] SKIP_ASSET_PREFLIGHT=1 — skipping host-asset preflight"; \
|
||||
fi
|
||||
@if [ -f $(MICROVM_OUT_DIR)/vmlinuz ]; then \
|
||||
$(MAKE) install-microvm; \
|
||||
else \
|
||||
echo "[deploy] microvm artifacts absent — run 'make install-microvm' for first-time setup"; \
|
||||
fi
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl restart spinifex.target
|
||||
|
||||
@@ -367,16 +379,25 @@ clean:
|
||||
rm -f ./bin/$(GO_PROJECT_NAME)
|
||||
rm -rf spinifex/runtime/roles/spinifexui/frontend/dist
|
||||
|
||||
# On dnf hosts the runtime packages and repositories come from setup.sh, so the
|
||||
# two cannot drift; nbdkit-devel, for building the plugin, is only in CodeReady.
|
||||
install-system:
|
||||
@echo -e "\n....Installing system dependencies for $(ARCH)...."
|
||||
sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \
|
||||
-o Dpkg::Options::="--force-confdef" \
|
||||
-o Dpkg::Options::="--force-confold" \
|
||||
nbdkit nbdkit-plugin-dev pkg-config qemu-system-x86 qemu-system-arm qemu-utils \
|
||||
ovmf qemu-efi-aarch64 \
|
||||
libvirt-daemon-system libvirt-clients libvirt-dev make gcc jq curl \
|
||||
iproute2 netcat-openbsd openssh-client wget git unzip sudo xz-utils file \
|
||||
ovn-central ovn-host openvswitch-switch dhcpcd-base
|
||||
@pm=$$(INSTALL_SPINIFEX_LIB_ONLY=1 bash -c '. scripts/setup.sh && distro_pkg_manager') || exit 1; \
|
||||
if [ "$$pm" = dnf ]; then \
|
||||
INSTALL_SPINIFEX_LIB_ONLY=1 bash -c '. scripts/setup.sh && setup_sudo && detect_os && install_dnf_deps' && \
|
||||
sudo dnf config-manager --set-enabled ol10_codeready_builder && \
|
||||
sudo dnf install -y nbdkit-devel pkgconf-pkg-config make gcc openssh-clients git sudo; \
|
||||
else \
|
||||
sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \
|
||||
-o Dpkg::Options::="--force-confdef" \
|
||||
-o Dpkg::Options::="--force-confold" \
|
||||
nbdkit nbdkit-plugin-dev pkg-config qemu-system-x86 qemu-system-arm qemu-utils \
|
||||
ovmf qemu-efi-aarch64 \
|
||||
libvirt-daemon-system libvirt-clients libvirt-dev make gcc jq curl \
|
||||
iproute2 netcat-openbsd openssh-client wget git unzip sudo xz-utils file \
|
||||
ovn-central ovn-host openvswitch-switch dhcpcd-base; \
|
||||
fi
|
||||
|
||||
install-go:
|
||||
@echo -e "\n....Installing Go 1.27.3 for $(ARCH) ($(GO_ARCH))...."
|
||||
@@ -438,10 +459,11 @@ distro-amd64:
|
||||
-f build/Dockerfile.distro \
|
||||
--output type=local,dest=dist/amd64/ \
|
||||
../
|
||||
@if [ -f $(MICROVM_OUT_DIR)/vmlinuz ] && [ -f $(MICROVM_OUT_DIR)/initramfs.cpio.gz ]; then \
|
||||
@if [ -f $(MICROVM_OUT_DIR)/vmlinuz ] && [ -f $(MICROVM_OUT_DIR)/initramfs.cpio.gz ] && [ -f $(MICROVM_OUT_DIR)/qboot.rom ] && [ -f $(QBOOT_SOURCE) ]; then \
|
||||
echo "[distro-amd64] staging microvm artifacts into tarball"; \
|
||||
mkdir -p dist/amd64/microvm; \
|
||||
cp $(MICROVM_OUT_DIR)/vmlinuz $(MICROVM_OUT_DIR)/initramfs.cpio.gz dist/amd64/microvm/; \
|
||||
cp $(MICROVM_OUT_DIR)/vmlinuz $(MICROVM_OUT_DIR)/initramfs.cpio.gz $(MICROVM_OUT_DIR)/qboot.rom $(MICROVM_OUT_DIR)/qboot.LICENSE dist/amd64/microvm/; \
|
||||
cp $(QBOOT_SOURCE) dist/; \
|
||||
else \
|
||||
echo "[distro-amd64] WARNING: microvm artifacts missing — tarball will not include them"; \
|
||||
echo "[distro-amd64] run 'make build-microvm-image' before 'make distro-amd64'"; \
|
||||
|
||||
@@ -45,6 +45,7 @@ RUN cd viperblock && GOFIPS140=v1.0.0 go build -o /out/nbdkit-viperblock-plugin.
|
||||
# Copy setup scripts and systemd units
|
||||
RUN cp spinifex/scripts/setup.sh /out/setup.sh 2>/dev/null || true
|
||||
RUN cp spinifex/scripts/setup-ovn.sh /out/setup-ovn.sh
|
||||
RUN mkdir -p /out/lib && cp spinifex/scripts/lib/distro.sh /out/lib/distro.sh
|
||||
RUN mkdir -p /out/systemd && cp spinifex/build/systemd/* /out/systemd/
|
||||
RUN mkdir -p /out/scripts && cp spinifex/build/scripts/* /out/scripts/
|
||||
RUN cp spinifex/build/logrotate/spinifex /out/logrotate-spinifex
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
# spinifex-unit-version: 3
|
||||
# spinifex-unit-version: 5
|
||||
[Unit]
|
||||
Description=Spinifex Daemon (VM Orchestration)
|
||||
PartOf=spinifex.target
|
||||
After=spinifex-nats.service spinifex-viperblock.service openvswitch-switch.service ovn-controller.service openvswitch-ipsec.service
|
||||
# Wants (not Requires): stay bootable when openvswitch-ipsec is absent
|
||||
Wants=openvswitch-ipsec.service
|
||||
After=spinifex-nats.service spinifex-viperblock.service ovsdb-server.service ovs-vswitchd.service ovn-controller.service openvswitch-ipsec.service spinifex-hostnet.service
|
||||
# Wants (not Requires): boot without openvswitch-ipsec, survive OVS restarts via hostnet
|
||||
Wants=openvswitch-ipsec.service spinifex-hostnet.service
|
||||
|
||||
[Service]
|
||||
EnvironmentFile=-/etc/spinifex/telemetry.env
|
||||
@@ -14,6 +14,7 @@ Type=simple
|
||||
Slice=spinifex-system.slice
|
||||
User=spinifex-daemon
|
||||
Group=spinifex
|
||||
ExecStartPre=/usr/local/bin/spx admin hostnet check
|
||||
ExecStart=/usr/local/bin/spx service spinifex start
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
# spinifex-unit-version: 1
|
||||
[Unit]
|
||||
Description=Spinifex host network wiring (veth pairs, mgmt address, encap route)
|
||||
# PartOf/WantedBy ovs-vswitchd: re-applied whenever OVS starts; daemon and vpcd only Want it
|
||||
Requires=ovs-vswitchd.service
|
||||
After=ovs-vswitchd.service NetworkManager-wait-online.service
|
||||
PartOf=ovs-vswitchd.service
|
||||
Before=spinifex.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
ExecStart=/usr/local/bin/spx admin hostnet apply
|
||||
# Covers the 120s wait behind setup-ovn.sh plus the 60s wait for links
|
||||
TimeoutStartSec=240
|
||||
TimeoutStopSec=10
|
||||
ProtectSystem=full
|
||||
ProtectHome=yes
|
||||
PrivateTmp=yes
|
||||
|
||||
[Install]
|
||||
WantedBy=spinifex.target ovs-vswitchd.service
|
||||
@@ -1,9 +1,9 @@
|
||||
# spinifex-unit-version: 1
|
||||
# spinifex-unit-version: 3
|
||||
[Unit]
|
||||
Description=Spinifex VPC Daemon (OVN Networking)
|
||||
PartOf=spinifex.target
|
||||
After=spinifex-nats.service spinifex-daemon.service systemd-networkd.service openvswitch-switch.service ovn-controller.service
|
||||
Wants=spinifex-nats.service ovn-controller.service
|
||||
After=spinifex-nats.service spinifex-daemon.service systemd-networkd.service ovsdb-server.service ovs-vswitchd.service ovn-controller.service spinifex-hostnet.service
|
||||
Wants=spinifex-nats.service ovn-controller.service spinifex-hostnet.service
|
||||
|
||||
[Service]
|
||||
EnvironmentFile=-/etc/spinifex/telemetry.env
|
||||
@@ -11,6 +11,7 @@ Type=simple
|
||||
Slice=spinifex-system.slice
|
||||
User=spinifex-vpcd
|
||||
Group=spinifex
|
||||
ExecStartPre=/usr/local/bin/spx admin hostnet check
|
||||
ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
|
||||
ExecStart=/usr/local/bin/spx service vpcd start
|
||||
Restart=on-failure
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
# spinifex-unit-version: 1
|
||||
# spinifex-unit-version: 2
|
||||
[Unit]
|
||||
Description=Spinifex Infrastructure Platform
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
Wants=spinifex-nats.service spinifex-predastore.service spinifex-northstar.service spinifex-viperblock.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service spinifex-ui.service spinifex-qmp-collector.service spinifex-shutdown.service spinifex-nats-watchdog.timer openvswitch-switch.service ovn-controller.service
|
||||
Wants=spinifex-nats.service spinifex-predastore.service spinifex-northstar.service spinifex-viperblock.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service spinifex-ui.service spinifex-qmp-collector.service spinifex-shutdown.service spinifex-nats-watchdog.timer ovsdb-server.service ovs-vswitchd.service ovn-controller.service
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
||||
+1
-1
@@ -126,7 +126,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
|
||||
|
||||
| Command | Flags | Description |
|
||||
|---------|-------|-------------|
|
||||
| `spx admin preflight` | `--quiet` (print only non-OK rows) | Reports whether this host has the privileged helper scripts and sudoers grants the installed build depends on. `spx version` knows the build but not the host: `setup.sh` and `setup-ovn.sh` install those assets, and a binary-swap deploy (`make deploy`, or a manual copy) skips them. It also reports any running service still executing a replaced binary — overwriting `/usr/local/bin/spx` while `spinifex.target` is up does not move services onto it, so each keeps executing the replaced inode until it restarts and a node can serve a mixture of two builds indefinitely. Checks compare content, not existence, so a present-but-stale asset is caught. Exits non-zero if any asset is Missing, Stale or Ungranted. |
|
||||
| `spx admin preflight` | `--quiet` (print only non-OK rows) | Reports whether this host has the privileged helper scripts and sudoers grants the installed build depends on. `spx version` knows the build but not the host: `setup.sh` and `setup-ovn.sh` install those assets, and a binary-swap deploy (`make deploy`, or a manual copy) skips them. It also reports any running service still executing a replaced binary — overwriting `/usr/local/bin/spx` while `spinifex.target` is up does not move services onto it, so each keeps executing the replaced inode until it restarts and a node can serve a mixture of two builds indefinitely. Checks compare content, not existence, so a present-but-stale asset is caught. On x86_64 it also checks that the QEMU binary the daemon will run lists the `microvm` machine type and that the microvm firmware `/usr/share/spinifex/microvm/qboot.rom` is installed, since load balancers boot on both. Exits non-zero if any asset is Missing, Stale, Ungranted or Unsupported. |
|
||||
| `spx admin node js-probe` | `--timeout` (default: 10s) | Round-trips a canary key through the cluster-state KV bucket and exits non-zero if the write or the read-back fails. JetStream can latch a write error per message block and keep serving reads — nats-server's filestore records the failure on the block and returns it on every later write, and nothing clears it short of a restart. The HTTP health endpoint reports the server-wide JetStream state, which that latch never touches, so only an actual write observes it. |
|
||||
| `spx admin banner` | `--boot-check` | Writes the node information banner to `/etc/issue` (shown before login on the physical/serial console) and `/etc/motd` (shown after SSH login). With `--boot-check` it also detects a management IP that has changed since last boot and updates `/etc/spinifex/node.conf` accordingly. |
|
||||
|
||||
|
||||
+39
-1
@@ -7,11 +7,45 @@ Node installation, dev-environment lifecycle, guest image builds, and verificati
|
||||
| Script | What it does |
|
||||
|---|---|
|
||||
| `setup.sh` | The production binary installer, served from `install.mulgadc.com`. Installs deps, users, sudoers, files, systemd units, logrotate and udev rules. `SETUP_STAGES` runs a subset; `ISO_BUILD=1` runs it inside the ISO builder's chroot. |
|
||||
| `setup-ovn.sh` | Bootstraps a compute node for OVN VPC networking: installs OVN/OVS, creates `br-int`, configures the WAN bridge (bridged, DHCP, or routed-NAT uplink), sets the chassis identity, and applies overlay sysctls. `--management` also starts the OVN central services. |
|
||||
| `setup-ovn.sh` | Bootstraps a compute node for OVN VPC networking: installs OVN/OVS, creates `br-int`, configures the WAN bridge (bridged, DHCP, or routed-NAT uplink), sets the chassis identity, and applies overlay sysctls. It runs OVS and OVN as root, setting `OVS_USER_ID` and `OVN_USER_ID` where `/etc/sysconfig` holds them, and hooks the socket-permission helper onto `ovs-vswitchd`, `ovn-controller`, `openvswitch-ipsec` and, on EL, `ovn-northd`. `--management` also starts the OVN central services. The links it creates persist through systemd-networkd where that is enabled, and otherwise through `/etc/spinifex/hostnet.json`, which `spinifex-hostnet.service` (`spx admin hostnet apply`) replays at boot and on every OVS start; the first run records the choice in `/etc/spinifex/hostnet-backend`. Where NetworkManager runs, it leaves the links Spinifex owns unmanaged. |
|
||||
| `install-node.sh` | Forms a multi-node cluster across hosts that are already running Spinifex: rebuilds the OVN database in clustered form, runs `spx admin init` on the first host and `spx admin join` on the rest, then verifies. The scripted form of [install-multi-node](../docs/install/install-multi-node/README.md). Destroys the joining nodes' logical network state and crypto — see its `--help`. |
|
||||
| `dev-install.sh` | Full local dev setup through the production installer: builds from source, packs a tarball, runs `setup.sh`, initialises the cluster, and starts services under systemd. |
|
||||
| `clone-deps.sh` | Clones or updates the viperblock and predastore checkouts alongside spinifex for cross-repo development. |
|
||||
|
||||
### `lib/distro.sh`: host-OS capability probes
|
||||
|
||||
The node scripts select host-OS behaviour by probing for the capability, not by reading `/etc/os-release`; only `setup.sh`'s supported-OS gate keys on the OS ID.
|
||||
Each probe prints its answer, or names every candidate it tried on stderr and returns 1, which it also does when it finds both candidates of a pair.
|
||||
|
||||
| Probe | Answers |
|
||||
|---|---|
|
||||
| `distro_pkg_manager` | `apt` or `dnf` |
|
||||
| `distro_ovs_unit` | `openvswitch-switch.service` (Debian, Ubuntu) or `openvswitch.service` (EL) |
|
||||
| `distro_ovn_central_layout` | `split` (Debian's `ovn-central` plus separate NB and SB units) or `lone-northd` (EL's single `ovn-northd.service`) |
|
||||
| `distro_ca_trust` | the trust-store update command and its anchor directory |
|
||||
| `distro_ovn_logrotate` | `/etc/logrotate.d/ovn-common` or `/etc/logrotate.d/ovn` |
|
||||
|
||||
`distro_missing_ovs_deps <unit-file>...` is a check rather than a probe: it prints each OVS or OVN unit the given unit files depend on or order against that is not installed, and fails only when a file cannot be read.
|
||||
`setup-ovn.sh` runs it against the installed Spinifex units and stops when one is missing, because systemd silently drops ordering against a unit that does not exist; a missing `openvswitch-ipsec.service` is only a warning.
|
||||
|
||||
`setup-ovn.sh`, `uninstall-spx.sh` and `dev-install.sh` source it from `lib/` beside themselves, then from `/usr/local/share/spinifex/lib/distro.sh`, where `setup.sh` installs it, and stop naming both paths when neither exists.
|
||||
`setup.sh` is piped from the install endpoint before the tarball is downloaded, and `install-node.sh --wipe` copies `node-reset.sh` alone to each host, so those two carry inline copies of the probes they need between `# --- BEGIN distro.sh probes ---` markers.
|
||||
`lib/distro_test.sh` fails when an inline copy differs from the library.
|
||||
Go code uses `spinifex/runtime/host/hostos` for the same purpose.
|
||||
|
||||
### System packages
|
||||
|
||||
`setup.sh` installs `APT_RUNTIME_PACKAGES` with apt, or on Oracle Linux `OL_RUNTIME_PACKAGES` with dnf, choosing by `distro_pkg_manager`.
|
||||
Before the dnf install it enables `ol10_UEKR8` and `ol10_kvm_utils`, excludes `qemu-kvm*` and `qemu-img*` from `ol10_appstream` so QEMU always comes from KVM Utilities, and writes `/etc/yum.repos.d/spinifex-nfv-sig.repo` for OVS 3.5 and OVN 25.03 from the CentOS NFV SIG.
|
||||
That repo carries an `includepkgs=` allowlist, and its key is refused unless its fingerprint matches `NFV_SIG_KEY_FPR`.
|
||||
It then installs `kernel-uek-modules` for the running kernel, stopping with a reboot instruction if that version has left the repositories, and loads `openvswitch`, `geneve`, `vhost_net` and `tun`.
|
||||
`make install-system` takes the same dnf path, adding the build packages from CodeReady Builder.
|
||||
|
||||
| Script | What it does |
|
||||
|---|---|
|
||||
| `lint-apt-packages.sh <image>` | Resolves the apt list against a Debian or Ubuntu image's real indices with `apt-get install -s`. |
|
||||
| `lint-dnf-packages.sh [image]` | Runs `setup.sh`'s Oracle Linux repository setup in an `oraclelinux:10` container, resolves the dnf list, and fails unless QEMU comes from KVM Utilities and OVS and OVN from the NFV SIG. |
|
||||
|
||||
## Teardown
|
||||
|
||||
Both ship to `/usr/local/share/spinifex/` on every node, so they are available on a host installed with `curl … | bash` and not only from a checkout. They differ in what they take away:
|
||||
@@ -21,6 +55,9 @@ Both ship to `/usr/local/share/spinifex/` on every node, so they are available o
|
||||
| `node-reset.sh` | Clears **state** and leaves the node installed: stops services, destroys guests, wipes `/etc/spinifex`, `/var/lib/spinifex`, `/var/log/spinifex` and the OVN databases, and removes the OVS bridges. The node is then ready for a fresh `spx admin init`. `--keep-data` preserves `/var/lib/spinifex`; `SPX_RESET_CONFIRM=reset` runs it unattended. |
|
||||
| `uninstall-spx.sh` | Removes the **software** — the inverse of `setup.sh`. Delegates the state teardown above to `node-reset.sh` rather than reimplementing it, then removes the binary, the nbdkit plugin, the systemd units and slices, the service users, the sudoers grant, the sysctl/logrotate/chrony/udev drop-ins, the `systemd-networkd` units, the host firewall policy and its loaded nft table, and the Spinifex CA from the host trust store. Verifies afterwards and exits non-zero naming anything that survived. |
|
||||
|
||||
`node-reset.sh` finds guests by process name, `qemu-system-*` or `qemu-kvm`, owned by the daemon's service user, and waits for them to exit before it deletes any state.
|
||||
`node-reset_test.sh` checks that match against real processes, and that `reset-dev-env.sh` uses the same pattern.
|
||||
|
||||
`uninstall-spx.sh` keeps `/var/lib/spinifex` and every apt package by default — uninstalling software should not destroy data, and a `curl … | bash` install lands on hosts that were already using openvswitch, libvirt and chrony. `--purge-data` and `--purge-deps` opt into each, and both confirm separately even under `--yes` (`SPX_PURGE_CONFIRM=destroy` for unattended data removal). `--dry-run` prints the whole plan and touches nothing.
|
||||
|
||||
**It refuses on a multi-node cluster, and `--yes` does not override that.** Uninstalling one node of a running cluster strands its guests and leaves the survivors reconciling against a chassis that will never answer. Run `spx admin cluster shutdown` first, then the uninstaller on each node.
|
||||
@@ -52,6 +89,7 @@ Both ship to `/usr/local/share/spinifex/` on every node, so they are available o
|
||||
| `build-system-image.sh` | Builds a minimal system AMI (Alpine or Ubuntu) from a manifest. All customisation happens inside the libguestfs appliance, so the build touches no host block device, no host mount, and needs no sudo. `--import` also registers it as an AMI. |
|
||||
| `publish-system-image.sh` | Converts a built raw image to compressed qcow2, writes the `.sha256` sidecar the catalog verifier expects, and uploads both to the R2 bucket behind `iso.mulgadc.com/system-ami/`. |
|
||||
| `build-microvm-image.sh` | Builds the minimal Alpine kernel + initramfs used by the QEMU microvm machine type and the lb-agent, carrying haproxy (ALB) and nginx-stream (NLB) data planes. |
|
||||
| `build-qboot.sh` | Builds qboot, the firmware the daemon passes to every microvm with `-bios`, from a pinned upstream commit in a digest-pinned Debian container. Fails unless the output matches the recorded sha256, which is also Debian 13's `qboot.rom`. Writes `qboot.rom` and its GPL-2.0 licence beside the microvm kernel, and the matching source tarball, which releases publish, under `build/qboot/`. `make build-microvm-image` runs it. |
|
||||
| `mkosi-build.sh` | Runs an mkosi system-image build inside the pinned builder container, so a runner needs only Docker and the toolchain cannot drift. `--image` names an output and expands to the right profile list. |
|
||||
| `mkosi-publish.sh` | Publishes an mkosi-built image to R2 from inside the same pinned container — qcow2 conversion, `.sha256` sidecar, and upload. |
|
||||
| `mkosi-common.sh` | Shared plumbing sourced by both mkosi scripts. Builds the toolchain image once, with the builder account matched to the invoking user so bind-mounted output is not written back as root. |
|
||||
|
||||
Executable
+75
@@ -0,0 +1,75 @@
|
||||
#!/bin/bash
|
||||
# build-qboot.sh — build qboot, the firmware the QEMU microvm machine boots
|
||||
# system instances (ALB/NLB) with, from a pinned upstream commit.
|
||||
#
|
||||
# Outputs:
|
||||
# $REPO_ROOT/build/microvm/qboot.rom firmware, shipped beside the microvm kernel
|
||||
# $REPO_ROOT/build/microvm/qboot.LICENSE qboot's GPL-2.0 licence text
|
||||
# $REPO_ROOT/build/qboot/qboot-<commit>.tar.gz the matching source, published with releases
|
||||
#
|
||||
# The build runs in a digest-pinned Debian container, so its output is
|
||||
# reproducible and is checked against a recorded hash. That hash is also the
|
||||
# sha256 of Debian 13's /usr/share/qemu/qboot.rom, which is built from the same commit.
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
OUT_DIR="$REPO_ROOT/build/microvm"
|
||||
SRC_DIR="$REPO_ROOT/build/qboot"
|
||||
|
||||
QBOOT_REPO="${QBOOT_REPO:-https://gitlab.com/qemu-project/qboot.git}"
|
||||
QBOOT_COMMIT="8ca302e86d685fa05b16e2b208888243da319941"
|
||||
QBOOT_SHA256="5c9c4b12c4438e3cf11d087263afe6519c8c05dd5896727b8b169b8f5dbf9877"
|
||||
BUILD_IMAGE="debian:trixie@sha256:4c25275446ca371766b8b093c696782eabce06263fccacb8d5d813b3dc44b308"
|
||||
|
||||
log() { echo "[build-qboot] $*"; }
|
||||
fail() { echo "ERROR: $*" >&2; exit 1; }
|
||||
|
||||
CONTAINER_TOOL=""
|
||||
for tool in docker podman; do
|
||||
if command -v "$tool" >/dev/null 2>&1 && "$tool" info >/dev/null 2>&1; then
|
||||
CONTAINER_TOOL=$tool
|
||||
break
|
||||
fi
|
||||
done
|
||||
[ -n "$CONTAINER_TOOL" ] || fail "qboot builds in a container; install docker or podman"
|
||||
command -v git >/dev/null 2>&1 || fail "required tool not found: git"
|
||||
|
||||
WORK=$(mktemp -d)
|
||||
trap 'rm -rf "$WORK"' EXIT
|
||||
|
||||
log "fetching qboot $QBOOT_COMMIT from $QBOOT_REPO"
|
||||
git init -q "$WORK/src"
|
||||
git -C "$WORK/src" fetch -q --depth 1 "$QBOOT_REPO" "$QBOOT_COMMIT"
|
||||
git -C "$WORK/src" checkout -q FETCH_HEAD
|
||||
got_commit=$(git -C "$WORK/src" rev-parse HEAD)
|
||||
[ "$got_commit" = "$QBOOT_COMMIT" ] || fail "fetched qboot $got_commit, not the pinned $QBOOT_COMMIT"
|
||||
|
||||
mkdir -p "$SRC_DIR"
|
||||
src_tarball="$SRC_DIR/qboot-${QBOOT_COMMIT}.tar.gz"
|
||||
git -C "$WORK/src" archive --format=tar.gz --prefix="qboot-${QBOOT_COMMIT}/" -o "$src_tarball" "$QBOOT_COMMIT"
|
||||
|
||||
log "building in $BUILD_IMAGE with $CONTAINER_TOOL"
|
||||
# The firmware comes back on stdout, so nothing the container writes is left
|
||||
# root-owned on the host.
|
||||
"$CONTAINER_TOOL" run --rm --platform linux/amd64 -v "$WORK/src:/src:ro" \
|
||||
-e DEBIAN_FRONTEND=noninteractive "$BUILD_IMAGE" sh -euc '
|
||||
{
|
||||
apt-get update -qq
|
||||
apt-get install -y -qq --no-install-recommends gcc libc6-dev meson ninja-build binutils
|
||||
meson setup /tmp/build /src
|
||||
ninja -C /tmp/build
|
||||
} >/dev/null
|
||||
cat /tmp/build/bios.bin
|
||||
' > "$WORK/qboot.rom"
|
||||
|
||||
got_sha=$(sha256sum "$WORK/qboot.rom" | cut -d' ' -f1)
|
||||
if [ "$got_sha" != "$QBOOT_SHA256" ]; then
|
||||
fail "qboot.rom has sha256 $got_sha, not the recorded $QBOOT_SHA256.
|
||||
The pinned commit or build image changed; rebuild, boot an ALB on it, then update QBOOT_SHA256."
|
||||
fi
|
||||
|
||||
mkdir -p "$OUT_DIR"
|
||||
install -m 0644 "$WORK/qboot.rom" "$OUT_DIR/qboot.rom"
|
||||
install -m 0644 "$WORK/src/LICENSE" "$OUT_DIR/qboot.LICENSE"
|
||||
log "qboot.rom sha256 $got_sha"
|
||||
log "source: $src_tarball"
|
||||
@@ -19,6 +19,19 @@ SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
MULGA_ROOT="$(cd "$PROJECT_ROOT/.." && pwd)"
|
||||
|
||||
# Host-OS capability probes: beside this script in a checkout and in the
|
||||
# installed share dir, else where setup.sh installs them.
|
||||
DISTRO_LIB=""
|
||||
for candidate in "$(dirname "${BASH_SOURCE[0]}")/lib/distro.sh" /usr/local/share/spinifex/lib/distro.sh; do
|
||||
if [ -f "$candidate" ]; then DISTRO_LIB="$candidate"; break; fi
|
||||
done
|
||||
if [ -z "$DISTRO_LIB" ]; then
|
||||
echo "ERROR: distro.sh not found at $(dirname "${BASH_SOURCE[0]}")/lib/distro.sh or /usr/local/share/spinifex/lib/distro.sh; re-run setup.sh to install it" >&2
|
||||
exit 1
|
||||
fi
|
||||
# shellcheck source=lib/distro.sh
|
||||
. "$DISTRO_LIB"
|
||||
|
||||
echo "=== Building ==="
|
||||
cd "$PROJECT_ROOT" && make build
|
||||
cd "$MULGA_ROOT/viperblock" && make go_build_nbd
|
||||
@@ -31,6 +44,8 @@ cp "$MULGA_ROOT/viperblock/lib/nbdkit-viperblock-plugin.so" "$STAGING/"
|
||||
cp "$PROJECT_ROOT/scripts/setup-ovn.sh" "$STAGING/"
|
||||
cp "$PROJECT_ROOT/scripts/node-reset.sh" "$STAGING/"
|
||||
cp "$PROJECT_ROOT/scripts/uninstall-spx.sh" "$STAGING/"
|
||||
mkdir -p "$STAGING/lib"
|
||||
cp "$PROJECT_ROOT/scripts/lib/distro.sh" "$STAGING/lib/"
|
||||
mkdir -p "$STAGING/systemd"
|
||||
cp "$PROJECT_ROOT/build/systemd/"* "$STAGING/systemd/"
|
||||
mkdir -p "$STAGING/scripts"
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
# shellcheck shell=bash
|
||||
# distro.sh — host-OS capability probes. Each prints its answer, or names every
|
||||
# candidate it tried on stderr and returns 1, also when it finds both of a pair.
|
||||
# setup.sh and node-reset.sh carry inline copies; distro_test.sh keeps them equal.
|
||||
|
||||
# Reads unit files off disk, not through systemctl, so it answers in a chroot.
|
||||
# DISTRO_ROOT prefixes every host path the probes read; tests set it.
|
||||
distro_unit_exists() {
|
||||
local dir
|
||||
for dir in /etc/systemd/system /run/systemd/system /usr/lib/systemd/system /lib/systemd/system; do
|
||||
if [ -e "${DISTRO_ROOT:-}$dir/$1" ] || [ -L "${DISTRO_ROOT:-}$dir/$1" ]; then
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
distro_pkg_manager() {
|
||||
local apt=false dnf=false
|
||||
command -v apt-get >/dev/null 2>&1 && apt=true
|
||||
command -v dnf >/dev/null 2>&1 && dnf=true
|
||||
if $apt && $dnf; then
|
||||
echo "distro.sh: package manager: found both apt-get and dnf; cannot tell which one manages this host" >&2
|
||||
return 1
|
||||
fi
|
||||
if $apt; then echo apt; return 0; fi
|
||||
if $dnf; then echo dnf; return 0; fi
|
||||
echo "distro.sh: package manager: found neither apt-get nor dnf on PATH" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
# The OVS umbrella unit: openvswitch-switch.service on Debian, openvswitch.service on EL.
|
||||
distro_ovs_unit() {
|
||||
local deb=false el=false
|
||||
distro_unit_exists openvswitch-switch.service && deb=true
|
||||
distro_unit_exists openvswitch.service && el=true
|
||||
if $deb && $el; then
|
||||
echo "distro.sh: OVS unit: found both openvswitch-switch.service and openvswitch.service; cannot tell which one owns OVS" >&2
|
||||
return 1
|
||||
fi
|
||||
if $deb; then echo openvswitch-switch.service; return 0; fi
|
||||
if $el; then echo openvswitch.service; return 0; fi
|
||||
echo "distro.sh: OVS unit: found neither openvswitch-switch.service nor openvswitch.service; is Open vSwitch installed?" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
# "split": Debian's ovn-central plus separate NB and SB DB units.
|
||||
# "lone-northd": EL's single ovn-northd.service, which starts both DB servers.
|
||||
distro_ovn_central_layout() {
|
||||
local split=false lone=false unit missing=""
|
||||
for unit in ovn-central.service ovn-ovsdb-server-nb.service ovn-ovsdb-server-sb.service; do
|
||||
if distro_unit_exists "$unit"; then
|
||||
split=true
|
||||
else
|
||||
missing="${missing:+$missing, }$unit"
|
||||
fi
|
||||
done
|
||||
if distro_unit_exists ovn-northd.service && [ -f "${DISTRO_ROOT:-}/etc/sysconfig/ovn" ]; then
|
||||
lone=true
|
||||
fi
|
||||
if $split && $lone; then
|
||||
echo "distro.sh: OVN central layout: found both the split units (ovn-central.service) and a lone ovn-northd.service with /etc/sysconfig/ovn; cannot tell which one owns the OVN databases" >&2
|
||||
return 1
|
||||
fi
|
||||
if $split; then
|
||||
if [ -n "$missing" ]; then
|
||||
echo "distro.sh: OVN central layout: ovn-central is installed but $missing is missing" >&2
|
||||
return 1
|
||||
fi
|
||||
echo split
|
||||
return 0
|
||||
fi
|
||||
if $lone; then echo lone-northd; return 0; fi
|
||||
echo "distro.sh: OVN central layout: found neither ovn-central.service nor ovn-northd.service with /etc/sysconfig/ovn; is OVN central installed?" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
# The trust-store update command and its local anchor directory, space-separated.
|
||||
distro_ca_trust() {
|
||||
local deb=false el=false
|
||||
command -v update-ca-certificates >/dev/null 2>&1 && deb=true
|
||||
command -v update-ca-trust >/dev/null 2>&1 && el=true
|
||||
if $deb && $el; then
|
||||
echo "distro.sh: CA trust: found both update-ca-certificates and update-ca-trust; cannot tell which trust store TLS clients read" >&2
|
||||
return 1
|
||||
fi
|
||||
if $deb; then echo "update-ca-certificates /usr/local/share/ca-certificates"; return 0; fi
|
||||
if $el; then echo "update-ca-trust /etc/pki/ca-trust/source/anchors"; return 0; fi
|
||||
echo "distro.sh: CA trust: found neither update-ca-certificates nor update-ca-trust on PATH" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
distro_ovn_logrotate() {
|
||||
local deb=false el=false
|
||||
[ -f "${DISTRO_ROOT:-}/etc/logrotate.d/ovn-common" ] && deb=true
|
||||
[ -f "${DISTRO_ROOT:-}/etc/logrotate.d/ovn" ] && el=true
|
||||
if $deb && $el; then
|
||||
echo "distro.sh: OVN logrotate: found both /etc/logrotate.d/ovn-common and /etc/logrotate.d/ovn; cannot tell which one the OVN packages own" >&2
|
||||
return 1
|
||||
fi
|
||||
if $deb; then echo /etc/logrotate.d/ovn-common; return 0; fi
|
||||
if $el; then echo /etc/logrotate.d/ovn; return 0; fi
|
||||
echo "distro.sh: OVN logrotate: found neither /etc/logrotate.d/ovn-common nor /etc/logrotate.d/ovn; is OVN installed?" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
# The OVS and OVN units the given unit files depend on or order against that
|
||||
# are not installed, one per line; prints nothing when all are. Fails only
|
||||
# when a unit file cannot be read.
|
||||
distro_missing_ovs_deps() {
|
||||
local file line dep missing=""
|
||||
for file in "$@"; do
|
||||
if [ ! -r "${DISTRO_ROOT:-}$file" ]; then
|
||||
echo "distro.sh: OVS dependencies: cannot read $file" >&2
|
||||
return 1
|
||||
fi
|
||||
while IFS= read -r line || [ -n "$line" ]; do
|
||||
case "$line" in
|
||||
After=* | Before=* | Wants=* | Requires=* | Requisite=* | BindsTo=* | PartOf=* | Upholds=*) ;;
|
||||
*) continue ;;
|
||||
esac
|
||||
for dep in ${line#*=}; do
|
||||
case "$dep" in
|
||||
ovs* | ovn* | openvswitch*) ;;
|
||||
*) continue ;;
|
||||
esac
|
||||
case " $missing " in
|
||||
*" $dep "*) continue ;;
|
||||
esac
|
||||
distro_unit_exists "$dep" || missing="${missing:+$missing }$dep"
|
||||
done
|
||||
done <"${DISTRO_ROOT:-}$file"
|
||||
done
|
||||
for dep in $missing; do echo "$dep"; done
|
||||
}
|
||||
@@ -0,0 +1,233 @@
|
||||
#!/bin/sh
|
||||
# distro_test.sh — tests for the host-OS probes and the inline copies of them.
|
||||
#
|
||||
# Every probe runs against a fixture tree (DISTRO_ROOT) and a PATH holding only
|
||||
# stub commands, so the result never depends on the distro running the test.
|
||||
set -eu
|
||||
|
||||
LIB_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
|
||||
SCRIPTS_DIR=$(dirname -- "$LIB_DIR")
|
||||
DISTRO="$LIB_DIR/distro.sh"
|
||||
TMP=$(mktemp -d)
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
|
||||
fails=0
|
||||
pass() { echo " ok - $1"; }
|
||||
fail() { echo " FAIL - $1"; fails=$((fails + 1)); }
|
||||
|
||||
echo "-- lib/distro.sh"
|
||||
|
||||
# fixture <name> — a fresh empty host root and stub bin dir.
|
||||
fixture() {
|
||||
ROOT="$TMP/$1/root"
|
||||
BIN="$TMP/$1/bin"
|
||||
mkdir -p "$ROOT" "$BIN"
|
||||
}
|
||||
unit() { mkdir -p "$ROOT/usr/lib/systemd/system" && : > "$ROOT/usr/lib/systemd/system/$1"; }
|
||||
file() { mkdir -p "$ROOT$(dirname "$1")" && : > "$ROOT$1"; }
|
||||
cmd() { printf '#!/bin/sh\nexit 0\n' > "$BIN/$1" && chmod +x "$BIN/$1"; }
|
||||
|
||||
# probe <fn> [arg...] — runs one probe in bash against the current fixture,
|
||||
# with only the stub dir on PATH. Sets OUT, ERR and RC.
|
||||
probe() {
|
||||
RC=0
|
||||
OUT=$(DISTRO_ROOT="$ROOT" PATH="$BIN" /bin/bash -c '. "$1"; shift; "$@"' _ "$DISTRO" "$@" 2>"$TMP/err") || RC=$?
|
||||
ERR=$(cat "$TMP/err")
|
||||
}
|
||||
|
||||
# expect_ok <desc> <want>; expect_fail <desc> <word>... — every word must appear
|
||||
# in the error, which is how "names every candidate" is checked.
|
||||
expect_ok() {
|
||||
if [ "$RC" = 0 ] && [ "$OUT" = "$2" ]; then pass "$1"; else fail "$1 (rc=$RC out='$OUT' err='$ERR')"; fi
|
||||
}
|
||||
expect_fail() {
|
||||
desc=$1; shift
|
||||
if [ "$RC" = 0 ]; then fail "$desc (succeeded with '$OUT')"; return; fi
|
||||
for word in "$@"; do
|
||||
case "$ERR" in
|
||||
*"$word"*) ;;
|
||||
*) fail "$desc (error does not name $word: '$ERR')"; return ;;
|
||||
esac
|
||||
done
|
||||
pass "$desc"
|
||||
}
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Package manager.
|
||||
# --------------------------------------------------------------------------
|
||||
fixture pm-apt; cmd apt-get; probe distro_pkg_manager
|
||||
expect_ok "package manager: apt-get alone is apt" apt
|
||||
fixture pm-dnf; cmd dnf; probe distro_pkg_manager
|
||||
expect_ok "package manager: dnf alone is dnf" dnf
|
||||
fixture pm-both; cmd apt-get; cmd dnf; probe distro_pkg_manager
|
||||
expect_fail "package manager: both fails naming both" apt-get dnf
|
||||
fixture pm-none; probe distro_pkg_manager
|
||||
expect_fail "package manager: neither fails naming both" apt-get dnf
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# OVS umbrella unit. Unit files are found under any systemd unit directory, and
|
||||
# a masked unit (a symlink to /dev/null) still counts as installed.
|
||||
# --------------------------------------------------------------------------
|
||||
fixture ovs-deb; unit openvswitch-switch.service; probe distro_ovs_unit
|
||||
expect_ok "OVS unit: Debian name" openvswitch-switch.service
|
||||
fixture ovs-el; unit openvswitch.service; probe distro_ovs_unit
|
||||
expect_ok "OVS unit: EL name" openvswitch.service
|
||||
fixture ovs-masked; mkdir -p "$ROOT/etc/systemd/system"
|
||||
ln -s /dev/null "$ROOT/etc/systemd/system/openvswitch.service"; probe distro_ovs_unit
|
||||
expect_ok "OVS unit: a masked unit in /etc counts" openvswitch.service
|
||||
fixture ovs-lib; mkdir -p "$ROOT/lib/systemd/system"
|
||||
: > "$ROOT/lib/systemd/system/openvswitch-switch.service"; probe distro_ovs_unit
|
||||
expect_ok "OVS unit: found under /lib on an unmerged-/usr host" openvswitch-switch.service
|
||||
fixture ovs-both; unit openvswitch-switch.service; unit openvswitch.service; probe distro_ovs_unit
|
||||
expect_fail "OVS unit: both fails naming both" openvswitch-switch.service openvswitch.service
|
||||
fixture ovs-none; probe distro_ovs_unit
|
||||
expect_fail "OVS unit: neither fails naming both" openvswitch-switch.service openvswitch.service
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# OVN central layout. Both distros ship ovn-northd.service, so the unit alone
|
||||
# must not decide it: Debian's split layout carries one too.
|
||||
# --------------------------------------------------------------------------
|
||||
split_units() { unit ovn-central.service; unit ovn-ovsdb-server-nb.service; unit ovn-ovsdb-server-sb.service; unit ovn-northd.service; }
|
||||
|
||||
fixture ovn-split; split_units; probe distro_ovn_central_layout
|
||||
expect_ok "OVN layout: Debian split units" split
|
||||
fixture ovn-lone; unit ovn-northd.service; file /etc/sysconfig/ovn; probe distro_ovn_central_layout
|
||||
expect_ok "OVN layout: EL lone ovn-northd" lone-northd
|
||||
fixture ovn-both; split_units; file /etc/sysconfig/ovn; probe distro_ovn_central_layout
|
||||
expect_fail "OVN layout: both fails naming both" ovn-central.service ovn-northd.service /etc/sysconfig/ovn
|
||||
fixture ovn-partial; unit ovn-central.service; unit ovn-ovsdb-server-nb.service; probe distro_ovn_central_layout
|
||||
expect_fail "OVN layout: a partial split install names the missing unit" ovn-ovsdb-server-sb.service
|
||||
fixture ovn-northd-only; unit ovn-northd.service; probe distro_ovn_central_layout
|
||||
expect_fail "OVN layout: ovn-northd without /etc/sysconfig/ovn is not EL" ovn-central.service ovn-northd.service /etc/sysconfig/ovn
|
||||
fixture ovn-none; probe distro_ovn_central_layout
|
||||
expect_fail "OVN layout: neither fails naming both" ovn-central.service ovn-northd.service
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# CA trust.
|
||||
# --------------------------------------------------------------------------
|
||||
fixture ca-deb; cmd update-ca-certificates; probe distro_ca_trust
|
||||
expect_ok "CA trust: Debian command and anchor dir" "update-ca-certificates /usr/local/share/ca-certificates"
|
||||
fixture ca-el; cmd update-ca-trust; probe distro_ca_trust
|
||||
expect_ok "CA trust: EL command and anchor dir" "update-ca-trust /etc/pki/ca-trust/source/anchors"
|
||||
fixture ca-both; cmd update-ca-certificates; cmd update-ca-trust; probe distro_ca_trust
|
||||
expect_fail "CA trust: both fails naming both" update-ca-certificates update-ca-trust
|
||||
fixture ca-none; probe distro_ca_trust
|
||||
expect_fail "CA trust: neither fails naming both" update-ca-certificates update-ca-trust
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# OVN logrotate file.
|
||||
# --------------------------------------------------------------------------
|
||||
fixture lr-deb; file /etc/logrotate.d/ovn-common; probe distro_ovn_logrotate
|
||||
expect_ok "OVN logrotate: Debian file" /etc/logrotate.d/ovn-common
|
||||
fixture lr-el; file /etc/logrotate.d/ovn; probe distro_ovn_logrotate
|
||||
expect_ok "OVN logrotate: EL file" /etc/logrotate.d/ovn
|
||||
fixture lr-both; file /etc/logrotate.d/ovn-common; file /etc/logrotate.d/ovn; probe distro_ovn_logrotate
|
||||
expect_fail "OVN logrotate: both fails naming both" /etc/logrotate.d/ovn-common /etc/logrotate.d/ovn
|
||||
fixture lr-none; probe distro_ovn_logrotate
|
||||
expect_fail "OVN logrotate: neither fails naming both" /etc/logrotate.d/ovn-common /etc/logrotate.d/ovn
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# OVS and OVN units the shipped Spinifex units name. Run against the real unit
|
||||
# files, so a unit that names a Debian-only OVS unit fails on the EL fixture.
|
||||
# --------------------------------------------------------------------------
|
||||
UNITS_SRC="$(dirname -- "$SCRIPTS_DIR")/build/systemd"
|
||||
spinifex_units() {
|
||||
mkdir -p "$ROOT/etc/systemd/system"
|
||||
cp "$UNITS_SRC"/spinifex.target "$UNITS_SRC"/spinifex-*.service "$ROOT/etc/systemd/system/"
|
||||
}
|
||||
deps_probe() {
|
||||
set --
|
||||
for f in "$ROOT"/etc/systemd/system/spinifex*; do set -- "$@" "${f#"$ROOT"}"; done
|
||||
probe distro_missing_ovs_deps "$@"
|
||||
}
|
||||
el_ovs_units() {
|
||||
for u in openvswitch.service ovsdb-server.service ovs-vswitchd.service ovn-controller.service ovn-northd.service openvswitch-ipsec.service; do unit "$u"; done
|
||||
}
|
||||
|
||||
fixture deps-el; spinifex_units; el_ovs_units; deps_probe
|
||||
expect_ok "OVS dependencies: every one the shipped units name is installed on EL" ""
|
||||
fixture deps-no-ipsec; spinifex_units; el_ovs_units; rm "$ROOT/usr/lib/systemd/system/openvswitch-ipsec.service"; deps_probe
|
||||
expect_ok "OVS dependencies: a missing unit is printed once" openvswitch-ipsec.service
|
||||
fixture deps-debian-only; el_ovs_units
|
||||
mkdir -p "$ROOT/etc/systemd/system"
|
||||
printf '[Unit]\n# After=ovs-old.service\nAfter=openvswitch-switch.service ovn-controller.service\nWants=openvswitch-switch.service\n' >"$ROOT/etc/systemd/system/spinifex-x.service"
|
||||
printf '[Unit]\nRequires=ovn-central.service' >"$ROOT/etc/systemd/system/spinifex-y.service"
|
||||
deps_probe
|
||||
expect_ok "OVS dependencies: Debian-only names are missing on EL, comments are not read" "openvswitch-switch.service
|
||||
ovn-central.service"
|
||||
fixture deps-unreadable; probe distro_missing_ovs_deps /etc/systemd/system/spinifex.target
|
||||
expect_fail "OVS dependencies: an unreadable unit file fails naming it" "cannot read /etc/systemd/system/spinifex.target"
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Inline copies. setup.sh and node-reset.sh cannot source the library, so they
|
||||
# carry copies between markers; each copied function must match the library's
|
||||
# byte for byte, or a fix to one silently misses the other.
|
||||
# --------------------------------------------------------------------------
|
||||
# fn_body <file> <name> — the function from "name() {" to the first "}" line.
|
||||
fn_body() {
|
||||
awk -v name="$2" '$0 == name "() {" { on = 1 } on { print } on && $0 == "}" { exit }' "$1"
|
||||
}
|
||||
|
||||
for copy in setup.sh node-reset.sh; do
|
||||
path="$SCRIPTS_DIR/$copy"
|
||||
block=$(sed -n '/^# --- BEGIN distro.sh probes ---$/,/^# --- END distro.sh probes ---$/p' "$path")
|
||||
if [ -z "$block" ]; then
|
||||
fail "$copy: no distro.sh probe block"
|
||||
continue
|
||||
fi
|
||||
names=$(printf '%s\n' "$block" | sed -n 's/^\([a-z_][a-z0-9_]*\)() {$/\1/p')
|
||||
if [ -z "$names" ]; then
|
||||
fail "$copy: the distro.sh probe block holds no functions"
|
||||
continue
|
||||
fi
|
||||
for name in $names; do
|
||||
want=$(fn_body "$DISTRO" "$name")
|
||||
got=$(fn_body "$path" "$name")
|
||||
if [ -z "$want" ]; then
|
||||
fail "$copy: $name is not in lib/distro.sh"
|
||||
elif [ "$got" = "$want" ]; then
|
||||
pass "$copy: $name matches lib/distro.sh"
|
||||
else
|
||||
fail "$copy: $name differs from lib/distro.sh"
|
||||
fi
|
||||
done
|
||||
|
||||
# A copied probe calling a probe helper the block lacks fails only when run.
|
||||
for called in $(printf '%s\n' "$block" | grep -v '^ *#' | grep -o 'distro_[a-z_]*' | sort -u); do
|
||||
case " $(echo $names) " in
|
||||
*" $called "*) ;;
|
||||
*) fail "$copy: copies a probe that calls $called without copying it" ;;
|
||||
esac
|
||||
done
|
||||
done
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# A consumer with no library beside it and none installed must stop naming
|
||||
# both paths, not run on with the probes undefined. Skipped on a host where
|
||||
# setup.sh has installed the library, since the fallback then finds it.
|
||||
# --------------------------------------------------------------------------
|
||||
if [ -e /usr/local/share/spinifex/lib/distro.sh ]; then
|
||||
echo " skip - consumers without distro.sh: /usr/local/share/spinifex/lib/distro.sh is installed here"
|
||||
else
|
||||
# uninstall-spx.sh sources the library after its --help, so it is run into
|
||||
# its multi-node refusal, which exits 1 without touching the host either way.
|
||||
mkdir -p "$TMP/alone/bin" "$TMP/alone/etc"
|
||||
printf '#!/bin/sh\nexec "$@"\n' > "$TMP/alone/bin/sudo"
|
||||
chmod +x "$TMP/alone/bin/sudo"
|
||||
printf '[nodes.node1]\n[nodes.node2]\n' > "$TMP/alone/etc/spinifex.toml"
|
||||
for consumer in "setup-ovn.sh --help" "uninstall-spx.sh --dry-run"; do
|
||||
script=${consumer%% *}
|
||||
cp "$SCRIPTS_DIR/$script" "$TMP/alone/"
|
||||
PATH="$TMP/alone/bin:$PATH" SPX_ETC_DIR="$TMP/alone/etc" "$TMP/alone/$script" "${consumer#* }" >"$TMP/out" 2>&1 && rc=0 || rc=$?
|
||||
if [ "$rc" != 0 ] && grep -q "distro.sh not found at $TMP/alone/lib/distro.sh or /usr/local/share/spinifex/lib/distro.sh" "$TMP/out"; then
|
||||
pass "$script: stops naming both paths when distro.sh is missing"
|
||||
else
|
||||
fail "$script: without distro.sh exited $rc: $(head -3 "$TMP/out")"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
if [ "$fails" -gt 0 ]; then
|
||||
echo "$fails failure(s)"
|
||||
exit 1
|
||||
fi
|
||||
Executable
+45
@@ -0,0 +1,45 @@
|
||||
#!/usr/bin/env bash
|
||||
# Runs setup.sh's Oracle Linux repository setup in a container, then resolves
|
||||
# its runtime dnf list against the real repositories with dnf install --assumeno.
|
||||
# Catches a renamed or dropped package, a SIG dependency the includepkgs
|
||||
# allowlist misses, a changed SIG key, and QEMU or OVS resolving from the wrong
|
||||
# repository.
|
||||
set -euo pipefail
|
||||
|
||||
IMAGE="${1:-oraclelinux:10}"
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
|
||||
echo "Resolving ${IMAGE}..."
|
||||
# The repository setup and the list both come from setup.sh itself, so the lint
|
||||
# cannot drift from what the installer does.
|
||||
docker run --rm -v "${SCRIPT_DIR}/setup.sh:/setup.sh:ro" "${IMAGE}" bash -c '
|
||||
set -euo pipefail
|
||||
INSTALL_SPINIFEX_LIB_ONLY=1 . /setup.sh
|
||||
SUDO=""
|
||||
ensure_ol_repos
|
||||
|
||||
# --assumeno always exits 1, so the transaction is judged from its output.
|
||||
# shellcheck disable=SC2086
|
||||
out=$(dnf install --assumeno $OL_RUNTIME_PACKAGES 2>&1) || true
|
||||
if ! grep -q "^Operation aborted" <<< "$out" || grep -qE "^(Error|Problem)|No match for argument" <<< "$out"; then
|
||||
echo "$out"
|
||||
echo "FAIL: the Oracle Linux package list does not resolve" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# repo_of <package> — the repository dnf would install the package from.
|
||||
repo_of() { awk -v p="$1" "\$1 == p { print \$4; exit }" <<< "$out"; }
|
||||
for want in "qemu-kvm $OL_KVM_UTILS_REPO" "qemu-img $OL_KVM_UTILS_REPO" \
|
||||
"openvswitch3.5 $NFV_SIG_REPO_ID" "ovn25.03-central $NFV_SIG_REPO_ID"; do
|
||||
pkg=${want% *} repo=${want#* }
|
||||
got=$(repo_of "$pkg")
|
||||
if [ "$got" != "$repo" ]; then
|
||||
echo "$out"
|
||||
echo "FAIL: $pkg resolves from ${got:-nowhere}, not $repo" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
'
|
||||
|
||||
echo "OK: every package resolves on ${IMAGE}, QEMU from KVM Utilities and OVS/OVN from the NFV SIG"
|
||||
+78
-3
@@ -57,6 +57,77 @@ run() {
|
||||
"$@"
|
||||
}
|
||||
|
||||
# --- BEGIN distro.sh probes ---
|
||||
# Copied from scripts/lib/distro.sh: install-node.sh --wipe copies this script
|
||||
# alone to each host, so it cannot source the library. distro_test.sh keeps them equal.
|
||||
distro_unit_exists() {
|
||||
local dir
|
||||
for dir in /etc/systemd/system /run/systemd/system /usr/lib/systemd/system /lib/systemd/system; do
|
||||
if [ -e "${DISTRO_ROOT:-}$dir/$1" ] || [ -L "${DISTRO_ROOT:-}$dir/$1" ]; then
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
distro_ovs_unit() {
|
||||
local deb=false el=false
|
||||
distro_unit_exists openvswitch-switch.service && deb=true
|
||||
distro_unit_exists openvswitch.service && el=true
|
||||
if $deb && $el; then
|
||||
echo "distro.sh: OVS unit: found both openvswitch-switch.service and openvswitch.service; cannot tell which one owns OVS" >&2
|
||||
return 1
|
||||
fi
|
||||
if $deb; then echo openvswitch-switch.service; return 0; fi
|
||||
if $el; then echo openvswitch.service; return 0; fi
|
||||
echo "distro.sh: OVS unit: found neither openvswitch-switch.service nor openvswitch.service; is Open vSwitch installed?" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
distro_ovn_central_layout() {
|
||||
local split=false lone=false unit missing=""
|
||||
for unit in ovn-central.service ovn-ovsdb-server-nb.service ovn-ovsdb-server-sb.service; do
|
||||
if distro_unit_exists "$unit"; then
|
||||
split=true
|
||||
else
|
||||
missing="${missing:+$missing, }$unit"
|
||||
fi
|
||||
done
|
||||
if distro_unit_exists ovn-northd.service && [ -f "${DISTRO_ROOT:-}/etc/sysconfig/ovn" ]; then
|
||||
lone=true
|
||||
fi
|
||||
if $split && $lone; then
|
||||
echo "distro.sh: OVN central layout: found both the split units (ovn-central.service) and a lone ovn-northd.service with /etc/sysconfig/ovn; cannot tell which one owns the OVN databases" >&2
|
||||
return 1
|
||||
fi
|
||||
if $split; then
|
||||
if [ -n "$missing" ]; then
|
||||
echo "distro.sh: OVN central layout: ovn-central is installed but $missing is missing" >&2
|
||||
return 1
|
||||
fi
|
||||
echo split
|
||||
return 0
|
||||
fi
|
||||
if $lone; then echo lone-northd; return 0; fi
|
||||
echo "distro.sh: OVN central layout: found neither ovn-central.service nor ovn-northd.service with /etc/sysconfig/ovn; is OVN central installed?" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
distro_ca_trust() {
|
||||
local deb=false el=false
|
||||
command -v update-ca-certificates >/dev/null 2>&1 && deb=true
|
||||
command -v update-ca-trust >/dev/null 2>&1 && el=true
|
||||
if $deb && $el; then
|
||||
echo "distro.sh: CA trust: found both update-ca-certificates and update-ca-trust; cannot tell which trust store TLS clients read" >&2
|
||||
return 1
|
||||
fi
|
||||
if $deb; then echo "update-ca-certificates /usr/local/share/ca-certificates"; return 0; fi
|
||||
if $el; then echo "update-ca-trust /etc/pki/ca-trust/source/anchors"; return 0; fi
|
||||
echo "distro.sh: CA trust: found neither update-ca-certificates nor update-ca-trust on PATH" >&2
|
||||
return 1
|
||||
}
|
||||
# --- END distro.sh probes ---
|
||||
|
||||
WIPE_DIRS=("$ETC_DIR" "$LOG_DIR" "$RUN_DIR")
|
||||
$KEEP_DATA || WIPE_DIRS+=("$DATA_DIR")
|
||||
|
||||
@@ -102,6 +173,9 @@ fi
|
||||
|
||||
log "stopping services"
|
||||
run sudo systemctl stop spinifex.target 2>/dev/null || true
|
||||
# spinifex-hostnet starts with OVS, which is started below to delete bridges; it
|
||||
# would rebuild the old veth pairs. setup-ovn.sh enables it again.
|
||||
run sudo systemctl disable --now spinifex-hostnet.service 2>/dev/null || true
|
||||
run sudo systemctl reset-failed 'spinifex-*' 2>/dev/null || true
|
||||
|
||||
# Stopping the target is not the same as its services being stopped.
|
||||
@@ -134,8 +208,9 @@ fi
|
||||
# `pgrep -x` and `pkill -x` match against comm, which the kernel truncates to 15
|
||||
# characters, so `qemu-system-x86_64` is stored as `qemu-system-x86` and an -x
|
||||
# match on the full name can never fire. Match the prefix instead: it is 11
|
||||
# characters, well under the limit, and covers both x86_64 and aarch64. This is
|
||||
# the same rule qemuProcessPrefix applies in spinifex/runtime/compute/vm/orphan_scan.go.
|
||||
# characters, well under the limit, and covers both x86_64 and aarch64. EL-family
|
||||
# hosts run /usr/libexec/qemu-kvm, whose comm is exactly `qemu-kvm`. This is the
|
||||
# rule hostos.IsQEMUProcess applies in spinifex/runtime/host/hostos.
|
||||
#
|
||||
# Deliberately not -f. That matches the whole command line, where argv[0] may
|
||||
# carry a path, so an anchored pattern misses the process it was written for.
|
||||
@@ -144,7 +219,7 @@ fi
|
||||
# ones has both under the same comm and an unscoped kill would take the wrong
|
||||
# set down. The daemon runs guests as its own service user on an installed node;
|
||||
# a dev install runs them as whoever invoked spx.
|
||||
QEMU_PGREP=('^qemu-system')
|
||||
QEMU_PGREP=('^(qemu-system|qemu-kvm$)')
|
||||
for qemu_user in spinifex-daemon spinifex "${SUDO_USER:-$(id -un)}"; do
|
||||
if id -u "$qemu_user" >/dev/null 2>&1; then
|
||||
QEMU_PGREP=(-u "$qemu_user" "${QEMU_PGREP[@]}")
|
||||
|
||||
Executable
+90
@@ -0,0 +1,90 @@
|
||||
#!/bin/sh
|
||||
# node-reset_test.sh — tests the QEMU process match node-reset.sh uses to stop
|
||||
# guests before it deletes Viperblock state, against real processes.
|
||||
#
|
||||
# Each fake guest is a copy of sleep named like a QEMU binary, so the kernel
|
||||
# gives it the same comm, including the 15-character truncation.
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
|
||||
TMP=$(mktemp -d)
|
||||
pids=""
|
||||
cleanup() {
|
||||
for pid in $pids; do kill "$pid" 2>/dev/null || true; done
|
||||
rm -rf "$TMP"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
fails=0
|
||||
pass() { echo " ok - $1"; }
|
||||
fail() { echo " FAIL - $1"; fails=$((fails + 1)); }
|
||||
|
||||
echo "-- node-reset.sh QEMU process match"
|
||||
|
||||
# pattern_in <script> — the pgrep pattern the script assigns to QEMU_PGREP.
|
||||
pattern_in() {
|
||||
sed -n "s/^QEMU_PGREP=('\(.*\)')$/\1/p" "$1"
|
||||
}
|
||||
|
||||
PATTERN=$(pattern_in "$SCRIPT_DIR/node-reset.sh")
|
||||
if [ -z "$PATTERN" ]; then
|
||||
echo " FAIL - no QEMU_PGREP=('...') line in node-reset.sh"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# reset-dev-env.sh stops guests the same way and must not drift from it.
|
||||
dev_pattern=$(pattern_in "$SCRIPT_DIR/reset-dev-env.sh")
|
||||
if [ "$dev_pattern" = "$PATTERN" ]; then
|
||||
pass "reset-dev-env.sh uses node-reset.sh's pattern"
|
||||
else
|
||||
fail "reset-dev-env.sh pattern '$dev_pattern' differs from node-reset.sh's '$PATTERN'"
|
||||
fi
|
||||
|
||||
# start <name> — runs a process whose comm is <name>, truncated to 15
|
||||
# characters. Its PID is left in $!.
|
||||
start() {
|
||||
cp "$(command -v sleep)" "$TMP/$1"
|
||||
"$TMP/$1" 300 &
|
||||
pids="$pids $!"
|
||||
}
|
||||
start qemu-system-x86_64; pid_x86=$!
|
||||
start qemu-system-aarch64; pid_arm=$!
|
||||
start qemu-kvm; pid_kvm=$!
|
||||
start qemu-img; pid_img=$!
|
||||
start qemu-kvm-helper; pid_helper=$!
|
||||
|
||||
me=$(id -un)
|
||||
found=" $(pgrep -u "$me" "$PATTERN" | tr '\n' ' ' || true) "
|
||||
|
||||
check_found() {
|
||||
case "$found" in
|
||||
*" $2 "*) pass "finds a $1 guest owned by the service user" ;;
|
||||
*) fail "misses a $1 guest (pid $2) owned by the service user" ;;
|
||||
esac
|
||||
}
|
||||
check_found qemu-system-x86_64 "$pid_x86"
|
||||
check_found qemu-system-aarch64 "$pid_arm"
|
||||
check_found qemu-kvm "$pid_kvm"
|
||||
|
||||
check_ignored() {
|
||||
case "$found" in
|
||||
*" $2 "*) fail "matches $1 (pid $2), which is not a guest" ;;
|
||||
*) pass "ignores $1" ;;
|
||||
esac
|
||||
}
|
||||
check_ignored qemu-img "$pid_img"
|
||||
check_ignored qemu-kvm-helper "$pid_helper"
|
||||
|
||||
# The owner scope: the same guests, asked for as another user, are not found.
|
||||
other=root
|
||||
[ "$me" = root ] && other=nobody
|
||||
other_found=" $(pgrep -u "$other" "$PATTERN" | tr '\n' ' ' || true) "
|
||||
case "$other_found" in
|
||||
*" $pid_kvm "* | *" $pid_x86 "*) fail "a scope of -u $other still matches guests owned by $me" ;;
|
||||
*) pass "ignores guests owned by another user" ;;
|
||||
esac
|
||||
|
||||
if [ "$fails" -gt 0 ]; then
|
||||
echo "$fails failure(s)"
|
||||
exit 1
|
||||
fi
|
||||
@@ -359,17 +359,29 @@ fi
|
||||
# --- Shutdown services ---
|
||||
echo "==> Stopping services"
|
||||
sudo systemctl stop spinifex.target 2>/dev/null || true
|
||||
# spinifex-hostnet starts with OVS, which is started below to delete bridges; it
|
||||
# would rebuild the old veth pairs. setup-ovn.sh enables it again.
|
||||
sudo systemctl disable --now spinifex-hostnet.service 2>/dev/null || true
|
||||
sudo systemctl reset-failed 'spinifex-*' 2>/dev/null || true
|
||||
sudo pkill -x qemu-system-x86_64 2>/dev/null || true
|
||||
sudo pkill -x qemu-system-aarch64 2>/dev/null || true
|
||||
# comm truncates to 15 characters, so match qemu-system-* by prefix; EL-family
|
||||
# hosts run qemu-kvm. Scoped to the user the daemon runs guests as, as in
|
||||
# node-reset.sh, so libvirt guests on the same host are left alone.
|
||||
QEMU_PGREP=('^(qemu-system|qemu-kvm$)')
|
||||
for qemu_user in spinifex-daemon spinifex "$INVOKING_USER"; do
|
||||
if id -u "$qemu_user" >/dev/null 2>&1; then
|
||||
QEMU_PGREP=(-u "$qemu_user" "${QEMU_PGREP[@]}")
|
||||
break
|
||||
fi
|
||||
done
|
||||
sudo pkill "${QEMU_PGREP[@]}" 2>/dev/null || true
|
||||
|
||||
# Wait for QEMU to fully exit before tearing down viperblock state.
|
||||
timeout=30
|
||||
elapsed=0
|
||||
while pgrep -x 'qemu-system-x86_64|qemu-system-aarch64' > /dev/null 2>&1; do
|
||||
while pgrep "${QEMU_PGREP[@]}" > /dev/null 2>&1; do
|
||||
if [ "$elapsed" -ge "$timeout" ]; then
|
||||
echo "❌ QEMU still running after ${timeout}s:"
|
||||
pgrep -af 'qemu-system-' || true
|
||||
pgrep -a "${QEMU_PGREP[@]}" || true
|
||||
echo " Kill them manually and re-run this script."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -66,7 +66,7 @@ MAX_QEMU_ATTEMPTS=60
|
||||
QEMU_CMD=""
|
||||
|
||||
while [ $QEMU_ATTEMPTS -lt $MAX_QEMU_ATTEMPTS ]; do
|
||||
QEMU_CMD=$(ps auxw | grep "$INSTANCE_ID" | grep qemu-system | grep -v grep)
|
||||
QEMU_CMD=$(ps auxw | grep "$INSTANCE_ID" | grep -E "qemu-system|qemu-kvm" | grep -v grep)
|
||||
|
||||
if [ -n "$QEMU_CMD" ]; then
|
||||
break
|
||||
|
||||
+376
-74
@@ -4,12 +4,20 @@
|
||||
#
|
||||
# This script bootstraps a compute node for OVN-based VPC networking:
|
||||
# 1. Installs OVN/OVS packages (if not present)
|
||||
# 2. Enables required services (openvswitch-switch, ovn-controller)
|
||||
# 2. Enables required services (Open vSwitch, ovn-controller)
|
||||
# 3. Creates br-int with secure fail-mode
|
||||
# 4. Configures WAN bridge for public subnet uplink (auto-detected or manual)
|
||||
# 5. Configures OVS external_ids for OVN chassis identity
|
||||
# 6. Applies sysctl tuning for overlay networking
|
||||
#
|
||||
# Host network persistence: the links this script creates are kernel-only. On
|
||||
# a host where systemd-networkd is enabled, or that already has Spinifex
|
||||
# networkd files, they persist in /etc/systemd/network. Otherwise (for example
|
||||
# a NetworkManager host) the topology goes to /etc/spinifex/hostnet.json and
|
||||
# spinifex-hostnet.service re-creates it at boot and on every OVS start. The
|
||||
# first run records which in /etc/spinifex/hostnet-backend; it never changes.
|
||||
# Where NetworkManager runs, the links Spinifex owns are made unmanaged.
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/setup-ovn.sh [options]
|
||||
#
|
||||
@@ -121,6 +129,19 @@
|
||||
|
||||
set -e
|
||||
|
||||
# Host-OS capability probes: beside this script in a checkout and in the
|
||||
# installed share dir, else where setup.sh installs them.
|
||||
DISTRO_LIB=""
|
||||
for candidate in "$(dirname "${BASH_SOURCE[0]}")/lib/distro.sh" /usr/local/share/spinifex/lib/distro.sh; do
|
||||
if [ -f "$candidate" ]; then DISTRO_LIB="$candidate"; break; fi
|
||||
done
|
||||
if [ -z "$DISTRO_LIB" ]; then
|
||||
echo "ERROR: distro.sh not found at $(dirname "${BASH_SOURCE[0]}")/lib/distro.sh or /usr/local/share/spinifex/lib/distro.sh; re-run setup.sh to install it" >&2
|
||||
exit 1
|
||||
fi
|
||||
# shellcheck source=lib/distro.sh
|
||||
. "$DISTRO_LIB"
|
||||
|
||||
# Defaults
|
||||
MANAGEMENT=false
|
||||
WAN_BRIDGE=""
|
||||
@@ -222,6 +243,152 @@ iface_master() {
|
||||
printf '%s' "$m"
|
||||
}
|
||||
|
||||
# --- Host network persistence ---
|
||||
#
|
||||
# Links this script creates are kernel-only and vanish on reboot. The first run
|
||||
# picks how they persist and records it: networkd where systemd-networkd is
|
||||
# enabled, else hostnet (spinifex-hostnet.service replays HOSTNET_STATE at boot
|
||||
# and on every ovs-vswitchd start). There is no switching; reset the node.
|
||||
HOSTNET_BACKEND_FILE=/etc/spinifex/hostnet-backend
|
||||
HOSTNET_STATE=/etc/spinifex/hostnet.json
|
||||
HOSTNET_LOCK=/run/spinifex/hostnet.lock
|
||||
NETWORKD_DIR=/etc/systemd/network
|
||||
NM_UNMANAGED_CONF=/etc/NetworkManager/conf.d/00-spinifex-unmanaged.conf
|
||||
NM_UNMANAGED_RULE=/etc/udev/rules.d/90-spinifex-nm-unmanaged.rules
|
||||
# System-instance management TAPs: "mg" plus 13 hex digits of the instance ID.
|
||||
# NetworkManager globs have no character classes, so this match goes to udev.
|
||||
MGMT_TAP_GLOB='mg[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]'
|
||||
|
||||
# Prints the recorded backend, or picks one and records it. A node with
|
||||
# Spinifex networkd files and no record was set up before the record existed.
|
||||
select_hostnet_backend() {
|
||||
local backend
|
||||
if sudo test -e "$HOSTNET_BACKEND_FILE"; then
|
||||
backend=$(sudo cat "$HOSTNET_BACKEND_FILE" | tr -d '[:space:]')
|
||||
case "$backend" in
|
||||
networkd|hostnet) printf '%s\n' "$backend"; return 0 ;;
|
||||
esac
|
||||
echo "ERROR: $HOSTNET_BACKEND_FILE says '$backend'; expected networkd or hostnet" >&2
|
||||
return 1
|
||||
fi
|
||||
if compgen -G "$NETWORKD_DIR/*-spinifex-*.network" >/dev/null || compgen -G "$NETWORKD_DIR/*-spinifex-*.netdev" >/dev/null; then
|
||||
backend=networkd
|
||||
elif systemctl is-enabled --quiet systemd-networkd.service 2>/dev/null; then
|
||||
backend=networkd
|
||||
else
|
||||
backend=hostnet
|
||||
fi
|
||||
sudo mkdir -p "$(dirname "$HOSTNET_BACKEND_FILE")"
|
||||
printf '%s\n' "$backend" | sudo tee "$HOSTNET_BACKEND_FILE" >/dev/null
|
||||
# Read by `spx admin hostnet check`, which the daemon and vpcd run unprivileged.
|
||||
sudo chmod 0644 "$HOSTNET_BACKEND_FILE"
|
||||
printf '%s\n' "$backend"
|
||||
}
|
||||
|
||||
# write_hostnet_state <true|false> — records the topology for
|
||||
# `spx admin hostnet apply`, atomically. true marks it pending: apply refuses
|
||||
# it until the run that wrote it finishes.
|
||||
write_hostnet_state() {
|
||||
local pending="$1" body
|
||||
body=$(printf '{\n "version": 1,\n "pending": %s,\n "int_bridge": "br-int"' "$pending")
|
||||
case "$WAN_BRIDGE_MODE" in
|
||||
existing)
|
||||
body="$body"$(printf ',\n "wan": {"mode": "existing", "ovs_bridge": "%s"}' "$WAN_BRIDGE") ;;
|
||||
veth)
|
||||
body="$body"$(printf ',\n "wan": {"mode": "veth", "ovs_bridge": "%s", "linux_bridge": "%s", "host_end": "veth-wan-br", "ovs_end": "veth-wan-ovs"}' "$WAN_BRIDGE" "$LINUX_BRIDGE") ;;
|
||||
direct)
|
||||
body="$body"$(printf ',\n "wan": {"mode": "direct", "ovs_bridge": "%s", "nic": "%s"}' "$WAN_BRIDGE" "$WAN_IFACE") ;;
|
||||
nat)
|
||||
body="$body"$(printf ',\n "wan": {"mode": "nat", "ovs_bridge": "%s", "host_end": "spx-nat-host", "ovs_end": "spx-nat-ovs", "host_mac": "%s", "host_cidr": "%s"}' "$WAN_BRIDGE" "$NAT_TRANSIT_HOST_MAC" "$NAT_TRANSIT_GW_CIDR") ;;
|
||||
esac
|
||||
if [ "$MGMT_BRIDGE_ENABLED" = true ]; then
|
||||
body="$body"$(printf ',\n "mgmt": {"bridge": "%s", "cidr": "%s", "nic": "%s"}' "$MGMT_BRIDGE" "$MGMT_CIDR" "$MGMT_IFACE")
|
||||
fi
|
||||
if [ -n "${ENCAP_ROUTE_DST:-}" ]; then
|
||||
body="$body"$(printf ',\n "encap_route": {"dst": "%s", "dev": "%s", "src": "%s", "metric": %s}' \
|
||||
"$ENCAP_ROUTE_DST" "$ENCAP_ROUTE_DEV" "$ENCAP_IP" "$ENCAP_ROUTE_METRIC")
|
||||
fi
|
||||
sudo mkdir -p "$(dirname "$HOSTNET_STATE")"
|
||||
printf '%s\n}\n' "$body" | sudo tee "$HOSTNET_STATE.tmp" >/dev/null
|
||||
sudo chmod 0644 "$HOSTNET_STATE.tmp"
|
||||
sudo mv -f "$HOSTNET_STATE.tmp" "$HOSTNET_STATE"
|
||||
}
|
||||
|
||||
# Holds the lock `spx admin hostnet apply` takes, from a root flock process fed
|
||||
# by a pipe, so an apply started by an OVS restart waits for this run's links.
|
||||
# Never run systemctl against OVS or spinifex-hostnet while holding it.
|
||||
hostnet_lock() {
|
||||
local reply=""
|
||||
sudo mkdir -p "$(dirname "$HOSTNET_LOCK")"
|
||||
coproc HOSTNET_LOCK_HOLDER { exec sudo flock -w 120 "$HOSTNET_LOCK" sh -c 'echo locked; read -r _'; }
|
||||
read -r reply <&"${HOSTNET_LOCK_HOLDER[0]}" || true
|
||||
if [ "$reply" != locked ]; then
|
||||
echo "ERROR: $HOSTNET_LOCK still held after 120 s; is spx admin hostnet apply stuck? (systemctl status spinifex-hostnet)" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
hostnet_unlock() {
|
||||
local fd="${HOSTNET_LOCK_HOLDER[1]}" pid="$HOSTNET_LOCK_HOLDER_PID"
|
||||
exec {fd}>&-
|
||||
wait "$pid" 2>/dev/null || true
|
||||
}
|
||||
|
||||
nm_running() {
|
||||
systemctl is-active --quiet NetworkManager.service 2>/dev/null
|
||||
}
|
||||
|
||||
# nm_release_nic <nic> — before a NIC moves into OVS, refuse the one carrying
|
||||
# the default route or this SSH session, then take it from NetworkManager. Its
|
||||
# profiles are left alone: once unmanaged, none can activate on it.
|
||||
nm_release_nic() {
|
||||
local nic="$1" ssh_ip state
|
||||
if sudo ovs-vsctl port-to-br "$nic" >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
if ip -4 route show default 2>/dev/null | awk '{for(i=1;i<NF;i++) if($i=="dev") print $(i+1)}' | grep -qx "$nic"; then
|
||||
echo "ERROR: $nic carries the default route; moving it into OVS would cut this host off" >&2
|
||||
exit 1
|
||||
fi
|
||||
ssh_ip=$(printf '%s\n' "${SSH_CONNECTION:-}" | awk '{print $3}')
|
||||
if [ -n "$ssh_ip" ] && ip -o -4 addr show dev "$nic" 2>/dev/null | awk '{print $4}' | cut -d/ -f1 | grep -qxF "$ssh_ip"; then
|
||||
echo "ERROR: this SSH session arrived on $nic ($ssh_ip); moving it into OVS would drop it" >&2
|
||||
exit 1
|
||||
fi
|
||||
state=$(nmcli -g GENERAL.STATE device show "$nic" 2>/dev/null || true)
|
||||
case "$state" in
|
||||
*"(connect"*)
|
||||
sudo nmcli device disconnect "$nic" >/dev/null
|
||||
echo " $nic: disconnected from NetworkManager"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Writes the NetworkManager config and udev rule that leave every link Spinifex
|
||||
# owns unmanaged, then has NetworkManager and udev reread them. The host uplink
|
||||
# (a Linux bridge such as br-wan, and its NIC) stays NetworkManager's.
|
||||
write_nm_unmanaged() {
|
||||
local names=(br-int "$WAN_BRIDGE" 'veth-wan-*' 'spx-nat-*' 'tap*' 'ime-*' 'imi-*' 'vhp-*' br-imds 'genev_sys_*')
|
||||
local list="" name
|
||||
if [ "$MGMT_BRIDGE_ENABLED" = true ]; then
|
||||
names+=("$MGMT_BRIDGE" "$MGMT_IFACE")
|
||||
fi
|
||||
if [ "$WAN_BRIDGE_MODE" = direct ]; then
|
||||
names+=("$WAN_IFACE")
|
||||
fi
|
||||
for name in "${names[@]}"; do
|
||||
[ -n "$name" ] && list="${list:+$list;}interface-name:$name"
|
||||
done
|
||||
sudo mkdir -p "$(dirname "$NM_UNMANAGED_CONF")" "$(dirname "$NM_UNMANAGED_RULE")"
|
||||
printf '# Written by setup-ovn.sh on every run: links Spinifex owns.\n[keyfile]\nunmanaged-devices+=%s\n' "$list" \
|
||||
| sudo tee "$NM_UNMANAGED_CONF" >/dev/null
|
||||
printf '# Written by setup-ovn.sh: system-instance management TAPs.\nSUBSYSTEM=="net", KERNEL=="%s", ENV{NM_UNMANAGED}="1"\n' "$MGMT_TAP_GLOB" \
|
||||
| sudo tee "$NM_UNMANAGED_RULE" >/dev/null
|
||||
sudo udevadm control --reload
|
||||
sudo udevadm trigger --action=change --subsystem-match=net --sysname-match='mg*'
|
||||
sudo nmcli general reload conf
|
||||
}
|
||||
|
||||
teardown() {
|
||||
echo "Tearing down the OVN host wiring built by this script"
|
||||
[ "$DRY_RUN" = true ] && echo " (dry run — nothing is changed)"
|
||||
@@ -469,6 +636,15 @@ else
|
||||
detect_wan_bridge
|
||||
fi
|
||||
|
||||
# Routed NAT transit, used by nat mode and recorded for spinifex-hostnet.
|
||||
NAT_TRANSIT_CIDR="100.127.0.0/24"
|
||||
NAT_TRANSIT_GW_CIDR="100.127.0.1/24"
|
||||
# Same on every node. OVN keeps one MAC binding for the transit nexthop
|
||||
# cluster-wide, so a random per-node MAC sends every chassis's egress to
|
||||
# whichever node seeded it; the pair is node-local, so one address everywhere
|
||||
# is unambiguous.
|
||||
NAT_TRANSIT_HOST_MAC="02:00:64:7f:00:01"
|
||||
|
||||
# Auto-detect encap IP if not specified
|
||||
if [ -z "$ENCAP_IP" ]; then
|
||||
# Prefer br-vpc IP if it exists (dedicated VPC data plane)
|
||||
@@ -575,19 +751,70 @@ ensure_clustered_db_storage() {
|
||||
# fresh SB and re-registers under whatever system-id it started with — which
|
||||
# on a renamed node is the old one, and that row then holds this node's encap
|
||||
# IP so the correctly-named chassis can never commit. Step 5 restarts it.
|
||||
sudo systemctl stop ovn-northd ovn-ovsdb-server-nb ovn-ovsdb-server-sb ovn-central ovn-controller 2>/dev/null || true
|
||||
sudo systemctl stop "${OVN_CENTRAL_UNITS[@]}" ovn-controller 2>/dev/null || true
|
||||
sudo rm -f "${standalone[@]}"
|
||||
}
|
||||
|
||||
# set_root_service_user <file> <var> — sets <var>="root:root" in <file> when
|
||||
# the file exists. Returns 0 only when it changed the file.
|
||||
set_root_service_user() {
|
||||
local file=$1 var=$2
|
||||
[ -f "$file" ] || return 1
|
||||
grep -qx "$var=\"root:root\"" "$file" && return 1
|
||||
if grep -q "^$var=" "$file"; then
|
||||
sudo sed -i "s|^$var=.*|$var=\"root:root\"|" "$file"
|
||||
else
|
||||
echo "$var=\"root:root\"" | sudo tee -a "$file" >/dev/null
|
||||
fi
|
||||
echo " $file: $var=\"root:root\""
|
||||
}
|
||||
|
||||
echo ""
|
||||
echo "Step 2: Enabling services..."
|
||||
|
||||
sudo systemctl enable openvswitch-switch
|
||||
sudo systemctl start openvswitch-switch
|
||||
echo " openvswitch-switch: started"
|
||||
# Unit names differ by distro: the OVS umbrella, and whether the OVN databases
|
||||
# have units of their own (Debian) or are started by ovn-northd.service (EL).
|
||||
OVS_UNIT=$(distro_ovs_unit) || exit 1
|
||||
OVN_CENTRAL_LAYOUT=""
|
||||
if [ "$MANAGEMENT" = true ] || distro_unit_exists ovn-central.service || distro_unit_exists ovn-northd.service; then
|
||||
OVN_CENTRAL_LAYOUT=$(distro_ovn_central_layout) || exit 1
|
||||
fi
|
||||
case "$OVN_CENTRAL_LAYOUT" in
|
||||
split)
|
||||
OVN_CENTRAL_UNIT=ovn-central.service
|
||||
OVN_DB_UNITS=(ovn-ovsdb-server-nb.service ovn-ovsdb-server-sb.service)
|
||||
OVN_SERVER_UNITS=("${OVN_DB_UNITS[@]}" ovn-northd.service)
|
||||
OVN_CENTRAL_UNITS=(ovn-central.service "${OVN_SERVER_UNITS[@]}")
|
||||
;;
|
||||
lone-northd)
|
||||
OVN_CENTRAL_UNIT=ovn-northd.service
|
||||
OVN_DB_UNITS=(ovn-northd.service)
|
||||
OVN_SERVER_UNITS=(ovn-northd.service)
|
||||
OVN_CENTRAL_UNITS=(ovn-northd.service)
|
||||
;;
|
||||
esac
|
||||
|
||||
# OVS and OVN run as root, as on Debian: the service users reach the sockets
|
||||
# through the spinifex group, a model tested only against root-owned daemons.
|
||||
# EL ships non-root users and re-chowns the run dirs to them on every start.
|
||||
OVS_USER_CHANGED=false
|
||||
if set_root_service_user /etc/sysconfig/openvswitch OVS_USER_ID; then
|
||||
OVS_USER_CHANGED=true
|
||||
fi
|
||||
# Takes effect below: ovn-northd and ovn-controller are restarted either way.
|
||||
set_root_service_user /etc/sysconfig/ovn OVN_USER_ID || true
|
||||
|
||||
sudo systemctl enable "$OVS_UNIT"
|
||||
if [ "$OVS_USER_CHANGED" = true ]; then
|
||||
# start is a no-op on a running OVS, which would keep its old user.
|
||||
sudo systemctl restart "$OVS_UNIT"
|
||||
else
|
||||
sudo systemctl start "$OVS_UNIT"
|
||||
fi
|
||||
echo " ${OVS_UNIT%.service}: started"
|
||||
|
||||
if [ "$MANAGEMENT" = true ]; then
|
||||
sudo systemctl enable ovn-central
|
||||
sudo systemctl enable "$OVN_CENTRAL_UNIT"
|
||||
|
||||
# LAN-plane NB/SB client listener, added to the loopback one set below.
|
||||
# Guard is load-bearing: --db-nb-addr defaults to 0.0.0.0, so
|
||||
@@ -651,10 +878,10 @@ ExecStop=/usr/share/ovn/scripts/ovn-ctl stop_northd --no-monitor --ovn-manage-ov
|
||||
EOF
|
||||
sudo systemctl daemon-reload
|
||||
|
||||
# The ovn-central aggregator is ExecStart=/bin/true, so restarting it
|
||||
# won't restart the children — restart the per-DB units directly to
|
||||
# pick up the new OVN_CTL_OPTS.
|
||||
sudo systemctl restart ovn-ovsdb-server-nb ovn-ovsdb-server-sb ovn-northd
|
||||
# Debian's ovn-central aggregator is ExecStart=/bin/true, so restarting
|
||||
# it won't restart the children — restart the DB and northd units
|
||||
# directly to pick up the new OVN_CTL_OPTS.
|
||||
sudo systemctl restart "${OVN_SERVER_UNITS[@]}"
|
||||
echo " ovn-central: started clustered (NB DB + SB DB + ovn-northd)"
|
||||
else
|
||||
# Always write the file, even when there is nothing to put in it. This
|
||||
@@ -664,12 +891,12 @@ EOF
|
||||
# peer that no longer exists, which looks like a hung leader election.
|
||||
echo "OVN_CTL_OPTS=\"$OVN_DB_LISTEN_OPTS\"" | sudo tee /etc/default/ovn-central >/dev/null
|
||||
echo " wrote /etc/default/ovn-central"
|
||||
sudo systemctl start ovn-central
|
||||
sudo systemctl start "$OVN_CENTRAL_UNIT"
|
||||
|
||||
# ovn-central is ExecStart=/bin/true; restarting it does not restart
|
||||
# its children, so a re-run with changed options needs the per-DB units
|
||||
# restarted directly to pick them up.
|
||||
sudo systemctl restart ovn-ovsdb-server-nb ovn-ovsdb-server-sb
|
||||
# Debian's ovn-central is ExecStart=/bin/true; restarting it does not
|
||||
# restart its children, so a re-run with changed options needs the DB
|
||||
# units restarted directly to pick them up.
|
||||
sudo systemctl restart "${OVN_DB_UNITS[@]}"
|
||||
echo " ovn-central: started (NB DB + SB DB + ovn-northd)"
|
||||
if [ -n "$LAN_ADDR" ]; then
|
||||
echo " NB/SB client listen: 127.0.0.1, $LAN_ADDR"
|
||||
@@ -812,10 +1039,10 @@ else
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if systemctl is-enabled ovn-central >/dev/null 2>&1 ||
|
||||
systemctl is-active ovn-central >/dev/null 2>&1; then
|
||||
sudo systemctl stop ovn-central ovn-northd ovn-ovsdb-server-nb ovn-ovsdb-server-sb 2>/dev/null || true
|
||||
sudo systemctl disable ovn-central ovn-northd ovn-ovsdb-server-nb ovn-ovsdb-server-sb 2>/dev/null || true
|
||||
if [ -n "$OVN_CENTRAL_LAYOUT" ] && { systemctl is-enabled "$OVN_CENTRAL_UNIT" >/dev/null 2>&1 ||
|
||||
systemctl is-active "$OVN_CENTRAL_UNIT" >/dev/null 2>&1; }; then
|
||||
sudo systemctl stop "${OVN_CENTRAL_UNITS[@]}" 2>/dev/null || true
|
||||
sudo systemctl disable "${OVN_CENTRAL_UNITS[@]}" 2>/dev/null || true
|
||||
echo " ovn-central: stopped and disabled (compute node runs no database)"
|
||||
fi
|
||||
for db in "$OVN_DBDIR/ovnnb_db.db" "$OVN_DBDIR/ovnsb_db.db"; do
|
||||
@@ -825,6 +1052,34 @@ else
|
||||
done
|
||||
fi
|
||||
|
||||
# --- Step 2b: Host network persistence and NetworkManager ---
|
||||
echo ""
|
||||
echo "Step 2b: Host network persistence..."
|
||||
HOSTNET_BACKEND=$(select_hostnet_backend)
|
||||
echo " backend: $HOSTNET_BACKEND (recorded in $HOSTNET_BACKEND_FILE)"
|
||||
|
||||
if nm_running; then
|
||||
if [ "$MGMT_BRIDGE_ENABLED" = true ] && [ -n "$MGMT_IFACE" ]; then
|
||||
nm_release_nic "$MGMT_IFACE"
|
||||
fi
|
||||
if [ "$WAN_BRIDGE_MODE" = direct ]; then
|
||||
nm_release_nic "$WAN_IFACE"
|
||||
fi
|
||||
write_nm_unmanaged
|
||||
echo " NetworkManager: Spinifex links unmanaged ($NM_UNMANAGED_CONF, $NM_UNMANAGED_RULE)"
|
||||
fi
|
||||
|
||||
# Held until the wiring below is complete; a run that dies before then leaves
|
||||
# the state pending, and spinifex-hostnet refuses it until this script is rerun.
|
||||
ENCAP_ROUTE_DST=""
|
||||
ENCAP_ROUTE_DEV=""
|
||||
ENCAP_ROUTE_METRIC=50
|
||||
if [ "$HOSTNET_BACKEND" = hostnet ]; then
|
||||
hostnet_lock
|
||||
write_hostnet_state true
|
||||
echo " $HOSTNET_STATE: pending until this run finishes wiring"
|
||||
fi
|
||||
|
||||
# --- Step 3: Create and configure br-int ---
|
||||
echo ""
|
||||
echo "Step 3: Configuring br-int..."
|
||||
@@ -842,7 +1097,7 @@ echo " br-int: created, fail-mode=secure, up"
|
||||
# still forwards, but ovn-controller flow programming and any tooling that
|
||||
# probes link state misbehave. Unmanaged=yes keeps OVS in sole control.
|
||||
OVS_INTERNAL_NET=/etc/systemd/network/05-spinifex-ovs-internal.network
|
||||
if [ ! -f "$OVS_INTERNAL_NET" ]; then
|
||||
if [ "$HOSTNET_BACKEND" = networkd ] && [ ! -f "$OVS_INTERNAL_NET" ]; then
|
||||
sudo tee "$OVS_INTERNAL_NET" >/dev/null <<'NETWORK'
|
||||
[Match]
|
||||
Name=br-int br-ext
|
||||
@@ -956,23 +1211,24 @@ if [ -n "$WAN_BRIDGE" ]; then
|
||||
# existing networkd config. networkd searches /etc, /run, /usr/lib;
|
||||
# check all three. Our own file is excluded so idempotent re-runs
|
||||
# still rewrite when needed.
|
||||
BR_WAN_NETDEV="/etc/systemd/network/14-spinifex-br-wan.netdev"
|
||||
VETH_NETDEV="/etc/systemd/network/15-spinifex-veth-wan.netdev"
|
||||
VETH_NETWORK="/etc/systemd/network/15-spinifex-veth-wan.network"
|
||||
VETH_OVS_NETWORK="/etc/systemd/network/16-spinifex-veth-wan-ovs.network"
|
||||
EXISTING_BR_NETDEV=$(grep -rls --include="*.netdev" -E "^\s*Name=$LINUX_BRIDGE\s*$" \
|
||||
/etc/systemd/network /run/systemd/network /usr/lib/systemd/network 2>/dev/null \
|
||||
| grep -v "^$BR_WAN_NETDEV$" || true)
|
||||
if [ -n "$EXISTING_BR_NETDEV" ]; then
|
||||
echo " skipping $BR_WAN_NETDEV — operator-managed NetDev already declares $LINUX_BRIDGE: $EXISTING_BR_NETDEV"
|
||||
else
|
||||
sudo tee "$BR_WAN_NETDEV" >/dev/null <<NETDEV
|
||||
if [ "$HOSTNET_BACKEND" = networkd ]; then
|
||||
BR_WAN_NETDEV="/etc/systemd/network/14-spinifex-br-wan.netdev"
|
||||
VETH_NETDEV="/etc/systemd/network/15-spinifex-veth-wan.netdev"
|
||||
VETH_NETWORK="/etc/systemd/network/15-spinifex-veth-wan.network"
|
||||
VETH_OVS_NETWORK="/etc/systemd/network/16-spinifex-veth-wan-ovs.network"
|
||||
EXISTING_BR_NETDEV=$(grep -rls --include="*.netdev" -E "^\s*Name=$LINUX_BRIDGE\s*$" \
|
||||
/etc/systemd/network /run/systemd/network /usr/lib/systemd/network 2>/dev/null \
|
||||
| grep -v "^$BR_WAN_NETDEV$" || true)
|
||||
if [ -n "$EXISTING_BR_NETDEV" ]; then
|
||||
echo " skipping $BR_WAN_NETDEV — operator-managed NetDev already declares $LINUX_BRIDGE: $EXISTING_BR_NETDEV"
|
||||
else
|
||||
sudo tee "$BR_WAN_NETDEV" >/dev/null <<NETDEV
|
||||
[NetDev]
|
||||
Name=$LINUX_BRIDGE
|
||||
Kind=bridge
|
||||
NETDEV
|
||||
fi
|
||||
sudo tee "$VETH_NETDEV" >/dev/null <<NETDEV
|
||||
fi
|
||||
sudo tee "$VETH_NETDEV" >/dev/null <<NETDEV
|
||||
[NetDev]
|
||||
Name=veth-wan-br
|
||||
Kind=veth
|
||||
@@ -980,7 +1236,7 @@ Kind=veth
|
||||
[Peer]
|
||||
Name=veth-wan-ovs
|
||||
NETDEV
|
||||
sudo tee "$VETH_NETWORK" >/dev/null <<NETWORK
|
||||
sudo tee "$VETH_NETWORK" >/dev/null <<NETWORK
|
||||
[Match]
|
||||
Name=veth-wan-br
|
||||
|
||||
@@ -988,12 +1244,12 @@ Name=veth-wan-br
|
||||
Bridge=$LINUX_BRIDGE
|
||||
ConfigureWithoutCarrier=yes
|
||||
NETWORK
|
||||
# Second unit admin-ups the OVS end of the pair. OVS owns the port
|
||||
# (enslaved via ovs-vsctl add-port above) but does not flip admin
|
||||
# state on external ports — that's networkd's job. Without this,
|
||||
# veth-wan-ovs stays DOWN after reboot, peer goes LOWERLAYERDOWN,
|
||||
# br-wan loses carrier.
|
||||
sudo tee "$VETH_OVS_NETWORK" >/dev/null <<NETWORK
|
||||
# Second unit admin-ups the OVS end of the pair. OVS owns the port
|
||||
# (enslaved via ovs-vsctl add-port above) but does not flip admin
|
||||
# state on external ports — that's networkd's job. Without this,
|
||||
# veth-wan-ovs stays DOWN after reboot, peer goes LOWERLAYERDOWN,
|
||||
# br-wan loses carrier.
|
||||
sudo tee "$VETH_OVS_NETWORK" >/dev/null <<NETWORK
|
||||
[Match]
|
||||
Name=veth-wan-ovs
|
||||
|
||||
@@ -1003,8 +1259,9 @@ RequiredForOnline=no
|
||||
[Network]
|
||||
ConfigureWithoutCarrier=yes
|
||||
NETWORK
|
||||
sudo networkctl reload 2>/dev/null || true
|
||||
echo " wrote $VETH_NETDEV + $VETH_NETWORK + $VETH_OVS_NETWORK (veth persists on reboot)"
|
||||
sudo networkctl reload 2>/dev/null || true
|
||||
echo " wrote $VETH_NETDEV + $VETH_NETWORK + $VETH_OVS_NETWORK (veth persists on reboot)"
|
||||
fi
|
||||
;;
|
||||
|
||||
direct)
|
||||
@@ -1038,14 +1295,6 @@ NETWORK
|
||||
# forwards and masquerades the transit /24 out whatever uplink it
|
||||
# has. OVN SNATs each VPC CIDR to its gateway LRP transit IP, so
|
||||
# the masquerade rule below is the only host-side NAT state.
|
||||
NAT_TRANSIT_CIDR="100.127.0.0/24"
|
||||
NAT_TRANSIT_GW_CIDR="100.127.0.1/24"
|
||||
# Same on every node. OVN keeps one MAC binding for the transit
|
||||
# nexthop cluster-wide, so a random per-node MAC sends every
|
||||
# chassis's egress to whichever node seeded it; the pair is
|
||||
# node-local, so one address everywhere is unambiguous.
|
||||
NAT_TRANSIT_HOST_MAC="02:00:64:7f:00:01"
|
||||
|
||||
if ! sudo ovs-vsctl br-exists "$WAN_BRIDGE" 2>/dev/null; then
|
||||
sudo ovs-vsctl --may-exist add-br "$WAN_BRIDGE"
|
||||
echo " created OVS bridge: $WAN_BRIDGE"
|
||||
@@ -1073,10 +1322,11 @@ NETWORK
|
||||
|
||||
# Persist the veth pair + transit IP across reboot (veths are
|
||||
# kernel-only; same rationale as veth mode above).
|
||||
NAT_NETDEV="/etc/systemd/network/17-spinifex-nat.netdev"
|
||||
NAT_NETWORK="/etc/systemd/network/17-spinifex-nat.network"
|
||||
NAT_OVS_NETWORK="/etc/systemd/network/18-spinifex-nat-ovs.network"
|
||||
sudo tee "$NAT_NETDEV" >/dev/null <<NETDEV
|
||||
if [ "$HOSTNET_BACKEND" = networkd ]; then
|
||||
NAT_NETDEV="/etc/systemd/network/17-spinifex-nat.netdev"
|
||||
NAT_NETWORK="/etc/systemd/network/17-spinifex-nat.network"
|
||||
NAT_OVS_NETWORK="/etc/systemd/network/18-spinifex-nat-ovs.network"
|
||||
sudo tee "$NAT_NETDEV" >/dev/null <<NETDEV
|
||||
[NetDev]
|
||||
Name=spx-nat-host
|
||||
Kind=veth
|
||||
@@ -1085,7 +1335,7 @@ MACAddress=$NAT_TRANSIT_HOST_MAC
|
||||
[Peer]
|
||||
Name=spx-nat-ovs
|
||||
NETDEV
|
||||
sudo tee "$NAT_NETWORK" >/dev/null <<NETWORK
|
||||
sudo tee "$NAT_NETWORK" >/dev/null <<NETWORK
|
||||
[Match]
|
||||
Name=spx-nat-host
|
||||
|
||||
@@ -1093,9 +1343,9 @@ Name=spx-nat-host
|
||||
Address=$NAT_TRANSIT_GW_CIDR
|
||||
ConfigureWithoutCarrier=yes
|
||||
NETWORK
|
||||
# Admin-up the OVS end after reboot (OVS owns the port but does
|
||||
# not flip admin state on external ports — same as veth mode).
|
||||
sudo tee "$NAT_OVS_NETWORK" >/dev/null <<NETWORK
|
||||
# Admin-up the OVS end after reboot (OVS owns the port but does
|
||||
# not flip admin state on external ports — same as veth mode).
|
||||
sudo tee "$NAT_OVS_NETWORK" >/dev/null <<NETWORK
|
||||
[Match]
|
||||
Name=spx-nat-ovs
|
||||
|
||||
@@ -1105,8 +1355,9 @@ RequiredForOnline=no
|
||||
[Network]
|
||||
ConfigureWithoutCarrier=yes
|
||||
NETWORK
|
||||
sudo networkctl reload 2>/dev/null || true
|
||||
echo " wrote $NAT_NETDEV + $NAT_NETWORK + $NAT_OVS_NETWORK (transit veth persists on reboot)"
|
||||
sudo networkctl reload 2>/dev/null || true
|
||||
echo " wrote $NAT_NETDEV + $NAT_NETWORK + $NAT_OVS_NETWORK (transit veth persists on reboot)"
|
||||
fi
|
||||
|
||||
# Kernel egress: masquerade the transit /24 out any uplink and
|
||||
# accept forwarded transit traffic. vpcd re-ensures these on every
|
||||
@@ -1261,8 +1512,9 @@ if [ "$MGMT_BRIDGE_ENABLED" = true ]; then
|
||||
# Persist the IP across reboots via a systemd-networkd drop-in. The
|
||||
# OVS bridge definition itself survives in ovsdb; only the L3 IP
|
||||
# needs re-applying on boot.
|
||||
MGMT_NETD_UNIT="/etc/systemd/network/10-spinifex-mgmt.network"
|
||||
sudo tee "$MGMT_NETD_UNIT" >/dev/null <<NETD
|
||||
if [ "$HOSTNET_BACKEND" = networkd ]; then
|
||||
MGMT_NETD_UNIT="/etc/systemd/network/10-spinifex-mgmt.network"
|
||||
sudo tee "$MGMT_NETD_UNIT" >/dev/null <<NETD
|
||||
[Match]
|
||||
Name=$MGMT_BRIDGE
|
||||
|
||||
@@ -1270,7 +1522,8 @@ Name=$MGMT_BRIDGE
|
||||
Address=$MGMT_CIDR
|
||||
ConfigureWithoutCarrier=yes
|
||||
NETD
|
||||
echo " wrote $MGMT_NETD_UNIT (IP persists on reboot via systemd-networkd)"
|
||||
echo " wrote $MGMT_NETD_UNIT (IP persists on reboot via systemd-networkd)"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
@@ -1401,8 +1654,10 @@ DATA_IFACE=$(ip -o -4 addr show | awk -v ip="$ENCAP_IP" '$0 ~ ip"/" {print $2}')
|
||||
if [ -n "$DATA_IFACE" ]; then
|
||||
SUBNET=$(ip -o -4 route show dev "$DATA_IFACE" proto kernel scope link | awk '{print $1}' | head -1)
|
||||
if [ -n "$SUBNET" ]; then
|
||||
sudo ip route replace "$SUBNET" dev "$DATA_IFACE" src "$ENCAP_IP" metric 50
|
||||
echo " data route: $SUBNET via $DATA_IFACE src $ENCAP_IP (metric 50)"
|
||||
sudo ip route replace "$SUBNET" dev "$DATA_IFACE" src "$ENCAP_IP" metric "$ENCAP_ROUTE_METRIC"
|
||||
echo " data route: $SUBNET via $DATA_IFACE src $ENCAP_IP (metric $ENCAP_ROUTE_METRIC)"
|
||||
ENCAP_ROUTE_DST="$SUBNET"
|
||||
ENCAP_ROUTE_DEV="$DATA_IFACE"
|
||||
else
|
||||
echo " skipped: no kernel route found for $DATA_IFACE"
|
||||
fi
|
||||
@@ -1410,6 +1665,12 @@ else
|
||||
echo " skipped: could not find interface for $ENCAP_IP"
|
||||
fi
|
||||
|
||||
if [ "$HOSTNET_BACKEND" = hostnet ]; then
|
||||
write_hostnet_state false
|
||||
hostnet_unlock
|
||||
echo " $HOSTNET_STATE: recorded for spinifex-hostnet"
|
||||
fi
|
||||
|
||||
# --- Step 7: Verify Geneve kernel support ---
|
||||
echo ""
|
||||
echo "Step 7: Verifying Geneve kernel module..."
|
||||
@@ -1509,17 +1770,24 @@ echo " OVS/OVN sockets: 0660 root:spinifex (group-scoped, no world access)"
|
||||
# file is the old 0666 override, and skipping would leave that exposure in place.
|
||||
# openvswitch-ipsec is included because Step 10 restarts it after this sweep,
|
||||
# so its ctl socket would otherwise be the one file left at the shipped mode.
|
||||
for unit in openvswitch-switch:/var/run/openvswitch/db.sock \
|
||||
"ovn-controller:/var/run/ovn/*.ctl" \
|
||||
"openvswitch-ipsec:/var/run/openvswitch/ovs-monitor-ipsec.*.ctl"; do
|
||||
# ovs-vswitchd starts after ovsdb-server and restarts with it (Requires=), so
|
||||
# its hook sees both daemons' sockets and fires on a restart of either.
|
||||
PERMS_UNITS=("ovs-vswitchd:/var/run/openvswitch/ovs-vswitchd.*.ctl"
|
||||
"ovn-controller:/var/run/ovn/*.ctl"
|
||||
"openvswitch-ipsec:/var/run/openvswitch/ovs-monitor-ipsec.*.ctl")
|
||||
# EL's ovn-northd.service starts the NB and SB databases as well as northd.
|
||||
if [ "$OVN_CENTRAL_LAYOUT" = lone-northd ]; then
|
||||
PERMS_UNITS+=("ovn-northd:/var/run/ovn/ovn-northd.*.ctl")
|
||||
fi
|
||||
for unit in "${PERMS_UNITS[@]}"; do
|
||||
UNIT="${unit%%:*}"
|
||||
WAIT_GLOB="${unit#*:}"
|
||||
OVERRIDE_DIR="/etc/systemd/system/${UNIT}.service.d"
|
||||
sudo mkdir -p "$OVERRIDE_DIR"
|
||||
# `ExecStartPost=-` so a failure here cannot stop the unit. This drop-in goes
|
||||
# into another package's service, and tightening a socket mode is hardening,
|
||||
# not a liveness requirement — it must never be the reason openvswitch-switch
|
||||
# or ovn-controller will not start. Without the dash, removing the helper
|
||||
# not a liveness requirement — it must never be the reason OVS or
|
||||
# ovn-controller will not start. Without the dash, removing the helper
|
||||
# (an uninstall, a partial re-run that has not reached this step yet) takes
|
||||
# OVS down across the whole host with `Failed at step EXEC`, which reads as
|
||||
# an OVS fault and is nothing of the sort.
|
||||
@@ -1570,7 +1838,7 @@ fi
|
||||
# to prevent log spam during those retries.
|
||||
echo ""
|
||||
echo "Step 10: Enabling OVN auto-start on boot..."
|
||||
sudo systemctl enable openvswitch-switch 2>/dev/null || true
|
||||
sudo systemctl enable "$OVS_UNIT" 2>/dev/null || true
|
||||
sudo systemctl enable ovn-controller 2>/dev/null || true
|
||||
# ovs-monitor-ipsec drives strongSwan from OVS DB cert pointers. The daemon's
|
||||
# enableOVNIPSec() flips ipsec_encapsulation=true at runtime and silently drops
|
||||
@@ -1583,9 +1851,43 @@ sudo systemctl enable openvswitch-ipsec.service 2>/dev/null || true
|
||||
# now that strongswan-starter is present.
|
||||
sudo systemctl reset-failed openvswitch-ipsec.service 2>/dev/null || true
|
||||
sudo systemctl restart openvswitch-ipsec.service 2>/dev/null || true
|
||||
echo " openvswitch-switch: enabled on boot"
|
||||
echo " ovn-controller: enabled on boot"
|
||||
echo " openvswitch-ipsec: enabled on boot"
|
||||
printf ' %-21s enabled on boot\n' "${OVS_UNIT%.service}:" ovn-controller: openvswitch-ipsec:
|
||||
|
||||
# Applied now, outside the hostnet lock, so the daemon and vpcd find its
|
||||
# readiness marker; on the networkd backend the apply does nothing.
|
||||
if [ -f /etc/systemd/system/spinifex-hostnet.service ]; then
|
||||
sudo systemctl enable spinifex-hostnet.service >/dev/null 2>&1
|
||||
if ! sudo systemctl restart spinifex-hostnet.service; then
|
||||
echo "ERROR: spinifex-hostnet could not apply the host network: journalctl -u spinifex-hostnet" >&2
|
||||
exit 1
|
||||
fi
|
||||
printf ' %-21s enabled on boot, applied (%s backend)\n' spinifex-hostnet: "$HOSTNET_BACKEND"
|
||||
else
|
||||
echo " spinifex-hostnet: not installed yet — run setup.sh, then re-run this script"
|
||||
fi
|
||||
|
||||
# systemd silently drops ordering against a unit that is not installed, so a
|
||||
# missing OVS or OVN unit would leave Spinifex starting unordered against OVS.
|
||||
SPINIFEX_UNITS=()
|
||||
for f in /etc/systemd/system/spinifex.target /etc/systemd/system/spinifex-*.service; do
|
||||
[ -f "$f" ] && SPINIFEX_UNITS+=("$f")
|
||||
done
|
||||
if [ ${#SPINIFEX_UNITS[@]} -eq 0 ]; then
|
||||
echo " Spinifex units: not installed yet — run setup.sh, then re-run this script"
|
||||
else
|
||||
MISSING_DEPS=$(distro_missing_ovs_deps "${SPINIFEX_UNITS[@]}") || exit 1
|
||||
DEPS_OK=true
|
||||
for dep in $MISSING_DEPS; do
|
||||
if [ "$dep" = openvswitch-ipsec.service ]; then
|
||||
echo " WARNING: $dep is not installed; IPsec cannot be turned on"
|
||||
else
|
||||
echo "ERROR: the Spinifex units order against $dep, which is not installed" >&2
|
||||
DEPS_OK=false
|
||||
fi
|
||||
done
|
||||
[ "$DEPS_OK" = true ] || exit 1
|
||||
echo " OVS/OVN units the Spinifex units name: installed"
|
||||
fi
|
||||
|
||||
# --- Step 11: Health check ---
|
||||
echo ""
|
||||
|
||||
+121
-1
@@ -131,7 +131,7 @@ esac
|
||||
|
||||
# The script and the Go constant must agree: vpcd repairs a veth made before
|
||||
# this existed, and a disagreement would have the two fighting every pass.
|
||||
script_mac=$(printf '%s\n' "$nat_mode_body" | sed -n 's/.*NAT_TRANSIT_HOST_MAC="\([^"]*\)".*/\1/p' | head -1)
|
||||
script_mac=$(sed -n 's/^NAT_TRANSIT_HOST_MAC="\([^"]*\)".*/\1/p' "$SETUP" | head -1)
|
||||
go_mac=$(sed -n 's/.*NATTransitHostMAC = "\([^"]*\)".*/\1/p' "$SCRIPT_DIR/../spinifex/domains/network/host/routed.go" | head -1)
|
||||
if [ -n "$script_mac" ] && [ "$script_mac" = "$go_mac" ]; then
|
||||
pass "the transit MAC matches host.NATTransitHostMAC"
|
||||
@@ -139,6 +139,126 @@ else
|
||||
fail "transit MAC disagrees: script=$script_mac go=$go_mac"
|
||||
fi
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# OVS and OVN service users.
|
||||
# --------------------------------------------------------------------------
|
||||
# EL runs OVS and OVN as non-root users and re-chowns their sockets on every
|
||||
# start. The function is run on its own against scratch files, through a sudo
|
||||
# that only runs its arguments.
|
||||
awk '$0 == "set_root_service_user() {" { on = 1 } on { print } on && $0 == "}" { exit }' "$SETUP" >"$TMP/user_fn.sh"
|
||||
mkdir -p "$TMP/bin"
|
||||
printf '#!/bin/sh\nexec "$@"\n' >"$TMP/bin/sudo"
|
||||
chmod +x "$TMP/bin/sudo"
|
||||
|
||||
# set_user <file> — runs the function on <file> for OVS_USER_ID; sets RC.
|
||||
set_user() {
|
||||
RC=0
|
||||
PATH="$TMP/bin:$PATH" bash -c '. "$1"; set_root_service_user "$2" OVS_USER_ID' _ "$TMP/user_fn.sh" "$1" >/dev/null || RC=$?
|
||||
}
|
||||
|
||||
set_user "$TMP/absent"
|
||||
if [ "$RC" != 0 ] && [ ! -e "$TMP/absent" ]; then pass "service user: no file, no change"; else fail "service user: absent file gave rc=$RC or was created"; fi
|
||||
|
||||
printf '# OVS_USER_ID="openvswitch:openvswitch"\nOPTIONS=""\nOVS_USER_ID="openvswitch:hugetlbfs"\n' >"$TMP/el"
|
||||
set_user "$TMP/el"
|
||||
want='# OVS_USER_ID="openvswitch:openvswitch"
|
||||
OPTIONS=""
|
||||
OVS_USER_ID="root:root"'
|
||||
if [ "$RC" = 0 ] && [ "$(cat "$TMP/el")" = "$want" ]; then pass "service user: EL default replaced in place, comments and other lines kept"; else fail "service user: EL file rc=$RC: $(cat "$TMP/el")"; fi
|
||||
|
||||
set_user "$TMP/el"
|
||||
if [ "$RC" != 0 ] && [ "$(cat "$TMP/el")" = "$want" ]; then pass "service user: already root reports no change"; else fail "service user: second run rc=$RC: $(cat "$TMP/el")"; fi
|
||||
|
||||
printf 'OPTIONS=""\n' >"$TMP/unset"
|
||||
set_user "$TMP/unset"
|
||||
if [ "$RC" = 0 ] && [ "$(cat "$TMP/unset")" = 'OPTIONS=""
|
||||
OVS_USER_ID="root:root"' ]; then pass "service user: appended when the file does not set it"; else fail "service user: unset file rc=$RC: $(cat "$TMP/unset")"; fi
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Host network persistence backend.
|
||||
# --------------------------------------------------------------------------
|
||||
# The backend is picked once and recorded; every later run reads the record.
|
||||
# The functions run on their own against scratch paths, with systemctl stubbed.
|
||||
for fn in select_hostnet_backend write_nm_unmanaged; do
|
||||
awk -v f="$fn() {" '$0 == f { on = 1 } on { print } on && $0 == "}" { exit }' "$SETUP" >>"$TMP/hostnet_fn.sh"
|
||||
done
|
||||
|
||||
# backend <networkd-enabled:yes|no> — runs select_hostnet_backend; sets RC, OUT.
|
||||
backend() {
|
||||
printf '#!/bin/sh
|
||||
[ "$1 $2" = "is-enabled --quiet" ] && [ "%s" = yes ]
|
||||
' "$1" >"$TMP/bin/systemctl"
|
||||
chmod +x "$TMP/bin/systemctl"
|
||||
RC=0
|
||||
OUT=$(PATH="$TMP/bin:$PATH" HOSTNET_BACKEND_FILE="$TMP/hn/etc/hostnet-backend" NETWORKD_DIR="$TMP/hn/network" \
|
||||
bash -c '. "$1"; select_hostnet_backend' _ "$TMP/hostnet_fn.sh" 2>&1) || RC=$?
|
||||
}
|
||||
|
||||
reset_hn() { rm -rf "$TMP/hn"; mkdir -p "$TMP/hn/network"; }
|
||||
|
||||
reset_hn
|
||||
backend no
|
||||
if [ "$RC" = 0 ] && [ "$OUT" = hostnet ] && [ "$(cat "$TMP/hn/etc/hostnet-backend")" = hostnet ]; then pass "backend: no networkd picks and records hostnet"; else fail "backend: no networkd gave rc=$RC out=$OUT"; fi
|
||||
|
||||
reset_hn
|
||||
backend yes
|
||||
if [ "$RC" = 0 ] && [ "$OUT" = networkd ] && [ "$(cat "$TMP/hn/etc/hostnet-backend")" = networkd ]; then pass "backend: networkd enabled picks networkd"; else fail "backend: networkd enabled gave rc=$RC out=$OUT"; fi
|
||||
|
||||
reset_hn
|
||||
: >"$TMP/hn/network/10-spinifex-mgmt.network"
|
||||
backend no
|
||||
if [ "$RC" = 0 ] && [ "$OUT" = networkd ]; then pass "backend: legacy Spinifex networkd files keep networkd"; else fail "backend: legacy files gave rc=$RC out=$OUT"; fi
|
||||
|
||||
reset_hn
|
||||
mkdir -p "$TMP/hn/etc"
|
||||
echo hostnet >"$TMP/hn/etc/hostnet-backend"
|
||||
backend yes
|
||||
if [ "$RC" = 0 ] && [ "$OUT" = hostnet ]; then pass "backend: the record wins over networkd being enabled"; else fail "backend: recorded hostnet gave rc=$RC out=$OUT"; fi
|
||||
|
||||
echo ifupdown >"$TMP/hn/etc/hostnet-backend"
|
||||
backend yes
|
||||
case "$RC:$OUT" in
|
||||
0:*) fail "backend: an unknown record was accepted: $OUT" ;;
|
||||
*"expected networkd or hostnet"*) pass "backend: an unknown record is refused" ;;
|
||||
*) fail "backend: an unknown record gave rc=$RC out=$OUT" ;;
|
||||
esac
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# NetworkManager unmanaged config.
|
||||
# --------------------------------------------------------------------------
|
||||
# NetworkManager globs have no character classes, so the management TAPs go to
|
||||
# a udev rule; the keyfile lists the rest. The host uplink bridge stays managed.
|
||||
printf '#!/bin/sh
|
||||
exit 0
|
||||
' >"$TMP/bin/udevadm"
|
||||
cp "$TMP/bin/udevadm" "$TMP/bin/nmcli"
|
||||
chmod +x "$TMP/bin/udevadm" "$TMP/bin/nmcli"
|
||||
glob=$(sed -n "s/^MGMT_TAP_GLOB='\([^']*\)'$/\1/p" "$SETUP")
|
||||
RC=0
|
||||
PATH="$TMP/bin:$PATH" NM_UNMANAGED_CONF="$TMP/nm/conf.d/00-spinifex-unmanaged.conf" NM_UNMANAGED_RULE="$TMP/nm/rules.d/90.rules" \
|
||||
MGMT_TAP_GLOB="$glob" WAN_BRIDGE=br-ext WAN_BRIDGE_MODE=direct WAN_IFACE=eth3 LINUX_BRIDGE=br-wan \
|
||||
MGMT_BRIDGE_ENABLED=true MGMT_BRIDGE=br-mgmt MGMT_IFACE=eth2 \
|
||||
bash -c '. "$1"; write_nm_unmanaged' _ "$TMP/hostnet_fn.sh" >/dev/null 2>&1 || RC=$?
|
||||
conf=$(cat "$TMP/nm/conf.d/00-spinifex-unmanaged.conf" 2>/dev/null || true)
|
||||
line=$(printf '%s\n' "$conf" | grep '^unmanaged-devices+=' || true)
|
||||
if [ "$RC" = 0 ] && printf '%s\n' "$conf" | grep -qx '\[keyfile\]' && [ -n "$line" ]; then pass "nm: keyfile section appends to unmanaged-devices"; else fail "nm: rc=$RC conf=$conf"; fi
|
||||
for want in br-int br-ext 'veth-wan-*' 'spx-nat-*' 'tap*' br-mgmt eth2 eth3; do
|
||||
case ";${line#unmanaged-devices+=};" in
|
||||
*";interface-name:$want;"*) pass "nm: $want unmanaged" ;;
|
||||
*) fail "nm: $want missing from $line" ;;
|
||||
esac
|
||||
done
|
||||
case "$line" in
|
||||
*br-wan*) fail "nm: the host uplink bridge br-wan was made unmanaged" ;;
|
||||
*'['*) fail "nm: keyfile uses a character class NetworkManager cannot match: $line" ;;
|
||||
*) pass "nm: host uplink left managed and no character classes" ;;
|
||||
esac
|
||||
rule=$(cat "$TMP/nm/rules.d/90.rules" 2>/dev/null || true)
|
||||
case "$rule" in
|
||||
*"KERNEL==\"$glob\", ENV{NM_UNMANAGED}=\"1\""*) pass "nm: udev rule unmanages management TAPs" ;;
|
||||
*) fail "nm: udev rule is $rule" ;;
|
||||
esac
|
||||
|
||||
if [ "$fails" -ne 0 ]; then
|
||||
echo " setup-ovn.sh: $fails failure(s)"
|
||||
exit 1
|
||||
|
||||
+288
-30
@@ -7,7 +7,7 @@
|
||||
# INSTALL_SPINIFEX_CHANNEL Release channel: latest (default), dev
|
||||
# INSTALL_SPINIFEX_VERSION Pin to specific version (overrides channel)
|
||||
# INSTALL_SPINIFEX_TARBALL Path to local tarball (skips download, for testing/air-gapped)
|
||||
# INSTALL_SPINIFEX_SKIP_APT Set to 1 to skip apt dependency install
|
||||
# INSTALL_SPINIFEX_SKIP_APT Set to 1 to skip the system package install (apt or dnf)
|
||||
# INSTALL_SPINIFEX_SKIP_AWS Set to 1 to skip AWS CLI install
|
||||
# INSTALL_SPINIFEX_SKIP_NEWGRP Set to 1 to skip newgrp exec at end (for callers like dev-install.sh)
|
||||
# ISO_BUILD Set to 1 when running inside a debootstrap chroot from the ISO
|
||||
@@ -58,6 +58,24 @@ info() { echo -e "${GREEN}[INFO]${NC} $*"; }
|
||||
warn() { echo -e "${YELLOW}[WARN]${NC} $*"; }
|
||||
fatal() { echo -e "${RED}[ERROR]${NC} $*"; exit 1; }
|
||||
|
||||
# --- BEGIN distro.sh probes ---
|
||||
# Copied from scripts/lib/distro.sh, which this script installs but cannot source
|
||||
# when piped from the install endpoint. distro_test.sh keeps the copies equal.
|
||||
distro_pkg_manager() {
|
||||
local apt=false dnf=false
|
||||
command -v apt-get >/dev/null 2>&1 && apt=true
|
||||
command -v dnf >/dev/null 2>&1 && dnf=true
|
||||
if $apt && $dnf; then
|
||||
echo "distro.sh: package manager: found both apt-get and dnf; cannot tell which one manages this host" >&2
|
||||
return 1
|
||||
fi
|
||||
if $apt; then echo apt; return 0; fi
|
||||
if $dnf; then echo dnf; return 0; fi
|
||||
echo "distro.sh: package manager: found neither apt-get nor dnf on PATH" >&2
|
||||
return 1
|
||||
}
|
||||
# --- END distro.sh probes ---
|
||||
|
||||
stage() {
|
||||
[ "${VERBOSE:-0}" = "1" ] && echo "[setup] $*"
|
||||
return 0
|
||||
@@ -92,14 +110,25 @@ setup_sudo() {
|
||||
}
|
||||
|
||||
# --- OS detection ---
|
||||
# os_release_field <key> <file> — the value of KEY=value, with one level of quotes removed.
|
||||
os_release_field() {
|
||||
sed -n "s/^$1=//p" "$2" | head -n 1 | sed -e 's/^"\(.*\)"$/\1/' -e "s/^'\(.*\)'$/\1/"
|
||||
}
|
||||
|
||||
# The only place that keys on the OS ID: the gate and, later, the package table.
|
||||
# Everything else probes for the capability (scripts/lib/distro.sh).
|
||||
detect_os() {
|
||||
if [ ! -f /etc/os-release ]; then
|
||||
fatal "Cannot detect OS: /etc/os-release not found"
|
||||
local os_release="${SPINIFEX_OS_RELEASE:-/etc/os-release}"
|
||||
if [ ! -f "$os_release" ]; then
|
||||
fatal "Cannot detect OS: $os_release not found"
|
||||
fi
|
||||
|
||||
. /etc/os-release
|
||||
# Parsed, not sourced, so a malformed file is refused instead of executed.
|
||||
ID=$(os_release_field ID "$os_release")
|
||||
VERSION_ID=$(os_release_field VERSION_ID "$os_release")
|
||||
PRETTY_NAME=$(os_release_field PRETTY_NAME "$os_release")
|
||||
|
||||
case "$ID" in
|
||||
case "${ID:-}" in
|
||||
debian)
|
||||
if [ "${VERSION_ID%%.*}" -lt 13 ] 2>/dev/null; then
|
||||
fatal "Debian $VERSION_ID is not supported. Minimum: Debian 13"
|
||||
@@ -111,12 +140,77 @@ detect_os() {
|
||||
fatal "Ubuntu $VERSION_ID is not supported. Minimum: Ubuntu 24.04"
|
||||
fi
|
||||
;;
|
||||
ol)
|
||||
check_oracle_linux
|
||||
;;
|
||||
*)
|
||||
fatal "Unsupported OS: $ID $VERSION_ID. Spinifex requires Debian 13+ or Ubuntu 24.04+"
|
||||
fatal "Unsupported OS: ${ID:-unknown} ${VERSION_ID:-}. Spinifex requires Debian 13+, Ubuntu 24.04+ or Oracle Linux 10.2+"
|
||||
;;
|
||||
esac
|
||||
|
||||
info "Detected OS: $PRETTY_NAME"
|
||||
info "Detected OS: ${PRETTY_NAME:-$ID $VERSION_ID}"
|
||||
}
|
||||
|
||||
# Oracle Linux 10.2+ on x86_64 only, running UEK R8: the KVM Utilities QEMU
|
||||
# needs UEK, and Oracle ships kernel modules per kernel, so installing a newer
|
||||
# kernel-uek alone leaves the running one without openvswitch.ko.
|
||||
check_oracle_linux() {
|
||||
local unsupported="Oracle Linux ${VERSION_ID:-<no VERSION_ID>} is not supported. Spinifex requires Oracle Linux 10.2 or a later 10.x"
|
||||
local major="${VERSION_ID%%.*}" minor="${VERSION_ID#*.}" kernel
|
||||
minor="${minor%%.*}"
|
||||
case "${VERSION_ID:-}" in *.*) ;; *) fatal "$unsupported" ;; esac
|
||||
case "$major" in "" | *[!0-9]*) fatal "$unsupported" ;; esac
|
||||
case "$minor" in "" | *[!0-9]*) fatal "$unsupported" ;; esac
|
||||
if [ "$major" -ne 10 ] || [ "$minor" -lt 2 ]; then
|
||||
fatal "$unsupported"
|
||||
fi
|
||||
|
||||
if [ "$(uname -m)" != "x86_64" ]; then
|
||||
fatal "Oracle Linux on $(uname -m) is not supported. Spinifex supports Oracle Linux on x86_64 only"
|
||||
fi
|
||||
|
||||
kernel=$(uname -r)
|
||||
case "$kernel" in
|
||||
6.12.0-*uek*) ;;
|
||||
*) fatal "The running kernel $kernel is not UEK R8 (6.12.0-*uek). Install kernel-uek, reboot into it, then re-run this installer" ;;
|
||||
esac
|
||||
|
||||
check_x86_64_v3
|
||||
check_selinux_not_enforcing
|
||||
}
|
||||
|
||||
# Spinifex ships no SELinux policy yet. Oracle's templates boot enforcing, and
|
||||
# setenforce alone reverts on reboot, so the config file is checked as well.
|
||||
check_selinux_not_enforcing() {
|
||||
local config="${SPINIFEX_SELINUX_CONFIG:-/etc/selinux/config}" now="" boot=""
|
||||
if command -v getenforce >/dev/null 2>&1; then
|
||||
now=$(getenforce 2>/dev/null | tr '[:upper:]' '[:lower:]')
|
||||
fi
|
||||
if [ -f "$config" ]; then
|
||||
boot=$(os_release_field SELINUX "$config" | tr '[:upper:]' '[:lower:]')
|
||||
fi
|
||||
if [ "$now" = "enforcing" ]; then
|
||||
fatal "SELinux is enforcing, and Spinifex has no SELinux policy. Run 'sudo setenforce 0', set SELINUX=permissive in $config, then re-run this installer"
|
||||
fi
|
||||
if [ "$boot" = "enforcing" ]; then
|
||||
fatal "SELinux is set to enforcing at next boot (SELINUX=enforcing in $config). Set SELINUX=permissive there, then re-run this installer"
|
||||
fi
|
||||
}
|
||||
|
||||
# The NFV SIG OVS/OVN and the KVM Utilities QEMU are built for x86-64-v3, so an
|
||||
# older CPU installs cleanly and then dies on an illegal instruction.
|
||||
check_x86_64_v3() {
|
||||
local cpuinfo="${SPINIFEX_CPUINFO:-/proc/cpuinfo}" flags flag missing=""
|
||||
flags=" $(grep -m1 '^flags' "$cpuinfo" 2>/dev/null | cut -d: -f2) "
|
||||
for flag in cx16 lahf_lm popcnt sse4_1 sse4_2 ssse3 avx avx2 bmi1 bmi2 f16c fma abm movbe xsave; do
|
||||
case "$flags" in
|
||||
*" $flag "*) ;;
|
||||
*) missing="${missing:+$missing }$flag" ;;
|
||||
esac
|
||||
done
|
||||
if [ -n "$missing" ]; then
|
||||
fatal "This CPU is below x86-64-v3 (missing: $missing). Oracle Linux's OVS, OVN and QEMU builds require x86-64-v3"
|
||||
fi
|
||||
}
|
||||
|
||||
# --- Architecture detection ---
|
||||
@@ -733,7 +827,7 @@ SUDOERS
|
||||
info "Scoped sudoers rules installed for spinifex-daemon (spinifex-vpcd needs none)"
|
||||
}
|
||||
|
||||
# --- Install apt dependencies ---
|
||||
# --- Install system dependencies ---
|
||||
# NOTE: Runtime deps must stay in sync with the ISO package list at
|
||||
# scripts/iso-builder/build/packages.list in the mulga repo. When you add,
|
||||
# remove, or change a runtime package here, review that file too — drift means
|
||||
@@ -750,29 +844,43 @@ jq curl iproute2 ethtool netcat-openbsd wget unzip xz-utils file
|
||||
ovn-central ovn-host openvswitch-switch openvswitch-ipsec strongswan-charon dhcpcd-base
|
||||
chrony nftables"
|
||||
|
||||
install_apt_deps() {
|
||||
stage "installing apt dependencies"
|
||||
# The Oracle Linux equivalents, plus what a minimal OL install lacks. OVS 3.5
|
||||
# and OVN 25.03 come from the CentOS NFV SIG; 25.03 is Debian 13's OVN, which
|
||||
# the hand-written OVN NB models are written against.
|
||||
OL_RUNTIME_PACKAGES="nbdkit nbdkit-basic-filters
|
||||
qemu-kvm qemu-img edk2-ovmf seabios-bin less
|
||||
pciutils
|
||||
jq curl iproute ethtool nmap-ncat wget unzip xz file
|
||||
ovn25.03-central ovn25.03-host openvswitch3.5 openvswitch3.5-ipsec libreswan dhcpcd
|
||||
chrony nftables iptables-nft systemd-resolved"
|
||||
|
||||
# Oracle ships no OVS or OVN for OL10. The allowlist stops the SIG supplying
|
||||
# anything but OVS, OVN and the two SIG-only packages they depend on.
|
||||
NFV_SIG_REPO_ID="spinifex-nfv-sig"
|
||||
NFV_SIG_KEY_URL="https://www.centos.org/keys/RPM-GPG-KEY-CentOS-SIG-NFV"
|
||||
NFV_SIG_KEY_FILE="/etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-NFV"
|
||||
NFV_SIG_KEY_FPR="35154228174901BEFA8E69A621465E289D2A76A7"
|
||||
NFV_SIG_INCLUDEPKGS="openvswitch3.5* ovn25.03* python3-openvswitch3.5 openvswitch-selinux-extra-policy"
|
||||
|
||||
# Only the KVM Utilities QEMU has the microvm machine. Excluding QEMU from
|
||||
# AppStream stops a later AppStream build ever replacing part of that stack.
|
||||
OL_KVM_UTILS_REPO="ol10_kvm_utils"
|
||||
OL_APPSTREAM_REPO="ol10_appstream"
|
||||
OL_APPSTREAM_EXCLUDES="qemu-kvm* qemu-img*"
|
||||
# The running kernel's modules come from here, and KVM Utilities requires kernel-uek.
|
||||
OL_UEK_REPO="ol10_UEKR8"
|
||||
|
||||
install_deps() {
|
||||
stage "installing system dependencies"
|
||||
if [ "${INSTALL_SPINIFEX_SKIP_APT}" = "1" ]; then
|
||||
info "Skipping apt dependencies (INSTALL_SPINIFEX_SKIP_APT=1)"
|
||||
info "Skipping system dependencies (INSTALL_SPINIFEX_SKIP_APT=1)"
|
||||
else
|
||||
info "Installing system dependencies..."
|
||||
$SUDO apt-get update -qq
|
||||
|
||||
# Unquoted on purpose: both variables are whitespace-separated package lists.
|
||||
# shellcheck disable=SC2086
|
||||
DEBIAN_FRONTEND=noninteractive $SUDO apt-get install -y -qq \
|
||||
$QEMU_PACKAGES $APT_RUNTIME_PACKAGES \
|
||||
> /dev/null
|
||||
|
||||
info "System dependencies installed"
|
||||
|
||||
# Only on a machine that already boots from ZFS. Pulling zfsutils-linux
|
||||
# in unconditionally would drag zfs-dkms onto every dev box and rebuild
|
||||
# the module on each kernel upgrade for no benefit.
|
||||
if [ "$(findmnt -no FSTYPE /)" = "zfs" ] && ! command -v zpool >/dev/null 2>&1; then
|
||||
info "ZFS root detected, installing zfsutils-linux..."
|
||||
DEBIAN_FRONTEND=noninteractive $SUDO apt-get install -y -qq zfsutils-linux > /dev/null
|
||||
fi
|
||||
local pm
|
||||
pm=$(distro_pkg_manager) || fatal "Cannot tell which package manager this host uses"
|
||||
case "$pm" in
|
||||
apt) install_apt_deps ;;
|
||||
dnf) install_dnf_deps ;;
|
||||
esac
|
||||
fi
|
||||
|
||||
# Mask the standalone dhcpcd.service auto-enabled on Debian Trixie when
|
||||
@@ -786,6 +894,149 @@ install_apt_deps() {
|
||||
$SUDO systemctl mask dhcpcd.service 2>/dev/null || true
|
||||
}
|
||||
|
||||
install_apt_deps() {
|
||||
info "Installing system dependencies..."
|
||||
$SUDO apt-get update -qq
|
||||
|
||||
# Unquoted on purpose: both variables are whitespace-separated package lists.
|
||||
# shellcheck disable=SC2086
|
||||
DEBIAN_FRONTEND=noninteractive $SUDO apt-get install -y -qq \
|
||||
$QEMU_PACKAGES $APT_RUNTIME_PACKAGES \
|
||||
> /dev/null
|
||||
|
||||
info "System dependencies installed"
|
||||
|
||||
# Only on a machine that already boots from ZFS. Pulling zfsutils-linux
|
||||
# in unconditionally would drag zfs-dkms onto every dev box and rebuild
|
||||
# the module on each kernel upgrade for no benefit.
|
||||
if [ "$(findmnt -no FSTYPE /)" = "zfs" ] && ! command -v zpool >/dev/null 2>&1; then
|
||||
info "ZFS root detected, installing zfsutils-linux..."
|
||||
DEBIAN_FRONTEND=noninteractive $SUDO apt-get install -y -qq zfsutils-linux > /dev/null
|
||||
fi
|
||||
}
|
||||
|
||||
# The package table is the other place keyed on the OS ID: dnf alone does not
|
||||
# say which repositories or package names a host has.
|
||||
install_dnf_deps() {
|
||||
case "${ID:-}" in
|
||||
ol) ;;
|
||||
*) fatal "No dnf package list for ${ID:-an unknown OS}. Spinifex installs with dnf on Oracle Linux only" ;;
|
||||
esac
|
||||
if [ "$(findmnt -no FSTYPE /)" = "zfs" ]; then
|
||||
fatal "A ZFS root is not supported on Oracle Linux"
|
||||
fi
|
||||
|
||||
ensure_ol_repos
|
||||
install_running_kernel_modules
|
||||
|
||||
info "Installing system dependencies..."
|
||||
# shellcheck disable=SC2086 # a whitespace-separated package list
|
||||
$SUDO dnf install -y -q $OL_RUNTIME_PACKAGES > /dev/null
|
||||
info "System dependencies installed"
|
||||
|
||||
load_kernel_modules openvswitch geneve vhost_net tun
|
||||
}
|
||||
|
||||
# Enables KVM Utilities, pins QEMU to it, and adds the NFV SIG repository with
|
||||
# a key whose fingerprint must match the one recorded above.
|
||||
ensure_ol_repos() {
|
||||
info "Configuring Oracle Linux repositories..."
|
||||
local repo
|
||||
$SUDO dnf install -y -q dnf-plugins-core gnupg2 curl > /dev/null
|
||||
|
||||
for repo in "$OL_UEK_REPO" "$OL_KVM_UTILS_REPO" "$OL_APPSTREAM_REPO"; do
|
||||
dnf config-manager --dump "$repo" > /dev/null 2>&1 \
|
||||
|| fatal "Repository $repo is not defined on this host. Oracle Linux's release packages define it; reinstall oraclelinux-release-el10"
|
||||
done
|
||||
$SUDO dnf config-manager --set-enabled "$OL_UEK_REPO" "$OL_KVM_UTILS_REPO" > /dev/null
|
||||
exclude_from_appstream
|
||||
|
||||
install_nfv_sig_key
|
||||
$SUDO tee "/etc/yum.repos.d/${NFV_SIG_REPO_ID}.repo" > /dev/null <<EOF
|
||||
# Written by the Spinifex installer. Re-running it rewrites this file.
|
||||
[${NFV_SIG_REPO_ID}]
|
||||
name=CentOS NFV SIG Open vSwitch and OVN, for Spinifex
|
||||
baseurl=https://mirror.stream.centos.org/SIGs/10-stream/nfv/\$basearch/openvswitch-2/
|
||||
enabled=1
|
||||
gpgcheck=1
|
||||
gpgkey=file://${NFV_SIG_KEY_FILE}
|
||||
includepkgs=${NFV_SIG_INCLUDEPKGS}
|
||||
EOF
|
||||
|
||||
for repo in "$OL_UEK_REPO" "$OL_APPSTREAM_REPO" "$OL_KVM_UTILS_REPO" "$NFV_SIG_REPO_ID"; do
|
||||
$SUDO dnf -q makecache --repo "$repo" > /dev/null \
|
||||
|| fatal "Repository $repo is unreachable. Check this host's network access and proxy settings, then re-run this installer"
|
||||
done
|
||||
}
|
||||
|
||||
# Adds the QEMU excludes to AppStream, keeping any exclude already set there.
|
||||
exclude_from_appstream() {
|
||||
local current pattern merged="" patterns
|
||||
current=$(dnf config-manager --dump "$OL_APPSTREAM_REPO" | sed -n 's/^excludepkgs = //p' | tr ',' ' ')
|
||||
# read, not an unquoted expansion, so the patterns are never globbed.
|
||||
read -ra patterns <<< "$current $OL_APPSTREAM_EXCLUDES"
|
||||
for pattern in "${patterns[@]}"; do
|
||||
case " $merged " in
|
||||
*" $pattern "*) ;;
|
||||
*) merged="${merged:+$merged }$pattern" ;;
|
||||
esac
|
||||
done
|
||||
$SUDO dnf config-manager --save --setopt="${OL_APPSTREAM_REPO}.excludepkgs=${merged// /,}" > /dev/null
|
||||
}
|
||||
|
||||
# gpg_fingerprints <file> — the primary-key fingerprint of each key in file.
|
||||
gpg_fingerprints() {
|
||||
local home
|
||||
home=$(mktemp -d)
|
||||
GNUPGHOME="$home" gpg --batch --with-colons --show-keys "$1" 2>/dev/null \
|
||||
| awk -F: '$1 == "pub" { want = 1; next } want && $1 == "fpr" { print $10; want = 0 }'
|
||||
rm -rf "$home"
|
||||
}
|
||||
|
||||
install_nfv_sig_key() {
|
||||
if [ -f "$NFV_SIG_KEY_FILE" ] && [ "$(gpg_fingerprints "$NFV_SIG_KEY_FILE")" = "$NFV_SIG_KEY_FPR" ]; then
|
||||
$SUDO rpm --import "$NFV_SIG_KEY_FILE"
|
||||
return 0
|
||||
fi
|
||||
local tmp got
|
||||
tmp=$(mktemp)
|
||||
curl -fsSL "$NFV_SIG_KEY_URL" -o "$tmp" \
|
||||
|| { rm -f "$tmp"; fatal "Cannot download the CentOS NFV SIG key from $NFV_SIG_KEY_URL"; }
|
||||
got=$(gpg_fingerprints "$tmp")
|
||||
if [ "$got" != "$NFV_SIG_KEY_FPR" ]; then
|
||||
rm -f "$tmp"
|
||||
fatal "The key at $NFV_SIG_KEY_URL has fingerprint ${got:-<unreadable>}, not the CentOS NFV SIG's $NFV_SIG_KEY_FPR. Refusing to trust it"
|
||||
fi
|
||||
$SUDO install -D -m 0644 "$tmp" "$NFV_SIG_KEY_FILE"
|
||||
rm -f "$tmp"
|
||||
$SUDO rpm --import "$NFV_SIG_KEY_FILE"
|
||||
}
|
||||
|
||||
# Oracle ships kernel modules per kernel version. The unversioned package
|
||||
# follows the newest kernel and leaves the running one without openvswitch.ko.
|
||||
install_running_kernel_modules() {
|
||||
local pkg
|
||||
pkg="kernel-uek-modules-$(uname -r)"
|
||||
if rpm -q "$pkg" > /dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
if ! $SUDO dnf -q list --available "$pkg" > /dev/null 2>&1; then
|
||||
fatal "$pkg, the modules for the running kernel, is no longer in the repositories. Run 'sudo dnf update kernel-uek', reboot into the new kernel, then re-run this installer"
|
||||
fi
|
||||
info "Installing $pkg (it pulls in linux-firmware, about 800 MB)..."
|
||||
$SUDO dnf install -y -q "$pkg" > /dev/null
|
||||
}
|
||||
|
||||
load_kernel_modules() {
|
||||
local mod failed=""
|
||||
for mod in "$@"; do
|
||||
$SUDO modprobe "$mod" || failed="${failed:+$failed }$mod"
|
||||
done
|
||||
if [ -n "$failed" ]; then
|
||||
fatal "Cannot load kernel modules for the running kernel $(uname -r): $failed"
|
||||
fi
|
||||
}
|
||||
|
||||
# --- Install AWS CLI ---
|
||||
install_aws_cli() {
|
||||
stage "installing AWS CLI v2"
|
||||
@@ -891,6 +1142,13 @@ install_files() {
|
||||
info " /usr/local/share/spinifex/setup-ovn.sh"
|
||||
fi
|
||||
|
||||
# Sourced, not run, by setup-ovn.sh and uninstall-spx.sh wherever they are
|
||||
# installed, including the ISO's /usr/local/bin/setup-ovn.sh.
|
||||
if [ -f "$EXTRACT_DIR/lib/distro.sh" ]; then
|
||||
$SUDO install -D -m 0644 "$EXTRACT_DIR/lib/distro.sh" /usr/local/share/spinifex/lib/distro.sh
|
||||
info " /usr/local/share/spinifex/lib/distro.sh"
|
||||
fi
|
||||
|
||||
# Install setup.sh itself so firstboot and future re-runs can find it at a
|
||||
# stable path on both ISO and curl|bash installs.
|
||||
if [ -f "$EXTRACT_DIR/setup.sh" ]; then
|
||||
@@ -1451,7 +1709,7 @@ main() {
|
||||
download_spinifex
|
||||
fi
|
||||
|
||||
stage_enabled deps && install_apt_deps
|
||||
stage_enabled deps && install_deps
|
||||
stage_enabled aws && install_aws_cli
|
||||
stage_enabled users && create_service_users
|
||||
stage_enabled sudoers && install_sudoers
|
||||
|
||||
@@ -0,0 +1,172 @@
|
||||
#!/bin/sh
|
||||
# setup_test.sh — tests for setup.sh's supported-OS gate.
|
||||
#
|
||||
# setup.sh is sourced with INSTALL_SPINIFEX_LIB_ONLY=1 so main never runs, and
|
||||
# detect_os reads fixture os-release, cpuinfo and SELinux config files, and stub
|
||||
# uname and getenforce commands.
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
|
||||
SETUP="$SCRIPT_DIR/setup.sh"
|
||||
TMP=$(mktemp -d)
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
|
||||
fails=0
|
||||
pass() { echo " ok - $1"; }
|
||||
fail() { echo " FAIL - $1"; fails=$((fails + 1)); }
|
||||
|
||||
echo "-- setup.sh OS gate"
|
||||
|
||||
V3_FLAGS="fpu cx16 lahf_lm popcnt sse4_1 sse4_2 ssse3 avx avx2 bmi1 bmi2 f16c fma abm movbe xsave"
|
||||
V2_FLAGS="fpu cx16 lahf_lm popcnt sse4_1 sse4_2 ssse3"
|
||||
UEK_R8="6.12.0-207.111.5.4.el10uek.x86_64"
|
||||
|
||||
# gate <os-release body> [machine] [kernel] [cpu flags] — runs detect_os and
|
||||
# sets RC and OUT (stdout and stderr together). SELINUX_NOW is what getenforce
|
||||
# prints; SELINUX_BOOT is the config file's SELINUX=, or "none" for no file.
|
||||
gate() {
|
||||
printf '%s\n' "$1" > "$TMP/os-release"
|
||||
printf 'processor\t: 0\nflags\t\t: %s\n' "${4:-$V3_FLAGS}" > "$TMP/cpuinfo"
|
||||
rm -f "$TMP/selinux-config"
|
||||
if [ "${SELINUX_BOOT:-permissive}" != none ]; then
|
||||
printf '# fixture\nSELINUX=%s\nSELINUXTYPE=targeted\n' "${SELINUX_BOOT:-permissive}" > "$TMP/selinux-config"
|
||||
fi
|
||||
mkdir -p "$TMP/bin"
|
||||
printf '#!/bin/sh\necho %s\n' "${SELINUX_NOW:-Permissive}" > "$TMP/bin/getenforce"
|
||||
chmod +x "$TMP/bin/getenforce"
|
||||
cat > "$TMP/bin/uname" <<EOF
|
||||
#!/bin/sh
|
||||
case "\$1" in
|
||||
-m) echo "${2:-x86_64}" ;;
|
||||
-r) echo "${3:-$UEK_R8}" ;;
|
||||
*) exit 1 ;;
|
||||
esac
|
||||
EOF
|
||||
chmod +x "$TMP/bin/uname"
|
||||
RC=0
|
||||
OUT=$(PATH="$TMP/bin:$PATH" SPINIFEX_OS_RELEASE="$TMP/os-release" SPINIFEX_CPUINFO="$TMP/cpuinfo" \
|
||||
SPINIFEX_SELINUX_CONFIG="$TMP/selinux-config" INSTALL_SPINIFEX_LIB_ONLY=1 bash -c '. "$1"; detect_os' _ "$SETUP" 2>&1) || RC=$?
|
||||
# Some shells keep a prefix assignment to a function call after it returns.
|
||||
unset SELINUX_NOW SELINUX_BOOT
|
||||
}
|
||||
accepts() {
|
||||
if [ "$RC" = 0 ]; then pass "accepts $1"; else fail "rejects $1: $OUT"; fi
|
||||
}
|
||||
# rejects <desc> <word> — the refusal must say why, not just exit.
|
||||
rejects() {
|
||||
if [ "$RC" = 0 ]; then
|
||||
fail "accepts $1"
|
||||
else
|
||||
case "$OUT" in
|
||||
*"$2"*) pass "rejects $1" ;;
|
||||
*) fail "rejects $1 without naming '$2': $OUT" ;;
|
||||
esac
|
||||
fi
|
||||
}
|
||||
|
||||
OL='ID="ol"
|
||||
NAME="Oracle Linux Server"
|
||||
PRETTY_NAME="Oracle Linux Server 10.2"'
|
||||
|
||||
gate 'ID=debian
|
||||
VERSION_ID="13"
|
||||
PRETTY_NAME="Debian GNU/Linux 13 (trixie)"'
|
||||
accepts "Debian 13"
|
||||
|
||||
# Debian testing and sid carry no VERSION_ID. The gate has always let them
|
||||
# through, and Debian behaviour is not this gate's to change.
|
||||
gate 'ID=debian
|
||||
PRETTY_NAME="Debian GNU/Linux forky/sid"'
|
||||
accepts "Debian testing with no VERSION_ID"
|
||||
|
||||
gate 'ID=debian
|
||||
VERSION_ID="12"'
|
||||
rejects "Debian 12" "Minimum: Debian 13"
|
||||
|
||||
gate 'ID=ubuntu
|
||||
VERSION_ID="26.04"
|
||||
PRETTY_NAME="Ubuntu 26.04 LTS"'
|
||||
accepts "Ubuntu 26.04"
|
||||
|
||||
gate 'ID=ubuntu
|
||||
VERSION_ID="22.04"'
|
||||
rejects "Ubuntu 22.04" "Minimum: Ubuntu 24.04"
|
||||
|
||||
gate "$OL
|
||||
VERSION_ID=\"10.2\""
|
||||
accepts "Oracle Linux 10.2 on UEK R8, x86_64, x86-64-v3"
|
||||
|
||||
gate "$OL
|
||||
VERSION_ID=\"10.4\""
|
||||
accepts "Oracle Linux 10.4"
|
||||
|
||||
gate "$OL
|
||||
VERSION_ID=\"10.1\""
|
||||
rejects "Oracle Linux 10.1" "Oracle Linux 10.2 or a later 10.x"
|
||||
|
||||
gate "$OL
|
||||
VERSION_ID=\"10\""
|
||||
rejects "Oracle Linux with no minor version" "Oracle Linux 10.2 or a later 10.x"
|
||||
|
||||
gate "$OL
|
||||
VERSION_ID=\"11.0\""
|
||||
rejects "Oracle Linux 11" "Oracle Linux 10.2 or a later 10.x"
|
||||
|
||||
gate "$OL
|
||||
VERSION_ID=\"9.6\"" x86_64 "5.15.0-310.184.5.2.el9uek.x86_64"
|
||||
rejects "Oracle Linux 9" "Oracle Linux 10.2 or a later 10.x"
|
||||
|
||||
gate "$OL" x86_64
|
||||
rejects "Oracle Linux with no VERSION_ID" "Oracle Linux 10.2 or a later 10.x"
|
||||
|
||||
gate "$OL
|
||||
VERSION_ID=\"10.2\"" aarch64 "6.12.0-207.111.5.4.el10uek.aarch64"
|
||||
rejects "Oracle Linux on aarch64" "x86_64 only"
|
||||
|
||||
gate "$OL
|
||||
VERSION_ID=\"10.2\"" x86_64 "6.12.0-55.9.1.el10_0.x86_64"
|
||||
rejects "Oracle Linux on the Red Hat compatible kernel" "is not UEK R8"
|
||||
|
||||
gate "$OL
|
||||
VERSION_ID=\"10.2\"" x86_64 "5.15.0-310.184.5.2.el9uek.x86_64"
|
||||
rejects "Oracle Linux on a UEK older than R8" "is not UEK R8"
|
||||
|
||||
gate "$OL
|
||||
VERSION_ID=\"10.2\"" x86_64 "$UEK_R8" "$V2_FLAGS"
|
||||
rejects "Oracle Linux on an x86-64-v2 CPU" "missing: avx avx2 bmi1 bmi2 f16c fma abm movbe xsave"
|
||||
|
||||
OL102="$OL
|
||||
VERSION_ID=\"10.2\""
|
||||
|
||||
SELINUX_NOW=Enforcing gate "$OL102"
|
||||
rejects "Oracle Linux with SELinux enforcing" "SELinux is enforcing"
|
||||
|
||||
SELINUX_BOOT=enforcing gate "$OL102"
|
||||
rejects "Oracle Linux with SELinux permissive now but enforcing at next boot" "enforcing at next boot"
|
||||
|
||||
SELINUX_BOOT='"enforcing"' gate "$OL102"
|
||||
rejects "Oracle Linux with a quoted SELINUX=\"enforcing\"" "enforcing at next boot"
|
||||
|
||||
SELINUX_NOW=Disabled SELINUX_BOOT=disabled gate "$OL102"
|
||||
accepts "Oracle Linux with SELinux disabled"
|
||||
|
||||
SELINUX_NOW=Disabled SELINUX_BOOT=none gate "$OL102"
|
||||
accepts "Oracle Linux with no SELinux config file"
|
||||
|
||||
# Spinifex has no SELinux policy on any OS, but refusing enforcing on Debian
|
||||
# would change Debian behaviour, so only the Oracle Linux gate checks it.
|
||||
SELINUX_NOW=Enforcing SELINUX_BOOT=enforcing gate 'ID=debian
|
||||
VERSION_ID="13"'
|
||||
accepts "Debian 13 regardless of SELinux"
|
||||
|
||||
gate 'ID="rocky"
|
||||
VERSION_ID="10.0"'
|
||||
rejects "Rocky Linux 10" "Unsupported OS: rocky"
|
||||
|
||||
gate 'this is not an os-release file'
|
||||
rejects "a malformed os-release" "Unsupported OS"
|
||||
|
||||
if [ "$fails" -gt 0 ]; then
|
||||
echo "$fails failure(s)"
|
||||
exit 1
|
||||
fi
|
||||
@@ -73,6 +73,19 @@ while [[ $# -gt 0 ]]; do
|
||||
shift
|
||||
done
|
||||
|
||||
# Host-OS capability probes: beside this script in a checkout and in the
|
||||
# installed share dir, else where setup.sh installs them.
|
||||
DISTRO_LIB=""
|
||||
for candidate in "$(dirname "${BASH_SOURCE[0]}")/lib/distro.sh" /usr/local/share/spinifex/lib/distro.sh; do
|
||||
if [ -f "$candidate" ]; then DISTRO_LIB="$candidate"; break; fi
|
||||
done
|
||||
if [ -z "$DISTRO_LIB" ]; then
|
||||
echo "ERROR: distro.sh not found at $(dirname "${BASH_SOURCE[0]}")/lib/distro.sh or /usr/local/share/spinifex/lib/distro.sh; re-run setup.sh to install it" >&2
|
||||
exit 1
|
||||
fi
|
||||
# shellcheck source=lib/distro.sh
|
||||
. "$DISTRO_LIB"
|
||||
|
||||
log() { echo "[uninstall-spx] $*"; }
|
||||
run() {
|
||||
if $DRY_RUN; then
|
||||
@@ -208,7 +221,7 @@ fi
|
||||
#
|
||||
# It handles the guests, the OVS bridges, the OVN databases and the JetStream
|
||||
# store, and it carries correctness that is invisible in its code: QEMU matched
|
||||
# as ^qemu-system because comm truncates at 15 characters, a wait for QEMU to
|
||||
# by a comm prefix because comm truncates at 15 characters, a wait for QEMU to
|
||||
# exit rather than for the signal, and the firewall-mode stash that stops an
|
||||
# interrupted run leaving the node unarmed. A second copy of that would drift,
|
||||
# and the failure mode of the drift is data loss.
|
||||
@@ -294,12 +307,14 @@ rm_path /etc/systemd/system/spinifex-*.service \
|
||||
# exactly that way on the first dev-oci round trip.
|
||||
log "removing drop-ins written into other units"
|
||||
rm_path /etc/systemd/system/openvswitch-switch.service.d/spinifex-perms.conf \
|
||||
/etc/systemd/system/ovs-vswitchd.service.d/spinifex-perms.conf \
|
||||
/etc/systemd/system/ovn-controller.service.d/spinifex-perms.conf \
|
||||
/etc/systemd/system/ovn-controller.service.d/log-level.conf \
|
||||
/etc/systemd/system/openvswitch-ipsec.service.d/spinifex-perms.conf \
|
||||
/etc/systemd/system/ovn-northd.service.d/spinifex-perms.conf \
|
||||
/etc/systemd/system/ovn-northd.service.d/no-manage-ovsdb.conf
|
||||
# Only if we emptied them — another package may own the same override dir.
|
||||
for d in openvswitch-switch ovn-controller openvswitch-ipsec ovn-northd; do
|
||||
for d in openvswitch-switch ovs-vswitchd ovn-controller openvswitch-ipsec ovn-northd; do
|
||||
dir="/etc/systemd/system/${d}.service.d"
|
||||
if sudo test -d "$dir" && [ -z "$(sudo ls -A "$dir" 2>/dev/null)" ]; then
|
||||
rm_path "$dir"
|
||||
@@ -379,8 +394,12 @@ rm_path /etc/sysctl.d/*spinifex* \
|
||||
/etc/sudoers.d/spinifex-network \
|
||||
/etc/chrony/conf.d/spinifex.conf \
|
||||
/etc/logrotate.d/spinifex \
|
||||
/etc/udev/rules.d/*spinifex*
|
||||
/etc/udev/rules.d/*spinifex* \
|
||||
/etc/NetworkManager/conf.d/*spinifex*
|
||||
run sudo sysctl --system >/dev/null 2>&1 || true
|
||||
if systemctl is-active --quiet NetworkManager.service 2>/dev/null; then
|
||||
run sudo nmcli general reload conf 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 10. Spinifex directories.
|
||||
@@ -503,6 +522,7 @@ if ! $DRY_RUN; then
|
||||
for p in /etc/systemd/system/spinifex-*.service /etc/systemd/system/spinifex*.slice \
|
||||
/etc/systemd/system/spinifex.target /etc/sysctl.d/*spinifex* \
|
||||
/etc/systemd/network/1[5-8]-spinifex-* \
|
||||
/etc/NetworkManager/conf.d/*spinifex* \
|
||||
/etc/systemd/system/*.service.d/spinifex-perms.conf \
|
||||
/etc/systemd/system/ovn-northd.service.d/no-manage-ovsdb.conf \
|
||||
/etc/systemd/system/ovn-controller.service.d/log-level.conf; do
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
package preflight
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/host/hostnet"
|
||||
)
|
||||
|
||||
// hostnetUnit is the unit whose failure this check surfaces.
|
||||
const hostnetUnit = "spinifex-hostnet.service"
|
||||
|
||||
// hostnetOVNRemote is a variable so tests never query a real OVS.
|
||||
var hostnetOVNRemote = hostnet.DefaultEnv().OVNRemote
|
||||
|
||||
// checkHostnet runs the check the daemon and vpcd run before they start, so a
|
||||
// failed host network apply shows here, not only as a refused service start.
|
||||
func checkHostnet(root string) Result {
|
||||
env := hostnet.DefaultEnv()
|
||||
env.Paths = hostnet.PathsAt(root)
|
||||
env.OVNRemote = hostnetOVNRemote
|
||||
if err := hostnet.Check(context.Background(), env); err != nil {
|
||||
return Result{Path: hostnetUnit, Kind: "hostnet", Status: Failed, Detail: err.Error()}
|
||||
}
|
||||
return Result{Path: hostnetUnit, Kind: "hostnet", Status: OK}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package preflight
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCheckHostAt_Hostnet(t *testing.T) {
|
||||
for name, tc := range map[string]struct {
|
||||
backend string
|
||||
state bool
|
||||
want Status
|
||||
detail string
|
||||
}{
|
||||
"no record, nothing set up": {want: OK},
|
||||
"networkd backend": {backend: "networkd", want: OK},
|
||||
"hostnet backend, no state": {backend: "hostnet", want: Failed, detail: "hostnet.json"},
|
||||
"hostnet backend, no marker": {backend: "hostnet", state: true, want: Failed, detail: "host network not applied"},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
etc := filepath.Join(root, "etc/spinifex")
|
||||
if err := os.MkdirAll(etc, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if tc.backend != "" {
|
||||
if err := os.WriteFile(filepath.Join(etc, "hostnet-backend"), []byte(tc.backend+"\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if tc.state {
|
||||
state := `{"version": 1, "pending": false, "int_bridge": "br-int"}`
|
||||
if err := os.WriteFile(filepath.Join(etc, "hostnet.json"), []byte(state), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
r := findResult(t, CheckHostAt(root), hostnetUnit)
|
||||
if r.Status != tc.want || !strings.Contains(r.Detail, tc.detail) {
|
||||
t.Errorf("hostnet result = %s %q, want %s containing %q", r.Status, r.Detail, tc.want, tc.detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package preflight
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/host/hostos"
|
||||
)
|
||||
|
||||
// microvmMachine is the QEMU machine type load balancer system instances boot on.
|
||||
const microvmMachine = "microvm"
|
||||
|
||||
// microvmChecked reports whether this host runs system instances. The microvm
|
||||
// machine type exists only in x86 QEMU.
|
||||
var microvmChecked = runtime.GOARCH == "amd64"
|
||||
|
||||
// resolveQEMU and machineHelp are variables so tests never run a real QEMU.
|
||||
var (
|
||||
resolveQEMU = func() (string, error) { return hostos.QEMUBinary("x86_64") }
|
||||
machineHelp = func(binary string) ([]byte, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
return exec.CommandContext(ctx, binary, "-machine", "help").Output()
|
||||
}
|
||||
)
|
||||
|
||||
// checkMicroVM checks the QEMU binary the daemon will run, and the firmware it
|
||||
// passes, can boot a microvm. A QEMU build without microvm, such as Oracle
|
||||
// Linux's AppStream qemu-kvm, otherwise surfaces only at the first ALB launch.
|
||||
func checkMicroVM(root string) []Result {
|
||||
if !microvmChecked {
|
||||
return nil
|
||||
}
|
||||
return []Result{checkQEMUMicroVM(), checkMicroVMFirmware(root)}
|
||||
}
|
||||
|
||||
func checkQEMUMicroVM() Result {
|
||||
binary, err := resolveQEMU()
|
||||
if err != nil {
|
||||
return Result{Path: "qemu", Kind: "qemu", Status: Missing, Detail: err.Error()}
|
||||
}
|
||||
out, err := machineHelp(binary)
|
||||
if err != nil {
|
||||
return Result{Path: binary, Kind: "qemu", Status: Missing, Detail: fmt.Sprintf("%s -machine help: %v", binary, err)}
|
||||
}
|
||||
if !listsMachine(out, microvmMachine) {
|
||||
return Result{
|
||||
Path: binary,
|
||||
Kind: "qemu",
|
||||
Status: Unsupported,
|
||||
Detail: fmt.Sprintf("no %s machine type, so load balancers cannot boot; install a QEMU build that has it (on Oracle Linux, qemu-kvm from ol10_kvm_utils)", microvmMachine),
|
||||
}
|
||||
}
|
||||
return Result{Path: binary, Kind: "qemu", Status: OK}
|
||||
}
|
||||
|
||||
// listsMachine reports whether `-machine help` output has a row for name. Rows
|
||||
// are "<name> <description>", after a "Supported machines are:" header.
|
||||
func listsMachine(help []byte, name string) bool {
|
||||
sc := bufio.NewScanner(bytes.NewReader(help))
|
||||
for sc.Scan() {
|
||||
if fields := strings.Fields(sc.Text()); len(fields) > 0 && fields[0] == name {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func checkMicroVMFirmware(root string) Result {
|
||||
path := filepath.Join(vm.MicroVMDir, vm.MicroVMFirmware)
|
||||
if _, err := os.Stat(filepath.Join(root, path)); err != nil {
|
||||
return Result{Path: path, Kind: "firmware", Status: Missing, Detail: err.Error()}
|
||||
}
|
||||
return Result{Path: path, Kind: "firmware", Status: OK}
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
package preflight
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/host/hostos"
|
||||
)
|
||||
|
||||
// Debian's qemu-system-x86_64 -machine help, trimmed.
|
||||
const helpWithMicroVM = `Supported machines are:
|
||||
microvm microvm (i386)
|
||||
pc Standard PC (i440FX + PIIX, 1996) (alias of pc-i440fx-10.0)
|
||||
q35 Standard PC (Q35 + ICH9, 2009) (alias of pc-q35-10.0)
|
||||
`
|
||||
|
||||
// Oracle Linux 10's AppStream qemu-kvm 10.1.0 -machine help, trimmed. It has
|
||||
// no microvm row.
|
||||
const helpWithoutMicroVM = `Supported machines are:
|
||||
pc RHEL 10.0.0 PC (i440FX + PIIX, 1996) (alias of pc-i440fx-rhel10.0.0)
|
||||
pc-i440fx-rhel10.0.0 RHEL 10.0.0 PC (i440FX + PIIX, 1996) (default) (deprecated)
|
||||
pc-q35-rhel9.8.0 RHEL-9.8.0 PC (Q35 + ICH9, 2009) (deprecated)
|
||||
`
|
||||
|
||||
var microvmFirmwarePath = filepath.Join(vm.MicroVMDir, vm.MicroVMFirmware)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
microvmChecked = true
|
||||
resolveQEMU = func() (string, error) { return hostos.QEMUKVMPath, nil }
|
||||
machineHelp = func(string) ([]byte, error) { return []byte(helpWithMicroVM), nil }
|
||||
hostnetOVNRemote = func(context.Context) (string, error) { return "", nil }
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
func stubQEMU(t *testing.T, binary string, resolveErr error, help string, helpErr error) {
|
||||
t.Helper()
|
||||
origResolve, origHelp := resolveQEMU, machineHelp
|
||||
t.Cleanup(func() { resolveQEMU, machineHelp = origResolve, origHelp })
|
||||
var asked string
|
||||
resolveQEMU = func() (string, error) { return binary, resolveErr }
|
||||
machineHelp = func(b string) ([]byte, error) {
|
||||
asked = b
|
||||
return []byte(help), helpErr
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if resolveErr == nil && asked != binary {
|
||||
t.Errorf("machine help ran %q, want the resolved binary %q", asked, binary)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func qemuResult(t *testing.T, results []Result) Result {
|
||||
t.Helper()
|
||||
for _, r := range results {
|
||||
if r.Kind == "qemu" {
|
||||
return r
|
||||
}
|
||||
}
|
||||
t.Fatalf("no qemu result in %+v", results)
|
||||
return Result{}
|
||||
}
|
||||
|
||||
func TestCheckHostAt_QEMUMicroVM(t *testing.T) {
|
||||
t.Run("resolved binary lists microvm", func(t *testing.T) {
|
||||
stubQEMU(t, hostos.QEMUKVMPath, nil, helpWithMicroVM, nil)
|
||||
r := qemuResult(t, CheckHostAt(t.TempDir()))
|
||||
if r.Status != OK || r.Path != hostos.QEMUKVMPath {
|
||||
t.Errorf("got %+v, want OK for %s", r, hostos.QEMUKVMPath)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("resolved binary has no microvm", func(t *testing.T) {
|
||||
stubQEMU(t, hostos.QEMUKVMPath, nil, helpWithoutMicroVM, nil)
|
||||
r := qemuResult(t, CheckHostAt(t.TempDir()))
|
||||
if r.Status != Unsupported || !strings.Contains(r.Detail, "ol10_kvm_utils") {
|
||||
t.Errorf("got %+v, want Unsupported naming ol10_kvm_utils", r)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("no QEMU resolves", func(t *testing.T) {
|
||||
noQEMU := errors.New("no QEMU for x86_64 guests: found neither /usr/libexec/qemu-kvm (package qemu-kvm) nor qemu-system-x86_64 (package qemu-system-x86) on PATH")
|
||||
stubQEMU(t, "", noQEMU, "", nil)
|
||||
r := qemuResult(t, CheckHostAt(t.TempDir()))
|
||||
if r.Status != Missing || r.Detail != noQEMU.Error() {
|
||||
t.Errorf("got %+v, want Missing carrying the resolver's error", r)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("machine help fails", func(t *testing.T) {
|
||||
stubQEMU(t, "/usr/bin/qemu-system-x86_64", nil, "", errors.New("exit status 1"))
|
||||
r := qemuResult(t, CheckHostAt(t.TempDir()))
|
||||
if r.Status != Missing || !strings.Contains(r.Detail, "exit status 1") {
|
||||
t.Errorf("got %+v, want Missing with the exec error", r)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestCheckHostAt_MicroVMFirmware(t *testing.T) {
|
||||
t.Run("absent", func(t *testing.T) {
|
||||
r := findResult(t, CheckHostAt(t.TempDir()), microvmFirmwarePath)
|
||||
if r.Status != Missing {
|
||||
t.Errorf("status = %v, want Missing", r.Status)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("present", func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
writeHelper(t, root, microvmFirmwarePath, []byte("qboot"))
|
||||
r := findResult(t, CheckHostAt(root), microvmFirmwarePath)
|
||||
if r.Status != OK {
|
||||
t.Errorf("status = %v, want OK (detail: %s)", r.Status, r.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestCheckHostAt_NoMicroVMChecksOffX86(t *testing.T) {
|
||||
orig := microvmChecked
|
||||
microvmChecked = false
|
||||
t.Cleanup(func() { microvmChecked = orig })
|
||||
|
||||
for _, r := range CheckHostAt(t.TempDir()) {
|
||||
if r.Kind == "qemu" || r.Kind == "firmware" {
|
||||
t.Errorf("unexpected %s result on a host without system instances: %+v", r.Kind, r)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -29,6 +29,10 @@ const (
|
||||
Stale
|
||||
// Ungranted means the sudoers grant for a helper is absent or missing its NOPASSWD line.
|
||||
Ungranted
|
||||
// Unsupported means the asset is present but lacks a capability this build needs.
|
||||
Unsupported
|
||||
// Failed means a host check that gates the services did not pass.
|
||||
Failed
|
||||
)
|
||||
|
||||
// String returns the human-readable name of the status, used in table output and log lines.
|
||||
@@ -42,6 +46,10 @@ func (s Status) String() string {
|
||||
return "Stale"
|
||||
case Ungranted:
|
||||
return "Ungranted"
|
||||
case Unsupported:
|
||||
return "Unsupported"
|
||||
case Failed:
|
||||
return "Failed"
|
||||
default:
|
||||
return "Unknown"
|
||||
}
|
||||
@@ -50,7 +58,7 @@ func (s Status) String() string {
|
||||
// Result is the outcome of checking one managed asset against this build's manifest.
|
||||
type Result struct {
|
||||
Path string
|
||||
Kind string // "helper" | "sudoers-grant" | "service"
|
||||
Kind string // "helper" | "sudoers-grant" | "service" | "qemu" | "firmware" | "hostnet"
|
||||
Status Status
|
||||
Detail string
|
||||
}
|
||||
@@ -82,6 +90,8 @@ func CheckHostAt(root string) []Result {
|
||||
}
|
||||
results = append(results, checkSudoersGrant(root))
|
||||
results = append(results, checkRunningBinaries(root)...)
|
||||
results = append(results, checkMicroVM(root)...)
|
||||
results = append(results, checkHostnet(root))
|
||||
return results
|
||||
}
|
||||
|
||||
|
||||
@@ -105,7 +105,7 @@ func TestHasProblem(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestStatusString(t *testing.T) {
|
||||
cases := map[Status]string{OK: "OK", Missing: "Missing", Stale: "Stale", Ungranted: "Ungranted"}
|
||||
cases := map[Status]string{OK: "OK", Missing: "Missing", Stale: "Stale", Ungranted: "Ungranted", Unsupported: "Unsupported", Failed: "Failed"}
|
||||
for status, want := range cases {
|
||||
if got := status.String(); got != want {
|
||||
t.Errorf("Status(%d).String() = %q, want %q", status, got, want)
|
||||
|
||||
@@ -774,10 +774,14 @@ func (d *Daemon) WaitForSystemInstance(instanceID string, timeout time.Duration)
|
||||
return fmt.Errorf("instance %s did not reach running state within %s", instanceID, timeout)
|
||||
}
|
||||
|
||||
// microvmDir holds the microvm kernel, initramfs and firmware. A variable so
|
||||
// tests can supply their own.
|
||||
var microvmDir = vm.MicroVMDir
|
||||
|
||||
// buildDirectBootConfig constructs a vm.Config for a direct-boot microVM
|
||||
// launch. It does not call buildBaseVMConfig — the microvm machine type
|
||||
// requires no PCIe root ports, no UEFI/BIOS, and no drives. Runtime paths
|
||||
// (PID file, console log, serial socket) are filled in later by startQEMU.
|
||||
// requires no PCIe root ports, no UEFI, and no drives; it boots qboot. Runtime
|
||||
// paths (PID file, console log, serial socket) are filled in later by startQEMU.
|
||||
//
|
||||
// Network topology from input.NICs is serialised to three fw_cfg tmpfiles
|
||||
// (netcfg, lb-agent-env, ca-cert) under hostprocess.RuntimeDir().
|
||||
@@ -794,7 +798,10 @@ func (d *Daemon) buildDirectBootConfig(instanceID string, input *handlers_elbv2.
|
||||
memMiB = int(instanceTypeMemoryMiB(it))
|
||||
}
|
||||
|
||||
const imagePath = "/usr/share/spinifex/microvm"
|
||||
firmware := filepath.Join(microvmDir, vm.MicroVMFirmware)
|
||||
if _, err := os.Stat(firmware); err != nil {
|
||||
return vm.Config{}, fmt.Errorf("microvm firmware %s is missing, so load balancer instances cannot boot; reinstall Spinifex to restore it: %w", firmware, err)
|
||||
}
|
||||
|
||||
fwCfg, err := d.writeFwCfgBlobs(instanceID, input)
|
||||
if err != nil {
|
||||
@@ -846,8 +853,9 @@ func (d *Daemon) buildDirectBootConfig(instanceID string, input *handlers_elbv2.
|
||||
CPUCount: vcpus,
|
||||
Memory: memMiB,
|
||||
MachineType: machineType,
|
||||
KernelImage: filepath.Join(imagePath, "vmlinuz"),
|
||||
Initrd: filepath.Join(imagePath, "initramfs.cpio.gz"),
|
||||
BIOS: firmware,
|
||||
KernelImage: filepath.Join(microvmDir, "vmlinuz"),
|
||||
Initrd: filepath.Join(microvmDir, "initramfs.cpio.gz"),
|
||||
KernelCmdline: cmdline,
|
||||
FwCfg: fwCfg,
|
||||
}
|
||||
|
||||
@@ -804,6 +804,37 @@ func TestBuildDirectBootConfig_InstanceTypeShape(t *testing.T) {
|
||||
assert.Len(t, cfg.FwCfg, 3)
|
||||
}
|
||||
|
||||
func TestBuildDirectBootConfig_BootsTheShippedQboot(t *testing.T) {
|
||||
t.Setenv("XDG_RUNTIME_DIR", t.TempDir())
|
||||
d := &Daemon{resourceMgr: &ResourceManager{}}
|
||||
|
||||
cfg, err := d.buildDirectBootConfig("i-qboot", &handlers_elbv2.SystemInstanceInput{
|
||||
NICs: []handlers_elbv2.NICConfig{{MAC: "02:00:00:00:00:05", IsDefault: true}},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, filepath.Join(microvmDir, vm.MicroVMFirmware), cfg.BIOS)
|
||||
for _, e := range cfg.FwCfg {
|
||||
assert.NoError(t, os.Remove(e.File))
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildDirectBootConfig_MissingFirmwareFailsByName(t *testing.T) {
|
||||
runtimeDir := t.TempDir()
|
||||
t.Setenv("XDG_RUNTIME_DIR", runtimeDir)
|
||||
orig := microvmDir
|
||||
microvmDir = t.TempDir()
|
||||
t.Cleanup(func() { microvmDir = orig })
|
||||
|
||||
_, err := (&Daemon{resourceMgr: &ResourceManager{}}).buildDirectBootConfig("i-nofw", &handlers_elbv2.SystemInstanceInput{
|
||||
NICs: []handlers_elbv2.NICConfig{{MAC: "02:00:00:00:00:06", IsDefault: true}},
|
||||
})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), filepath.Join(microvmDir, vm.MicroVMFirmware))
|
||||
entries, readErr := os.ReadDir(runtimeDir)
|
||||
require.NoError(t, readErr)
|
||||
assert.Empty(t, entries, "a launch refused for missing firmware must leave no fw_cfg blobs behind")
|
||||
}
|
||||
|
||||
func TestBuildDirectBootConfig_FwCfgWriteError(t *testing.T) {
|
||||
notADir := filepath.Join(t.TempDir(), "file")
|
||||
require.NoError(t, os.WriteFile(notADir, nil, 0o600))
|
||||
|
||||
@@ -3,10 +3,12 @@ package daemon
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/clustersize"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
)
|
||||
|
||||
@@ -31,6 +33,19 @@ func TestMain(m *testing.M) {
|
||||
// has its own tests that inject a reader.
|
||||
os.Setenv("SPINIFEX_ADMISSION_LIVE_MEM", "0")
|
||||
|
||||
// System-instance launches refuse to build a config without the microvm
|
||||
// firmware, which a test host does not have installed.
|
||||
fwDir, err := os.MkdirTemp("", "microvm-*")
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Failed to create microvm fixture dir: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(fwDir, vm.MicroVMFirmware), []byte("qboot"), 0o644); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Failed to write microvm firmware fixture: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
microvmDir = fwDir
|
||||
|
||||
// Start a plain NATS server (no JetStream)
|
||||
ns, err := server.NewServer(&server.Options{
|
||||
Host: "127.0.0.1",
|
||||
@@ -88,6 +103,7 @@ func TestMain(m *testing.M) {
|
||||
ns.Shutdown()
|
||||
jsNS.Shutdown()
|
||||
os.RemoveAll(jsTmpDir)
|
||||
os.RemoveAll(fwDir)
|
||||
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/host/hostnet"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var adminHostnetCmd = &cobra.Command{
|
||||
Use: "hostnet",
|
||||
Short: "Re-create or check this node's host network wiring",
|
||||
Long: `On nodes whose host network wiring is not persisted by systemd-networkd,
|
||||
setup-ovn.sh records it in /etc/spinifex/hostnet.json and spinifex-hostnet.service
|
||||
re-creates it at boot and whenever ovs-vswitchd restarts.`,
|
||||
}
|
||||
|
||||
var adminHostnetApplyCmd = &cobra.Command{
|
||||
Use: "apply",
|
||||
Short: "Make the host network match the recorded wiring (run by spinifex-hostnet.service)",
|
||||
Long: `Creates the uplink veth pairs and sets the links, addresses and encap route
|
||||
named in /etc/spinifex/hostnet.json. It never deletes a link, and never touches a
|
||||
link the file does not name. Does nothing on a node whose wiring systemd-networkd
|
||||
persists. Writes /run/spinifex/hostnet-applied only on success.`,
|
||||
Args: cobra.NoArgs,
|
||||
SilenceUsage: true,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
return hostnet.Apply(cmd.Context(), hostnet.DefaultEnv())
|
||||
},
|
||||
}
|
||||
|
||||
var adminHostnetCheckCmd = &cobra.Command{
|
||||
Use: "check",
|
||||
Short: "Fail unless the host network is ready for spinifex-daemon and spinifex-vpcd",
|
||||
Long: `Runs the same backend and state-file checks as apply, and on the hostnet backend
|
||||
also needs the marker apply writes on success. The daemon and vpcd run it before
|
||||
they start, so neither starts on a node whose wiring failed to apply.`,
|
||||
Args: cobra.NoArgs,
|
||||
SilenceUsage: true,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
return hostnet.Check(cmd.Context(), hostnet.DefaultEnv())
|
||||
},
|
||||
}
|
||||
|
||||
func init() {
|
||||
adminCmd.AddCommand(adminHostnetCmd)
|
||||
adminHostnetCmd.AddCommand(adminHostnetApplyCmd)
|
||||
adminHostnetCmd.AddCommand(adminHostnetCheckCmd)
|
||||
}
|
||||
@@ -24,8 +24,16 @@ Overwriting /usr/local/bin/spx while spinifex.target is up does not move the
|
||||
services onto it: each keeps executing the replaced inode until it restarts,
|
||||
so a node can serve a mixture of two builds indefinitely.
|
||||
|
||||
On x86_64 it checks that the QEMU binary the daemon will run has the microvm
|
||||
machine type, and that the microvm firmware is installed: load balancers boot
|
||||
on both.
|
||||
|
||||
It runs the host network check spinifex-daemon and spinifex-vpcd run before they
|
||||
start, so a failed spinifex-hostnet apply shows here.
|
||||
|
||||
Checks compare content, not mere existence, so a present-but-stale asset is
|
||||
caught too. Exits non-zero if any asset is Missing, Stale, or Ungranted.`,
|
||||
caught too. Exits non-zero if any asset is Missing, Stale, Ungranted, Unsupported or
|
||||
Failed.`,
|
||||
RunE: runAdminPreflight,
|
||||
SilenceUsage: true,
|
||||
}
|
||||
|
||||
@@ -8,9 +8,10 @@ package systemd
|
||||
var Units = map[string]string{
|
||||
"regenerate-ssh-host-keys.service": "# spinifex-unit-version: 1\n[Unit]\nDescription=Regenerate SSH host keys\nConditionPathExists=!/etc/ssh/ssh_host_ed25519_key\nBefore=ssh.service\n\n[Service]\nType=oneshot\nExecStart=/usr/bin/ssh-keygen -A\nRemainAfterExit=yes\n# No ExecStop: the keygen process has already exited by the time this unit\n# is stopped, so there is nothing to signal.\nTimeoutStopSec=5\n\n[Install]\nWantedBy=multi-user.target\n",
|
||||
"spinifex-awsgw.service": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex AWS Gateway\nPartOf=spinifex.target\nAfter=spinifex-nats.service spinifex-daemon.service\nWants=spinifex-nats.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-gw\nGroup=spinifex\nExecStartPre=/var/lib/spinifex/wait-for-nats.sh\nExecStart=/usr/local/bin/spx service awsgw start\nRestart=on-failure\nRestartSec=5\n# No SIGTERM handler in this process: it exits immediately on signal, so this\n# only bounds an unexpectedly wedged process before SIGKILL.\nTimeoutStopSec=10\n\nOOMScoreAdjust=-500\n# Per-service ceiling bounds a leak in this service alone rather than\n# tripping the slice-wide cap, which would pick an arbitrary victim.\nMemoryHigh=10%\nMemoryMax=20%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_BASE_DIR=/var/lib/spinifex/awsgw/\nEnvironment=SPINIFEX_AWSGW_TLS_CERT=/etc/spinifex/server.pem\nEnvironment=SPINIFEX_AWSGW_TLS_KEY=/etc/spinifex/server.key\n\n# Filesystem access\nNoNewPrivileges=yes\nProtectSystem=strict\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/master.key /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem\nReadWritePaths=/var/lib/spinifex/awsgw /var/log/spinifex\n\n# Hardening\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nCapabilityBoundingSet=\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n",
|
||||
"spinifex-daemon.service": "# spinifex-unit-version: 3\n[Unit]\nDescription=Spinifex Daemon (VM Orchestration)\nPartOf=spinifex.target\nAfter=spinifex-nats.service spinifex-viperblock.service openvswitch-switch.service ovn-controller.service openvswitch-ipsec.service\n# Wants (not Requires): stay bootable when openvswitch-ipsec is absent\nWants=openvswitch-ipsec.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\n# ZFS root: installer raises SPINIFEX_RESERVED_MEM_GB by the ARC ceiling the admission gate cannot see\nEnvironmentFile=-/etc/spinifex/host.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-daemon\nGroup=spinifex\nExecStart=/usr/local/bin/spx service spinifex start\nRestart=on-failure\nRestartSec=5\n# KillMode=process: guest QEMU survives daemon restart; the new daemon reattaches (DDIL)\nKillMode=process\nTimeoutStopSec=15\n# RG-4: infra tier — reaped only after every guest\nOOMScoreAdjust=-500\n# Guest QEMU shares this cgroup: the ceiling tracks the slice budget, not daemon overhead\nMemoryHigh=75%\nMemoryMax=85%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_BASE_DIR=/var/lib/spinifex/spinifex/\nEnvironment=SPINIFEX_WAL_DIR=/var/lib/spinifex/spinifex/\n# Pin to host tmpfs: QEMU pidfiles/QMP sockets must survive PrivateTmp namespace churn (DDIL)\nEnvironment=XDG_RUNTIME_DIR=/run/spinifex\n\n# RG-9: daemon tier — privileged by necessity (GPU vfio). No CAP_NET_ADMIN here,\n# so unlike vpcd it still escalates ip/sysctl and NoNewPrivileges stays omitted.\nAmbientCapabilities=CAP_SYS_ADMIN CAP_DAC_OVERRIDE\nProtectSystem=full\nPrivateDevices=no\nDevicePolicy=closed\nDeviceAllow=/dev/kvm rw\nDeviceAllow=/dev/net/tun rw\nDeviceAllow=/dev/vhost-net rw\nDeviceAllow=char-vfio rw\nDeviceAllow=/dev/vfio/vfio rw\nReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem\n# /proc/sys/net/ipv4/conf: ProtectKernelTunables would keep the per-tap IMDS\n# rp_filter/accept_local read-only; /etc/spinifex/firewall: the root helper writes peers.nft.\nReadWritePaths=/var/lib/spinifex/spinifex /var/log/spinifex /run/spinifex /run/spinifex/nbd /proc/sys/net/ipv4/conf -/etc/spinifex/firewall\n\n# RG-9: hardening baseline\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n# LimitMEMLOCK=infinity: VFIO pins all guest RAM\nLimitMEMLOCK=infinity\n\n[Install]\nWantedBy=spinifex.target\n",
|
||||
"spinifex-daemon.service": "# spinifex-unit-version: 5\n[Unit]\nDescription=Spinifex Daemon (VM Orchestration)\nPartOf=spinifex.target\nAfter=spinifex-nats.service spinifex-viperblock.service ovsdb-server.service ovs-vswitchd.service ovn-controller.service openvswitch-ipsec.service spinifex-hostnet.service\n# Wants (not Requires): boot without openvswitch-ipsec, survive OVS restarts via hostnet\nWants=openvswitch-ipsec.service spinifex-hostnet.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\n# ZFS root: installer raises SPINIFEX_RESERVED_MEM_GB by the ARC ceiling the admission gate cannot see\nEnvironmentFile=-/etc/spinifex/host.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-daemon\nGroup=spinifex\nExecStartPre=/usr/local/bin/spx admin hostnet check\nExecStart=/usr/local/bin/spx service spinifex start\nRestart=on-failure\nRestartSec=5\n# KillMode=process: guest QEMU survives daemon restart; the new daemon reattaches (DDIL)\nKillMode=process\nTimeoutStopSec=15\n# RG-4: infra tier — reaped only after every guest\nOOMScoreAdjust=-500\n# Guest QEMU shares this cgroup: the ceiling tracks the slice budget, not daemon overhead\nMemoryHigh=75%\nMemoryMax=85%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_BASE_DIR=/var/lib/spinifex/spinifex/\nEnvironment=SPINIFEX_WAL_DIR=/var/lib/spinifex/spinifex/\n# Pin to host tmpfs: QEMU pidfiles/QMP sockets must survive PrivateTmp namespace churn (DDIL)\nEnvironment=XDG_RUNTIME_DIR=/run/spinifex\n\n# RG-9: daemon tier — privileged by necessity (GPU vfio). No CAP_NET_ADMIN here,\n# so unlike vpcd it still escalates ip/sysctl and NoNewPrivileges stays omitted.\nAmbientCapabilities=CAP_SYS_ADMIN CAP_DAC_OVERRIDE\nProtectSystem=full\nPrivateDevices=no\nDevicePolicy=closed\nDeviceAllow=/dev/kvm rw\nDeviceAllow=/dev/net/tun rw\nDeviceAllow=/dev/vhost-net rw\nDeviceAllow=char-vfio rw\nDeviceAllow=/dev/vfio/vfio rw\nReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem\n# /proc/sys/net/ipv4/conf: ProtectKernelTunables would keep the per-tap IMDS\n# rp_filter/accept_local read-only; /etc/spinifex/firewall: the root helper writes peers.nft.\nReadWritePaths=/var/lib/spinifex/spinifex /var/log/spinifex /run/spinifex /run/spinifex/nbd /proc/sys/net/ipv4/conf -/etc/spinifex/firewall\n\n# RG-9: hardening baseline\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n# LimitMEMLOCK=infinity: VFIO pins all guest RAM\nLimitMEMLOCK=infinity\n\n[Install]\nWantedBy=spinifex.target\n",
|
||||
"spinifex-firewall.service": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex host firewall (nftables input policy)\nDocumentation=file:/etc/spinifex/firewall/spinifex.nft\nWants=network-pre.target\nBefore=network-pre.target shutdown.target\nDefaultDependencies=no\nConditionPathExists=/etc/spinifex/firewall/peers.nft\n\n[Service]\nType=oneshot\nRemainAfterExit=yes\nStandardInput=null\nProtectSystem=full\nProtectHome=true\nExecStart=/usr/local/lib/spinifex/spinifex-firewall-apply\nExecReload=/usr/local/lib/spinifex/spinifex-firewall-apply\nTimeoutStopSec=10s\n\n# No ExecStop: flushing on stop would take vpcd's MASQUERADE and per-EIP FORWARD\n# rules with it. Remove the policy deliberately with\n# `nft delete table inet spinifex_filter`.\n\n[Install]\nWantedBy=sysinit.target\n",
|
||||
"spinifex-guests.slice": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex guest VM tier\nBefore=slices.target\n\n[Slice]\n# RG-3: child of spinifex.slice (spinifex-guests -> spinifex), sibling of\n# spinifex-system. Parent for the per-instance spinifex-instance-<id>.slice\n# leaves (0002 RG-7). Inherits the spinifex.slice ceiling; no extra limit here.\n",
|
||||
"spinifex-hostnet.service": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex host network wiring (veth pairs, mgmt address, encap route)\n# PartOf/WantedBy ovs-vswitchd: re-applied whenever OVS starts; daemon and vpcd only Want it\nRequires=ovs-vswitchd.service\nAfter=ovs-vswitchd.service NetworkManager-wait-online.service\nPartOf=ovs-vswitchd.service\nBefore=spinifex.target\n\n[Service]\nType=oneshot\nRemainAfterExit=yes\nExecStart=/usr/local/bin/spx admin hostnet apply\n# Covers the 120s wait behind setup-ovn.sh plus the 60s wait for links\nTimeoutStartSec=240\nTimeoutStopSec=10\nProtectSystem=full\nProtectHome=yes\nPrivateTmp=yes\n\n[Install]\nWantedBy=spinifex.target ovs-vswitchd.service\n",
|
||||
"spinifex-nats-watchdog.service": "# spinifex-unit-version: 2\n[Unit]\nDescription=Spinifex NATS JetStream liveness watchdog\nPartOf=spinifex.target\nAfter=spinifex-nats.service\n\n[Service]\nType=oneshot\n# Runs as root: the probes are a local curl to the NATS monitoring port and a\n# canary KV round-trip via spx, and the remedy is `systemctl try-restart\n# spinifex-nats` when JetStream stops accepting writes while the process stays\n# alive (which Restart=on-failure cannot catch).\nExecStart=/var/lib/spinifex/nats-js-watchdog.sh\n# Sized for the probe loop's own worst case (3 samples x (5s curl + 5s canary)\n# + 2 gaps x 2s = 34s): a stop mid-cycle lets the script finish rather than\n# cutting it off. Only the failing path runs that long; a healthy node exits on\n# the first sample.\nTimeoutStopSec=45\n# Same 34s worst case bounds the start side: without this the unit inherits\n# oneshot's infinite default, so a genuinely hung run would sit in\n# activating(start) forever and could not be retriggered by its timer.\nTimeoutStartSec=120\n\n# Hardening (kept compatible with talking to PID1 over /run/systemd).\nNoNewPrivileges=yes\nProtectHome=yes\nPrivateTmp=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectControlGroups=yes\nRestrictRealtime=yes\nLockPersonality=yes\nRestrictSUIDSGID=yes\n",
|
||||
"spinifex-nats-watchdog.timer": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex NATS JetStream liveness watchdog timer\nPartOf=spinifex.target\n\n[Timer]\n# Delay the first probe so NATS startup + JetStream cluster quorum settle (a\n# forming cluster reports JS not-current transiently, which is not a fault).\nOnBootSec=60s\nOnUnitActiveSec=30s\nAccuracySec=5s\n\n[Install]\nWantedBy=timers.target\n",
|
||||
"spinifex-nats.service": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex NATS Message Bus\nPartOf=spinifex.target\nBefore=spinifex-predastore.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-nats\nGroup=spinifex\nExecStart=/usr/local/bin/spx service nats start\nRestart=on-failure\nRestartSec=5\n# nats-server installs its own SIGTERM handler (JetStream store shutdown, raft\n# stepdown, client drain) with no internal deadline; every dependent service\n# stops before nats does, so size this for nats's own graceful-stop needs.\nTimeoutStopSec=20\nOOMScoreAdjust=-500\n# Per-service ceiling bounds a leak in this service alone rather than\n# tripping the slice-wide cap, which would pick an arbitrary victim.\nMemoryHigh=10%\nMemoryMax=20%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/nats/nats.conf\n\n# Filesystem access\nNoNewPrivileges=yes\nProtectSystem=strict\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/nats /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem\nReadWritePaths=/var/lib/spinifex/nats /var/log/spinifex\n\n# Hardening\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nCapabilityBoundingSet=\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n",
|
||||
@@ -21,7 +22,7 @@ var Units = map[string]string{
|
||||
"spinifex-system.slice": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex control/infra tier (NATS, predastore, viperblock, daemon, awsgw, vpcd, ui)\nBefore=slices.target\n\n[Slice]\n# RG-3: child of spinifex.slice (spinifex-system -> spinifex), inherits the\n# parent ceiling, no extra limit. Holds the 7 control/infra services. Guest QEMU\n# moves to the sibling spinifex-guests.slice (per-instance placement lands in\n# 0002 RG-7); until then the daemon-forked guests stay under spinifex-daemon.service.\n",
|
||||
"spinifex-ui.service": "# spinifex-unit-version: 2\n[Unit]\nDescription=Spinifex Web UI\nPartOf=spinifex.target\nAfter=spinifex-awsgw.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-ui\nGroup=spinifex\nExecStart=/usr/local/bin/spx service spinifex-ui start\nRestart=on-failure\nRestartSec=5\n# server.Shutdown() uses an internal 30s context (WriteTimeout=120s covers\n# long streaming proxy responses, e.g. console passthrough); add margin above it.\nTimeoutStopSec=40\nOOMScoreAdjust=-500\n# Per-service ceiling bounds a leak in this service alone rather than\n# tripping the slice-wide cap, which would pick an arbitrary victim.\nMemoryHigh=10%\nMemoryMax=20%\nEnvironment=SPINIFEX_UI_TLS_CERT=/etc/spinifex/server.pem\nEnvironment=SPINIFEX_UI_TLS_KEY=/etc/spinifex/server.key\nEnvironment=SPINIFEX_UI_BASE_DIR=/var/lib/spinifex/spinifex-ui/\nEnvironment=SPINIFEX_UI_OCHRE_ENABLED=false\n\n# Filesystem access\nNoNewPrivileges=yes\nProtectSystem=strict\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/server.pem /etc/spinifex/server.key\nReadWritePaths=/var/lib/spinifex/spinifex-ui /var/log/spinifex\n\n# Hardening\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nCapabilityBoundingSet=\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n",
|
||||
"spinifex-viperblock.service": "# spinifex-unit-version: 2\n[Unit]\nDescription=Spinifex Viperblock (Block Storage)\nPartOf=spinifex.target\nAfter=spinifex-nats.service spinifex-predastore.service\nWants=spinifex-nats.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-viperblock\nGroup=spinifex\nExecStartPre=/var/lib/spinifex/wait-for-nats.sh\nExecStart=/usr/local/bin/spx service viperblock start\nRestart=on-failure\nRestartSec=5\nOOMScoreAdjust=-500\n# Per-service ceiling bounds a leak in this service alone rather than\n# tripping the slice-wide cap, which would pick an arbitrary victim.\nMemoryHigh=15%\nMemoryMax=25%\n# KillMode=process: nbdkit still serving a guest survives a restart here, the\n# way guest QEMU survives a daemon restart. Tearing one out mid-write corrupts\n# that guest's filesystem; idle nbdkit is reaped explicitly on shutdown.\nKillMode=process\n# One worst-case nbdkit reap: killProcessGracePeriod (120s, utils.KillProcess)\n# plus its own shutdownDrainTimeout (20s, viperblock/nbd) and headroom — must\n# strictly exceed 120s now shutdownVolumes reaps concurrently, not serially.\nTimeoutStopSec=150\nEnvironmentFile=/etc/spinifex/systemd.env\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_BASE_DIR=/var/lib/spinifex/viperblock/\n\n# Filesystem access\nNoNewPrivileges=yes\nProtectSystem=strict\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/systemd.env /etc/spinifex/ca.pem\nReadWritePaths=/var/lib/spinifex/viperblock /var/log/spinifex /run/spinifex/nbd\n\n# Hardening\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\n# ProtectProc must stay default: hidepid=invisible hides spinifex-daemon's\n# QEMU /proc entries (different uid) orphan nbdkit reaping must see.\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nCapabilityBoundingSet=\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n",
|
||||
"spinifex-vpcd.service": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex VPC Daemon (OVN Networking)\nPartOf=spinifex.target\nAfter=spinifex-nats.service spinifex-daemon.service systemd-networkd.service openvswitch-switch.service ovn-controller.service\nWants=spinifex-nats.service ovn-controller.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-vpcd\nGroup=spinifex\nExecStartPre=/var/lib/spinifex/wait-for-nats.sh\nExecStart=/usr/local/bin/spx service vpcd start\nRestart=on-failure\nRestartSec=5\n# Waits for an in-flight OVN drift-reconcile pass to return before exiting;\n# no sub-timeout bounds that call today, so this covers a normal pass.\nTimeoutStopSec=20\nOOMScoreAdjust=-500\n# Per-service ceiling: bounds a leak here instead of the slice-wide cap.\nMemoryHigh=10%\nMemoryMax=20%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_BASE_DIR=/var/lib/spinifex/vpcd/\n\n# RG-9: network tier. Per-tap IMDS removed the in-process setns, so vpcd no\n# longer enters a per-VPC netns: CAP_SYS_ADMIN is dropped and the filesystem\n# sandbox is restored.\n#\n# RG-10: these are ambient so ip/iptables/sysctl/arping inherit them on exec and\n# vpcd holds no sudoers rules at all. NoNewPrivileges stays off pending the ADR\n# update that retires the last escalation path.\nAmbientCapabilities=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID\nCapabilityBoundingSet=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID\nNoNewPrivileges=no\nProtectSystem=full\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem\nReadWritePaths=/var/lib/spinifex/vpcd /var/log/spinifex\n\n# RG-9: hardening baseline\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\n# AF_PACKET: raw L2 BOOTP for the in-process DHCP client before the iface has an IP\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK AF_PACKET\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n",
|
||||
"spinifex-vpcd.service": "# spinifex-unit-version: 3\n[Unit]\nDescription=Spinifex VPC Daemon (OVN Networking)\nPartOf=spinifex.target\nAfter=spinifex-nats.service spinifex-daemon.service systemd-networkd.service ovsdb-server.service ovs-vswitchd.service ovn-controller.service spinifex-hostnet.service\nWants=spinifex-nats.service ovn-controller.service spinifex-hostnet.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-vpcd\nGroup=spinifex\nExecStartPre=/usr/local/bin/spx admin hostnet check\nExecStartPre=/var/lib/spinifex/wait-for-nats.sh\nExecStart=/usr/local/bin/spx service vpcd start\nRestart=on-failure\nRestartSec=5\n# Waits for an in-flight OVN drift-reconcile pass to return before exiting;\n# no sub-timeout bounds that call today, so this covers a normal pass.\nTimeoutStopSec=20\nOOMScoreAdjust=-500\n# Per-service ceiling: bounds a leak here instead of the slice-wide cap.\nMemoryHigh=10%\nMemoryMax=20%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_BASE_DIR=/var/lib/spinifex/vpcd/\n\n# RG-9: network tier. Per-tap IMDS removed the in-process setns, so vpcd no\n# longer enters a per-VPC netns: CAP_SYS_ADMIN is dropped and the filesystem\n# sandbox is restored.\n#\n# RG-10: these are ambient so ip/iptables/sysctl/arping inherit them on exec and\n# vpcd holds no sudoers rules at all. NoNewPrivileges stays off pending the ADR\n# update that retires the last escalation path.\nAmbientCapabilities=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID\nCapabilityBoundingSet=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID\nNoNewPrivileges=no\nProtectSystem=full\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem\nReadWritePaths=/var/lib/spinifex/vpcd /var/log/spinifex\n\n# RG-9: hardening baseline\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\n# AF_PACKET: raw L2 BOOTP for the in-process DHCP client before the iface has an IP\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK AF_PACKET\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n",
|
||||
"spinifex.slice": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex resource-bounded slice (host ceiling)\nBefore=slices.target\n\n[Slice]\n# RG-1: no CPUWeight — slice roots are siblings; capping below the default would\n# let user.slice out-prioritize the VM fleet. Operator CPU priority lives on\n# system.slice (system.slice.d/spinifex-reserve.conf).\n# RG-1/RG-5: throttle+reclaim at High, hard wall at Max (% of physical RAM).\nMemoryHigh=80%\nMemoryMax=90%\n# RG-5: spill to the full swap device before the MemoryMax wall; matched to swap size.\nMemorySwapMax=8G\n",
|
||||
"spinifex.target": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex Infrastructure Platform\nAfter=network-online.target\nWants=network-online.target\nWants=spinifex-nats.service spinifex-predastore.service spinifex-northstar.service spinifex-viperblock.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service spinifex-ui.service spinifex-qmp-collector.service spinifex-shutdown.service spinifex-nats-watchdog.timer openvswitch-switch.service ovn-controller.service\n\n[Install]\nWantedBy=multi-user.target\n",
|
||||
"spinifex.target": "# spinifex-unit-version: 2\n[Unit]\nDescription=Spinifex Infrastructure Platform\nAfter=network-online.target\nWants=network-online.target\nWants=spinifex-nats.service spinifex-predastore.service spinifex-northstar.service spinifex-viperblock.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service spinifex-ui.service spinifex-qmp-collector.service spinifex-shutdown.service spinifex-nats-watchdog.timer ovsdb-server.service ovs-vswitchd.service ovn-controller.service\n\n[Install]\nWantedBy=multi-user.target\n",
|
||||
}
|
||||
|
||||
@@ -423,9 +423,12 @@ var unitBodyHashes = map[string]map[int]string{
|
||||
1: "04b0f23b9cda322e4012d589acc196abe466b619dec5ac0d832897b9af2f926c",
|
||||
2: "97a2d4c72967a9bdf9545ff74593de532e8469568bb717c9958d33b77e62d0ed",
|
||||
3: "8059fe13a678b8cb2a383ae75c069560cf39adece9a9394e17781998c711cd57",
|
||||
4: "f66e1240cc5655f6a13f3b56ca282c73e2fe289f2e51d0ce624d14b90f12f838",
|
||||
5: "d3d25d7eb0051ccf97f70838ddff763bade38bef934b8f85733e048151ac9319",
|
||||
},
|
||||
"spinifex-firewall.service": {1: "01d0a7a79f47eedca02aaab3ff97a0b6462d61b7834e16c7336a7b96034dc392"},
|
||||
"spinifex-guests.slice": {1: "22dac772e81a9a716db98415a4cf590885c9cf19d4290cc21e645e5fe15bc793"},
|
||||
"spinifex-hostnet.service": {1: "4a110daac63b652aef49e358f22ac8a27a7797261b231d5b1184e171cccb8f87"},
|
||||
"spinifex-nats-watchdog.service": {
|
||||
1: "b778e398da37c4cb3c170781f074c76555cb6a137a7f96f8dc692457b12aa9b8",
|
||||
2: "c31eb4fa964060e1681113a3de4f8f3ee4336aee96469380aac5fc77480c964a",
|
||||
@@ -449,9 +452,16 @@ var unitBodyHashes = map[string]map[int]string{
|
||||
1: "5c8cdd2004abf8e5725cbce0200565b9b29be7ab3210a4e0a2cdfe37ec5facb9",
|
||||
2: "5d691bf5ce4a5a636d5dcc07f7bfa36d0da9eb0d7fb26079f89358ca0e5440cd",
|
||||
},
|
||||
"spinifex-vpcd.service": {1: "1b722640310145767cd34e87f4804852e63f46d2145ed20f8cc5f5400ebc5965"},
|
||||
"spinifex.slice": {1: "f73d9343e0e1bedd647835c8bb0c80fb3a3bd66474661234ecac23a4caafc24f"},
|
||||
"spinifex.target": {1: "0ffba9faee5a477f8ff7466a6bccb4dc7e04f5cf92a405553c242e6548402078"},
|
||||
"spinifex-vpcd.service": {
|
||||
1: "1b722640310145767cd34e87f4804852e63f46d2145ed20f8cc5f5400ebc5965",
|
||||
2: "6ca01c8104ce0fc96ee4fdd8c0f2cd2ae907211b58b6f868fdc6a26f55a579f8",
|
||||
3: "3ac639bf8017f6eeb8777096edc40aaeeda9ca2bcc0a8313b93c51d20847ea20",
|
||||
},
|
||||
"spinifex.slice": {1: "f73d9343e0e1bedd647835c8bb0c80fb3a3bd66474661234ecac23a4caafc24f"},
|
||||
"spinifex.target": {
|
||||
1: "0ffba9faee5a477f8ff7466a6bccb4dc7e04f5cf92a405553c242e6548402078",
|
||||
2: "0dc31d0e415277d10e4d90959f40d574f5c2319560299448002f701453737d5b",
|
||||
},
|
||||
}
|
||||
|
||||
// TestUnitBodyPinnedToVersion asserts each unit's body hash matches the
|
||||
@@ -493,3 +503,98 @@ func stripVersionMarkerLine(content string) string {
|
||||
}
|
||||
return content
|
||||
}
|
||||
|
||||
// crossDistroOVSUnits are the OVS and OVN units Debian and EL both ship under
|
||||
// the same name. The umbrella differs (openvswitch-switch against openvswitch),
|
||||
// and systemd silently drops ordering against a unit that does not exist.
|
||||
var crossDistroOVSUnits = []string{
|
||||
"ovsdb-server.service",
|
||||
"ovs-vswitchd.service",
|
||||
"ovn-controller.service",
|
||||
"ovn-northd.service",
|
||||
"openvswitch-ipsec.service",
|
||||
}
|
||||
|
||||
// TestOVSDependenciesExistOnEveryDistro asserts every OVS or OVN unit a
|
||||
// Spinifex unit depends on or orders against is one both distros ship.
|
||||
func TestOVSDependenciesExistOnEveryDistro(t *testing.T) {
|
||||
dir := unitsDir(t)
|
||||
keys := []string{"After", "Before", "Wants", "Requires", "Requisite", "BindsTo", "PartOf", "Upholds"}
|
||||
for _, name := range allUnitFiles(t, dir) {
|
||||
u := readUnit(t, dir, name)
|
||||
for line := range strings.SplitSeq(u, "\n") {
|
||||
key, value, ok := strings.Cut(strings.TrimSpace(line), "=")
|
||||
if !ok || !slices.Contains(keys, key) {
|
||||
continue
|
||||
}
|
||||
for dep := range strings.FieldsSeq(value) {
|
||||
if !strings.HasPrefix(dep, "ovs") && !strings.HasPrefix(dep, "ovn") && !strings.HasPrefix(dep, "openvswitch") {
|
||||
continue
|
||||
}
|
||||
if !slices.Contains(crossDistroOVSUnits, dep) {
|
||||
t.Errorf("%s: %s=%s is not shipped under that name on every distro; use one of %v", name, key, dep, crossDistroOVSUnits)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestHostnetIsWantedNotRequired asserts the daemon and vpcd start after
|
||||
// spinifex-hostnet without Requires=: hostnet is PartOf ovs-vswitchd, so a
|
||||
// Requires= would restart both on every OVS restart or package upgrade.
|
||||
func TestHostnetIsWantedNotRequired(t *testing.T) {
|
||||
dir := unitsDir(t)
|
||||
for _, name := range []string{"spinifex-daemon.service", "spinifex-vpcd.service"} {
|
||||
u := readUnit(t, dir, name)
|
||||
for _, key := range []string{"Wants", "After"} {
|
||||
if !directiveContains(u, key, "spinifex-hostnet.service") {
|
||||
t.Errorf("%s: %s= must include spinifex-hostnet.service", name, key)
|
||||
}
|
||||
}
|
||||
for _, key := range []string{"Requires", "Requisite", "BindsTo", "PartOf"} {
|
||||
if directiveContains(u, key, "spinifex-hostnet.service") {
|
||||
t.Errorf("%s: %s=spinifex-hostnet.service would restart it with OVS; use Wants=", name, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestHostnetFollowsOVS asserts spinifex-hostnet re-applies whenever
|
||||
// ovs-vswitchd starts, and runs even when its state file is gone: a unit
|
||||
// skipped by a condition still satisfies the daemon and vpcd ordering on it.
|
||||
func TestHostnetFollowsOVS(t *testing.T) {
|
||||
dir := unitsDir(t)
|
||||
u := readUnit(t, dir, "spinifex-hostnet.service")
|
||||
for key, values := range map[string][]string{
|
||||
"Requires": {"ovs-vswitchd.service"},
|
||||
"After": {"ovs-vswitchd.service", "NetworkManager-wait-online.service"},
|
||||
"PartOf": {"ovs-vswitchd.service"},
|
||||
"Before": {"spinifex.target"},
|
||||
"WantedBy": {"spinifex.target", "ovs-vswitchd.service"},
|
||||
} {
|
||||
for _, v := range values {
|
||||
if !directiveContains(u, key, v) {
|
||||
t.Errorf("spinifex-hostnet.service: %s= must include %s", key, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, want := range []string{"Type=oneshot", "RemainAfterExit=yes", "ExecStart=/usr/local/bin/spx admin hostnet apply"} {
|
||||
if !hasDirective(u, want) {
|
||||
t.Errorf("spinifex-hostnet.service must carry %q", want)
|
||||
}
|
||||
}
|
||||
if strings.Contains(u, "\nCondition") {
|
||||
t.Error("spinifex-hostnet.service must not be conditional: a skipped unit lets the daemon and vpcd start unwired")
|
||||
}
|
||||
}
|
||||
|
||||
// TestDaemonAndVPCDCheckHostnet asserts both refuse to start, including on a
|
||||
// crash restart, while the host network is not applied.
|
||||
func TestDaemonAndVPCDCheckHostnet(t *testing.T) {
|
||||
dir := unitsDir(t)
|
||||
for _, name := range []string{"spinifex-daemon.service", "spinifex-vpcd.service"} {
|
||||
if !hasDirective(readUnit(t, dir, name), "ExecStartPre=/usr/local/bin/spx admin hostnet check") {
|
||||
t.Errorf("%s must run `spx admin hostnet check` before it starts", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
package vm
|
||||
|
||||
// MicroVMDir is where Spinifex installs the kernel, initramfs and firmware
|
||||
// that system instances boot on the microvm machine.
|
||||
const MicroVMDir = "/usr/share/spinifex/microvm"
|
||||
|
||||
// MicroVMFirmware is the qboot build Spinifex ships in MicroVMDir and passes
|
||||
// with -bios on every host OS.
|
||||
const MicroVMFirmware = "qboot.rom"
|
||||
@@ -8,14 +8,15 @@ import (
|
||||
)
|
||||
|
||||
// TestMicroVMDirectBoot asserts that a microvm Config with direct-boot fields
|
||||
// set produces the expected QEMU args and does not include PCI-specific flags
|
||||
// such as -bios or pcie-root-port.
|
||||
// set produces the expected QEMU args, including the explicit firmware, and
|
||||
// does not include PCI-specific devices such as pcie-root-port.
|
||||
func TestMicroVMDirectBoot(t *testing.T) {
|
||||
cfg := Config{
|
||||
CPUCount: 2,
|
||||
Memory: 512,
|
||||
Architecture: "x86_64",
|
||||
MachineType: "microvm,x-option-roms=off,pic=off,pit=off,rtc=on,acpi=on,isa-serial=on",
|
||||
BIOS: "/path/qboot.rom",
|
||||
KernelImage: "/path/vmlinuz",
|
||||
Initrd: "/path/initramfs.cpio.gz",
|
||||
KernelCmdline: "console=ttyS0 panic=1",
|
||||
@@ -34,13 +35,13 @@ func TestMicroVMDirectBoot(t *testing.T) {
|
||||
assert.Equal(t, "microvm,x-option-roms=off,pic=off,pit=off,rtc=on,acpi=on,isa-serial=on", argValue(args, "-M"))
|
||||
|
||||
// Direct-boot flags must be present
|
||||
assert.Equal(t, "/path/qboot.rom", argValue(args, "-bios"))
|
||||
assert.Equal(t, "/path/vmlinuz", argValue(args, "-kernel"))
|
||||
assert.Equal(t, "/path/initramfs.cpio.gz", argValue(args, "-initrd"))
|
||||
assert.Equal(t, "console=ttyS0 panic=1", argValue(args, "-append"))
|
||||
assert.Equal(t, "name=opt/spinifex/netcfg,file=/tmp/nc.tmp", argValue(args, "-fw_cfg"))
|
||||
|
||||
// PCI-specific flags must NOT appear
|
||||
assert.False(t, argExists(args, "-bios"), "-bios must not appear for microvm direct-boot")
|
||||
// PCI-specific devices must NOT appear
|
||||
for _, a := range args {
|
||||
assert.NotContains(t, a, "pcie-root-port", "pcie-root-port must not appear for microvm")
|
||||
}
|
||||
|
||||
@@ -3,12 +3,16 @@ package vm
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
|
||||
)
|
||||
|
||||
// fakeNetworkPlumber records calls so tests can assert per-spec behaviour.
|
||||
@@ -103,6 +107,32 @@ func TestMgmtTapName(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestMgmtTapNameMatchesNMUnmanagedGlob checks the udev glob setup-ovn.sh uses
|
||||
// to keep NetworkManager off management TAPs matches every real TAP name, and
|
||||
// not a host NIC that merely starts with "mg".
|
||||
func TestMgmtTapNameMatchesNMUnmanagedGlob(t *testing.T) {
|
||||
data, err := os.ReadFile("../../../../scripts/setup-ovn.sh")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := regexp.MustCompile(`(?m)^MGMT_TAP_GLOB='([^']+)'$`).FindSubmatch(data)
|
||||
if m == nil {
|
||||
t.Fatal("setup-ovn.sh: no MGMT_TAP_GLOB")
|
||||
}
|
||||
glob := string(m[1])
|
||||
for range 1000 {
|
||||
tap := MgmtTapName(identifiers.GenerateResourceID("i"))
|
||||
if ok, err := path.Match(glob, tap); err != nil || !ok {
|
||||
t.Fatalf("glob %q does not match TAP %q (err %v)", glob, tap, err)
|
||||
}
|
||||
}
|
||||
for _, name := range []string{"mgmt0", "mgmt", "br-mgmt", "mg0123456789abcdef"} {
|
||||
if ok, _ := path.Match(glob, name); ok {
|
||||
t.Errorf("glob %q matches %q, which is not a management TAP", glob, name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestOVSIfaceID(t *testing.T) {
|
||||
tests := []struct {
|
||||
eniID string
|
||||
|
||||
@@ -7,20 +7,17 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/host/hostos"
|
||||
hostprocess "github.com/mulgadc/spinifex/spinifex/runtime/host/process"
|
||||
)
|
||||
|
||||
// qemuProcessPrefix matches both qemu-system-x86_64 and qemu-system-aarch64;
|
||||
// /proc/<pid>/comm truncates at 15 bytes but this prefix is only 11.
|
||||
const qemuProcessPrefix = "qemu-system"
|
||||
|
||||
// qemuProcRoot is the /proc mount scanned for live qemu-system processes.
|
||||
// qemuProcRoot is the /proc mount scanned for live QEMU processes.
|
||||
// Overridden in tests to avoid depending on the real host process table.
|
||||
var qemuProcRoot = "/proc"
|
||||
|
||||
// scanLiveQEMUPIDs walks procRoot for processes whose comm starts with
|
||||
// qemuProcessPrefix. procRoot is a parameter so tests point it at a
|
||||
// fabricated directory instead of spawning a real qemu-system binary.
|
||||
// scanLiveQEMUPIDs walks procRoot for processes whose comm names a QEMU
|
||||
// guest. procRoot is a parameter so tests point it at a fabricated directory
|
||||
// instead of spawning a real QEMU binary.
|
||||
func scanLiveQEMUPIDs(procRoot string) ([]int, error) {
|
||||
entries, err := os.ReadDir(procRoot)
|
||||
if err != nil {
|
||||
@@ -37,7 +34,7 @@ func scanLiveQEMUPIDs(procRoot string) ([]int, error) {
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(strings.TrimSpace(string(comm)), qemuProcessPrefix) {
|
||||
if hostos.IsQEMUProcess(string(comm)) {
|
||||
pids = append(pids, pid)
|
||||
}
|
||||
}
|
||||
@@ -67,7 +64,7 @@ func pidFileOwners(runtimeDir string) map[int]string {
|
||||
return owners
|
||||
}
|
||||
|
||||
// qemuOrphan is a live qemu-system PID with no matching instance in
|
||||
// qemuOrphan is a live QEMU PID with no matching instance in
|
||||
// Snapshot(). hasPidFile distinguishes two very different situations: a
|
||||
// pidfile this daemon itself wrote (instanceID is that pidfile's claim,
|
||||
// positively this daemon's process) versus no pidfile at all (could be
|
||||
@@ -78,7 +75,7 @@ type qemuOrphan struct {
|
||||
hasPidFile bool
|
||||
}
|
||||
|
||||
// classifyQEMUOrphans returns every live qemu-system PID not vouched for by
|
||||
// classifyQEMUOrphans returns every live QEMU PID not vouched for by
|
||||
// a pidfile naming a known instance. This is the gap classifyRestoredInstances
|
||||
// cannot see, since it only walks Snapshot().
|
||||
func classifyQEMUOrphans(procRoot, runtimeDir string, known map[string]bool) ([]qemuOrphan, error) {
|
||||
@@ -102,7 +99,7 @@ func classifyQEMUOrphans(procRoot, runtimeDir string, known map[string]bool) ([]
|
||||
return orphans, nil
|
||||
}
|
||||
|
||||
// reportRecordlessQEMUOrphans logs, but never kills, a qemu-system process
|
||||
// reportRecordlessQEMUOrphans logs, but never kills, a QEMU process
|
||||
// with no instance record. A pidfile in runtimeDir naming an unknown
|
||||
// instance positively identifies the process as this daemon's own stale
|
||||
// orphan (logged distinctly, safe for an operator to reap by hand); no
|
||||
@@ -123,11 +120,11 @@ func (m *Manager) reportRecordlessQEMUOrphans() {
|
||||
}
|
||||
for _, o := range orphans {
|
||||
if o.hasPidFile {
|
||||
slog.Warn("qemu-system process has a stale spinifex pidfile with no matching instance record; not killing, safe to reap manually",
|
||||
slog.Warn("QEMU process has a stale spinifex pidfile with no matching instance record; not killing, safe to reap manually",
|
||||
"pid", o.pid, "instanceId", o.instanceID)
|
||||
continue
|
||||
}
|
||||
slog.Warn("qemu-system process has no spinifex pidfile at all; not killing, ownership unconfirmed",
|
||||
slog.Warn("QEMU process has no spinifex pidfile at all; not killing, ownership unconfirmed",
|
||||
"pid", o.pid)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,13 +24,16 @@ func TestScanLiveQEMUPIDs(t *testing.T) {
|
||||
fakeProcEntry(t, procRoot, 111, "qemu-system-x86_64")
|
||||
fakeProcEntry(t, procRoot, 222, "qemu-system-aarch64")
|
||||
fakeProcEntry(t, procRoot, 333, "bash")
|
||||
// EL-family hosts run QEMU as /usr/libexec/qemu-kvm.
|
||||
fakeProcEntry(t, procRoot, 444, "qemu-kvm")
|
||||
fakeProcEntry(t, procRoot, 555, "qemu-img")
|
||||
// Non-numeric /proc entries (self, curproc, etc.) must not crash the scan.
|
||||
require.NoError(t, os.MkdirAll(filepath.Join(procRoot, "self"), 0o755))
|
||||
|
||||
pids, err := scanLiveQEMUPIDs(procRoot)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.ElementsMatch(t, []int{111, 222}, pids, "only qemu-system* processes are returned")
|
||||
assert.ElementsMatch(t, []int{111, 222, 444}, pids, "only QEMU guest processes are returned")
|
||||
}
|
||||
|
||||
func TestScanLiveQEMUPIDs_MissingProcRoot(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
package vm
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/host/hostos"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
resolveQEMUBinary = fakeQEMUBinary
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
// fakeQEMUBinary stands in for hostos.QEMUBinary so Execute tests need no QEMU
|
||||
// installed on the host.
|
||||
func fakeQEMUBinary(arch string) (string, error) {
|
||||
switch arch {
|
||||
case "x86_64":
|
||||
return "/usr/bin/qemu-system-x86_64", nil
|
||||
case "arm64":
|
||||
return "/usr/bin/qemu-system-aarch64", nil
|
||||
}
|
||||
return "", fmt.Errorf("no QEMU for guest architecture %q", arch)
|
||||
}
|
||||
|
||||
func withQEMUResolver(t *testing.T, fn func(string) (string, error)) {
|
||||
t.Helper()
|
||||
orig := resolveQEMUBinary
|
||||
resolveQEMUBinary = fn
|
||||
t.Cleanup(func() { resolveQEMUBinary = orig })
|
||||
}
|
||||
|
||||
func minimalConfig() Config {
|
||||
return Config{
|
||||
CPUCount: 1,
|
||||
Memory: 512,
|
||||
Architecture: "x86_64",
|
||||
Drives: []Drive{{File: "disk.img", Format: "raw"}},
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecute_RunsTheResolvedQEMUBinary(t *testing.T) {
|
||||
var asked string
|
||||
withQEMUResolver(t, func(arch string) (string, error) {
|
||||
asked = arch
|
||||
return hostos.QEMUKVMPath, nil
|
||||
})
|
||||
|
||||
cfg := minimalConfig()
|
||||
cmd, err := cfg.Execute()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "x86_64", asked)
|
||||
assert.Equal(t, hostos.QEMUKVMPath, cmd.Path)
|
||||
assert.Equal(t, hostos.QEMUKVMPath, cmd.Args[0])
|
||||
}
|
||||
|
||||
func TestExecute_FailsWhenNoQEMUResolves(t *testing.T) {
|
||||
noQEMU := errors.New("no QEMU for x86_64 guests: found neither /usr/libexec/qemu-kvm (package qemu-kvm) nor qemu-system-x86_64 (package qemu-system-x86) on PATH")
|
||||
withQEMUResolver(t, func(string) (string, error) { return "", noQEMU })
|
||||
|
||||
cfg := minimalConfig()
|
||||
cmd, err := cfg.Execute()
|
||||
assert.Nil(t, cmd)
|
||||
assert.ErrorIs(t, err, noQEMU)
|
||||
}
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"github.com/mulgadc/spinifex/contracts/ec2/v1"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/compute/gpu"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/compute/qmp"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/host/hostos"
|
||||
)
|
||||
|
||||
// InstanceHealthState tracks crash detection and auto-restart metadata for a VM.
|
||||
@@ -408,6 +409,11 @@ type Config struct {
|
||||
SMBIOSManufacturer string `json:"smbios_manufacturer,omitempty"`
|
||||
SMBIOSAssetTag string `json:"smbios_asset_tag,omitempty"`
|
||||
|
||||
// BIOS is a firmware image passed with -bios. The microvm machine needs
|
||||
// one: its default, bios-microvm.bin, is not shipped under that name by
|
||||
// every distro.
|
||||
BIOS string `json:"bios,omitempty"`
|
||||
|
||||
KernelImage string `json:"kernel_image,omitempty"` // path to vmlinuz; emits -kernel when set
|
||||
Initrd string `json:"initrd,omitempty"` // path to initramfs; emits -initrd when set
|
||||
KernelCmdline string `json:"kernel_cmdline,omitempty"` // emits -append when set
|
||||
@@ -420,6 +426,9 @@ type FwCfgEntry struct {
|
||||
File string `json:"file"`
|
||||
}
|
||||
|
||||
// resolveQEMUBinary is a variable so tests run without QEMU installed.
|
||||
var resolveQEMUBinary = hostos.QEMUBinary
|
||||
|
||||
func (cfg *Config) Execute() (*exec.Cmd, error) {
|
||||
args := []string{}
|
||||
|
||||
@@ -593,6 +602,10 @@ func (cfg *Config) Execute() (*exec.Cmd, error) {
|
||||
args = append(args, "-netdev", netdev.Value)
|
||||
}
|
||||
|
||||
if cfg.BIOS != "" {
|
||||
args = append(args, "-bios", cfg.BIOS)
|
||||
}
|
||||
|
||||
if cfg.KernelImage != "" {
|
||||
args = append(args, "-kernel", cfg.KernelImage)
|
||||
}
|
||||
@@ -609,18 +622,13 @@ func (cfg *Config) Execute() (*exec.Cmd, error) {
|
||||
args = append(args, "-fw_cfg", fmt.Sprintf("name=%s,file=%s", fw.Name, fw.File))
|
||||
}
|
||||
|
||||
var qemuArchitecture string
|
||||
|
||||
switch cfg.Architecture {
|
||||
case "arm64":
|
||||
qemuArchitecture = "qemu-system-aarch64"
|
||||
|
||||
case "x86_64":
|
||||
qemuArchitecture = "qemu-system-x86_64"
|
||||
|
||||
default:
|
||||
if cfg.Architecture == "" {
|
||||
return nil, fmt.Errorf("architecture missing")
|
||||
}
|
||||
qemuBinary, err := resolveQEMUBinary(cfg.Architecture)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// arm64 q35 is incompatible with the q35 PC machine; override to virt.
|
||||
// Firmware loading is unified: UseUEFI emits pflash drives above; no
|
||||
@@ -632,9 +640,9 @@ func (cfg *Config) Execute() (*exec.Cmd, error) {
|
||||
args = append(args, "-M", cfg.MachineType)
|
||||
}
|
||||
|
||||
slog.Info("Executing QEMU command:", "cmd", qemuArchitecture, "args", args)
|
||||
slog.Info("Executing QEMU command:", "cmd", qemuBinary, "args", args)
|
||||
|
||||
cmd := exec.Command(qemuArchitecture, args...)
|
||||
cmd := exec.Command(qemuBinary, args...)
|
||||
|
||||
//cmd.Stdout = os.Stdout
|
||||
//cmd.Stderr = os.Stderr
|
||||
|
||||
@@ -0,0 +1,423 @@
|
||||
package hostnet
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/netip"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
// DefaultLinkWait bounds the wait for links other components create: OVS
|
||||
// bridges, and NetworkManager's uplink bridge and NICs.
|
||||
DefaultLinkWait = 60 * time.Second
|
||||
// DefaultLockWait bounds the wait behind setup-ovn.sh, so a stuck setup
|
||||
// cannot hang boot.
|
||||
DefaultLockWait = 120 * time.Second
|
||||
pollInterval = 500 * time.Millisecond
|
||||
)
|
||||
|
||||
// Env is everything Apply and Check touch on the host.
|
||||
type Env struct {
|
||||
Paths Paths
|
||||
Host Host
|
||||
LinkWait time.Duration
|
||||
LockWait time.Duration
|
||||
// OVNRemote returns external_ids:ovn-remote, or "" when unset.
|
||||
OVNRemote func(ctx context.Context) (string, error)
|
||||
// Sleep waits between polls; tests replace it to advance a fake clock.
|
||||
Sleep func(time.Duration)
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
// DefaultEnv returns the Env for a live host.
|
||||
func DefaultEnv() Env {
|
||||
return Env{
|
||||
Paths: DefaultPaths(),
|
||||
Host: NewIPHost(),
|
||||
LinkWait: DefaultLinkWait,
|
||||
LockWait: DefaultLockWait,
|
||||
OVNRemote: func(ctx context.Context) (string, error) { return ovnRemote(ctx, execRunner{}) },
|
||||
Sleep: time.Sleep,
|
||||
Now: time.Now,
|
||||
}
|
||||
}
|
||||
|
||||
// resolve decides what this node needs: nil state means nothing to apply.
|
||||
// Apply and Check share it, so every case that fails Apply fails Check.
|
||||
func resolve(ctx context.Context, env Env) (*State, error) {
|
||||
backend, err := ReadBackend(env.Paths.Backend)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch backend {
|
||||
case BackendNetworkd:
|
||||
return nil, nil
|
||||
case BackendHostnet:
|
||||
return LoadState(env.Paths.State)
|
||||
}
|
||||
|
||||
// No record: a node set up before the record existed has networkd files.
|
||||
legacy, err := hasNetworkdFiles(env.Paths.NetworkdDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("look for networkd files in %s: %w", env.Paths.NetworkdDir, err)
|
||||
}
|
||||
if legacy {
|
||||
return nil, nil
|
||||
}
|
||||
remote, err := env.OVNRemote(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("no backend record at %s, and cannot tell whether this node is set up: %w", env.Paths.Backend, err)
|
||||
}
|
||||
if remote != "" {
|
||||
return nil, fmt.Errorf("this node is set up (external_ids:ovn-remote=%s) but has no backend record at %s and no networkd files; rerun setup-ovn.sh", remote, env.Paths.Backend)
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Check reports whether the daemon and vpcd may start: the same decision as
|
||||
// Apply, plus, on the hostnet backend, proof the last Apply succeeded.
|
||||
func Check(ctx context.Context, env Env) error {
|
||||
s, err := resolve(ctx, env)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if s == nil {
|
||||
return nil
|
||||
}
|
||||
if _, err := os.Stat(env.Paths.Marker); err != nil {
|
||||
return fmt.Errorf("host network not applied (no %s): see `systemctl status spinifex-hostnet` and `journalctl -u spinifex-hostnet`", env.Paths.Marker)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Apply makes the host match the state file, under the hostnet lock. It
|
||||
// removes the readiness marker first and writes it only on success.
|
||||
func Apply(ctx context.Context, env Env) error {
|
||||
unlock, err := acquireLock(env)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer unlock()
|
||||
|
||||
if err := os.Remove(env.Paths.Marker); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return fmt.Errorf("remove %s: %w", env.Paths.Marker, err)
|
||||
}
|
||||
s, err := resolve(ctx, env)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if s == nil {
|
||||
slog.Info("hostnet: nothing to apply on this node")
|
||||
return nil
|
||||
}
|
||||
if err := reconcile(ctx, env, s); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(env.Paths.Marker), 0o770); err != nil { //nolint:gosec // G301: the mode setup.sh's tmpfiles entry gives /run/spinifex, which the daemon and vpcd traverse to check the marker. Holds no secret material.
|
||||
return fmt.Errorf("create %s: %w", filepath.Dir(env.Paths.Marker), err)
|
||||
}
|
||||
stamp := env.Now().UTC().Format(time.RFC3339) + "\n"
|
||||
if err := os.WriteFile(env.Paths.Marker, []byte(stamp), 0o600); err != nil {
|
||||
return fmt.Errorf("write %s: %w", env.Paths.Marker, err)
|
||||
}
|
||||
slog.Info("hostnet: host network applied", "state", env.Paths.State)
|
||||
return nil
|
||||
}
|
||||
|
||||
func reconcile(ctx context.Context, env Env, s *State) error {
|
||||
if err := waitForLinks(ctx, env, s); err != nil {
|
||||
return err
|
||||
}
|
||||
h := env.Host
|
||||
if err := ensureUp(ctx, h, s.IntBridge); err != nil {
|
||||
return err
|
||||
}
|
||||
if w := s.WAN; w != nil {
|
||||
if err := ensureUp(ctx, h, w.OVSBridge); err != nil {
|
||||
return err
|
||||
}
|
||||
switch w.Mode {
|
||||
case ModeVeth:
|
||||
if err := ensureVethPair(ctx, h, w.HostEnd, w.OVSEnd); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ensureMaster(ctx, h, w.HostEnd, w.LinuxBridge); err != nil {
|
||||
return err
|
||||
}
|
||||
case ModeNAT:
|
||||
if err := ensureVethPair(ctx, h, w.HostEnd, w.OVSEnd); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ensureMAC(ctx, h, w.HostEnd, w.HostMAC); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ensureAddr(ctx, h, w.HostEnd, w.HostCIDR); err != nil {
|
||||
return err
|
||||
}
|
||||
case ModeDirect:
|
||||
if err := ensurePortNIC(ctx, h, w.NIC); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if w.Mode == ModeVeth || w.Mode == ModeNAT {
|
||||
for _, end := range []string{w.HostEnd, w.OVSEnd} {
|
||||
if err := ensureUp(ctx, h, end); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if m := s.Mgmt; m != nil {
|
||||
if err := ensureUp(ctx, h, m.Bridge); err != nil {
|
||||
return err
|
||||
}
|
||||
if m.CIDR != "" {
|
||||
if err := ensureAddr(ctx, h, m.Bridge, m.CIDR); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if m.NIC != "" {
|
||||
if err := ensurePortNIC(ctx, h, m.NIC); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
if r := s.EncapRoute; r != nil {
|
||||
if err := ensureRoute(ctx, h, *r); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// waitForLinks waits for the links other components create and the recorded
|
||||
// topology needs. The veth pairs are not waited for: Apply creates them.
|
||||
func waitForLinks(ctx context.Context, env Env, s *State) error {
|
||||
needed := []string{s.IntBridge}
|
||||
if w := s.WAN; w != nil {
|
||||
needed = append(needed, w.OVSBridge)
|
||||
switch w.Mode {
|
||||
case ModeVeth:
|
||||
needed = append(needed, w.LinuxBridge)
|
||||
case ModeDirect:
|
||||
needed = append(needed, w.NIC)
|
||||
}
|
||||
}
|
||||
if m := s.Mgmt; m != nil {
|
||||
needed = append(needed, m.Bridge)
|
||||
if m.NIC != "" {
|
||||
needed = append(needed, m.NIC)
|
||||
}
|
||||
}
|
||||
if r := s.EncapRoute; r != nil {
|
||||
needed = append(needed, r.Dev)
|
||||
}
|
||||
needed = slices.Compact(slices.Sorted(slices.Values(needed)))
|
||||
|
||||
deadline := env.Now().Add(env.LinkWait)
|
||||
for {
|
||||
var missing []string
|
||||
for _, name := range needed {
|
||||
l, err := env.Host.Link(ctx, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if l == nil {
|
||||
missing = append(missing, name)
|
||||
}
|
||||
}
|
||||
// The encap route's source address must be up before the route can
|
||||
// name it; on a DHCP uplink it can arrive after the link does.
|
||||
if len(missing) == 0 && s.EncapRoute != nil {
|
||||
has, err := hasAddr(ctx, env.Host, s.EncapRoute.Dev, s.EncapRoute.Src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !has {
|
||||
missing = append(missing, s.EncapRoute.Src+" on "+s.EncapRoute.Dev)
|
||||
}
|
||||
}
|
||||
if len(missing) == 0 {
|
||||
return nil
|
||||
}
|
||||
if !env.Now().Before(deadline) {
|
||||
return fmt.Errorf("still missing after %s: %s", env.LinkWait, strings.Join(missing, ", "))
|
||||
}
|
||||
env.Sleep(pollInterval)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func ensureUp(ctx context.Context, h Host, name string) error {
|
||||
l, err := h.Link(ctx, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if l == nil {
|
||||
return fmt.Errorf("%s: link missing", name)
|
||||
}
|
||||
if l.Up {
|
||||
return nil
|
||||
}
|
||||
if err := h.SetUp(ctx, name); err != nil {
|
||||
return fmt.Errorf("set %s up: %w", name, err)
|
||||
}
|
||||
slog.Info("hostnet: set link up", "link", name)
|
||||
return nil
|
||||
}
|
||||
|
||||
// ensureVethPair creates the pair when neither end exists, and refuses any
|
||||
// other link already holding either name.
|
||||
func ensureVethPair(ctx context.Context, h Host, a, b string) error {
|
||||
la, err := h.Link(ctx, a)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
lb, err := h.Link(ctx, b)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if la == nil && lb == nil {
|
||||
if err := h.AddVethPair(ctx, a, b); err != nil {
|
||||
return fmt.Errorf("create veth pair %s/%s: %w", a, b, err)
|
||||
}
|
||||
slog.Info("hostnet: created veth pair", "link", a, "peer", b)
|
||||
return nil
|
||||
}
|
||||
for _, l := range []*Link{la, lb} {
|
||||
if l != nil && l.Kind != "veth" {
|
||||
return fmt.Errorf("refusing to use %s: it is a %s link, not one end of the %s/%s veth pair", l.Name, kindName(l.Kind), a, b)
|
||||
}
|
||||
}
|
||||
if la == nil || lb == nil || la.Peer != b || lb.Peer != a {
|
||||
return fmt.Errorf("refusing to use veth pair %s/%s: %s", a, b, describePeers(a, b, la, lb))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func describePeers(a, b string, la, lb *Link) string {
|
||||
switch {
|
||||
case la == nil:
|
||||
return fmt.Sprintf("%s is missing and %s is peered with %q", a, b, lb.Peer)
|
||||
case lb == nil:
|
||||
return fmt.Sprintf("%s is missing and %s is peered with %q", b, a, la.Peer)
|
||||
default:
|
||||
return fmt.Sprintf("%s is peered with %q and %s with %q", a, la.Peer, b, lb.Peer)
|
||||
}
|
||||
}
|
||||
|
||||
func kindName(kind string) string {
|
||||
if kind == "" {
|
||||
return "physical"
|
||||
}
|
||||
return kind
|
||||
}
|
||||
|
||||
func ensureMaster(ctx context.Context, h Host, name, master string) error {
|
||||
l, err := h.Link(ctx, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if l == nil {
|
||||
return fmt.Errorf("%s: link missing", name)
|
||||
}
|
||||
if l.Master == master {
|
||||
return nil
|
||||
}
|
||||
if err := h.SetMaster(ctx, name, master); err != nil {
|
||||
return fmt.Errorf("attach %s to %s: %w", name, master, err)
|
||||
}
|
||||
slog.Info("hostnet: attached link to bridge", "link", name, "bridge", master)
|
||||
return nil
|
||||
}
|
||||
|
||||
func ensureMAC(ctx context.Context, h Host, name, mac string) error {
|
||||
l, err := h.Link(ctx, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if l == nil {
|
||||
return fmt.Errorf("%s: link missing", name)
|
||||
}
|
||||
if strings.EqualFold(l.MAC, mac) {
|
||||
return nil
|
||||
}
|
||||
if err := h.SetMAC(ctx, name, mac); err != nil {
|
||||
return fmt.Errorf("set %s MAC %s: %w", name, mac, err)
|
||||
}
|
||||
slog.Info("hostnet: set MAC", "link", name, "mac", mac)
|
||||
return nil
|
||||
}
|
||||
|
||||
func ensureAddr(ctx context.Context, h Host, name, cidr string) error {
|
||||
addrs, err := h.IPv4Addrs(ctx, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if slices.Contains(addrs, cidr) {
|
||||
return nil
|
||||
}
|
||||
if err := h.AddAddr(ctx, name, cidr); err != nil {
|
||||
return fmt.Errorf("add %s to %s: %w", cidr, name, err)
|
||||
}
|
||||
slog.Info("hostnet: added address", "link", name, "cidr", cidr)
|
||||
return nil
|
||||
}
|
||||
|
||||
func hasAddr(ctx context.Context, h Host, name, ip string) (bool, error) {
|
||||
addrs, err := h.IPv4Addrs(ctx, name)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, a := range addrs {
|
||||
if p, err := netip.ParsePrefix(a); err == nil && p.Addr().String() == ip {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// ensurePortNIC brings up a NIC that is an OVS port and strips any IPv4
|
||||
// address NetworkManager restored to it; the address belongs on the bridge.
|
||||
func ensurePortNIC(ctx context.Context, h Host, name string) error {
|
||||
if err := ensureUp(ctx, h, name); err != nil {
|
||||
return err
|
||||
}
|
||||
addrs, err := h.IPv4Addrs(ctx, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, a := range addrs {
|
||||
if err := h.DelAddr(ctx, name, a); err != nil {
|
||||
return fmt.Errorf("remove %s from %s: %w", a, name, err)
|
||||
}
|
||||
slog.Info("hostnet: removed address from OVS port", "link", name, "cidr", a)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ensureRoute(ctx context.Context, h Host, r Route) error {
|
||||
routes, err := h.Routes(ctx, r.Dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, e := range routes {
|
||||
if e.Metric == r.Metric && e.Dev == r.Dev && e.Src == r.Src {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
if err := h.ReplaceRoute(ctx, r); err != nil {
|
||||
return fmt.Errorf("encap route %s dev %s src %s metric %d: %w", r.Dst, r.Dev, r.Src, r.Metric, err)
|
||||
}
|
||||
slog.Info("hostnet: set encap route", "dst", r.Dst, "dev", r.Dev, "src", r.Src, "metric", r.Metric)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
package hostnet
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Link is the part of a kernel link Apply compares against the state file.
|
||||
type Link struct {
|
||||
Name string
|
||||
// Kind is the rtnetlink link kind ("veth", "bridge", "openvswitch"), or
|
||||
// "" for a physical device.
|
||||
Kind string
|
||||
// Peer is the other end of a veth in the same namespace.
|
||||
Peer string
|
||||
Master string
|
||||
MAC string
|
||||
Up bool
|
||||
}
|
||||
|
||||
// RouteEntry is one IPv4 route to a destination.
|
||||
type RouteEntry struct {
|
||||
Dev string
|
||||
Src string
|
||||
Metric int
|
||||
}
|
||||
|
||||
// Host reads and changes kernel network state. It has no way to delete a
|
||||
// link: Apply only creates and adjusts, so a link it does not name, or one it
|
||||
// wrongly matched, is never destroyed.
|
||||
type Host interface {
|
||||
// Link returns nil, nil when no link has this name.
|
||||
Link(ctx context.Context, name string) (*Link, error)
|
||||
AddVethPair(ctx context.Context, name, peer string) error
|
||||
SetUp(ctx context.Context, name string) error
|
||||
SetMaster(ctx context.Context, name, master string) error
|
||||
SetMAC(ctx context.Context, name, mac string) error
|
||||
// IPv4Addrs returns the link's IPv4 addresses as CIDR prefixes.
|
||||
IPv4Addrs(ctx context.Context, name string) ([]string, error)
|
||||
AddAddr(ctx context.Context, name, cidr string) error
|
||||
DelAddr(ctx context.Context, name, cidr string) error
|
||||
Routes(ctx context.Context, dst string) ([]RouteEntry, error)
|
||||
ReplaceRoute(ctx context.Context, r Route) error
|
||||
}
|
||||
|
||||
// Runner runs a host command and returns its stdout.
|
||||
type Runner interface {
|
||||
Run(ctx context.Context, name string, args ...string) ([]byte, error)
|
||||
}
|
||||
|
||||
type execRunner struct{}
|
||||
|
||||
func (execRunner) Run(ctx context.Context, name string, args ...string) ([]byte, error) {
|
||||
cmd := exec.CommandContext(ctx, name, args...)
|
||||
var stderr strings.Builder
|
||||
cmd.Stderr = &stderr
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("%s %s: %w: %s", name, strings.Join(args, " "), err, strings.TrimSpace(stderr.String()))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// IPHost is the Host backed by iproute2's JSON output.
|
||||
type IPHost struct {
|
||||
Runner Runner
|
||||
}
|
||||
|
||||
var _ Host = (*IPHost)(nil)
|
||||
|
||||
// NewIPHost returns an IPHost that runs ip directly; Apply runs as root.
|
||||
func NewIPHost() *IPHost { return &IPHost{Runner: execRunner{}} }
|
||||
|
||||
func (h *IPHost) ip(ctx context.Context, args ...string) ([]byte, error) {
|
||||
return h.Runner.Run(ctx, "ip", args...)
|
||||
}
|
||||
|
||||
type ipLink struct {
|
||||
IfName string `json:"ifname"`
|
||||
Link string `json:"link"`
|
||||
Master string `json:"master"`
|
||||
Address string `json:"address"`
|
||||
Flags []string `json:"flags"`
|
||||
LinkInfo struct {
|
||||
InfoKind string `json:"info_kind"`
|
||||
} `json:"linkinfo"`
|
||||
}
|
||||
|
||||
// Link lists every link rather than asking for one by name, so a missing link
|
||||
// is told apart from a failed ip by the output, not by matching error text.
|
||||
func (h *IPHost) Link(ctx context.Context, name string) (*Link, error) {
|
||||
out, err := h.ip(ctx, "-j", "-d", "link", "show")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var links []ipLink
|
||||
if err := json.Unmarshal(out, &links); err != nil {
|
||||
return nil, fmt.Errorf("parse ip link output: %w", err)
|
||||
}
|
||||
for _, l := range links {
|
||||
if l.IfName != name {
|
||||
continue
|
||||
}
|
||||
return &Link{
|
||||
Name: l.IfName,
|
||||
Kind: l.LinkInfo.InfoKind,
|
||||
Peer: l.Link,
|
||||
Master: l.Master,
|
||||
MAC: strings.ToLower(l.Address),
|
||||
Up: slices.Contains(l.Flags, "UP"),
|
||||
}, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (h *IPHost) AddVethPair(ctx context.Context, name, peer string) error {
|
||||
_, err := h.ip(ctx, "link", "add", name, "type", "veth", "peer", "name", peer)
|
||||
return err
|
||||
}
|
||||
|
||||
func (h *IPHost) SetUp(ctx context.Context, name string) error {
|
||||
_, err := h.ip(ctx, "link", "set", "dev", name, "up")
|
||||
return err
|
||||
}
|
||||
|
||||
func (h *IPHost) SetMaster(ctx context.Context, name, master string) error {
|
||||
_, err := h.ip(ctx, "link", "set", "dev", name, "master", master)
|
||||
return err
|
||||
}
|
||||
|
||||
func (h *IPHost) SetMAC(ctx context.Context, name, mac string) error {
|
||||
_, err := h.ip(ctx, "link", "set", "dev", name, "address", mac)
|
||||
return err
|
||||
}
|
||||
|
||||
func (h *IPHost) IPv4Addrs(ctx context.Context, name string) ([]string, error) {
|
||||
out, err := h.ip(ctx, "-j", "-4", "addr", "show", "dev", name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var links []struct {
|
||||
AddrInfo []struct {
|
||||
Local string `json:"local"`
|
||||
PrefixLen int `json:"prefixlen"`
|
||||
} `json:"addr_info"`
|
||||
}
|
||||
if err := json.Unmarshal(out, &links); err != nil {
|
||||
return nil, fmt.Errorf("parse ip addr output for %s: %w", name, err)
|
||||
}
|
||||
var addrs []string
|
||||
for _, l := range links {
|
||||
for _, a := range l.AddrInfo {
|
||||
addrs = append(addrs, a.Local+"/"+strconv.Itoa(a.PrefixLen))
|
||||
}
|
||||
}
|
||||
return addrs, nil
|
||||
}
|
||||
|
||||
func (h *IPHost) AddAddr(ctx context.Context, name, cidr string) error {
|
||||
_, err := h.ip(ctx, "addr", "add", cidr, "dev", name)
|
||||
return err
|
||||
}
|
||||
|
||||
func (h *IPHost) DelAddr(ctx context.Context, name, cidr string) error {
|
||||
_, err := h.ip(ctx, "addr", "del", cidr, "dev", name)
|
||||
return err
|
||||
}
|
||||
|
||||
func (h *IPHost) Routes(ctx context.Context, dst string) ([]RouteEntry, error) {
|
||||
out, err := h.ip(ctx, "-j", "-4", "route", "show", "exact", dst)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var routes []struct {
|
||||
Dev string `json:"dev"`
|
||||
PrefSrc string `json:"prefsrc"`
|
||||
Metric int `json:"metric"`
|
||||
Protocol string `json:"protocol"`
|
||||
}
|
||||
if err := json.Unmarshal(out, &routes); err != nil {
|
||||
return nil, fmt.Errorf("parse ip route output for %s: %w", dst, err)
|
||||
}
|
||||
entries := make([]RouteEntry, 0, len(routes))
|
||||
for _, r := range routes {
|
||||
entries = append(entries, RouteEntry{Dev: r.Dev, Src: r.PrefSrc, Metric: r.Metric})
|
||||
}
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
func (h *IPHost) ReplaceRoute(ctx context.Context, r Route) error {
|
||||
_, err := h.ip(ctx, "route", "replace", r.Dst, "dev", r.Dev, "src", r.Src, "metric", strconv.Itoa(r.Metric))
|
||||
return err
|
||||
}
|
||||
|
||||
// ovnRemote returns the node's external_ids:ovn-remote, or "" when it is unset
|
||||
// or OVS is not installed.
|
||||
func ovnRemote(ctx context.Context, r Runner) (string, error) {
|
||||
if !onPath("ovs-vsctl") {
|
||||
return "", nil
|
||||
}
|
||||
out, err := r.Run(ctx, "ovs-vsctl", "--timeout=5", "--if-exists", "get", "Open_vSwitch", ".", "external_ids:ovn-remote")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read external_ids:ovn-remote: %w", err)
|
||||
}
|
||||
return strings.Trim(strings.TrimSpace(string(out)), `"`), nil
|
||||
}
|
||||
|
||||
func onPath(name string) bool {
|
||||
_, err := exec.LookPath(name)
|
||||
return err == nil
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
package hostnet
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// scriptRunner answers ip commands from canned output keyed by the joined
|
||||
// command line, and records every command it is asked to run.
|
||||
type scriptRunner struct {
|
||||
out map[string]string
|
||||
runs []string
|
||||
}
|
||||
|
||||
func (r *scriptRunner) Run(_ context.Context, name string, args ...string) ([]byte, error) {
|
||||
line := name + " " + strings.Join(args, " ")
|
||||
r.runs = append(r.runs, line)
|
||||
if out, ok := r.out[line]; ok {
|
||||
return []byte(out), nil
|
||||
}
|
||||
if strings.HasPrefix(line, "ip -j") {
|
||||
return nil, fmt.Errorf("unexpected query %q", line)
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Captured from iproute2 on Debian 13.
|
||||
const ipLinkJSON = `[
|
||||
{"ifindex":13,"link":"veth-wan-ovs","ifname":"veth-wan-br","flags":["BROADCAST","MULTICAST","UP","LOWER_UP"],"master":"br-wan","address":"1E:90:24:9F:95:0C","linkinfo":{"info_kind":"veth","info_slave_kind":"bridge"}},
|
||||
{"ifindex":12,"link":"veth-wan-br","ifname":"veth-wan-ovs","flags":["BROADCAST","MULTICAST","UP","LOWER_UP"],"master":"ovs-system","address":"5a:1c:00:00:00:01","linkinfo":{"info_kind":"veth","info_slave_kind":"openvswitch"}},
|
||||
{"ifindex":9,"ifname":"br-int","flags":["BROADCAST","MULTICAST"],"address":"6e:00:00:00:00:02","linkinfo":{"info_kind":"openvswitch"}},
|
||||
{"ifindex":2,"ifname":"eth1","flags":["BROADCAST","MULTICAST","UP"],"address":"52:54:00:00:00:03"}
|
||||
]`
|
||||
|
||||
func TestIPHostLink(t *testing.T) {
|
||||
r := &scriptRunner{out: map[string]string{"ip -j -d link show": ipLinkJSON}}
|
||||
h := &IPHost{Runner: r}
|
||||
ctx := context.Background()
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
want *Link
|
||||
}{
|
||||
{"veth-wan-br", &Link{Name: "veth-wan-br", Kind: "veth", Peer: "veth-wan-ovs", Master: "br-wan", MAC: "1e:90:24:9f:95:0c", Up: true}},
|
||||
{"br-int", &Link{Name: "br-int", Kind: "openvswitch", MAC: "6e:00:00:00:00:02"}},
|
||||
{"eth1", &Link{Name: "eth1", MAC: "52:54:00:00:00:03", Up: true}},
|
||||
{"br-wan-missing", nil},
|
||||
} {
|
||||
got, err := h.Link(ctx, tc.name)
|
||||
if err != nil {
|
||||
t.Fatalf("Link(%s): %v", tc.name, err)
|
||||
}
|
||||
if (got == nil) != (tc.want == nil) || (got != nil && *got != *tc.want) {
|
||||
t.Errorf("Link(%s) = %+v, want %+v", tc.name, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIPHostAddrsAndRoutes(t *testing.T) {
|
||||
r := &scriptRunner{out: map[string]string{
|
||||
"ip -j -4 addr show dev br-mgmt": `[{"ifname":"br-mgmt","addr_info":[{"family":"inet","local":"10.15.8.1","prefixlen":24}]}]`,
|
||||
"ip -j -4 route show exact 10.15.8.0/24": `[{"dst":"10.15.8.0/24","dev":"br-mgmt","protocol":"kernel","scope":"link","prefsrc":"10.15.8.1","flags":[]},{"dst":"10.15.8.0/24","dev":"br-mgmt","prefsrc":"10.15.8.1","metric":50,"flags":[]}]`,
|
||||
}}
|
||||
h := &IPHost{Runner: r}
|
||||
ctx := context.Background()
|
||||
|
||||
addrs, err := h.IPv4Addrs(ctx, "br-mgmt")
|
||||
if err != nil || !slices.Equal(addrs, []string{"10.15.8.1/24"}) {
|
||||
t.Errorf("IPv4Addrs = %q, %v", addrs, err)
|
||||
}
|
||||
routes, err := h.Routes(ctx, "10.15.8.0/24")
|
||||
want := []RouteEntry{{Dev: "br-mgmt", Src: "10.15.8.1"}, {Dev: "br-mgmt", Src: "10.15.8.1", Metric: 50}}
|
||||
if err != nil || !slices.Equal(routes, want) {
|
||||
t.Errorf("Routes = %+v, %v; want %+v", routes, err, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestApplyNeverDeletesALink runs a full apply through IPHost, so the check is
|
||||
// on the ip commands themselves, not on the Host interface.
|
||||
func TestApplyNeverDeletesALink(t *testing.T) {
|
||||
r := &scriptRunner{out: map[string]string{
|
||||
"ip -j -d link show": `[
|
||||
{"ifname":"br-int","flags":[],"linkinfo":{"info_kind":"openvswitch"}},
|
||||
{"ifname":"br-ext","flags":[],"linkinfo":{"info_kind":"openvswitch"}},
|
||||
{"ifname":"br-wan","flags":["UP"],"linkinfo":{"info_kind":"bridge"}},
|
||||
{"ifname":"br-mgmt","flags":[],"linkinfo":{"info_kind":"openvswitch"}},
|
||||
{"ifname":"br-vpc","flags":["UP"],"linkinfo":{"info_kind":"bridge"}},
|
||||
{"ifname":"mgmt","flags":[]}
|
||||
]`,
|
||||
"ip -j -4 addr show dev br-mgmt": `[]`,
|
||||
"ip -j -4 addr show dev mgmt": `[{"addr_info":[{"local":"169.254.3.4","prefixlen":16}]}]`,
|
||||
"ip -j -4 addr show dev br-vpc": `[{"addr_info":[{"local":"10.1.0.5","prefixlen":16}]}]`,
|
||||
"ip -j -4 route show exact 10.1.0.0/16": `[]`,
|
||||
}}
|
||||
te := newTestEnv(t, nil)
|
||||
te.Host = &IPHost{Runner: r}
|
||||
te.hostnetBackend(t, vethState)
|
||||
|
||||
// The canned link list never gains the veth pair, so this apply fails
|
||||
// once it looks for it again; every command up to then is still checked.
|
||||
_ = Apply(context.Background(), te.Env)
|
||||
if !slices.Contains(r.runs, "ip link add veth-wan-br type veth peer name veth-wan-ovs") {
|
||||
t.Fatalf("apply did not reach the veth pair: %q", r.runs)
|
||||
}
|
||||
for _, run := range r.runs {
|
||||
f := strings.Fields(run)
|
||||
if len(f) >= 3 && f[1] == "link" && (f[2] == "del" || f[2] == "delete") {
|
||||
t.Errorf("apply deleted a link: %q", run)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyTakesLockBeforeReadingState(t *testing.T) {
|
||||
te := newTestEnv(t, newFakeHost())
|
||||
te.LockWait = 2 * time.Second
|
||||
te.hostnetBackend(t, "{")
|
||||
|
||||
held := holdLock(t, te.Paths.Lock)
|
||||
// Replace the broken state while Apply waits on the lock: it must read the
|
||||
// replacement, so the wait is where the state file changes.
|
||||
te.Sleep = func(time.Duration) {
|
||||
te.write(t, te.Paths.State, natState)
|
||||
te.host.add(Link{Name: "br-int", Kind: "openvswitch"}).add(Link{Name: "br-ext", Kind: "openvswitch"})
|
||||
held.Close()
|
||||
}
|
||||
if err := Apply(context.Background(), te.Env); err != nil {
|
||||
t.Fatalf("Apply read the state before taking the lock: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyLockTimeoutNamesHolder(t *testing.T) {
|
||||
te := newTestEnv(t, newFakeHost())
|
||||
te.hostnetBackend(t, natState)
|
||||
held := holdLock(t, te.Paths.Lock)
|
||||
defer held.Close()
|
||||
|
||||
err := Apply(context.Background(), te.Env)
|
||||
if err == nil || !strings.Contains(err.Error(), fmt.Sprintf("still held after 2m0s by PID %d", os.Getpid())) {
|
||||
t.Fatalf("Apply error = %v, want it to name this test's PID", err)
|
||||
}
|
||||
if te.slept < DefaultLockWait {
|
||||
t.Errorf("gave up after %s, before the %s wait", te.slept, DefaultLockWait)
|
||||
}
|
||||
}
|
||||
|
||||
// holdLock takes the hostnet flock on a separate open file description, as
|
||||
// setup-ovn.sh does from another process.
|
||||
func holdLock(t *testing.T, path string) *os.File {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, err := os.OpenFile(path, os.O_RDONLY|os.O_CREATE, 0o644)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := unix.Flock(int(f.Fd()), unix.LOCK_EX|unix.LOCK_NB); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return f
|
||||
}
|
||||
@@ -0,0 +1,645 @@
|
||||
package hostnet
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// fakeHost models kernel links, addresses and routes, records every change,
|
||||
// and can make links appear after a number of polls.
|
||||
type fakeHost struct {
|
||||
links map[string]*Link
|
||||
addrs map[string][]string
|
||||
routes map[string][]RouteEntry
|
||||
changes []string
|
||||
// appearAfter makes a link exist only from the nth Link lookup of it.
|
||||
appearAfter map[string]int
|
||||
lookups map[string]int
|
||||
pending map[string]*Link
|
||||
failOn string
|
||||
}
|
||||
|
||||
func newFakeHost() *fakeHost {
|
||||
return &fakeHost{
|
||||
links: map[string]*Link{},
|
||||
addrs: map[string][]string{},
|
||||
routes: map[string][]RouteEntry{},
|
||||
appearAfter: map[string]int{},
|
||||
lookups: map[string]int{},
|
||||
pending: map[string]*Link{},
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fakeHost) add(l Link) *fakeHost {
|
||||
f.links[l.Name] = &l
|
||||
return f
|
||||
}
|
||||
|
||||
func (f *fakeHost) later(l Link, lookups int) *fakeHost {
|
||||
f.pending[l.Name] = &l
|
||||
f.appearAfter[l.Name] = lookups
|
||||
return f
|
||||
}
|
||||
|
||||
func (f *fakeHost) change(format string, args ...any) error {
|
||||
c := fmt.Sprintf(format, args...)
|
||||
if f.failOn != "" && strings.HasPrefix(c, f.failOn) {
|
||||
return errors.New("injected failure")
|
||||
}
|
||||
f.changes = append(f.changes, c)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeHost) Link(_ context.Context, name string) (*Link, error) {
|
||||
f.lookups[name]++
|
||||
if p, ok := f.pending[name]; ok && f.lookups[name] >= f.appearAfter[name] {
|
||||
f.links[name] = p
|
||||
delete(f.pending, name)
|
||||
}
|
||||
l, ok := f.links[name]
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
c := *l
|
||||
return &c, nil
|
||||
}
|
||||
|
||||
func (f *fakeHost) AddVethPair(_ context.Context, name, peer string) error {
|
||||
if err := f.change("add-veth %s %s", name, peer); err != nil {
|
||||
return err
|
||||
}
|
||||
f.links[name] = &Link{Name: name, Kind: "veth", Peer: peer, MAC: "aa:aa:aa:aa:aa:01"}
|
||||
f.links[peer] = &Link{Name: peer, Kind: "veth", Peer: name, MAC: "aa:aa:aa:aa:aa:02"}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeHost) SetUp(_ context.Context, name string) error {
|
||||
if err := f.change("up %s", name); err != nil {
|
||||
return err
|
||||
}
|
||||
f.links[name].Up = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeHost) SetMaster(_ context.Context, name, master string) error {
|
||||
if err := f.change("master %s %s", name, master); err != nil {
|
||||
return err
|
||||
}
|
||||
f.links[name].Master = master
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeHost) SetMAC(_ context.Context, name, mac string) error {
|
||||
if err := f.change("mac %s %s", name, mac); err != nil {
|
||||
return err
|
||||
}
|
||||
f.links[name].MAC = mac
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeHost) IPv4Addrs(_ context.Context, name string) ([]string, error) {
|
||||
return slices.Clone(f.addrs[name]), nil
|
||||
}
|
||||
|
||||
func (f *fakeHost) AddAddr(_ context.Context, name, cidr string) error {
|
||||
if err := f.change("addr-add %s %s", name, cidr); err != nil {
|
||||
return err
|
||||
}
|
||||
f.addrs[name] = append(f.addrs[name], cidr)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeHost) DelAddr(_ context.Context, name, cidr string) error {
|
||||
if err := f.change("addr-del %s %s", name, cidr); err != nil {
|
||||
return err
|
||||
}
|
||||
f.addrs[name] = slices.DeleteFunc(f.addrs[name], func(a string) bool { return a == cidr })
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeHost) Routes(_ context.Context, dst string) ([]RouteEntry, error) {
|
||||
return slices.Clone(f.routes[dst]), nil
|
||||
}
|
||||
|
||||
func (f *fakeHost) ReplaceRoute(_ context.Context, r Route) error {
|
||||
if err := f.change("route %s %s %s %d", r.Dst, r.Dev, r.Src, r.Metric); err != nil {
|
||||
return err
|
||||
}
|
||||
f.routes[r.Dst] = slices.DeleteFunc(f.routes[r.Dst], func(e RouteEntry) bool { return e.Metric == r.Metric })
|
||||
f.routes[r.Dst] = append(f.routes[r.Dst], RouteEntry{Dev: r.Dev, Src: r.Src, Metric: r.Metric})
|
||||
return nil
|
||||
}
|
||||
|
||||
var _ Host = (*fakeHost)(nil)
|
||||
|
||||
// testEnv is a temp root with a fake clock that Sleep advances.
|
||||
type testEnv struct {
|
||||
Env
|
||||
|
||||
host *fakeHost
|
||||
now time.Time
|
||||
slept time.Duration
|
||||
// remote is what OVNRemote returns; remoteErr fails it.
|
||||
remote string
|
||||
remoteErr error
|
||||
}
|
||||
|
||||
func newTestEnv(t *testing.T, h *fakeHost) *testEnv {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
te := &testEnv{host: h, now: time.Date(2026, 10, 21, 0, 0, 0, 0, time.UTC)}
|
||||
te.Env = Env{
|
||||
Paths: PathsAt(root),
|
||||
Host: h,
|
||||
LinkWait: DefaultLinkWait,
|
||||
LockWait: DefaultLockWait,
|
||||
OVNRemote: func(context.Context) (string, error) { return te.remote, te.remoteErr },
|
||||
Sleep: func(d time.Duration) {
|
||||
te.now = te.now.Add(d)
|
||||
te.slept += d
|
||||
},
|
||||
Now: func() time.Time { return te.now },
|
||||
}
|
||||
for _, dir := range []string{filepath.Dir(te.Paths.Backend), filepath.Dir(te.Paths.Marker), te.Paths.NetworkdDir} {
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return te
|
||||
}
|
||||
|
||||
func (te *testEnv) write(t *testing.T, path, content string) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (te *testEnv) hostnetBackend(t *testing.T, state string) {
|
||||
t.Helper()
|
||||
te.write(t, te.Paths.Backend, "hostnet\n")
|
||||
te.write(t, te.Paths.State, state)
|
||||
}
|
||||
|
||||
func (te *testEnv) markerExists() bool {
|
||||
_, err := os.Stat(te.Paths.Marker)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
const vethState = `{
|
||||
"version": 1,
|
||||
"pending": false,
|
||||
"int_bridge": "br-int",
|
||||
"wan": {"mode": "veth", "ovs_bridge": "br-ext", "linux_bridge": "br-wan", "host_end": "veth-wan-br", "ovs_end": "veth-wan-ovs"},
|
||||
"mgmt": {"bridge": "br-mgmt", "cidr": "10.15.8.1/24", "nic": "mgmt"},
|
||||
"encap_route": {"dst": "10.1.0.0/16", "dev": "br-vpc", "src": "10.1.0.5", "metric": 50}
|
||||
}`
|
||||
|
||||
const natState = `{
|
||||
"version": 1,
|
||||
"pending": false,
|
||||
"int_bridge": "br-int",
|
||||
"wan": {"mode": "nat", "ovs_bridge": "br-ext", "host_end": "spx-nat-host", "ovs_end": "spx-nat-ovs", "host_mac": "02:00:64:7f:00:01", "host_cidr": "100.127.0.1/24"}
|
||||
}`
|
||||
|
||||
const directState = `{
|
||||
"version": 1,
|
||||
"pending": false,
|
||||
"int_bridge": "br-int",
|
||||
"wan": {"mode": "direct", "ovs_bridge": "br-ext", "nic": "eth1"}
|
||||
}`
|
||||
|
||||
// rebootedVethHost is a veth-mode node just after a reboot: OVS and
|
||||
// NetworkManager have brought back their own links, nothing else.
|
||||
func rebootedVethHost() *fakeHost {
|
||||
h := newFakeHost().
|
||||
add(Link{Name: "br-int", Kind: "openvswitch"}).
|
||||
add(Link{Name: "br-ext", Kind: "openvswitch"}).
|
||||
add(Link{Name: "br-mgmt", Kind: "openvswitch"}).
|
||||
add(Link{Name: "br-wan", Kind: "bridge", Up: true}).
|
||||
add(Link{Name: "br-vpc", Kind: "bridge", Up: true}).
|
||||
add(Link{Name: "mgmt", Up: false})
|
||||
h.addrs["br-vpc"] = []string{"10.1.0.5/16"}
|
||||
h.addrs["mgmt"] = []string{"169.254.3.4/16"}
|
||||
return h
|
||||
}
|
||||
|
||||
func TestApplyRebuildsVethModeAfterReboot(t *testing.T) {
|
||||
h := rebootedVethHost()
|
||||
te := newTestEnv(t, h)
|
||||
te.hostnetBackend(t, vethState)
|
||||
|
||||
if err := Apply(context.Background(), te.Env); err != nil {
|
||||
t.Fatalf("Apply: %v", err)
|
||||
}
|
||||
want := []string{
|
||||
"up br-int",
|
||||
"up br-ext",
|
||||
"add-veth veth-wan-br veth-wan-ovs",
|
||||
"master veth-wan-br br-wan",
|
||||
"up veth-wan-br",
|
||||
"up veth-wan-ovs",
|
||||
"up br-mgmt",
|
||||
"addr-add br-mgmt 10.15.8.1/24",
|
||||
"up mgmt",
|
||||
"addr-del mgmt 169.254.3.4/16",
|
||||
"route 10.1.0.0/16 br-vpc 10.1.0.5 50",
|
||||
}
|
||||
if !slices.Equal(h.changes, want) {
|
||||
t.Errorf("changes:\n got %q\nwant %q", h.changes, want)
|
||||
}
|
||||
if !te.markerExists() {
|
||||
t.Error("marker not written after a successful apply")
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyIsIdempotent(t *testing.T) {
|
||||
for name, tc := range map[string]struct {
|
||||
state string
|
||||
host func() *fakeHost
|
||||
}{
|
||||
"veth": {vethState, rebootedVethHost},
|
||||
"nat": {natState, func() *fakeHost {
|
||||
return newFakeHost().add(Link{Name: "br-int", Kind: "openvswitch"}).add(Link{Name: "br-ext", Kind: "openvswitch"})
|
||||
}},
|
||||
"direct": {directState, func() *fakeHost {
|
||||
return newFakeHost().add(Link{Name: "br-int", Kind: "openvswitch"}).add(Link{Name: "br-ext", Kind: "openvswitch"}).add(Link{Name: "eth1"})
|
||||
}},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
h := tc.host()
|
||||
te := newTestEnv(t, h)
|
||||
te.hostnetBackend(t, tc.state)
|
||||
if err := Apply(context.Background(), te.Env); err != nil {
|
||||
t.Fatalf("first Apply: %v", err)
|
||||
}
|
||||
h.changes = nil
|
||||
if err := Apply(context.Background(), te.Env); err != nil {
|
||||
t.Fatalf("second Apply: %v", err)
|
||||
}
|
||||
if len(h.changes) != 0 {
|
||||
t.Errorf("second Apply changed the host: %q", h.changes)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyNATSetsFixedMACAndTransitAddress(t *testing.T) {
|
||||
h := newFakeHost().
|
||||
add(Link{Name: "br-int", Kind: "openvswitch", Up: true}).
|
||||
add(Link{Name: "br-ext", Kind: "openvswitch", Up: true}).
|
||||
add(Link{Name: "spx-nat-host", Kind: "veth", Peer: "spx-nat-ovs", MAC: "be:ef:00:00:00:01"}).
|
||||
add(Link{Name: "spx-nat-ovs", Kind: "veth", Peer: "spx-nat-host", Up: true})
|
||||
te := newTestEnv(t, h)
|
||||
te.hostnetBackend(t, natState)
|
||||
|
||||
if err := Apply(context.Background(), te.Env); err != nil {
|
||||
t.Fatalf("Apply: %v", err)
|
||||
}
|
||||
want := []string{
|
||||
"mac spx-nat-host 02:00:64:7f:00:01",
|
||||
"addr-add spx-nat-host 100.127.0.1/24",
|
||||
"up spx-nat-host",
|
||||
}
|
||||
if !slices.Equal(h.changes, want) {
|
||||
t.Errorf("changes:\n got %q\nwant %q", h.changes, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyMovesVethBridgeEndToRecordedBridge(t *testing.T) {
|
||||
h := rebootedVethHost().
|
||||
add(Link{Name: "veth-wan-br", Kind: "veth", Peer: "veth-wan-ovs", Master: "br-old", Up: true}).
|
||||
add(Link{Name: "veth-wan-ovs", Kind: "veth", Peer: "veth-wan-br", Up: true})
|
||||
te := newTestEnv(t, h)
|
||||
te.hostnetBackend(t, vethState)
|
||||
|
||||
if err := Apply(context.Background(), te.Env); err != nil {
|
||||
t.Fatalf("Apply: %v", err)
|
||||
}
|
||||
if !slices.Contains(h.changes, "master veth-wan-br br-wan") {
|
||||
t.Errorf("veth bridge end not moved to br-wan: %q", h.changes)
|
||||
}
|
||||
if slices.ContainsFunc(h.changes, func(c string) bool { return strings.HasPrefix(c, "add-veth") }) {
|
||||
t.Errorf("recreated a veth pair that already existed: %q", h.changes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyReplacesEncapRouteOnOtherDevice(t *testing.T) {
|
||||
h := rebootedVethHost()
|
||||
h.routes["10.1.0.0/16"] = []RouteEntry{
|
||||
{Dev: "br-vpc", Src: "10.1.0.5", Metric: 0},
|
||||
{Dev: "br-wan", Src: "10.1.0.5", Metric: 50},
|
||||
}
|
||||
te := newTestEnv(t, h)
|
||||
te.hostnetBackend(t, vethState)
|
||||
|
||||
if err := Apply(context.Background(), te.Env); err != nil {
|
||||
t.Fatalf("Apply: %v", err)
|
||||
}
|
||||
if !slices.Contains(h.changes, "route 10.1.0.0/16 br-vpc 10.1.0.5 50") {
|
||||
t.Errorf("encap route on the wrong device not replaced: %q", h.changes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyRefusesForeignLinks(t *testing.T) {
|
||||
for name, tc := range map[string]struct {
|
||||
links []Link
|
||||
want string
|
||||
}{
|
||||
"non-veth under the bridge end's name": {
|
||||
[]Link{{Name: "veth-wan-br", Kind: "dummy"}},
|
||||
"veth-wan-br: it is a dummy link",
|
||||
},
|
||||
"physical NIC under the OVS end's name": {
|
||||
[]Link{{Name: "veth-wan-ovs"}},
|
||||
"veth-wan-ovs: it is a physical link",
|
||||
},
|
||||
"ends peered elsewhere": {
|
||||
[]Link{
|
||||
{Name: "veth-wan-br", Kind: "veth", Peer: "other0"},
|
||||
{Name: "veth-wan-ovs", Kind: "veth", Peer: "other1"},
|
||||
},
|
||||
`veth-wan-br is peered with "other0" and veth-wan-ovs with "other1"`,
|
||||
},
|
||||
"one end alone": {
|
||||
[]Link{{Name: "veth-wan-ovs", Kind: "veth", Peer: "x"}},
|
||||
"veth-wan-br is missing",
|
||||
},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
h := rebootedVethHost()
|
||||
for _, l := range tc.links {
|
||||
h.add(l)
|
||||
}
|
||||
te := newTestEnv(t, h)
|
||||
te.hostnetBackend(t, vethState)
|
||||
|
||||
err := Apply(context.Background(), te.Env)
|
||||
if err == nil || !strings.Contains(err.Error(), tc.want) {
|
||||
t.Fatalf("Apply error = %v, want it to contain %q", err, tc.want)
|
||||
}
|
||||
if te.markerExists() {
|
||||
t.Error("marker written after a refused apply")
|
||||
}
|
||||
if slices.ContainsFunc(h.changes, func(c string) bool { return strings.Contains(c, "veth-wan") }) {
|
||||
t.Errorf("touched a refused link: %q", h.changes)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyWaitsForLinksOthersCreate(t *testing.T) {
|
||||
h := rebootedVethHost()
|
||||
delete(h.links, "br-wan")
|
||||
h.later(Link{Name: "br-wan", Kind: "bridge", Up: true}, 5)
|
||||
te := newTestEnv(t, h)
|
||||
te.hostnetBackend(t, vethState)
|
||||
|
||||
if err := Apply(context.Background(), te.Env); err != nil {
|
||||
t.Fatalf("Apply: %v", err)
|
||||
}
|
||||
if te.slept == 0 {
|
||||
t.Error("Apply did not wait for br-wan")
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFailsNamingLinksStillMissing(t *testing.T) {
|
||||
h := rebootedVethHost()
|
||||
delete(h.links, "br-wan")
|
||||
delete(h.links, "br-mgmt")
|
||||
te := newTestEnv(t, h)
|
||||
te.hostnetBackend(t, vethState)
|
||||
|
||||
err := Apply(context.Background(), te.Env)
|
||||
if err == nil || !strings.Contains(err.Error(), "still missing after 1m0s: br-mgmt, br-wan") {
|
||||
t.Fatalf("Apply error = %v, want it to name br-mgmt and br-wan", err)
|
||||
}
|
||||
if te.slept < DefaultLinkWait {
|
||||
t.Errorf("gave up after %s, before the %s wait", te.slept, DefaultLinkWait)
|
||||
}
|
||||
if len(h.changes) != 0 {
|
||||
t.Errorf("changed the host before every link was there: %q", h.changes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyWaitsOnlyForLinksTheModeNeeds(t *testing.T) {
|
||||
// A NAT node has no Linux uplink bridge and no mgmt NIC: their absence
|
||||
// must not cost a wait.
|
||||
h := newFakeHost().
|
||||
add(Link{Name: "br-int", Kind: "openvswitch"}).
|
||||
add(Link{Name: "br-ext", Kind: "openvswitch"})
|
||||
te := newTestEnv(t, h)
|
||||
te.hostnetBackend(t, natState)
|
||||
|
||||
if err := Apply(context.Background(), te.Env); err != nil {
|
||||
t.Fatalf("Apply: %v", err)
|
||||
}
|
||||
if te.slept != 0 {
|
||||
t.Errorf("waited %s on a NAT node", te.slept)
|
||||
}
|
||||
for name := range h.lookups {
|
||||
if name == "br-wan" || name == "mgmt" {
|
||||
t.Errorf("looked up %s, which a NAT node does not have", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyWaitsForEncapSourceAddress(t *testing.T) {
|
||||
h := rebootedVethHost()
|
||||
h.addrs["br-vpc"] = nil
|
||||
te := newTestEnv(t, h)
|
||||
te.hostnetBackend(t, vethState)
|
||||
|
||||
err := Apply(context.Background(), te.Env)
|
||||
if err == nil || !strings.Contains(err.Error(), "10.1.0.5 on br-vpc") {
|
||||
t.Fatalf("Apply error = %v, want it to name the missing encap source", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyLeavesUnnamedLinksAlone(t *testing.T) {
|
||||
h := rebootedVethHost().add(Link{Name: "wg0", Kind: "wireguard"})
|
||||
h.addrs["wg0"] = []string{"10.99.0.1/24"}
|
||||
h.addrs["br-wan"] = []string{"192.168.0.13/24"}
|
||||
te := newTestEnv(t, h)
|
||||
te.hostnetBackend(t, vethState)
|
||||
|
||||
if err := Apply(context.Background(), te.Env); err != nil {
|
||||
t.Fatalf("Apply: %v", err)
|
||||
}
|
||||
for _, c := range h.changes {
|
||||
if strings.Contains(c, "wg0") || strings.HasPrefix(c, "addr-del br-wan") {
|
||||
t.Errorf("touched something the state file does not name: %q", c)
|
||||
}
|
||||
}
|
||||
if h.links["wg0"] == nil || len(h.addrs["wg0"]) != 1 || len(h.addrs["br-wan"]) != 1 {
|
||||
t.Error("an operator link or the uplink address changed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFailureRemovesMarker(t *testing.T) {
|
||||
h := rebootedVethHost()
|
||||
te := newTestEnv(t, h)
|
||||
te.hostnetBackend(t, vethState)
|
||||
if err := Apply(context.Background(), te.Env); err != nil {
|
||||
t.Fatalf("first Apply: %v", err)
|
||||
}
|
||||
if err := Check(context.Background(), te.Env); err != nil {
|
||||
t.Fatalf("Check after a successful apply: %v", err)
|
||||
}
|
||||
|
||||
delete(h.links, "veth-wan-br")
|
||||
delete(h.links, "veth-wan-ovs")
|
||||
h.failOn = "add-veth"
|
||||
if err := Apply(context.Background(), te.Env); err == nil {
|
||||
t.Fatal("second Apply succeeded despite the injected failure")
|
||||
}
|
||||
if te.markerExists() {
|
||||
t.Error("marker survived a failed apply")
|
||||
}
|
||||
if err := Check(context.Background(), te.Env); err == nil {
|
||||
t.Error("Check passed after a successful apply followed by a failed one")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackendTable(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
for name, tc := range map[string]struct {
|
||||
setup func(t *testing.T, te *testEnv)
|
||||
// applyErr and checkErr are substrings; "" means success.
|
||||
applyErr string
|
||||
checkErr string
|
||||
}{
|
||||
"networkd record": {
|
||||
setup: func(t *testing.T, te *testEnv) { te.write(t, te.Paths.Backend, "networkd\n") },
|
||||
},
|
||||
"networkd record ignores a broken state file": {
|
||||
setup: func(t *testing.T, te *testEnv) {
|
||||
te.write(t, te.Paths.Backend, "networkd\n")
|
||||
te.write(t, te.Paths.State, "{")
|
||||
},
|
||||
},
|
||||
"no record, legacy networkd files": {
|
||||
setup: func(t *testing.T, te *testEnv) {
|
||||
te.write(t, filepath.Join(te.Paths.NetworkdDir, "10-spinifex-mgmt.network"), "")
|
||||
te.remote = "tcp:127.0.0.1:6642"
|
||||
},
|
||||
},
|
||||
"no record, nothing set up": {},
|
||||
"no record, set up without networkd files": {
|
||||
setup: func(_ *testing.T, te *testEnv) { te.remote = "tcp:127.0.0.1:6642" },
|
||||
applyErr: "has no backend record",
|
||||
checkErr: "has no backend record",
|
||||
},
|
||||
"no record, OVS unreachable": {
|
||||
setup: func(_ *testing.T, te *testEnv) { te.remoteErr = errors.New("database connection failed") },
|
||||
applyErr: "cannot tell whether this node is set up",
|
||||
checkErr: "cannot tell whether this node is set up",
|
||||
},
|
||||
"unknown backend": {
|
||||
setup: func(t *testing.T, te *testEnv) { te.write(t, te.Paths.Backend, "netplan\n") },
|
||||
applyErr: `unknown backend "netplan"`,
|
||||
checkErr: `unknown backend "netplan"`,
|
||||
},
|
||||
"hostnet, state missing": {
|
||||
setup: func(t *testing.T, te *testEnv) { te.write(t, te.Paths.Backend, "hostnet\n") },
|
||||
applyErr: "no such file",
|
||||
checkErr: "no such file",
|
||||
},
|
||||
"hostnet, state unreadable": {
|
||||
setup: func(t *testing.T, te *testEnv) {
|
||||
te.write(t, te.Paths.Backend, "hostnet\n")
|
||||
if err := os.Mkdir(te.Paths.State, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
},
|
||||
applyErr: "is a directory",
|
||||
checkErr: "is a directory",
|
||||
},
|
||||
"hostnet, state truncated": {
|
||||
setup: func(t *testing.T, te *testEnv) { te.hostnetBackend(t, vethState[:40]) },
|
||||
applyErr: "not valid JSON",
|
||||
checkErr: "not valid JSON",
|
||||
},
|
||||
"hostnet, unknown version": {
|
||||
setup: func(t *testing.T, te *testEnv) {
|
||||
te.hostnetBackend(t, strings.Replace(natState, `"version": 1`, `"version": 2`, 1))
|
||||
},
|
||||
applyErr: "unknown version 2",
|
||||
checkErr: "unknown version 2",
|
||||
},
|
||||
"hostnet, pending": {
|
||||
setup: func(t *testing.T, te *testEnv) {
|
||||
te.hostnetBackend(t, strings.Replace(natState, `"pending": false`, `"pending": true`, 1))
|
||||
},
|
||||
applyErr: "pending",
|
||||
checkErr: "pending",
|
||||
},
|
||||
"hostnet, never applied": {
|
||||
setup: func(t *testing.T, te *testEnv) { te.hostnetBackend(t, natState) },
|
||||
checkErr: "host network not applied",
|
||||
},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
h := newFakeHost().
|
||||
add(Link{Name: "br-int", Kind: "openvswitch"}).
|
||||
add(Link{Name: "br-ext", Kind: "openvswitch"})
|
||||
// Check runs before Apply so "never applied" sees no marker.
|
||||
te := newTestEnv(t, h)
|
||||
if tc.setup != nil {
|
||||
tc.setup(t, te)
|
||||
}
|
||||
wantErr(t, "Check", Check(ctx, te.Env), tc.checkErr)
|
||||
wantErr(t, "Apply", Apply(ctx, te.Env), tc.applyErr)
|
||||
if tc.applyErr != "" && te.markerExists() {
|
||||
t.Error("marker written after a failed apply")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func wantErr(t *testing.T, what string, err error, want string) {
|
||||
t.Helper()
|
||||
switch {
|
||||
case want == "" && err != nil:
|
||||
t.Errorf("%s: unexpected error %v", what, err)
|
||||
case want != "" && err == nil:
|
||||
t.Errorf("%s: succeeded, want an error containing %q", what, want)
|
||||
case want != "" && !strings.Contains(err.Error(), want):
|
||||
t.Errorf("%s: error %q does not contain %q", what, err, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadStateRejectsInvalidFields(t *testing.T) {
|
||||
for name, tc := range map[string]struct {
|
||||
from, to, want string
|
||||
}{
|
||||
"unknown mode": {`"mode": "nat"`, `"mode": "bonded"`, `wan.mode: unknown mode "bonded"`},
|
||||
"long name": {`"ovs_bridge": "br-ext"`, `"ovs_bridge": "br-ext-much-too-long"`, "wan.ovs_bridge"},
|
||||
"slash in name": {`"host_end": "spx-nat-host"`, `"host_end": "spx/nat"`, "wan.host_end"},
|
||||
"bad MAC": {`"host_mac": "02:00:64:7f:00:01"`, `"host_mac": "02:00"`, "wan.host_mac"},
|
||||
"bad CIDR": {`"host_cidr": "100.127.0.1/24"`, `"host_cidr": "100.127.0.1"`, "wan.host_cidr"},
|
||||
"unknown field": {`"pending": false`, `"pending": false, "extra": 1`, `unknown field "extra"`},
|
||||
"missing int bridge": {`"int_bridge": "br-int"`, `"int_bridge": ""`, "int_bridge: missing"},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if !strings.Contains(natState, tc.from) {
|
||||
t.Fatalf("fixture lacks %q", tc.from)
|
||||
}
|
||||
path := filepath.Join(t.TempDir(), "hostnet.json")
|
||||
if err := os.WriteFile(path, []byte(strings.Replace(natState, tc.from, tc.to, 1)), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err := LoadState(path)
|
||||
if err == nil || !strings.Contains(err.Error(), tc.want) || !strings.Contains(err.Error(), path) {
|
||||
t.Fatalf("LoadState error = %v, want it to name %s and contain %q", err, path, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package hostnet
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// acquireLock takes the exclusive flock setup-ovn.sh also takes, waiting at
|
||||
// most env.LockWait, and returns the function that releases it.
|
||||
func acquireLock(env Env) (func(), error) {
|
||||
path := env.Paths.Lock
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o770); err != nil { //nolint:gosec // G301: the mode setup.sh's tmpfiles entry gives /run/spinifex, which the daemon and vpcd traverse to check the marker. Holds no secret material.
|
||||
return nil, fmt.Errorf("create %s: %w", filepath.Dir(path), err)
|
||||
}
|
||||
f, err := os.OpenFile(path, os.O_RDONLY|os.O_CREATE, 0o600)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open lock %s: %w", path, err)
|
||||
}
|
||||
deadline := env.Now().Add(env.LockWait)
|
||||
for {
|
||||
err := unix.Flock(int(f.Fd()), unix.LOCK_EX|unix.LOCK_NB)
|
||||
if err == nil {
|
||||
return func() { _ = f.Close() }, nil
|
||||
}
|
||||
if !errors.Is(err, unix.EWOULDBLOCK) {
|
||||
_ = f.Close()
|
||||
return nil, fmt.Errorf("lock %s: %w", path, err)
|
||||
}
|
||||
if !env.Now().Before(deadline) {
|
||||
holder := lockHolder(f, "/proc/locks")
|
||||
_ = f.Close()
|
||||
return nil, fmt.Errorf("lock %s still held after %s by %s; is setup-ovn.sh running?", path, env.LockWait, holder)
|
||||
}
|
||||
env.Sleep(pollInterval)
|
||||
}
|
||||
}
|
||||
|
||||
// lockHolder names the process holding a flock on f, from procLocks, whose
|
||||
// rows read "1: FLOCK ADVISORY WRITE <pid> <maj>:<min>:<inode> 0 EOF".
|
||||
func lockHolder(f *os.File, procLocks string) string {
|
||||
var st unix.Stat_t
|
||||
if err := unix.Fstat(int(f.Fd()), &st); err != nil {
|
||||
return "an unknown process"
|
||||
}
|
||||
id := fmt.Sprintf("%02x:%02x:%d", unix.Major(uint64(st.Dev)), unix.Minor(uint64(st.Dev)), st.Ino) //nolint:unconvert // Dev is not uint64 on every arch
|
||||
locks, err := os.Open(procLocks)
|
||||
if err != nil {
|
||||
return "an unknown process"
|
||||
}
|
||||
defer locks.Close()
|
||||
sc := bufio.NewScanner(locks)
|
||||
for sc.Scan() {
|
||||
fields := strings.Fields(sc.Text())
|
||||
if len(fields) < 6 || fields[1] != "FLOCK" || fields[5] != id {
|
||||
continue
|
||||
}
|
||||
pid, err := strconv.Atoi(fields[4])
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
return describePID(pid)
|
||||
}
|
||||
return "an unknown process"
|
||||
}
|
||||
|
||||
func describePID(pid int) string {
|
||||
cmdline, err := os.ReadFile(fmt.Sprintf("/proc/%d/cmdline", pid))
|
||||
if err != nil || len(cmdline) == 0 {
|
||||
return "PID " + strconv.Itoa(pid)
|
||||
}
|
||||
args := strings.TrimRight(strings.ReplaceAll(string(cmdline), "\x00", " "), " ")
|
||||
return fmt.Sprintf("PID %d (%s)", pid, args)
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
package hostnet
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const setupOVN = "../../../../scripts/setup-ovn.sh"
|
||||
|
||||
// shellFunc returns the body of a top-level function in setup-ovn.sh.
|
||||
func shellFunc(t *testing.T, name string) string {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(setupOVN)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
src := string(data)
|
||||
start := strings.Index(src, "\n"+name+"() {\n")
|
||||
if start < 0 {
|
||||
t.Fatalf("%s: no function %s", setupOVN, name)
|
||||
}
|
||||
end := strings.Index(src[start:], "\n}\n")
|
||||
if end < 0 {
|
||||
t.Fatalf("%s: function %s is not closed", setupOVN, name)
|
||||
}
|
||||
return src[start : start+end+3]
|
||||
}
|
||||
|
||||
// TestScriptStateLoads runs setup-ovn.sh's state writer for every WAN mode and
|
||||
// loads what it wrote, so the script and LoadState cannot drift apart.
|
||||
func TestScriptStateLoads(t *testing.T) {
|
||||
if _, err := exec.LookPath("bash"); err != nil {
|
||||
t.Skip("bash not installed")
|
||||
}
|
||||
fn := shellFunc(t, "write_hostnet_state")
|
||||
bin := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(bin, "sudo"), []byte("#!/bin/sh\nexec \"$@\"\n"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mgmt := &Mgmt{Bridge: "br-mgmt", CIDR: "10.15.8.1/24", NIC: "eth2"}
|
||||
route := &Route{Dst: "10.1.0.0/16", Dev: "eth1", Src: "10.1.0.5", Metric: 50}
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
env []string
|
||||
want State
|
||||
}{
|
||||
{"existing", []string{"WAN_BRIDGE_MODE=existing", "WAN_BRIDGE=br-ext"},
|
||||
State{WAN: &WAN{Mode: ModeExisting, OVSBridge: "br-ext"}}},
|
||||
{"veth", []string{"WAN_BRIDGE_MODE=veth", "WAN_BRIDGE=br-ext", "LINUX_BRIDGE=br-wan"},
|
||||
State{WAN: &WAN{Mode: ModeVeth, OVSBridge: "br-ext", LinuxBridge: "br-wan", HostEnd: "veth-wan-br", OVSEnd: "veth-wan-ovs"}}},
|
||||
{"direct", []string{"WAN_BRIDGE_MODE=direct", "WAN_BRIDGE=br-ext", "WAN_IFACE=eth3"},
|
||||
State{WAN: &WAN{Mode: ModeDirect, OVSBridge: "br-ext", NIC: "eth3"}}},
|
||||
{"nat", []string{"WAN_BRIDGE_MODE=nat", "WAN_BRIDGE=br-ext", "NAT_TRANSIT_HOST_MAC=02:00:64:7f:00:01", "NAT_TRANSIT_GW_CIDR=100.127.0.1/24"},
|
||||
State{WAN: &WAN{Mode: ModeNAT, OVSBridge: "br-ext", HostEnd: "spx-nat-host", OVSEnd: "spx-nat-ovs", HostMAC: "02:00:64:7f:00:01", HostCIDR: "100.127.0.1/24"}}},
|
||||
{"no wan", nil, State{}},
|
||||
{"mgmt and route", []string{"MGMT_BRIDGE_ENABLED=true", "MGMT_BRIDGE=br-mgmt", "MGMT_CIDR=10.15.8.1/24", "MGMT_IFACE=eth2",
|
||||
"ENCAP_ROUTE_DST=10.1.0.0/16", "ENCAP_ROUTE_DEV=eth1", "ENCAP_IP=10.1.0.5", "ENCAP_ROUTE_METRIC=50"},
|
||||
State{Mgmt: mgmt, EncapRoute: route}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
state := filepath.Join(t.TempDir(), "etc", "hostnet.json")
|
||||
write := func(pending string) {
|
||||
cmd := exec.Command("bash", "-c", fn+`write_hostnet_state "$1"`, "_", pending)
|
||||
cmd.Env = append([]string{"PATH=" + bin + ":" + os.Getenv("PATH"), "HOSTNET_STATE=" + state}, tc.env...)
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
t.Fatalf("write_hostnet_state %s: %v: %s", pending, err, out)
|
||||
}
|
||||
}
|
||||
write("false")
|
||||
got, err := LoadState(state)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := tc.want
|
||||
want.Version, want.IntBridge = StateVersion, "br-int"
|
||||
if !reflect.DeepEqual(*got, want) {
|
||||
t.Errorf("loaded %+v\nwant %+v", *got, want)
|
||||
}
|
||||
if _, err := os.Stat(state + ".tmp"); !os.IsNotExist(err) {
|
||||
t.Errorf("temporary file left behind: %v", err)
|
||||
}
|
||||
|
||||
write("true")
|
||||
if _, err := LoadState(state); err == nil || !strings.Contains(err.Error(), "is pending") {
|
||||
t.Errorf("pending write loaded as %v, want a pending error", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,271 @@
|
||||
// Package hostnet re-creates the host network wiring setup-ovn.sh builds, on
|
||||
// hosts where no systemd-networkd files persist it: the uplink veth pairs, the
|
||||
// management bridge address and the encap route. spinifex-hostnet.service runs
|
||||
// Apply at boot and whenever ovs-vswitchd restarts; the daemon and vpcd run
|
||||
// Check before they start, so neither starts on a node left unwired.
|
||||
package hostnet
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/netip"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Backend is how a node persists its host network wiring across reboots.
|
||||
type Backend string
|
||||
|
||||
const (
|
||||
// BackendNetworkd persists the wiring in systemd-networkd files.
|
||||
BackendNetworkd Backend = "networkd"
|
||||
// BackendHostnet persists it in the state file, applied by this package.
|
||||
BackendHostnet Backend = "hostnet"
|
||||
)
|
||||
|
||||
// StateVersion is the only state file version this build reads.
|
||||
const StateVersion = 1
|
||||
|
||||
// WAN uplink modes, as setup-ovn.sh names them.
|
||||
const (
|
||||
ModeExisting = "existing"
|
||||
ModeVeth = "veth"
|
||||
ModeDirect = "direct"
|
||||
ModeNAT = "nat"
|
||||
)
|
||||
|
||||
// Paths locates the files hostnet reads and writes.
|
||||
type Paths struct {
|
||||
// Backend holds the backend setup-ovn.sh chose on its first run.
|
||||
Backend string
|
||||
// State is the topology setup-ovn.sh recorded for Apply.
|
||||
State string
|
||||
// Marker exists only while the last Apply succeeded.
|
||||
Marker string
|
||||
// Lock serialises Apply against setup-ovn.sh.
|
||||
Lock string
|
||||
// NetworkdDir is where setup-ovn.sh writes its systemd-networkd files.
|
||||
NetworkdDir string
|
||||
}
|
||||
|
||||
// DefaultPaths returns the paths on a live host.
|
||||
func DefaultPaths() Paths { return PathsAt("/") }
|
||||
|
||||
// PathsAt returns the paths with root prefixed, for tests and preflight.
|
||||
func PathsAt(root string) Paths {
|
||||
return Paths{
|
||||
Backend: filepath.Join(root, "/etc/spinifex/hostnet-backend"),
|
||||
State: filepath.Join(root, "/etc/spinifex/hostnet.json"),
|
||||
Marker: filepath.Join(root, "/run/spinifex/hostnet-applied"),
|
||||
Lock: filepath.Join(root, "/run/spinifex/hostnet.lock"),
|
||||
NetworkdDir: filepath.Join(root, "/etc/systemd/network"),
|
||||
}
|
||||
}
|
||||
|
||||
// State is the host network topology setup-ovn.sh recorded. Every name in it
|
||||
// comes from setup-ovn.sh, so custom bridge names need nothing here.
|
||||
type State struct {
|
||||
Version int `json:"version"`
|
||||
// Pending is set while setup-ovn.sh is changing live wiring; Apply and Check
|
||||
// refuse it, since the recorded topology may no longer match OVSDB.
|
||||
Pending bool `json:"pending"`
|
||||
IntBridge string `json:"int_bridge"`
|
||||
WAN *WAN `json:"wan,omitempty"`
|
||||
Mgmt *Mgmt `json:"mgmt,omitempty"`
|
||||
EncapRoute *Route `json:"encap_route,omitempty"`
|
||||
}
|
||||
|
||||
// WAN is the public-subnet uplink.
|
||||
type WAN struct {
|
||||
Mode string `json:"mode"`
|
||||
// OVSBridge is the OVS bridge ovn-bridge-mappings names.
|
||||
OVSBridge string `json:"ovs_bridge"`
|
||||
// LinuxBridge is the host's bridge the veth pair joins (veth mode).
|
||||
LinuxBridge string `json:"linux_bridge,omitempty"`
|
||||
// NIC is the physical NIC moved into OVSBridge (direct mode).
|
||||
NIC string `json:"nic,omitempty"`
|
||||
// HostEnd and OVSEnd name the veth pair (veth and nat modes); OVSEnd is
|
||||
// the end that is a port on OVSBridge.
|
||||
HostEnd string `json:"host_end,omitempty"`
|
||||
OVSEnd string `json:"ovs_end,omitempty"`
|
||||
// HostMAC and HostCIDR are set on HostEnd (nat mode).
|
||||
HostMAC string `json:"host_mac,omitempty"`
|
||||
HostCIDR string `json:"host_cidr,omitempty"`
|
||||
}
|
||||
|
||||
// Mgmt is the system-instance management bridge.
|
||||
type Mgmt struct {
|
||||
Bridge string `json:"bridge"`
|
||||
CIDR string `json:"cidr,omitempty"`
|
||||
// NIC is moved into Bridge and keeps no IPv4 address of its own.
|
||||
NIC string `json:"nic,omitempty"`
|
||||
}
|
||||
|
||||
// Route is the Geneve encap route: Dst via Dev from Src, at Metric.
|
||||
type Route struct {
|
||||
Dst string `json:"dst"`
|
||||
Dev string `json:"dev"`
|
||||
Src string `json:"src"`
|
||||
Metric int `json:"metric"`
|
||||
}
|
||||
|
||||
// ReadBackend returns the recorded backend, or "" when there is no record.
|
||||
func ReadBackend(path string) (Backend, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return "", nil
|
||||
}
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read backend record %s: %w", path, err)
|
||||
}
|
||||
switch b := Backend(strings.TrimSpace(string(data))); b {
|
||||
case BackendNetworkd, BackendHostnet:
|
||||
return b, nil
|
||||
default:
|
||||
return "", fmt.Errorf("backend record %s: unknown backend %q (want %q or %q)", path, b, BackendNetworkd, BackendHostnet)
|
||||
}
|
||||
}
|
||||
|
||||
// LoadState reads and validates the state file. Every error names the file.
|
||||
func LoadState(path string) (*State, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("state file %s: %w", path, err)
|
||||
}
|
||||
// Read the version alone first, so a future layout is reported as an
|
||||
// unknown version rather than as whichever field it renamed.
|
||||
var head struct {
|
||||
Version *int `json:"version"`
|
||||
Pending bool `json:"pending"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &head); err != nil {
|
||||
return nil, fmt.Errorf("state file %s: not valid JSON (truncated?): %w", path, err)
|
||||
}
|
||||
if head.Version == nil {
|
||||
return nil, fmt.Errorf("state file %s: no version field", path)
|
||||
}
|
||||
if *head.Version != StateVersion {
|
||||
return nil, fmt.Errorf("state file %s: unknown version %d (this build reads version %d)", path, *head.Version, StateVersion)
|
||||
}
|
||||
// A pending file may describe a topology setup-ovn.sh never finished
|
||||
// building, so it is refused before its contents are judged.
|
||||
if head.Pending {
|
||||
return nil, fmt.Errorf("state file %s is pending: setup-ovn.sh stopped part-way through changing the host network; rerun it with the same arguments", path)
|
||||
}
|
||||
var s State
|
||||
dec := json.NewDecoder(bytes.NewReader(data))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&s); err != nil {
|
||||
return nil, fmt.Errorf("state file %s: %w", path, err)
|
||||
}
|
||||
if err := s.validate(); err != nil {
|
||||
return nil, fmt.Errorf("state file %s: %w", path, err)
|
||||
}
|
||||
return &s, nil
|
||||
}
|
||||
|
||||
func (s *State) validate() error {
|
||||
if err := checkName("int_bridge", s.IntBridge); err != nil {
|
||||
return err
|
||||
}
|
||||
if w := s.WAN; w != nil {
|
||||
if err := checkName("wan.ovs_bridge", w.OVSBridge); err != nil {
|
||||
return err
|
||||
}
|
||||
switch w.Mode {
|
||||
case ModeExisting:
|
||||
case ModeVeth:
|
||||
if err := checkNames(map[string]string{"wan.linux_bridge": w.LinuxBridge, "wan.host_end": w.HostEnd, "wan.ovs_end": w.OVSEnd}); err != nil {
|
||||
return err
|
||||
}
|
||||
case ModeDirect:
|
||||
if err := checkName("wan.nic", w.NIC); err != nil {
|
||||
return err
|
||||
}
|
||||
case ModeNAT:
|
||||
if err := checkNames(map[string]string{"wan.host_end": w.HostEnd, "wan.ovs_end": w.OVSEnd}); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := net.ParseMAC(w.HostMAC); err != nil {
|
||||
return fmt.Errorf("wan.host_mac: %w", err)
|
||||
}
|
||||
if _, err := netip.ParsePrefix(w.HostCIDR); err != nil {
|
||||
return fmt.Errorf("wan.host_cidr: %w", err)
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("wan.mode: unknown mode %q", w.Mode)
|
||||
}
|
||||
}
|
||||
if m := s.Mgmt; m != nil {
|
||||
if err := checkName("mgmt.bridge", m.Bridge); err != nil {
|
||||
return err
|
||||
}
|
||||
if m.CIDR != "" {
|
||||
if _, err := netip.ParsePrefix(m.CIDR); err != nil {
|
||||
return fmt.Errorf("mgmt.cidr: %w", err)
|
||||
}
|
||||
}
|
||||
if m.NIC != "" {
|
||||
if err := checkName("mgmt.nic", m.NIC); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
if r := s.EncapRoute; r != nil {
|
||||
if _, err := netip.ParsePrefix(r.Dst); err != nil {
|
||||
return fmt.Errorf("encap_route.dst: %w", err)
|
||||
}
|
||||
if err := checkName("encap_route.dev", r.Dev); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := netip.ParseAddr(r.Src); err != nil {
|
||||
return fmt.Errorf("encap_route.src: %w", err)
|
||||
}
|
||||
if r.Metric < 0 {
|
||||
return fmt.Errorf("encap_route.metric: %d is negative", r.Metric)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkNames(fields map[string]string) error {
|
||||
for field, name := range fields {
|
||||
if err := checkName(field, name); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkName applies the kernel's interface name rules (IFNAMSIZ, no '/' or
|
||||
// whitespace), so a bad name is reported against its field, not by ip.
|
||||
func checkName(field, name string) error {
|
||||
switch {
|
||||
case name == "":
|
||||
return fmt.Errorf("%s: missing", field)
|
||||
case len(name) > 15:
|
||||
return fmt.Errorf("%s: %q is longer than 15 characters", field, name)
|
||||
case name == "." || name == ".." || strings.ContainsAny(name, "/: \t\n"):
|
||||
return fmt.Errorf("%s: %q is not a valid interface name", field, name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// hasNetworkdFiles reports whether setup-ovn.sh has written systemd-networkd
|
||||
// files here, which every node set up before the backend record existed has.
|
||||
func hasNetworkdFiles(dir string) (bool, error) {
|
||||
for _, pattern := range []string{"*-spinifex-*.network", "*-spinifex-*.netdev"} {
|
||||
matches, err := filepath.Glob(filepath.Join(dir, pattern))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if len(matches) > 0 {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
@@ -0,0 +1,221 @@
|
||||
// Package hostos resolves the host tools whose names or paths differ between
|
||||
// host operating systems. Each resolver probes for the tool itself, never reads
|
||||
// /etc/os-release, and names every candidate it tried when nothing matches.
|
||||
package hostos
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// QEMUKVMPath is where EL-family distributions install QEMU. It only runs
|
||||
// guests of the host's own architecture.
|
||||
const QEMUKVMPath = "/usr/libexec/qemu-kvm"
|
||||
|
||||
// QEMU guest processes are named qemu-system-<arch> (comm truncates it to 15
|
||||
// bytes, so qemu-system-x86_64 reads qemu-system-x86) or qemu-kvm.
|
||||
const (
|
||||
qemuSystemPrefix = "qemu-system"
|
||||
qemuKVMName = "qemu-kvm"
|
||||
// QEMUProcessPattern is an extended regular expression, for grep -E and
|
||||
// pgrep, that matches a QEMU guest's process name or command line.
|
||||
QEMUProcessPattern = qemuSystemPrefix + "|" + qemuKVMName
|
||||
)
|
||||
|
||||
// IsQEMUProcess reports whether a /proc/<pid>/comm value names a QEMU guest.
|
||||
func IsQEMUProcess(comm string) bool {
|
||||
comm = strings.TrimSpace(comm)
|
||||
return strings.HasPrefix(comm, qemuSystemPrefix) || comm == qemuKVMName
|
||||
}
|
||||
|
||||
// Tool is a host command found by a resolver.
|
||||
type Tool struct {
|
||||
// Name is the candidate that matched, such as "grubby" or "update-grub".
|
||||
Name string
|
||||
// Path is where it was found on PATH.
|
||||
Path string
|
||||
}
|
||||
|
||||
// CATrust is the host's system trust store.
|
||||
type CATrust struct {
|
||||
// Update rebuilds the trust bundle from the anchor directory.
|
||||
Update Tool
|
||||
// AnchorDir is where locally added CA certificates go.
|
||||
AnchorDir string
|
||||
}
|
||||
|
||||
// BootTools are the commands that change the kernel command line and rebuild
|
||||
// the initramfs.
|
||||
type BootTools struct {
|
||||
Bootloader Tool
|
||||
Initramfs Tool
|
||||
}
|
||||
|
||||
type qemuArch struct {
|
||||
binary string
|
||||
pkg string
|
||||
}
|
||||
|
||||
// qemuArches maps the guest architectures the VM config uses to the upstream
|
||||
// QEMU binary and the Debian package that ships it.
|
||||
var qemuArches = map[string]qemuArch{
|
||||
"x86_64": {binary: "qemu-system-x86_64", pkg: "qemu-system-x86"},
|
||||
"arm64": {binary: "qemu-system-aarch64", pkg: "qemu-system-arm"},
|
||||
}
|
||||
|
||||
var caTrustAnchorDirs = map[string]string{
|
||||
"update-ca-certificates": "/usr/local/share/ca-certificates",
|
||||
"update-ca-trust": "/etc/pki/ca-trust/source/anchors",
|
||||
}
|
||||
|
||||
var std = newResolver()
|
||||
|
||||
// QEMUBinary returns the QEMU system emulator for guests of arch ("x86_64" or
|
||||
// "arm64"). QEMUKVMPath wins for host-arch guests, so a stray qemu-system-* on
|
||||
// PATH never replaces the distro's packaged build.
|
||||
func QEMUBinary(arch string) (string, error) {
|
||||
return std.qemuBinary(arch)
|
||||
}
|
||||
|
||||
// SystemCATrust returns the host's trust-store update command and anchor directory.
|
||||
func SystemCATrust() (CATrust, error) {
|
||||
return std.caTrust()
|
||||
}
|
||||
|
||||
// SystemBootTools returns the host's bootloader and initramfs tools.
|
||||
func SystemBootTools() (BootTools, error) {
|
||||
return std.bootTools()
|
||||
}
|
||||
|
||||
// resolver holds the host lookups behind fields so tests run against a temp
|
||||
// root and a stubbed PATH. Only successes are cached, so a tool installed
|
||||
// after a failed lookup is found on the next call.
|
||||
type resolver struct {
|
||||
root string
|
||||
hostArch string
|
||||
lookPath func(string) (string, error)
|
||||
|
||||
mu sync.Mutex
|
||||
qemu map[string]string
|
||||
ca *CATrust
|
||||
boots *BootTools
|
||||
}
|
||||
|
||||
func newResolver() *resolver {
|
||||
return &resolver{
|
||||
hostArch: hostArch(runtime.GOARCH),
|
||||
lookPath: exec.LookPath,
|
||||
qemu: map[string]string{},
|
||||
}
|
||||
}
|
||||
|
||||
// hostArch maps a GOARCH to the architecture names the VM config uses.
|
||||
func hostArch(goarch string) string {
|
||||
switch goarch {
|
||||
case "amd64":
|
||||
return "x86_64"
|
||||
case "arm64":
|
||||
return "arm64"
|
||||
default:
|
||||
return goarch
|
||||
}
|
||||
}
|
||||
|
||||
func (r *resolver) qemuBinary(arch string) (string, error) {
|
||||
want, ok := qemuArches[arch]
|
||||
if !ok {
|
||||
return "", fmt.Errorf("no QEMU for guest architecture %q: supported architectures are x86_64 and arm64", arch)
|
||||
}
|
||||
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if path, ok := r.qemu[arch]; ok {
|
||||
return path, nil
|
||||
}
|
||||
|
||||
haveKVM := r.isExecutable(QEMUKVMPath)
|
||||
if haveKVM && arch == r.hostArch {
|
||||
r.qemu[arch] = QEMUKVMPath
|
||||
return QEMUKVMPath, nil
|
||||
}
|
||||
|
||||
path, err := r.lookPath(want.binary)
|
||||
if err == nil {
|
||||
r.qemu[arch] = path
|
||||
return path, nil
|
||||
}
|
||||
if haveKVM {
|
||||
return "", fmt.Errorf("no QEMU for %s guests: %s (package qemu-kvm) runs only %s guests, and %s (package %s) is not on PATH",
|
||||
arch, QEMUKVMPath, r.hostArch, want.binary, want.pkg)
|
||||
}
|
||||
if arch != r.hostArch {
|
||||
return "", fmt.Errorf("no QEMU for %s guests: %s (package %s) is not on PATH", arch, want.binary, want.pkg)
|
||||
}
|
||||
return "", fmt.Errorf("no QEMU for %s guests: found neither %s (package qemu-kvm) nor %s (package %s) on PATH",
|
||||
arch, QEMUKVMPath, want.binary, want.pkg)
|
||||
}
|
||||
|
||||
func (r *resolver) caTrust() (CATrust, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.ca != nil {
|
||||
return *r.ca, nil
|
||||
}
|
||||
update, err := r.pickOne("CA trust store update command", "update-ca-certificates", "update-ca-trust")
|
||||
if err != nil {
|
||||
return CATrust{}, err
|
||||
}
|
||||
r.ca = &CATrust{Update: update, AnchorDir: caTrustAnchorDirs[update.Name]}
|
||||
return *r.ca, nil
|
||||
}
|
||||
|
||||
func (r *resolver) bootTools() (BootTools, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.boots != nil {
|
||||
return *r.boots, nil
|
||||
}
|
||||
bootloader, err := r.pickOne("bootloader tool", "update-grub", "grubby")
|
||||
if err != nil {
|
||||
return BootTools{}, err
|
||||
}
|
||||
initramfs, err := r.pickOne("initramfs tool", "update-initramfs", "dracut")
|
||||
if err != nil {
|
||||
return BootTools{}, err
|
||||
}
|
||||
r.boots = &BootTools{Bootloader: bootloader, Initramfs: initramfs}
|
||||
return *r.boots, nil
|
||||
}
|
||||
|
||||
// pickOne returns the one candidate on PATH. Finding several is an error too:
|
||||
// a host carrying both tools of a pair cannot tell us which one it uses.
|
||||
func (r *resolver) pickOne(what string, candidates ...string) (Tool, error) {
|
||||
var found []Tool
|
||||
for _, name := range candidates {
|
||||
if path, err := r.lookPath(name); err == nil {
|
||||
found = append(found, Tool{Name: name, Path: path})
|
||||
}
|
||||
}
|
||||
switch len(found) {
|
||||
case 1:
|
||||
return found[0], nil
|
||||
case 0:
|
||||
return Tool{}, fmt.Errorf("no %s: found none of %s on PATH", what, strings.Join(candidates, ", "))
|
||||
default:
|
||||
names := make([]string, len(found))
|
||||
for i, t := range found {
|
||||
names[i] = fmt.Sprintf("%s (%s)", t.Name, t.Path)
|
||||
}
|
||||
return Tool{}, fmt.Errorf("ambiguous %s: found %s; cannot tell which one this host uses", what, strings.Join(names, " and "))
|
||||
}
|
||||
}
|
||||
|
||||
func (r *resolver) isExecutable(path string) bool {
|
||||
info, err := os.Stat(filepath.Join(r.root, path))
|
||||
return err == nil && info.Mode().IsRegular() && info.Mode().Perm()&0o111 != 0
|
||||
}
|
||||
@@ -0,0 +1,330 @@
|
||||
package hostos
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// fakeHost is a temp root plus a PATH made of a name-to-path map, counting
|
||||
// lookups so the caching tests can see whether the host was probed again.
|
||||
type fakeHost struct {
|
||||
root string
|
||||
path map[string]string
|
||||
lookups int
|
||||
}
|
||||
|
||||
func newFakeHost(t *testing.T) *fakeHost {
|
||||
t.Helper()
|
||||
return &fakeHost{root: t.TempDir(), path: map[string]string{}}
|
||||
}
|
||||
|
||||
func (h *fakeHost) resolver(hostArch string) *resolver {
|
||||
r := newResolver()
|
||||
r.root = h.root
|
||||
r.hostArch = hostArch
|
||||
r.lookPath = func(name string) (string, error) {
|
||||
h.lookups++
|
||||
if p, ok := h.path[name]; ok {
|
||||
return p, nil
|
||||
}
|
||||
return "", &exec.Error{Name: name, Err: exec.ErrNotFound}
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func (h *fakeHost) onPath(names ...string) {
|
||||
for _, n := range names {
|
||||
h.path[n] = "/usr/bin/" + n
|
||||
}
|
||||
}
|
||||
|
||||
func (h *fakeHost) installQEMUKVM(t *testing.T, mode os.FileMode) {
|
||||
t.Helper()
|
||||
p := filepath.Join(h.root, QEMUKVMPath)
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, nil, mode); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// requireErrNames fails unless err is non-nil and mentions every word.
|
||||
func requireErrNames(t *testing.T, err error, words ...string) {
|
||||
t.Helper()
|
||||
if err == nil {
|
||||
t.Fatalf("want an error naming %v, got nil", words)
|
||||
}
|
||||
for _, w := range words {
|
||||
if !strings.Contains(err.Error(), w) {
|
||||
t.Errorf("error does not name %q: %v", w, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestQEMUBinary_QEMUKVMWinsOverQEMUSystemOnPath(t *testing.T) {
|
||||
h := newFakeHost(t)
|
||||
h.installQEMUKVM(t, 0o755)
|
||||
h.onPath("qemu-system-x86_64")
|
||||
|
||||
got, err := h.resolver("x86_64").qemuBinary("x86_64")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != QEMUKVMPath {
|
||||
t.Fatalf("got %q, want %q", got, QEMUKVMPath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQEMUBinary_QEMUSystemWithoutQEMUKVM(t *testing.T) {
|
||||
h := newFakeHost(t)
|
||||
h.onPath("qemu-system-x86_64")
|
||||
|
||||
got, err := h.resolver("x86_64").qemuBinary("x86_64")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != "/usr/bin/qemu-system-x86_64" {
|
||||
t.Fatalf("got %q, want the qemu-system-x86_64 on PATH", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQEMUBinary_NonExecutableQEMUKVMIsIgnored(t *testing.T) {
|
||||
h := newFakeHost(t)
|
||||
h.installQEMUKVM(t, 0o644)
|
||||
h.onPath("qemu-system-x86_64")
|
||||
|
||||
got, err := h.resolver("x86_64").qemuBinary("x86_64")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != "/usr/bin/qemu-system-x86_64" {
|
||||
t.Fatalf("got %q, want the qemu-system-x86_64 on PATH", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQEMUBinary_NeitherNamesBothCandidatesAndPackages(t *testing.T) {
|
||||
h := newFakeHost(t)
|
||||
|
||||
_, err := h.resolver("x86_64").qemuBinary("x86_64")
|
||||
requireErrNames(t, err, QEMUKVMPath, "qemu-kvm", "qemu-system-x86_64", "qemu-system-x86")
|
||||
}
|
||||
|
||||
func TestQEMUBinary_ForeignArchIsNeverQEMUKVM(t *testing.T) {
|
||||
h := newFakeHost(t)
|
||||
h.installQEMUKVM(t, 0o755)
|
||||
|
||||
_, err := h.resolver("x86_64").qemuBinary("arm64")
|
||||
requireErrNames(t, err, QEMUKVMPath, "runs only x86_64 guests", "qemu-system-aarch64", "qemu-system-arm")
|
||||
}
|
||||
|
||||
func TestQEMUBinary_ForeignArchUsesQEMUSystemBesideQEMUKVM(t *testing.T) {
|
||||
h := newFakeHost(t)
|
||||
h.installQEMUKVM(t, 0o755)
|
||||
h.onPath("qemu-system-aarch64")
|
||||
|
||||
got, err := h.resolver("x86_64").qemuBinary("arm64")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != "/usr/bin/qemu-system-aarch64" {
|
||||
t.Fatalf("got %q, want the qemu-system-aarch64 on PATH", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQEMUBinary_ForeignArchWithoutQEMUKVMNamesOnlyQEMUSystem(t *testing.T) {
|
||||
h := newFakeHost(t)
|
||||
|
||||
_, err := h.resolver("x86_64").qemuBinary("arm64")
|
||||
requireErrNames(t, err, "qemu-system-aarch64", "qemu-system-arm")
|
||||
if strings.Contains(err.Error(), QEMUKVMPath) {
|
||||
t.Errorf("error offers qemu-kvm, which cannot run arm64 guests on x86_64: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQEMUBinary_UnknownArch(t *testing.T) {
|
||||
h := newFakeHost(t)
|
||||
h.onPath("qemu-system-riscv64")
|
||||
|
||||
_, err := h.resolver("x86_64").qemuBinary("riscv64")
|
||||
requireErrNames(t, err, `"riscv64"`, "x86_64", "arm64")
|
||||
}
|
||||
|
||||
func TestQEMUBinary_CachesSuccessPerArch(t *testing.T) {
|
||||
h := newFakeHost(t)
|
||||
h.onPath("qemu-system-x86_64", "qemu-system-aarch64")
|
||||
r := h.resolver("x86_64")
|
||||
|
||||
first, err := r.qemuBinary("x86_64")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lookups := h.lookups
|
||||
h.installQEMUKVM(t, 0o755)
|
||||
|
||||
again, err := r.qemuBinary("x86_64")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if again != first || h.lookups != lookups {
|
||||
t.Fatalf("second call re-probed: got %q (first %q), lookups %d -> %d", again, first, lookups, h.lookups)
|
||||
}
|
||||
|
||||
other, err := r.qemuBinary("arm64")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if other != "/usr/bin/qemu-system-aarch64" {
|
||||
t.Fatalf("arm64 got %q from the x86_64 cache entry", other)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQEMUBinary_FailureIsNotCached(t *testing.T) {
|
||||
h := newFakeHost(t)
|
||||
r := h.resolver("x86_64")
|
||||
|
||||
if _, err := r.qemuBinary("x86_64"); err == nil {
|
||||
t.Fatal("want an error before QEMU is installed")
|
||||
}
|
||||
h.installQEMUKVM(t, 0o755)
|
||||
|
||||
got, err := r.qemuBinary("x86_64")
|
||||
if err != nil {
|
||||
t.Fatalf("QEMU installed after a failed lookup is not found: %v", err)
|
||||
}
|
||||
if got != QEMUKVMPath {
|
||||
t.Fatalf("got %q, want %q", got, QEMUKVMPath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostArch(t *testing.T) {
|
||||
for goarch, want := range map[string]string{"amd64": "x86_64", "arm64": "arm64"} {
|
||||
if got := hostArch(goarch); got != want {
|
||||
t.Errorf("hostArch(%q) = %q, want %q", goarch, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSystemCATrust(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
onPath []string
|
||||
wantTool string
|
||||
wantDir string
|
||||
wantError []string
|
||||
}{
|
||||
{name: "debian", onPath: []string{"update-ca-certificates"}, wantTool: "update-ca-certificates", wantDir: "/usr/local/share/ca-certificates"},
|
||||
{name: "el", onPath: []string{"update-ca-trust"}, wantTool: "update-ca-trust", wantDir: "/etc/pki/ca-trust/source/anchors"},
|
||||
{name: "both", onPath: []string{"update-ca-certificates", "update-ca-trust"}, wantError: []string{"/usr/bin/update-ca-certificates", "/usr/bin/update-ca-trust"}},
|
||||
{name: "neither", wantError: []string{"update-ca-certificates", "update-ca-trust"}},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
h := newFakeHost(t)
|
||||
h.onPath(tc.onPath...)
|
||||
|
||||
got, err := h.resolver("x86_64").caTrust()
|
||||
if tc.wantError != nil {
|
||||
requireErrNames(t, err, tc.wantError...)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.Update.Name != tc.wantTool || got.Update.Path != "/usr/bin/"+tc.wantTool || got.AnchorDir != tc.wantDir {
|
||||
t.Fatalf("got %+v, want %s in %s", got, tc.wantTool, tc.wantDir)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSystemCATrust_CachesSuccess(t *testing.T) {
|
||||
h := newFakeHost(t)
|
||||
h.onPath("update-ca-certificates")
|
||||
r := h.resolver("x86_64")
|
||||
|
||||
if _, err := r.caTrust(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h.onPath("update-ca-trust")
|
||||
|
||||
got, err := r.caTrust()
|
||||
if err != nil {
|
||||
t.Fatalf("cached result was re-probed: %v", err)
|
||||
}
|
||||
if got.Update.Name != "update-ca-certificates" {
|
||||
t.Fatalf("got %q, want the cached update-ca-certificates", got.Update.Name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSystemBootTools(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
onPath []string
|
||||
wantBootloader string
|
||||
wantInitramfs string
|
||||
wantError []string
|
||||
}{
|
||||
{name: "debian", onPath: []string{"update-grub", "update-initramfs"}, wantBootloader: "update-grub", wantInitramfs: "update-initramfs"},
|
||||
{name: "el", onPath: []string{"grubby", "dracut"}, wantBootloader: "grubby", wantInitramfs: "dracut"},
|
||||
{name: "both bootloader tools", onPath: []string{"update-grub", "grubby", "dracut"}, wantError: []string{"update-grub", "grubby"}},
|
||||
{name: "both initramfs tools", onPath: []string{"grubby", "update-initramfs", "dracut"}, wantError: []string{"update-initramfs", "dracut"}},
|
||||
{name: "no bootloader tool", onPath: []string{"dracut"}, wantError: []string{"update-grub", "grubby"}},
|
||||
{name: "no initramfs tool", onPath: []string{"grubby"}, wantError: []string{"update-initramfs", "dracut"}},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
h := newFakeHost(t)
|
||||
h.onPath(tc.onPath...)
|
||||
|
||||
got, err := h.resolver("x86_64").bootTools()
|
||||
if tc.wantError != nil {
|
||||
requireErrNames(t, err, tc.wantError...)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.Bootloader.Name != tc.wantBootloader || got.Initramfs.Name != tc.wantInitramfs {
|
||||
t.Fatalf("got %+v, want %s and %s", got, tc.wantBootloader, tc.wantInitramfs)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsQEMUProcess(t *testing.T) {
|
||||
for comm, want := range map[string]bool{
|
||||
// comm truncates to 15 bytes, so qemu-system-x86_64 reads qemu-system-x86.
|
||||
"qemu-system-x86\n": true,
|
||||
"qemu-system-aar": true,
|
||||
"qemu-kvm\n": true,
|
||||
"qemu-img": false,
|
||||
"qemu-kvm-helper": false,
|
||||
"qemu-nbd": false,
|
||||
"nbdkit": false,
|
||||
"": false,
|
||||
} {
|
||||
if got := IsQEMUProcess(comm); got != want {
|
||||
t.Errorf("IsQEMUProcess(%q) = %v, want %v", comm, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestQEMUProcessPatternMatchesEveryQEMUName(t *testing.T) {
|
||||
re := regexp.MustCompile(QEMUProcessPattern)
|
||||
for _, line := range []string{
|
||||
"spinifex 4242 qemu-system-x86_64 -name guest=i-0abc,debug-threads=on",
|
||||
"spinifex 4243 /usr/libexec/qemu-kvm -name guest=i-0def,debug-threads=on",
|
||||
} {
|
||||
if !re.MatchString(line) {
|
||||
t.Errorf("QEMUProcessPattern %q does not match %q", QEMUProcessPattern, line)
|
||||
}
|
||||
}
|
||||
if re.MatchString("spinifex 4244 nbdkit --unix /run/spinifex/nbd.sock") {
|
||||
t.Errorf("QEMUProcessPattern %q matches an nbdkit line", QEMUProcessPattern)
|
||||
}
|
||||
}
|
||||
@@ -10,10 +10,12 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/host/hostos"
|
||||
)
|
||||
|
||||
// InstanceHostingNode returns the cluster node running the named instance's
|
||||
// qemu process (ps auxw | grep qemu-system). Returns nil if no node owns it.
|
||||
// QEMU process (ps auxw | grep -E hostos.QEMUProcessPattern). Returns nil if no node owns it.
|
||||
func InstanceHostingNode(t *testing.T, c *Cluster, instanceID string) *Node {
|
||||
t.Helper()
|
||||
hosting := InstanceHostingNodes(t, c, instanceID, nil)
|
||||
@@ -56,7 +58,7 @@ func InstanceHostingNodes(t *testing.T, c *Cluster, instanceID string, skip []st
|
||||
|
||||
// runPSGrep runs the ps+grep pipeline on a node and returns combined output.
|
||||
func runPSGrep(ctx context.Context, ssh *PeerSSH, n Node, instanceID string) (string, error) {
|
||||
cmd := fmt.Sprintf("ps auxw | grep %q | grep qemu-system | grep -v grep", instanceID)
|
||||
cmd := fmt.Sprintf("ps auxw | grep %q | grep -E %s | grep -v grep", instanceID, ShellQuote(hostos.QEMUProcessPattern))
|
||||
out, err := ssh.Run(ctx, n.Addr, cmd)
|
||||
if err != nil {
|
||||
// exit 1 from grep means no match, not a transport error.
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/host/hostos"
|
||||
"golang.org/x/crypto/ssh"
|
||||
)
|
||||
|
||||
@@ -380,7 +381,7 @@ func (w *Witness) waitForRunning(ctx context.Context, id string, timeout time.Du
|
||||
// findHost walks cluster nodes looking for the QEMU process that owns instanceID
|
||||
// (the instance ID appears in -pidfile, -qmp, and -name on the QEMU command line).
|
||||
func (w *Witness) findHost(ctx context.Context, instanceID string) (Node, error) {
|
||||
cmd := fmt.Sprintf("ps auxw | grep %s | grep qemu-system | grep -v grep", ShellQuote(instanceID))
|
||||
cmd := fmt.Sprintf("ps auxw | grep %s | grep -E %s | grep -v grep", ShellQuote(instanceID), ShellQuote(hostos.QEMUProcessPattern))
|
||||
// EC2 "running" state is reported before QEMU is forked; give it a window.
|
||||
deadline := time.Now().Add(30 * time.Second)
|
||||
const interval = 1 * time.Second
|
||||
|
||||
@@ -274,9 +274,9 @@ find_instance_node() {
|
||||
for ip in "${NODE_IPS[@]}"; do
|
||||
local found
|
||||
if [ "$ip" = "$LOCAL_IP" ]; then
|
||||
found=$(ps auxw | grep "$instance_id" | grep qemu-system | grep -v grep || true)
|
||||
found=$(ps auxw | grep "$instance_id" | grep -E "qemu-system|qemu-kvm" | grep -v grep || true)
|
||||
else
|
||||
found=$(peer_ssh "$ip" "ps auxw | grep '$instance_id' | grep qemu-system | grep -v grep" 2>/dev/null || true)
|
||||
found=$(peer_ssh "$ip" "ps auxw | grep '$instance_id' | grep -E 'qemu-system|qemu-kvm' | grep -v grep" 2>/dev/null || true)
|
||||
fi
|
||||
if [ -n "$found" ]; then
|
||||
echo "$ip"
|
||||
@@ -298,9 +298,9 @@ get_remote_ssh_port() {
|
||||
while [ $attempt -lt $max_attempts ]; do
|
||||
local qemu_cmd
|
||||
if [ "$node_ip" = "$LOCAL_IP" ]; then
|
||||
qemu_cmd=$(ps auxw | grep "$instance_id" | grep qemu-system | grep -v grep || true)
|
||||
qemu_cmd=$(ps auxw | grep "$instance_id" | grep -E "qemu-system|qemu-kvm" | grep -v grep || true)
|
||||
else
|
||||
qemu_cmd=$(peer_ssh "$node_ip" "ps auxw | grep '$instance_id' | grep qemu-system | grep -v grep" 2>/dev/null || true)
|
||||
qemu_cmd=$(peer_ssh "$node_ip" "ps auxw | grep '$instance_id' | grep -E 'qemu-system|qemu-kvm' | grep -v grep" 2>/dev/null || true)
|
||||
fi
|
||||
|
||||
if [ -n "$qemu_cmd" ]; then
|
||||
|
||||
Reference in New Issue
Block a user