fix: write MariaDB's startup spelling for time_zone into the option file

mariadbd has no time_zone startup option — the name only exists for SET — so
the resolved parameter group aborted the server while it parsed defaults, before
it opened the datadir. No pidfile was ever written and the reconciler failed
every instance at the 20-minute engine-health deadline.

ParameterSpec gains an option-file spelling, used only when rendering the
include. time_zone is the one setting that needs it, as default_time_zone, which
resolves @@global.time_zone to the same value. The customer-facing name is
unchanged: the API still reports time_zone and a live apply still issues
SET GLOBAL time_zone. Names read back out of a generated file return to the
catalog's namespace first, so a startup spelling cannot read as an unknown name
and report a pending restart nothing could clear.

setup.sh now feeds the catalog's resolved defaults to the image's own mariadbd
at build time, so a name the server refuses fails the build rather than every
instance built from it.

.github/workflows/e2e.yml

@@ -52,7 +52,7 @@ jobs:
     # inputs.suites.
     if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-single') || contains(github.event.inputs.suites, 'e2e-iam') || contains(github.event.inputs.suites, 'e2e-cert') || contains(github.event.inputs.suites, 'e2e-eks') || contains(github.event.inputs.suites, 'e2e-ecs') || contains(github.event.inputs.suites, 'e2e-storagegrowth') || contains(github.event.inputs.suites, 'e2e-partialblock') || contains(github.event.inputs.suites, 'e2e-rds')
     runs-on: [self-hosted, ci-single]
-    timeout-minutes: 50
+    timeout-minutes: 60
     env:
       TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
       ARTIFACT_DIR: /tmp/spinifex-e2e-artifacts-${{ github.run_id }}
@@ -250,53 +250,58 @@ jobs:
           EOF
           ok "kubectl installed on VM"
 
-      # CreateDBInstance boots the DB VM from the rds-postgres AMI, so that image
+      # CreateDBInstance boots the DB VM from an RDS AMI, so both engines' images
       # must be in the VM's image store before the suite runs — same split as the
       # eks-node pair above: build on the runner (needs source + network), import
-      # on the VM (needs predastore/NATS). The suite's psql client needs no step
-      # here: it runs inside a client VM that installs postgresql-client from its
-      # own user-data, because a private-only endpoint is unreachable from the
-      # runner and from the cluster host alike.
-      - name: Build rds-postgres AMI (libguestfs, runner)
+      # on the VM (needs predastore/NATS). The suite's psql and mariadb clients
+      # need no step here: they run inside a client VM that installs them from
+      # its own user-data, because a private-only endpoint is unreachable from
+      # the runner and from the cluster host alike.
+      - name: Build RDS AMIs (libguestfs, runner)
         if: contains(env.SUITES, 'e2e-rds')
         working-directory: spinifex
         run: |
           set -euo pipefail
           source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
-          banner "build rds-postgres AMI (libguestfs — no host devices)"
           sudo apt-get update -qq
           sudo apt-get install -y --no-install-recommends libguestfs-tools qemu-utils
           export LIBGUESTFS_BACKEND=direct
           sudo chmod 0644 /boot/vmlinuz-* || true
-          group_open "make build-rds-postgres-image"
-          make build-rds-postgres-image
-          group_close
-          ok "raw: $(du -h /tmp/rds-postgres-image-build/rds-postgres-alpine.raw | cut -f1)"
+          for IMAGE in rds-postgres rds-mariadb; do
+            banner "build ${IMAGE} AMI (libguestfs — no host devices)"
+            group_open "make build-${IMAGE}-image"
+            make "build-${IMAGE}-image"
+            group_close
+            ok "raw: $(du -h "/tmp/${IMAGE}-image-build/${IMAGE}-alpine.raw" | cut -f1)"
+          done
 
-      - name: Import rds-postgres AMI on VM
+      - name: Import RDS AMIs on VM
         if: contains(env.SUITES, 'e2e-rds')
         run: |
           set -euo pipefail
           source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
-          eval "$(grep -E '^(ALPINE_VERSION|AMI_NAME|SYSTEM_TAG)=' \
-            spinifex/scripts/images/rds-postgres/manifest.conf)"
-          # SYSTEM_TAG carries three tags, not one: managed-by marks it a system
-          # image and engine/engine-version are what an EngineVersion request
-          # resolves against. --tag is repeatable, so expand rather than pass the
-          # whole string as a single value.
-          TAG_ARGS=""
-          for tag in ${SYSTEM_TAG}; do TAG_ARGS="${TAG_ARGS} --tag ${tag}"; done
           SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
           VM=tf-user@${{ steps.vm.outputs.wan_ip }}
-          banner "import rds-postgres AMI to VM image store"
-          scp "${SSH_OPTS[@]}" \
-            /tmp/rds-postgres-image-build/rds-postgres-alpine.raw \
-            "$VM":/tmp/rds-postgres-alpine.raw
-          ssh "${SSH_OPTS[@]}" "$VM" \
-            "spx admin images import --file /tmp/rds-postgres-alpine.raw \
-               --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
-               --boot-mode bios --ami-name ${AMI_NAME}${TAG_ARGS}" >/dev/null 2>&1
-          ok "rds-postgres AMI imported"
+          for IMAGE in rds-postgres rds-mariadb; do
+            eval "$(grep -E '^(ALPINE_VERSION|AMI_NAME|SYSTEM_TAG)=' \
+              "spinifex/scripts/images/${IMAGE}/manifest.conf")"
+            # SYSTEM_TAG carries four tags, not one: managed-by marks it a system
+            # image, engine/engine-version are what an EngineVersion request
+            # resolves against, and the contract tag gates the data-volume format
+            # grant. --tag is repeatable, so expand rather than pass the whole
+            # string as a single value.
+            TAG_ARGS=""
+            for tag in ${SYSTEM_TAG}; do TAG_ARGS="${TAG_ARGS} --tag ${tag}"; done
+            banner "import ${IMAGE} AMI to VM image store"
+            scp "${SSH_OPTS[@]}" \
+              "/tmp/${IMAGE}-image-build/${IMAGE}-alpine.raw" \
+              "$VM":"/tmp/${IMAGE}-alpine.raw"
+            ssh "${SSH_OPTS[@]}" "$VM" \
+              "spx admin images import --file /tmp/${IMAGE}-alpine.raw \
+                 --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
+                 --boot-mode bios --ami-name ${AMI_NAME}${TAG_ARGS}" >/dev/null 2>&1
+            ok "${IMAGE} AMI imported"
+          done
 
       - name: Build E2E test binaries on runner
         working-directory: spinifex/tests/e2e
@@ -383,9 +388,10 @@ jobs:
                echo \"::group::suite \$SUITE\"; \
                printf '\n%s%s━━ suite %s ━━%s\n' \"\$B\" \"\$C\" \"\$SUITE\" \"\$R\"; \
                date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
+               case \$SUITE in rds) TMO=45m ;; *) TMO=30m ;; esac; \
                GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
                  ARTIFACT_DIR='${ARTIFACT_DIR}' \
-                 ./\$SUITE.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \
+                 ./\$SUITE.test -test.v -test.timeout=\$TMO ${TEST_RUN_FLAG} \
                  2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log' \
                       | sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \
                rc=\${PIPESTATUS[0]}; \

.github/workflows/publish-system-ami.yml

@@ -2,18 +2,20 @@ name: Publish System AMIs
 
 # Builds the system AMIs (libguestfs, on the runner) and publishes each
 # compressed qcow2 + .sha256 sidecar to the Cloudflare R2 bucket that backs
-# iso.mulgadc.com/system-ami/. Five images ship:
+# iso.mulgadc.com/system-ami/. Six images ship:
 #   spinifex-eks-node       Alpine EKS K3s node (server|agent, role at boot)
 #   spinifex-ecs-node       Alpine ECS container instance
 #   spinifex-eks-node-gpu   Ubuntu EKS K3s GPU worker (NVIDIA driver + toolkit)
 #   spinifex-ecs-node-gpu   Ubuntu ECS GPU container instance
 #   spinifex-rds-postgres   Alpine RDS PostgreSQL instance
+#   spinifex-rds-mariadb    Alpine RDS MariaDB instance
 # Operators then install them on any cluster with:
 #   spx admin images import --name spinifex-eks-node
 #   spx admin images import --name spinifex-ecs-node
 #   spx admin images import --name spinifex-eks-node-gpu
 #   spx admin images import --name spinifex-ecs-node-gpu
 #   spx admin images import --name spinifex-rds-postgres
+#   spx admin images import --name spinifex-rds-mariadb
 #
 # Manual-only: publishing overwrites the mirror artifact, so it is a deliberate
 # action, not something every push should trigger. The `variant` input scopes
@@ -59,7 +61,7 @@ jobs:
           set -euo pipefail
           cpu='{"name":"spinifex-eks-node","manifest":"scripts/images/eks-node/manifest.conf"},{"name":"spinifex-ecs-node","manifest":"scripts/images/ecs-agent/manifest.conf"}'
           gpu='{"name":"spinifex-eks-node-gpu","manifest":"scripts/images/eks-node-gpu/manifest.conf"},{"name":"spinifex-ecs-node-gpu","manifest":"scripts/images/ecs-agent-gpu/manifest.conf"}'
-          rds='{"name":"spinifex-rds-postgres","manifest":"scripts/images/rds-postgres/manifest.conf"}'
+          rds='{"name":"spinifex-rds-postgres","manifest":"scripts/images/rds-postgres/manifest.conf"},{"name":"spinifex-rds-mariadb","manifest":"scripts/images/rds-mariadb/manifest.conf"}'
           case "${VARIANT}" in
             cpu) items="${cpu}" ;;
             gpu) items="${gpu}" ;;

Makefile

@@ -111,6 +111,12 @@ build-rds-postgres-image: ## Build the spinifex-rds-postgres AMI (Alpine + Postg
 import-rds-postgres-image: ## Build + register the rds-postgres AMI (requires a running cluster)
 	$(MAKE) build-system-image IMAGE=rds-postgres IMPORT=1
 
+build-rds-mariadb-image: ## Build the spinifex-rds-mariadb AMI (Alpine + MariaDB 11.8 + rds-init; IMPORT=1 to register)
+	$(MAKE) build-system-image IMAGE=rds-mariadb
+
+import-rds-mariadb-image: ## Build + register the rds-mariadb AMI (requires a running cluster)
+	$(MAKE) build-system-image IMAGE=rds-mariadb IMPORT=1
+
 MICROVM_OUT_DIR := build/microvm
 MICROVM_ARTIFACTS := $(MICROVM_OUT_DIR)/vmlinuz $(MICROVM_OUT_DIR)/initramfs.cpio.gz
 MICROVM_INPUTS := scripts/build-microvm-image.sh $(MICROVM_OUT_DIR)/init.sh $(MICROVM_OUT_DIR)/inittab bin/lb-agent
@@ -388,7 +394,7 @@ distro-arm64:
 distro-clean:
 	rm -rf dist/
 
-.PHONY: build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-images test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle manifest-check manifest-lint manifest-lint-update \
+.PHONY: build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-images test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle manifest-check manifest-lint manifest-lint-update \
 	deploy reinstall clean \
 	install-system install-go install-aws quickinstall \
 	lint fix govulncheck nilaway \

README.md

@@ -12,7 +12,7 @@
   <a href="#what-is-spinifex">What is Spinifex?</a> ·
   <a href="#the-platform">Platform</a> ·
   <a href="#three-ways-to-deploy">Deploy</a> ·
-  <a href="#aws-compatibility">AWS Compatibility</a> ·
+  <a href="#aws-compatibility">AWS services</a> ·
   <a href="#core-components">Components</a> ·
   <a href="#architecture-at-a-glance">Architecture</a> ·
   <a href="#installation">Installation</a> ·
@@ -48,7 +48,7 @@ You change *where* your software runs, not *what* it is.
 
 From commodity hardware up to unmodified AWS tooling, every layer is replaceable and yours to own.
 
-## Three Ways To Deploy
+## Three ways to deploy
 
 One AWS-compatible surface, three deployment shapes. Pick the one that matches your reality; the platform on top is the same.
 
@@ -64,21 +64,21 @@ Bring your own hardware and host Spinifex in your own data centre. A real multi-
 
 Cloud where the cloud can't reach. Air-gapped sites, vehicles, vessels, factories, and clinics. Compute next to the data, running through disconnection, on hardware you own. The same AWS APIs your software already uses.
 
-## AWS Compatibility
+## AWS compatibility
 
 Speak the AWS API surface, natively. The AWS SDKs, AWS CLI, and Terraform: everything you deploy on AWS deploys on Spinifex unchanged. At the edge, on-premise, or on a partner Neocloud.
 
 | Service | What it is | Status |
 | --- | --- | --- |
 | **EC2** | Compute | Available |
-| **EBS** | Block Storage | Available |
-| **S3** | Object Storage | Available |
+| **EBS** | Block storage | Available |
+| **S3** | Object storage | Available |
 | **VPC** | Networking | Available |
-| **IAM** | Identity & Auth | Available |
-| **ALB / NLB** | Load Balancers | Available |
+| **IAM** | Identity & auth | Available |
+| **ALB / NLB** | Load balancers | Available |
 | **EKS** | Kubernetes | Available |
-| **ECR** | Container Registry | Available |
-| **ECS** | Container Service | Available |
+| **ECR** | Container registry | Available |
+| **ECS** | Container service | Available |
 | **RDS** | Databases | Available |
 | **Bedrock** | AI Deployment | Q3 2026 |
 
@@ -112,14 +112,6 @@ Spinifex is a minimal VM orchestration layer built on top of QEMU, exposing APIs
 - Multipart uploads, streaming reads/writes
 - Data redundancy with Reed-Solomon encoding
 
-### Northstar (Authoritative DNS)
-
-[Northstar](https://github.com/mulgadc/northstar) is a lightweight authoritative DNS server that handles both internal service discovery and public-facing DNS for a Spinifex cluster. Zones are human-readable TOML files, served from the local filesystem or synced from Predastore, keeping DNS configuration in the same object storage as the rest of your infrastructure.
-
-- Authoritative DNS over UDP, TCP, and DNS-over-TLS
-- Service discovery for cluster components via SRV records
-- Zone files stored locally or in S3-compatible storage
-
 ## Architecture at a Glance
 
 <p align="center">
@@ -139,7 +131,7 @@ Every AWS API call is authenticated at the gateway, published to a NATS subject,
 
 ## Installation
 
-Installation requires an Ubuntu 26.04 or Debian 13 system. See the detailed documentation at [docs.mulgadc.com](https://docs.mulgadc.com) for installing Spinifex.
+Installation requires an Ubuntu / Debian system. See the detailed documentation at [docs.mulgadc.com](https://docs.mulgadc.com) for maintaining and installing Spinifex.
 
 ### Bare Metal ISO
 
@@ -149,11 +141,16 @@ The recommended installation is a [bootable x86 installer](https://iso.mulgadc.c
 curl -fLO https://iso.mulgadc.com/spinifex.iso
 ```
 
-Follow the [USB install guide](https://docs.mulgadc.com/docs/install-usb) to write the ISO to USB and install on your hardware.
+Follow the [USB install guide](https://docs.mulgadc.com/docs/install-usb) to write the ISO to USB and install on your hardware. The install guide walks through the full process.
+
 
 ### Single Node Install
 
->*Prerequisite:* Spinifex requires a Linux bridge configured on the host for VM networking. See the [single-node install guide](https://docs.mulgadc.com/docs/install#prerequisites) for setup details.
+The installation is straightforward to set up and running on a single node for testing purposes. Debian 13 is currently supported, additional Linux distributions are on the immediate roadmap.
+
+>*Prerequisite:* Linux bridge for networking.
+
+Spinifex requires a Linux bridge configured on the host for VM networking. See the [single-node install guide](https://docs.mulgadc.com/docs/install#prerequisites) prerequisites for setup details.
 
 ```bash
 curl -fsSL https://install.mulgadc.com | bash
@@ -171,7 +168,7 @@ aws ec2 describe-instance-types
 
 ### Development Setup
 
-For a complete development environment see the [Source Install](https://docs.mulgadc.com/docs/install-source) documentation.
+For a complete development environment see the [Source Install](https://docs.mulgadc.com/docs/install-source) documentation
 
 ### Component Repositories
 
@@ -179,7 +176,6 @@ Spinifex coordinates these independent components:
 
 - **[Predastore](https://github.com/mulgadc/predastore)** - S3-compatible object storage
 - **[Viperblock](https://github.com/mulgadc/viperblock)** - EBS-compatible block storage
-- **[Northstar](https://github.com/mulgadc/northstar)** - Authoritative DNS server
 
 Each component can be developed independently. See component-specific documentation for focused development guides.
 
@@ -193,6 +189,10 @@ Spinifex ships with a built-in web console — an optional alternative to the AW
 
 The console is served by each node on port `3000` over TLS, and becomes available as soon as `spinifex.target` is up:
 
+```bash
+open https://YOUR_NODE_IP:3000
+```
+
 - **Same API, different surface.** Every action in the UI is the same AWS SigV4 call the CLI makes — so RBAC, audit trails, and IAM policies apply uniformly.
 - **Single sign-on against your AWS credentials.** Log in with the access keys from `~/.aws/credentials` on the node where Spinifex is installed — no separate user database.
 - **Self-hosted, works offline.** The UI is embedded in the Spinifex binary and served from the node itself. No external CDN, no analytics calls, no cloud dependency.

cmd/installer/install/blockdev.go

@@ -45,7 +45,6 @@ var (
 	sysClassBlockDir = "/sys/class/block"
 	devByIDDir       = "/dev/disk/by-id"
 	procMountsPath   = "/proc/mounts"
-	procSwapsPath    = "/proc/swaps"
 )
 
 // ListDisks enumerates install-candidate disks from /sys/block. It reads sysfs
@@ -133,144 +132,6 @@ func partitionPath(disk string, n int) string {
 	return fmt.Sprintf("%s%d", disk, n)
 }
 
-// readKernelPartitions returns the partition device names the kernel currently
-// publishes for a disk, sorted.
-//
-// It reads sysfs rather than /dev deliberately. A partition device node
-// outlives the table that created it, so a stale node is indistinguishable
-// from a fresh one by existence alone — which is how a disk whose old table
-// the kernel never dropped passes every check and then gets formatted at the
-// previous layout's offsets.
-func readKernelPartitions(disk string) ([]string, error) {
-	name := filepath.Base(disk)
-	entries, err := os.ReadDir(filepath.Join(sysBlockDir, name))
-	if err != nil {
-		return nil, fmt.Errorf("read the kernel's partition list for %s: %w", disk, err)
-	}
-	var out []string
-	for _, e := range entries {
-		// Partitions are the child directories carrying a "partition" attribute.
-		// "queue", "device" and the plain attribute files sit alongside them.
-		if _, err := os.Stat(filepath.Join(sysBlockDir, name, e.Name(), "partition")); err != nil {
-			continue
-		}
-		out = append(out, e.Name())
-	}
-	slices.Sort(out)
-	return out, nil
-}
-
-// kernelPartitions is indirected so tests can drive the partition-table
-// assertions without a real block device.
-var kernelPartitions = readKernelPartitions
-
-// partitionDevice maps a kernel partition name back to its device node,
-// resolved alongside the disk it belongs to rather than assuming /dev.
-func partitionDevice(disk, part string) string {
-	return filepath.Join(filepath.Dir(disk), part)
-}
-
-// diskHolders names the md, LVM or device-mapper devices claiming this disk or
-// one of its partitions. A holder is the usual reason the kernel refuses to
-// re-read a partition table, so it is what the operator needs told when the
-// installer gives up on a disk.
-func diskHolders(disk string) []string {
-	name := filepath.Base(disk)
-	dirs := []string{filepath.Join(sysBlockDir, name, "holders")}
-	parts, _ := kernelPartitions(disk)
-	for _, p := range parts {
-		dirs = append(dirs, filepath.Join(sysBlockDir, name, p, "holders"))
-	}
-
-	seen := map[string]bool{}
-	var out []string
-	for _, dir := range dirs {
-		entries, err := os.ReadDir(dir)
-		if err != nil {
-			continue
-		}
-		for _, e := range entries {
-			if seen[e.Name()] {
-				continue
-			}
-			seen[e.Name()] = true
-			out = append(out, e.Name())
-		}
-	}
-	slices.Sort(out)
-	return out
-}
-
-// diskMount is one filesystem mounted from a disk the installer is about to
-// erase.
-type diskMount struct {
-	Device     string
-	Mountpoint string
-}
-
-// devicesOnDisk is the set of device paths belonging to a disk: the whole disk
-// and every partition the kernel currently publishes. Matching /proc tables
-// against a set rather than a string prefix keeps /dev/nvme0n1 from claiming
-// /dev/nvme0n11.
-func devicesOnDisk(disk string) map[string]bool {
-	out := map[string]bool{disk: true}
-	parts, _ := kernelPartitions(disk)
-	for _, p := range parts {
-		out[partitionDevice(disk, p)] = true
-	}
-	return out
-}
-
-// mountsOnDisk returns the filesystems mounted from a disk, deepest mountpoint
-// first so a nested mount is released before the one it sits inside.
-func mountsOnDisk(disk string) []diskMount {
-	devs := devicesOnDisk(disk)
-	var out []diskMount
-	for _, fields := range procTable(procMountsPath, 0) {
-		if len(fields) >= 2 && devs[fields[0]] {
-			out = append(out, diskMount{Device: fields[0], Mountpoint: fields[1]})
-		}
-	}
-	slices.SortFunc(out, func(a, b diskMount) int {
-		return strings.Count(b.Mountpoint, "/") - strings.Count(a.Mountpoint, "/")
-	})
-	return out
-}
-
-// swapsOnDisk returns the active swap areas backed by a disk.
-func swapsOnDisk(disk string) []string {
-	devs := devicesOnDisk(disk)
-	var out []string
-	// The first line of /proc/swaps is a column header, not an entry.
-	for _, fields := range procTable(procSwapsPath, 1) {
-		if len(fields) >= 1 && devs[fields[0]] {
-			out = append(out, fields[0])
-		}
-	}
-	return out
-}
-
-// procTable splits a whitespace-delimited /proc table into fields per line,
-// dropping the first skip lines and any blank ones.
-func procTable(path string, skip int) [][]string {
-	data, err := os.ReadFile(path)
-	if err != nil {
-		return nil
-	}
-	var out [][]string
-	n := 0
-	for line := range strings.SplitSeq(string(data), "\n") {
-		n++
-		if n <= skip {
-			continue
-		}
-		if fields := strings.Fields(line); len(fields) > 0 {
-			out = append(out, fields)
-		}
-	}
-	return out
-}
-
 // byIDPreference orders the /dev/disk/by-id/ namespaces we prefer to name a
 // pool member by. wwn- ids are last: they are stable but opaque, so a degraded
 // pool reports a number nobody can map to a drive bay.

cmd/installer/install/blockdev_test.go

@@ -5,7 +5,6 @@ import (
 	"path/filepath"
 	"slices"
 	"strconv"
-	"strings"
 	"testing"
 )
 
@@ -41,7 +40,6 @@ func newFakeSysfs(t *testing.T, disks ...fakeDisk) *fakeSysfs {
 	swap(&sysClassBlockDir, filepath.Join(f.root, "sys/class/block"))
 	swap(&devByIDDir, filepath.Join(f.root, "dev/disk/by-id"))
 	swap(&procMountsPath, filepath.Join(f.root, "proc/mounts"))
-	swap(&procSwapsPath, filepath.Join(f.root, "proc/swaps"))
 
 	f.mkdirAll(sysBlockDir)
 	f.mkdirAll(sysClassBlockDir)
@@ -115,18 +113,6 @@ func (f *fakeSysfs) setMounts(content string) {
 	f.write(procMountsPath, content)
 }
 
-func (f *fakeSysfs) setSwaps(content string) {
-	f.t.Helper()
-	f.write(procSwapsPath, content)
-}
-
-// addHolder marks a partition as claimed by md, LVM or device-mapper — the
-// state that makes the kernel refuse to re-read the disk's partition table.
-func (f *fakeSysfs) addHolder(disk, part, holder string) {
-	f.t.Helper()
-	f.mkdirAll(filepath.Join(sysBlockDir, disk, part, "holders", holder))
-}
-
 func boolAttr(b bool) string {
 	if b {
 		return "1\n"
@@ -366,119 +352,3 @@ func TestHasAnyPrefix(t *testing.T) {
 		t.Error("sda should not match")
 	}
 }
-
-func TestReadKernelPartitionsListsOnlyPartitions(t *testing.T) {
-	newFakeSysfs(t, fakeDisk{
-		name: "nvme0n1", sectors: 1 << 22,
-		partitions: []string{"nvme0n1p3", "nvme0n1p1", "nvme0n1p2"},
-	})
-
-	got, err := readKernelPartitions("/dev/nvme0n1")
-	if err != nil {
-		t.Fatalf("readKernelPartitions: %v", err)
-	}
-	// Sorted, and "queue" and "device" alongside them are not partitions.
-	want := []string{"nvme0n1p1", "nvme0n1p2", "nvme0n1p3"}
-	if !slices.Equal(got, want) {
-		t.Errorf("got %v, want %v", got, want)
-	}
-}
-
-func TestReadKernelPartitionsIsEmptyForAnUnpartitionedDisk(t *testing.T) {
-	newFakeSysfs(t, fakeDisk{name: "nvme2n1", sectors: 1 << 22})
-
-	got, err := readKernelPartitions("/dev/nvme2n1")
-	if err != nil {
-		t.Fatalf("readKernelPartitions: %v", err)
-	}
-	if len(got) != 0 {
-		t.Errorf("got %v, want none", got)
-	}
-}
-
-func TestReadKernelPartitionsFailsOnAnUnknownDisk(t *testing.T) {
-	newFakeSysfs(t)
-
-	if _, err := readKernelPartitions("/dev/nvme9n1"); err == nil {
-		t.Fatal("expected an error for a disk with no sysfs entry")
-	}
-}
-
-func TestDiskHoldersNamesEveryClaimant(t *testing.T) {
-	f := newFakeSysfs(t, fakeDisk{
-		name: "sda", sectors: 1 << 22, partitions: []string{"sda1", "sda2"},
-	})
-	f.addHolder("sda", "sda1", "md0")
-	f.addHolder("sda", "sda2", "dm-0")
-
-	if got, want := diskHolders("/dev/sda"), []string{"dm-0", "md0"}; !slices.Equal(got, want) {
-		t.Errorf("got %v, want %v", got, want)
-	}
-}
-
-func TestDiskHoldersIsEmptyWhenNothingClaimsTheDisk(t *testing.T) {
-	newFakeSysfs(t, fakeDisk{name: "sda", sectors: 1 << 22, partitions: []string{"sda1"}})
-
-	if got := diskHolders("/dev/sda"); len(got) != 0 {
-		t.Errorf("got %v, want none", got)
-	}
-}
-
-func TestMountsOnDiskFindsOnlyThisDiskDeepestFirst(t *testing.T) {
-	f := newFakeSysfs(t,
-		fakeDisk{name: "sda", sectors: 1 << 22, partitions: []string{"sda1", "sda2"}},
-		fakeDisk{name: "sdb", sectors: 1 << 22, partitions: []string{"sdb1"}},
-	)
-	f.setMounts(strings.Join([]string{
-		"/dev/sda1 /oldroot ext4 rw 0 0",
-		"/dev/sda2 /oldroot/boot/efi vfat rw 0 0",
-		"/dev/sdb1 /other ext4 rw 0 0",
-		"proc /proc proc rw 0 0",
-	}, "\n") + "\n")
-
-	got := mountsOnDisk("/dev/sda")
-	want := []diskMount{
-		{Device: "/dev/sda2", Mountpoint: "/oldroot/boot/efi"},
-		{Device: "/dev/sda1", Mountpoint: "/oldroot"},
-	}
-	if !slices.Equal(got, want) {
-		t.Errorf("got %v, want %v", got, want)
-	}
-}
-
-// /dev/nvme0n1 must not claim /dev/nvme0n11, which a string-prefix match would.
-func TestMountsOnDiskDoesNotMatchASiblingNamespace(t *testing.T) {
-	f := newFakeSysfs(t,
-		fakeDisk{name: "nvme0n1", sectors: 1 << 22, partitions: []string{"nvme0n1p1"}},
-		fakeDisk{name: "nvme0n11", sectors: 1 << 22, partitions: []string{"nvme0n11p1"}},
-	)
-	f.setMounts("/dev/nvme0n11p1 /elsewhere ext4 rw 0 0\n")
-
-	if got := mountsOnDisk("/dev/nvme0n1"); len(got) != 0 {
-		t.Errorf("got %v, want none", got)
-	}
-}
-
-func TestSwapsOnDiskSkipsTheHeaderAndOtherDisks(t *testing.T) {
-	f := newFakeSysfs(t,
-		fakeDisk{name: "sda", sectors: 1 << 22, partitions: []string{"sda1"}},
-		fakeDisk{name: "sdb", sectors: 1 << 22, partitions: []string{"sdb1"}},
-	)
-	f.setSwaps(strings.Join([]string{
-		"Filename\t\t\t\tType\t\tSize\tUsed\tPriority",
-		"/dev/sda1 partition 8388604 0 -2",
-		"/dev/sdb1 partition 8388604 0 -3",
-	}, "\n") + "\n")
-
-	if got, want := swapsOnDisk("/dev/sda"), []string{"/dev/sda1"}; !slices.Equal(got, want) {
-		t.Errorf("got %v, want %v", got, want)
-	}
-}
-
-func TestSwapsOnDiskIsEmptyWithoutASwapsTable(t *testing.T) {
-	newFakeSysfs(t, fakeDisk{name: "sda", sectors: 1 << 22, partitions: []string{"sda1"}})
-
-	if got := swapsOnDisk("/dev/sda"); len(got) != 0 {
-		t.Errorf("got %v, want none", got)
-	}
-}

cmd/installer/install/disk.go

@@ -4,9 +4,6 @@ import (
 	"fmt"
 	"log/slog"
 	"os"
-	"path/filepath"
-	"slices"
-	"strings"
 	"time"
 )
 
@@ -50,95 +47,6 @@ const (
 	tailReserveMiB = 1024
 )
 
-// partitionSettleTimeout bounds the wait for the kernel and udev to publish a
-// disk's new partition table. A var so tests do not pay it.
-var partitionSettleTimeout = 15 * time.Second
-
-// releaseDisk detaches whatever the live environment has attached to a disk
-// before it is erased. A mounted filesystem or an active swap area makes
-// BLKRRPART return EBUSY, and the kernel then keeps serving the disk's old
-// partition table while sgdisk writes the new one and reports success.
-func releaseDisk(d Disk) {
-	for _, m := range mountsOnDisk(d.Path) {
-		err := runQuiet("umount", m.Device)
-		if err != nil {
-			// Lazy detach: a busy mount must not stall an install that is about to
-			// erase the filesystem underneath it anyway.
-			err = runQuiet("umount", "-l", m.Device)
-		}
-		if err != nil {
-			slog.Warn("could not unmount before erase", "device", m.Device, "mountpoint", m.Mountpoint, "err", err)
-			continue
-		}
-		slog.Info("released mount before erase", "device", m.Device, "mountpoint", m.Mountpoint)
-	}
-	for _, s := range swapsOnDisk(d.Path) {
-		if err := runQuiet("swapoff", s); err != nil {
-			slog.Warn("could not disable swap before erase", "device", s, "err", err)
-			continue
-		}
-		slog.Info("disabled swap before erase", "device", s)
-	}
-}
-
-// settlePartitions forces the kernel to re-read a disk's partition table and
-// blocks until its view matches want, naming the partitions by number.
-//
-// An empty want asserts the disk has no partitions at all. That is the only
-// state no leftover can satisfy: a disk whose previous table had the same
-// partition numbers as the one being written — an old OS install is p1, p2, p3,
-// exactly what the boot layout writes — passes an "are the expected partitions
-// there" check on stale nodes alone.
-func settlePartitions(d Disk, want ...int) error {
-	if err := runQuiet("partprobe", d.Path); err != nil {
-		// partprobe rescans the whole disk and fails if it cannot open any part of
-		// it; the ioctl on its own sometimes still gets through.
-		if err := runQuiet("blockdev", "--rereadpt", d.Path); err != nil {
-			slog.Warn("could not force a partition table re-read", "disk", d.Path, "err", err)
-		}
-	}
-	_ = runQuiet("udevadm", "settle", "--timeout=10")
-
-	wanted := make([]string, 0, len(want))
-	for _, n := range want {
-		wanted = append(wanted, filepath.Base(d.PartitionPath(n)))
-	}
-	slices.Sort(wanted)
-
-	deadline := time.Now().Add(partitionSettleTimeout)
-	for {
-		got, err := kernelPartitions(d.Path)
-		if err != nil {
-			return err
-		}
-		if slices.Equal(got, wanted) {
-			return nil
-		}
-		if time.Now().After(deadline) {
-			return staleTableError(d, wanted, got)
-		}
-		time.Sleep(200 * time.Millisecond)
-	}
-}
-
-// staleTableError explains a kernel partition view that never caught up. The
-// erase case gets its own wording because it is the dangerous one — proceeding
-// would format the previous layout — and because the holder is usually the
-// whole diagnosis.
-func staleTableError(d Disk, wanted, got []string) error {
-	if len(wanted) == 0 {
-		msg := fmt.Sprintf("%s still shows partitions [%s] after its table was erased — the kernel is using the old table, so formatting would write at the previous layout's offsets",
-			d.Path, strings.Join(got, ", "))
-		if h := diskHolders(d.Path); len(h) > 0 {
-			return fmt.Errorf("%s; it is held by %s, which must be stopped before this disk can be installed to",
-				msg, strings.Join(h, ", "))
-		}
-		return fmt.Errorf("%s; something in the live environment is holding it open", msg)
-	}
-	return fmt.Errorf("%s: the kernel shows partitions [%s] but the new table has [%s] — the table was written but never picked up",
-		d.Path, strings.Join(got, ", "), strings.Join(wanted, ", "))
-}
-
 // espSizeMiB picks the ESP size for a disk.
 func espSizeMiB(d Disk) int {
 	if d.Bytes >= espLargeDiskGiB<<30 {
@@ -190,7 +98,7 @@ func partitionDataDisk(rm RoleMount) error {
 	if err := run("sgdisk", "-G", rm.Disk.Path); err != nil {
 		slog.Warn("could not randomise GPT GUIDs", "disk", rm.Disk.Path, "err", err)
 	}
-	return settlePartitions(rm.Disk, dataPartNum)
+	return nil
 }
 
 // partitionOne writes the GPT for a single disk.
@@ -221,15 +129,25 @@ func partitionOne(d Disk, rootType string) error {
 	if err := run("sgdisk", "-G", d.Path); err != nil {
 		slog.Warn("could not randomise GPT GUIDs", "disk", d.Path, "err", err)
 	}
-	return settlePartitions(d, biosPartNum, espPartNum, rootPartNum)
+	return nil
 }
 
 // waitForPartitions ensures every partition device node the installer is about
-// to use exists. The kernel's own view was already asserted per disk as each
-// table was written; udev publishes the /dev nodes and by-id symlinks after
-// that, and Trixie is slow enough about it that mkfs races them.
+// to use exists. Trixie's udev is slow enough after a table rewrite that mkfs
+// races it and fails with ENOENT on a device the kernel has already accepted.
 func waitForPartitions(cfg DiskConfig) error {
-	deadline := time.Now().Add(partitionSettleTimeout)
+	for _, d := range cfg.Disks {
+		// Best-effort: partprobe failing is not fatal, since udev may still act
+		// on the BLKRRPART ioctl sgdisk issued itself.
+		if err := run("partprobe", d.Path); err != nil {
+			slog.Warn("partprobe failed, continuing", "disk", d.Path, "err", err)
+		}
+	}
+	if err := run("udevadm", "settle", "--timeout=10"); err != nil {
+		slog.Warn("udevadm settle failed, continuing", "err", err)
+	}
+
+	deadline := time.Now().Add(15 * time.Second)
 	for _, d := range cfg.bootDisks() {
 		wanted := []string{d.PartitionPath(espPartNum), d.PartitionPath(rootPartNum)}
 		// The pool is built from by-id paths, so their symlinks have to exist

cmd/installer/install/steps_test.go

@@ -5,8 +5,6 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
-	"slices"
-	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -18,69 +16,17 @@ import (
 type fakeExec struct {
 	calls []string
 	fail  func(name string, args []string) error
-
-	// parts is the kernel's partition view per disk. Seeded by the test and then
-	// moved by the sgdisk calls the installer makes, so the partition-table
-	// assertions can run without a block device.
-	parts map[string][]string
-
-	// stale marks disks whose view never moves however many tables are written
-	// to them: something holds the disk open, BLKRRPART returns EBUSY, and the
-	// kernel keeps serving the old table while sgdisk reports success.
-	stale map[string]bool
-}
-
-// holdOpen makes a disk keep its current partition view forever.
-func (f *fakeExec) holdOpen(disk string) { f.stale[disk] = true }
-
-// seedPartitions gives a disk a pre-existing partition table — the state a
-// drive is in when the node has been installed on before.
-func (f *fakeExec) seedPartitions(disk string, nums ...int) {
-	for _, n := range nums {
-		f.parts[disk] = append(f.parts[disk], filepath.Base(partitionPath(disk, n)))
-	}
-}
-
-// applySgdisk moves the fake kernel view in step with a table rewrite, the way
-// a kernel that honoured every BLKRRPART would: -Z empties it, each -n adds the
-// partition it creates.
-func (f *fakeExec) applySgdisk(args []string) {
-	if len(args) == 0 {
-		return
-	}
-	disk := args[len(args)-1]
-	if f.stale[disk] {
-		return
-	}
-	for i, a := range args {
-		switch {
-		case a == "-Z":
-			f.parts[disk] = nil
-		case a == "-n" && i+1 < len(args):
-			num, _, _ := strings.Cut(args[i+1], ":")
-			n, err := strconv.Atoi(num)
-			if err != nil {
-				continue
-			}
-			f.parts[disk] = append(f.parts[disk], filepath.Base(partitionPath(disk, n)))
-		}
-	}
 }
 
 func fakeCommands(t *testing.T) *fakeExec {
 	t.Helper()
-	f := &fakeExec{parts: map[string][]string{}, stale: map[string]bool{}}
-	prevRun, prevQuiet, prevEnv, prevParts := run, runQuiet, runEnv, kernelPartitions
+	f := &fakeExec{}
+	prevRun, prevQuiet, prevEnv := run, runQuiet, runEnv
 
 	record := func(name string, args ...string) error {
 		f.calls = append(f.calls, strings.TrimSpace(name+" "+strings.Join(args, " ")))
 		if f.fail != nil {
-			if err := f.fail(name, args); err != nil {
-				return err
-			}
-		}
-		if name == "sgdisk" {
-			f.applySgdisk(args)
+			return f.fail(name, args)
 		}
 		return nil
 	}
@@ -89,21 +35,8 @@ func fakeCommands(t *testing.T) *fakeExec {
 	runEnv = func(env []string, name string, args ...string) error {
 		return record(name, append(env, args...)...)
 	}
-	kernelPartitions = func(disk string) ([]string, error) {
-		got := slices.Clone(f.parts[disk])
-		slices.Sort(got)
-		return got, nil
-	}
 
-	// A short settle so a test asserting the stale-table failure does not wait
-	// out the install's real budget.
-	prevTimeout := partitionSettleTimeout
-	partitionSettleTimeout = 10 * time.Millisecond
-
-	t.Cleanup(func() {
-		run, runQuiet, runEnv, kernelPartitions = prevRun, prevQuiet, prevEnv, prevParts
-		partitionSettleTimeout = prevTimeout
-	})
+	t.Cleanup(func() { run, runQuiet, runEnv = prevRun, prevQuiet, prevEnv })
 	return f
 }
 
@@ -181,24 +114,6 @@ func targetSkeleton(t *testing.T, root string) {
 
 // tempDisks builds disks whose device nodes are real files, so the steps that
 // wait for partition nodes to appear can complete.
-// fakeProcTables points the mount and swap readers at fixture tables for the
-// duration of a test.
-func fakeProcTables(t *testing.T, mounts, swaps string) {
-	t.Helper()
-	dir := t.TempDir()
-	point := func(dst *string, name, content string) {
-		prev := *dst
-		p := filepath.Join(dir, name)
-		if err := os.WriteFile(p, []byte(content), 0o644); err != nil {
-			t.Fatalf("write %s: %v", p, err)
-		}
-		*dst = p
-		t.Cleanup(func() { *dst = prev })
-	}
-	point(&procMountsPath, "mounts", mounts)
-	point(&procSwapsPath, "swaps", swaps)
-}
-
 func tempDisks(t *testing.T, n int, gib int64) []Disk {
 	t.Helper()
 	dir := t.TempDir()
@@ -322,154 +237,21 @@ func TestEspSizeMiB(t *testing.T) {
 	}
 }
 
-// A drive arriving with a previous install's layout must have every partition
-// cleared, not just the one the root filesystem used to be on: a data drive's
-// partition is p1, so a p3-only labelclear leaves a former pool member's ZFS
-// label in place at both ends of the device.
-func TestClearDiskWipesEveryPartitionItFinds(t *testing.T) {
+func TestClearDiskOnlyLabelclearsAnExistingPartition(t *testing.T) {
 	f := fakeCommands(t)
-	d := tempDisks(t, 1, 40)[0]
-	f.seedPartitions(d.Path, biosPartNum, espPartNum, rootPartNum)
-
-	if err := clearDisk(d); err != nil {
+	disks := tempDisks(t, 1, 40)
+	if err := clearDisk(disks[0]); err != nil {
 		t.Fatalf("clearDisk: %v", err)
 	}
-	for _, n := range []int{biosPartNum, espPartNum, rootPartNum} {
-		f.mustRun(t, "zpool labelclear -f "+d.PartitionPath(n))
-		f.mustRun(t, "wipefs -a "+d.PartitionPath(n))
-	}
-	f.mustRun(t, "wipefs -a "+d.Path)
-	f.mustRun(t, "sgdisk -Z "+d.Path)
-}
+	f.mustRun(t, "zpool labelclear -f "+disks[0].PartitionPath(rootPartNum))
 
-func TestClearDiskProbesNoPartitionsOnABlankDrive(t *testing.T) {
-	f := fakeCommands(t)
+	f2 := fakeCommands(t)
 	blank := Disk{Path: filepath.Join(t.TempDir(), "sdb")}
-
 	if err := clearDisk(blank); err != nil {
 		t.Fatalf("clearDisk: %v", err)
 	}
-	f.mustNotRun(t, "labelclear")
-	f.mustRun(t, "wipefs -a "+blank.Path)
-}
-
-// The reported field failure: a drive whose old table the kernel never dropped.
-// Proceeding would format the previous layout's offsets, so the install must
-// stop, and the message must say the table is stale rather than blaming the
-// device node that did not appear.
-func TestClearDiskFailsWhenTheKernelKeepsTheOldTable(t *testing.T) {
-	f := fakeCommands(t)
-	d := tempDisks(t, 1, 40)[0]
-	f.seedPartitions(d.Path, biosPartNum, espPartNum, rootPartNum)
-	// sgdisk writes to the platters and exits 0, but BLKRRPART returned EBUSY,
-	// so the kernel's view never changes.
-	f.holdOpen(d.Path)
-
-	err := clearDisk(d)
-	if err == nil {
-		t.Fatal("clearDisk succeeded against a stale partition table")
-	}
-	for _, want := range []string{d.Path, "still shows partitions", "previous layout"} {
-		if !strings.Contains(err.Error(), want) {
-			t.Errorf("err = %q, want it to mention %q", err, want)
-		}
-	}
-}
-
-func TestClearDiskReleasesMountsAndSwapFirst(t *testing.T) {
-	f := fakeCommands(t)
-	d := tempDisks(t, 1, 40)[0]
-	f.seedPartitions(d.Path, biosPartNum, espPartNum, rootPartNum)
-
-	root, esp := d.PartitionPath(rootPartNum), d.PartitionPath(espPartNum)
-	swapDev := d.PartitionPath(biosPartNum)
-	fakeProcTables(t,
-		fmt.Sprintf("%s /oldroot ext4 rw 0 0\n%s /oldroot/boot/efi vfat rw 0 0\n", root, esp),
-		fmt.Sprintf("Filename\t\t\t\tType\t\tSize\tUsed\tPriority\n%s partition 1000 0 -2\n", swapDev),
-	)
-
-	if err := clearDisk(d); err != nil {
-		t.Fatalf("clearDisk: %v", err)
-	}
-	f.mustRun(t, "swapoff "+swapDev)
-
-	// Deepest mountpoint first: unmounting /oldroot while /oldroot/boot/efi is
-	// still attached fails with EBUSY and leaves the disk held.
-	var order []string
-	for _, c := range f.calls {
-		if c == "umount "+root || c == "umount "+esp {
-			order = append(order, c)
-		}
-	}
-	if want := []string{"umount " + esp, "umount " + root}; !slices.Equal(order, want) {
-		t.Errorf("unmount order:\n got %v\nwant %v", order, want)
-	}
-}
-
-// A drive held by md or LVM cannot be released — the ISO ships neither tool —
-// so the operator has to be told what is holding it.
-func TestStaleTableErrorNamesTheHolder(t *testing.T) {
-	f := newFakeSysfs(t, fakeDisk{name: "sda", sectors: 1 << 20, partitions: []string{"sda1"}})
-	f.addHolder("sda", "sda1", "md0")
-
-	prev := kernelPartitions
-	kernelPartitions = readKernelPartitions
-	t.Cleanup(func() { kernelPartitions = prev })
-
-	err := staleTableError(Disk{Path: "/dev/sda"}, nil, []string{"sda1"})
-	if err == nil || !strings.Contains(err.Error(), "held by md0") {
-		t.Fatalf("err = %v, want it to name the md0 holder", err)
-	}
-}
-
-// The reported field failure as a unit test: two drives arrive carrying a
-// previous install's partitions and the third has never been partitioned. All
-// three must be taken over, and the blank one must not be the only one to get
-// a new table.
-func TestPartitionDisksTakesOverDrivesThatAlreadyHavePartitions(t *testing.T) {
-	f := fakeCommands(t)
-	nodes := tempDisks(t, 3, 200)
-	f.seedPartitions(nodes[0].Path, biosPartNum, espPartNum, rootPartNum) // held an OS
-	f.seedPartitions(nodes[1].Path, dataPartNum)                          // held data
-	// nodes[2] arrives blank.
-
-	cfg := DiskConfig{FS: FSExt4}.WithRoles([]RoleMount{
-		{Role: RoleOS, Disk: nodes[0]},
-		{Role: RoleSpinifex, Disk: nodes[1]},
-		{Role: RolePredastore, Disk: nodes[2]},
-	})
-	if err := partitionDisks(cfg); err != nil {
-		t.Fatalf("partitionDisks: %v", err)
-	}
-	for _, d := range nodes {
-		f.mustRun(t, "sgdisk -Z "+d.Path)
-	}
-	f.mustRun(t, "sgdisk", nodes[0].Path, "-n 3:0:-1024M")
-	for _, d := range nodes[1:] {
-		f.mustRun(t, "sgdisk", d.Path, "-n 1:1M:-1024M", "-t 1:8300")
-	}
-}
-
-// A previous OS install leaves p1, p2, p3 — exactly the numbering the boot
-// layout writes. Asserting the expected partitions exist would pass on those
-// stale nodes and the install would format the old offsets, so the erase-stage
-// assertion on an empty table is what has to catch it.
-func TestPartitionOneStopsWhenAnOldTableHasTheSameLayout(t *testing.T) {
-	f := fakeCommands(t)
-	d := tempDisks(t, 1, 200)[0]
-	f.seedPartitions(d.Path, biosPartNum, espPartNum, rootPartNum)
-	f.holdOpen(d.Path)
-
-	err := partitionOne(d, typeLinuxFS)
-	if err == nil {
-		t.Fatal("partitionOne succeeded against a stale partition table")
-	}
-	// Specifically the erase-stage message: reaching the post-write check would
-	// mean the stale nodes had already been accepted.
-	if !strings.Contains(err.Error(), "still shows partitions") {
-		t.Errorf("err = %q, want the erase-stage stale-table message", err)
-	}
-	f.mustNotRun(t, "sgdisk -n")
+	f2.mustNotRun(t, "labelclear")
+	f2.mustRun(t, "wipefs -a "+blank.Path)
 }
 
 func TestFormatESPsLabelsEveryMember(t *testing.T) {

cmd/installer/install/zfs.go

@@ -340,35 +340,23 @@ func exportPool() {
 	}
 }
 
-// clearDisk takes a disk over unconditionally, whatever it holds: everything
-// attached to it is released, every signature on it is wiped, its partition
-// table is erased, and the kernel is made to agree the disk is now empty.
-//
-// The last part is the point. sgdisk writes to the platters and exits 0 even
-// when the kernel refuses to drop the old table, so without the assertion an
-// install proceeds against partition nodes describing the previous layout.
+// clearDisk removes any prior filesystem, partition table and ZFS label from a
+// disk so a retry after a partial install is not rejected for stale metadata —
+// a failure that otherwise reads as mysterious rather than as leftover debris.
 //
 // The probes are quiet because a blank disk makes all of them fail, and their
 // stderr on the install console reads like the install is going wrong.
 func clearDisk(d Disk) error {
-	releaseDisk(d)
-
-	// Every partition, not just the root slice. ZFS labels live at both ends of
-	// a device and wipefs does not know about them, so a data drive that was
-	// once a pool member keeps its label and fails a later zpool create.
-	parts, err := kernelPartitions(d.Path)
-	if err != nil {
-		return err
+	// ZFS labels live at both ends of the device and wipefs does not know about
+	// them, so an old pool member has to be cleared before its partition goes.
+	if old := d.PartitionPath(rootPartNum); exists(old) {
+		_ = runQuiet("zpool", "labelclear", "-f", old)
 	}
-	for _, p := range parts {
-		dev := partitionDevice(d.Path, p)
-		_ = runQuiet("zpool", "labelclear", "-f", dev)
-		_ = runQuiet("wipefs", "-a", dev)
-	}
-
 	_ = runQuiet("wipefs", "-a", d.Path)
-	if err := runQuiet("sgdisk", "-Z", d.Path); err != nil {
-		return fmt.Errorf("erase partition table: %w", err)
-	}
-	return settlePartitions(d)
+	return runQuiet("sgdisk", "-Z", d.Path)
+}
+
+func exists(path string) bool {
+	_, err := os.Stat(path)
+	return err == nil
 }

cmd/rds-agent/agent.go

@@ -180,7 +180,7 @@ type Agent struct {
 	id              identity
 	cp              controlPlane
 	probe           *engineProbe
-	engine          *postgresEngine
+	engine          engine
 	handoffWriter   func(string, *handlers_rds.GetDBBootstrapConfigOutput) error
 	dataMountWaiter func(context.Context, string, string) error
 
@@ -195,10 +195,7 @@ type Agent struct {
 
 // Assembles from already-built parts, so tests can pass fakes; New builds the
 // production control plane and delegates here.
-func newAgent(cfg config, cp controlPlane, probe *engineProbe) *Agent {
-	if cfg.DataMount == "" {
-		cfg.DataMount = defaultDataMount
-	}
+func newAgent(cfg config, cp controlPlane, probe *engineProbe) (*Agent, error) {
 	if cfg.MountsFile == "" {
 		cfg.MountsFile = defaultMountsFile
 	}
@@ -211,11 +208,15 @@ func newAgent(cfg config, cp controlPlane, probe *engineProbe) *Agent {
 		cfg: cfg, id: id, cp: cp, probe: probe,
 		handoffWriter: writeHandoff, dataMountWaiter: waitForDataMount,
 	}
-	a.engine = newPostgresEngine(cfg, execCommandRunner, execSessionRunner, probe)
+	eng, err := newEngine(cfg, execCommandRunner, execSessionRunner, probe)
+	if err != nil {
+		return nil, err
+	}
+	a.engine = eng
 	a.hb = newHeartbeater(cp, probe, a.engine, handlers_rds.HeartbeatInterval)
 	a.cmd = newCommander(cp, newCommandRegistry(a.engine, newGuestStorage(cfg, execCommandRunner)), cfg.PollWait)
 	a.guard = newParamGuard(a.engine, probe, cp)
-	return a
+	return a, nil
 }
 
 // Does not wait for IMDS: the register loop rides out a datapath still coming up.
@@ -227,7 +228,11 @@ func New(cfg config) (*Agent, error) {
 	if err != nil {
 		return nil, err
 	}
-	return newAgent(cfg, cp, newEngineProbe(cfg, execProbeRunner)), nil
+	probe, err := newProbe(cfg, execProbeRunner)
+	if err != nil {
+		return nil, err
+	}
+	return newAgent(cfg, cp, probe)
 }
 
 func (a *Agent) Run(ctx context.Context) error {
@@ -245,6 +250,13 @@ func (a *Agent) Run(ctx context.Context) error {
 	}
 	a.hb.id, a.cmd.id, a.guard.id = a.id, a.id, a.id
 
+	// Ahead of the handoff, so rds-init never unblocks and no datadir is touched.
+	// An agent running on the wrong image would initialise one engine over
+	// another's data, coming up healthy and empty beside data nothing references.
+	if err := a.refuseEngineMismatch(ctx); err != nil {
+		return err
+	}
+
 	// Beating before the bootstrap keeps a stuck boot visible as a live VM with
 	// a down engine rather than as silence.
 	go a.hb.Run(ctx)
@@ -279,6 +291,26 @@ func (a *Agent) Run(ctx context.Context) error {
 	return nil
 }
 
+// The engine cloud-init says this VM was launched as, against the engine the
+// image bakes. An empty expectation is no assertion rather than a disagreement:
+// a VM launched before the control plane emitted RDS_ENGINE carries none.
+func (a *Agent) refuseEngineMismatch(ctx context.Context) error {
+	expected := strings.ToLower(strings.TrimSpace(a.cfg.Engine))
+	if expected == "" || expected == a.cfg.BakedEngine {
+		return nil
+	}
+
+	message := fmt.Sprintf("this VM was launched as engine %q but its image bakes %q; refusing to bootstrap",
+		expected, a.cfg.BakedEngine)
+	slog.ErrorContext(ctx, "rds-agent: "+message)
+	// Reported before returning, so the control plane learns why the instance
+	// never came up rather than only that it never did.
+	if _, err := a.cp.SubmitState(ctx, a.id, handlers_rds.EngineHealthUnhealthy, message); err != nil {
+		slog.ErrorContext(ctx, "rds-agent: reporting the engine mismatch failed", "err", err)
+	}
+	return errors.New(message)
+}
+
 const dataMountPollInterval = time.Second
 
 // waitForDataMount blocks the stateful agent loops until the kernel reports the

cmd/rds-agent/agent_test.go

@@ -168,18 +168,26 @@ func (f *fakeControlPlane) snapshotReplies() [][]handlers_rds.CommandReply {
 	return out
 }
 
-// testConfig points the agent's file output at a temp dir and its probe at a
-// name no real binary answers to.
+// testConfig stamps the engine a real rds-postgres image bakes, so the agent
+// resolves the same guest layout it would in a guest, and points its file output
+// at a temp dir.
 func testConfig(t *testing.T) config {
 	t.Helper()
-	return config{
-		GatewayURL: "https://gw.test",
-		HandoffDir: filepath.Join(t.TempDir(), "spinifex-rds"),
-		EngineHost: defaultEngineHost,
-		EnginePort: defaultEnginePort,
-		PGIsReady:  "pg_isready",
-		PollWait:   time.Second,
+	cfg := testLoadConfig(t, enginePostgres)
+	cfg.GatewayURL = "https://gw.test"
+	cfg.HandoffDir = filepath.Join(t.TempDir(), "spinifex-rds")
+	cfg.PollWait = time.Second
+	return cfg
+}
+
+// newTestAgent builds the agent over a probe the test drives directly.
+func newTestAgent(t *testing.T, cfg config, cp controlPlane, run probeRunner) *Agent {
+	t.Helper()
+	a, err := newAgent(cfg, cp, newPostgresProbe(cfg, run))
+	if err != nil {
+		t.Fatalf("newAgent: %v", err)
 	}
+	return a
 }
 
 // staticProbe is a probeRunner that always reports the same exit code.
@@ -272,7 +280,7 @@ func TestRun_RegistersThenDeliversBootstrap(t *testing.T) {
 	cp.bootstrapOut = bootstrapOutput(&password)
 
 	cfg := testConfig(t)
-	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
 	if err := runAgent(t, a); err != nil {
 		t.Fatalf("Run: %v", err)
 	}
@@ -303,7 +311,7 @@ func TestRun_AttachModeNeedsNoPassword(t *testing.T) {
 	cp.bootstrapOut = bootstrapOutput(nil)
 
 	cfg := testConfig(t)
-	if err := runAgent(t, newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))); err != nil {
+	if err := runAgent(t, newTestAgent(t, cfg, cp, staticProbe(0))); err != nil {
 		t.Fatalf("Run: %v", err)
 	}
 
@@ -325,7 +333,7 @@ func TestRun_RetriesRegisterAndBootstrap(t *testing.T) {
 	cp.bootstrapErrs = []error{errors.New("ServerInternal")}
 
 	cfg := testConfig(t)
-	if err := runAgent(t, newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))); err != nil {
+	if err := runAgent(t, newTestAgent(t, cfg, cp, staticProbe(0))); err != nil {
 		t.Fatalf("Run: %v", err)
 	}
 
@@ -344,7 +352,7 @@ func TestBootstrap_RetriesHandoffWithoutRefetching(t *testing.T) {
 	password := "staged-secret"
 	cp.bootstrapOut = bootstrapOutput(&password)
 	cfg := testConfig(t)
-	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
 
 	writes := 0
 	a.handoffWriter = func(_ string, got *handlers_rds.GetDBBootstrapConfigOutput) error {
@@ -385,7 +393,7 @@ func TestBootstrap_RetriesInitializeWithNoPassword(t *testing.T) {
 	}}
 	cp.bootstrapOut = bootstrapOutput(&password)
 	cfg := testConfig(t)
-	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
 
 	var handed []*handlers_rds.GetDBBootstrapConfigOutput
 	a.handoffWriter = func(_ string, got *handlers_rds.GetDBBootstrapConfigOutput) error {
@@ -408,7 +416,7 @@ func TestAcknowledgeBootstrap_WaitsForTheReceiptThenRetiresIt(t *testing.T) {
 	cp := newFakeControlPlane()
 	cfg := testConfig(t)
 	cfg.DataMount = t.TempDir()
-	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
 	a.id = identity{DBInstanceIdentifier: "db-resolved"}
 	a.pending = &pendingBootstrap{payloadID: testPayloadID, vmGeneration: 1, dataVolumeID: "vol-data-01"}
 
@@ -447,7 +455,7 @@ func TestAcknowledgeBootstrap_IgnoresAForeignReceipt(t *testing.T) {
 			cp := newFakeControlPlane()
 			cfg := testConfig(t)
 			cfg.DataMount = t.TempDir()
-			a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+			a := newTestAgent(t, cfg, cp, staticProbe(0))
 			a.id = identity{DBInstanceIdentifier: "db-resolved"}
 			a.pending = &pendingBootstrap{payloadID: testPayloadID, vmGeneration: 1}
 			writeReceipt(t, a, tc.payloadID, tc.dbID)
@@ -470,7 +478,7 @@ func TestAcknowledgeBootstrap_IsANoOpOnAttach(t *testing.T) {
 	cp := newFakeControlPlane()
 	cfg := testConfig(t)
 	cfg.DataMount = t.TempDir()
-	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
 
 	if err := a.acknowledgeBootstrap(t.Context()); err != nil {
 		t.Fatalf("acknowledgeBootstrap: %v", err)
@@ -480,6 +488,52 @@ func TestAcknowledgeBootstrap_IsANoOpOnAttach(t *testing.T) {
 	}
 }
 
+// The image is the authority on which engine the agent runs. A VM launched as
+// another engine has to stop ahead of the handoff: rds-init would otherwise
+// initialise one engine over the other's datadir and come up healthy and empty
+// beside data nothing references.
+func TestRun_RefusesAnEngineTheImageDoesNotBake(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.bootstrapOut = bootstrapOutput(nil)
+
+	cfg := testConfig(t)
+	cfg.Engine = "mariadb"
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
+
+	err := a.Run(t.Context())
+	if err == nil {
+		t.Fatal("Run returned nil for a VM launched as an engine this image does not bake")
+	}
+	if !strings.Contains(err.Error(), "mariadb") {
+		t.Errorf("error = %v, want it to name the engine the VM was launched as", err)
+	}
+	if _, statErr := os.Stat(filepath.Join(cfg.HandoffDir, handoffEnvFile)); !os.IsNotExist(statErr) {
+		t.Error("the agent wrote a bootstrap handoff for an engine it refused to run")
+	}
+	if n := len(cp.bootstrapReqs); n != 0 {
+		t.Errorf("bootstrap fetches = %d, want none before the refusal", n)
+	}
+	// Reported rather than only logged, so the control plane learns why the
+	// instance never came up rather than only that it never did.
+	states := cp.snapshotStates()
+	if len(states) != 1 || states[0].health != handlers_rds.EngineHealthUnhealthy {
+		t.Fatalf("submitted states = %+v, want a single unhealthy report", states)
+	}
+}
+
+// The same assertion when it agrees is not a gate: the boot goes on exactly as
+// it does for a VM launched before the control plane sent one at all.
+func TestRun_ProceedsWhenTheDeliveredEngineMatchesTheImage(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.bootstrapOut = bootstrapOutput(nil)
+
+	cfg := testConfig(t)
+	cfg.Engine = enginePostgres
+	if err := runAgent(t, newTestAgent(t, cfg, cp, staticProbe(0))); err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+}
+
 // A configured identifier is asserted on the wire so a mis-provisioned VM is
 // rejected by the gateway instead of adopting whatever it resolves to.
 func TestRun_SendsConfiguredIdentifier(t *testing.T) {
@@ -489,7 +543,7 @@ func TestRun_SendsConfiguredIdentifier(t *testing.T) {
 	cfg := testConfig(t)
 	cfg.DBInstanceIdentifier = "db-configured"
 	cfg.EngineVersion = "18.1"
-	if err := runAgent(t, newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))); err != nil {
+	if err := runAgent(t, newTestAgent(t, cfg, cp, staticProbe(0))); err != nil {
 		t.Fatalf("Run: %v", err)
 	}
 
@@ -503,12 +557,12 @@ func TestRun_WaitsForDataMountBeforePollingCommands(t *testing.T) {
 	cp := newFakeControlPlane()
 	cp.bootstrapOut = bootstrapOutput(nil)
 	cfg := testConfig(t)
-	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(2)))
+	a := newTestAgent(t, cfg, cp, staticProbe(2))
 
 	waiting := make(chan struct{})
 	release := make(chan struct{})
 	a.dataMountWaiter = func(ctx context.Context, mountsFile, target string) error {
-		if mountsFile != defaultMountsFile || target != defaultDataMount {
+		if mountsFile != defaultMountsFile || target != cfg.DataMount {
 			t.Errorf("mount waiter got %q/%q, want configured defaults", mountsFile, target)
 		}
 		close(waiting)
@@ -580,7 +634,7 @@ func TestRun_HeartbeatIncludesBootstrapFailure(t *testing.T) {
 	cp.registerOut.HeartbeatIntervalSeconds = 0
 
 	cfg := testConfig(t)
-	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(2)))
+	a := newTestAgent(t, cfg, cp, staticProbe(2))
 	a.hb.interval = 10 * time.Millisecond
 
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
@@ -609,7 +663,7 @@ func TestRun_HeartbeatsEngineHealth(t *testing.T) {
 	cp.registerOut.HeartbeatIntervalSeconds = 0
 
 	cfg := testConfig(t)
-	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
 	a.hb.interval = 10 * time.Millisecond
 
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
@@ -639,7 +693,7 @@ func TestRegister_AdoptsControlPlaneCadence(t *testing.T) {
 	cp.registerOut.HeartbeatIntervalSeconds = 45
 
 	cfg := testConfig(t)
-	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
 	if err := runAgent(t, a); err != nil {
 		t.Fatalf("Run: %v", err)
 	}
@@ -656,12 +710,15 @@ func TestRun_HeartbeatReportsDownEngine(t *testing.T) {
 	cp.registerOut.HeartbeatIntervalSeconds = 0
 
 	cfg := testConfig(t)
-	probe := newEngineProbe(cfg, staticProbe(2))
+	probe := newPostgresProbe(cfg, staticProbe(2))
 	// The engine has served before, so a silent one now is a failure rather
 	// than a boot still in progress.
 	probe.seenHealthy = true
 
-	a := newAgent(cfg, cp, probe)
+	a, err := newAgent(cfg, cp, probe)
+	if err != nil {
+		t.Fatalf("newAgent: %v", err)
+	}
 	a.hb.interval = 10 * time.Millisecond
 
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
@@ -752,7 +809,7 @@ func TestAcknowledgeBootstrap_StopsOnATerminalDenial(t *testing.T) {
 	}}
 	cfg := testConfig(t)
 	cfg.DataMount = t.TempDir()
-	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
 	a.id = identity{DBInstanceIdentifier: "db-resolved"}
 	a.pending = &pendingBootstrap{payloadID: testPayloadID, vmGeneration: 1}
 	writeReceipt(t, a, testPayloadID, "db-resolved")
@@ -780,7 +837,7 @@ func TestAcknowledgeBootstrap_DoesNotReportAnAbsentReceiptAsAFailure(t *testing.
 	cp := newFakeControlPlane()
 	cfg := testConfig(t)
 	cfg.DataMount = t.TempDir()
-	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
 	a.id = identity{DBInstanceIdentifier: "db-resolved"}
 	a.pending = &pendingBootstrap{payloadID: testPayloadID, vmGeneration: 1}
 

cmd/rds-agent/bootstrap.go

@@ -107,7 +107,11 @@ func writeHandoff(dir string, cfg *handlers_rds.GetDBBootstrapConfigOutput) erro
 		return fmt.Errorf("secure handoff dir %s: %w", dir, err)
 	}
 
-	if err := writeHandoffFile(dir, handoffParamsFile, renderParameters(cfg.Parameters)); err != nil {
+	params, err := renderParameters(cfg.Engine, cfg.Parameters)
+	if err != nil {
+		return err
+	}
+	if err := writeHandoffFile(dir, handoffParamsFile, params); err != nil {
 		return err
 	}
 	// Cert and key are written as a pair; half of one would start the engine
@@ -177,18 +181,27 @@ func shellQuote(s string) string {
 	return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
 }
 
-// postgresql.conf syntax. Values are quoted so a setting with a space or unit
-// suffix survives; the engine accepts quoted numerics and booleans too.
-func renderParameters(params []handlers_rds.Parameter) string {
+// postgresql.conf syntax, which MariaDB's option files also parse. Values are
+// quoted so a setting with a space or unit suffix survives; the engine accepts
+// quoted numerics and booleans too.
+//
+// Names are the engine's rather than the customer's: a setting the server takes
+// only through SET has a different spelling at startup, and writing the wrong one
+// leaves the engine refusing to boot on a file already on the data volume.
+func renderParameters(engineName string, params []handlers_rds.Parameter) (string, error) {
+	engine, err := handlers_rds.LookupEngine(engineName)
+	if err != nil {
+		return "", fmt.Errorf("render the resolved parameters: %w", err)
+	}
 	var b strings.Builder
 	b.WriteString("# Resolved parameter group, written by rds-agent.\n")
 	for _, p := range params {
 		if p.Name == "" {
 			continue
 		}
-		fmt.Fprintf(&b, "%s = '%s'\n", p.Name, strings.ReplaceAll(p.Value, "'", "''"))
+		fmt.Fprintf(&b, "%s = '%s'\n", engine.OptionFileName(p.Name), strings.ReplaceAll(p.Value, "'", "''"))
 	}
-	return b.String()
+	return b.String(), nil
 }
 
 // Atomic: a temp file in the same directory, renamed over the target. The mode

cmd/rds-agent/bootstrap_test.go

@@ -16,6 +16,7 @@ func TestWriteHandoff_WritesEveryFileRootOnly(t *testing.T) {
 
 	err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
 		Mode:               handlers_rds.BootstrapModeInitialize,
+		Engine:             "postgres",
 		MasterUsername:     "master",
 		MasterUserPassword: &password,
 		DBName:             "appdb",
@@ -82,6 +83,7 @@ func TestWriteHandoff_AttachOmitsPassword(t *testing.T) {
 	dir := filepath.Join(t.TempDir(), "handoff")
 	if err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
 		Mode:           handlers_rds.BootstrapModeAttach,
+		Engine:         "postgres",
 		MasterUsername: "master",
 		Port:           5432,
 	}); err != nil {
@@ -112,6 +114,7 @@ func TestWriteHandoff_QuotesShellMetacharacters(t *testing.T) {
 
 	if err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
 		Mode:               handlers_rds.BootstrapModeInitialize,
+		Engine:             "postgres",
 		MasterUsername:     "master",
 		MasterUserPassword: &password,
 		Port:               5432,
@@ -132,6 +135,7 @@ func TestWriteHandoff_SkipsHalfATLSPair(t *testing.T) {
 	dir := filepath.Join(t.TempDir(), "handoff")
 	if err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
 		Mode:               handlers_rds.BootstrapModeAttach,
+		Engine:             "postgres",
 		MasterUsername:     "master",
 		Port:               5432,
 		ServingCertificate: "CERT",
@@ -149,8 +153,9 @@ func TestWriteHandoff_SkipsHalfATLSPair(t *testing.T) {
 func TestWriteHandoff_RejectsConfigWithNoMasterUser(t *testing.T) {
 	dir := filepath.Join(t.TempDir(), "handoff")
 	err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
-		Mode: handlers_rds.BootstrapModeAttach,
-		Port: 5432,
+		Mode:   handlers_rds.BootstrapModeAttach,
+		Engine: "postgres",
+		Port:   5432,
 	})
 	if err == nil {
 		t.Fatal("writeHandoff accepted a config with no master username, want an error")
@@ -170,6 +175,7 @@ func TestWriteHandoff_TightensExistingDirectory(t *testing.T) {
 
 	if err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
 		Mode:           handlers_rds.BootstrapModeAttach,
+		Engine:         "postgres",
 		MasterUsername: "master",
 		Port:           5432,
 	}); err != nil {
@@ -219,3 +225,46 @@ func findLine(t *testing.T, body, prefix string) string {
 	t.Fatalf("no line starting %q in:\n%s", prefix, body)
 	return ""
 }
+
+// The include rds-init installs is what the engine starts on, so the spelling
+// written here is the one that has to boot.
+func TestRenderParameters_UsesTheEngineStartupSpelling(t *testing.T) {
+	params := []handlers_rds.Parameter{
+		{Name: "time_zone", Value: "SYSTEM"},
+		{Name: "max_connections", Value: "85"},
+	}
+
+	rendered, err := renderParameters("mariadb", params)
+	if err != nil {
+		t.Fatalf("renderParameters: %v", err)
+	}
+	if !strings.Contains(rendered, "default_time_zone = 'SYSTEM'\n") {
+		t.Errorf("time_zone was not written under its startup spelling:\n%s", rendered)
+	}
+	if strings.Contains(rendered, "\ntime_zone = ") {
+		t.Errorf("time_zone reached the option file, which mariadbd refuses:\n%s", rendered)
+	}
+	if !strings.Contains(rendered, "max_connections = '85'\n") {
+		t.Errorf("an unmapped parameter was not written unchanged:\n%s", rendered)
+	}
+}
+
+func TestRenderParameters_PostgresKeepsEveryName(t *testing.T) {
+	rendered, err := renderParameters("postgres", []handlers_rds.Parameter{
+		{Name: "shared_buffers", Value: "262144"},
+	})
+	if err != nil {
+		t.Fatalf("renderParameters: %v", err)
+	}
+	if !strings.Contains(rendered, "shared_buffers = '262144'\n") {
+		t.Errorf("postgres name was rewritten:\n%s", rendered)
+	}
+}
+
+// Failing closed beats writing a file the engine may refuse: the bad include
+// would already be on the data volume by the time the server rejected it.
+func TestRenderParameters_RefusesAnUnknownEngine(t *testing.T) {
+	if _, err := renderParameters("", nil); err == nil {
+		t.Fatal("renderParameters accepted a config naming no engine, want an error")
+	}
+}

cmd/rds-agent/config.go

@@ -1,7 +1,9 @@
 package main
 
 import (
+	"os"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/mulgadc/spinifex/internal/guestenv"
@@ -12,17 +14,11 @@ const (
 	defaultGatewayCA  = "/etc/spinifex-rds/gateway-ca.pem"
 	defaultHandoffDir = "/run/spinifex-rds"
 	defaultEngineHost = "127.0.0.1"
-	defaultEnginePort = 5432
-	defaultPGIsReady  = "pg_isready"
-
-	// The guest layout rds-init lays down. Overridable for the same reason it is
-	// there: a sibling engine preset points at its own paths.
-	defaultPGBin         = "/usr/libexec/postgresql18"
-	defaultPGData        = "/var/lib/postgresql/18/data"
-	defaultSocketDir     = "/run/postgresql"
-	defaultPGUser        = "postgres"
-	defaultRCService     = "rc-service"
-	defaultEngineSvcName = "postgresql"
+	// Where setup.sh stamps the engine the image bakes. The agent builds its
+	// engine implementation from this file rather than from anything delivered.
+	defaultEngineFile         = "/etc/spinifex-rds/engine"
+	defaultRCService          = "rc-service"
+	defaultEngineProbeTimeout = 5 * time.Second
 	// The long-poll window the agent asks the gateway to hold a request open
 	// for. The gateway caps it at 20s.
 	defaultPollWait = 20 * time.Second
@@ -44,19 +40,29 @@ type config struct {
 	HandoffDir    string
 	EngineHost    string
 	EnginePort    int
-	// Overridable so a test or a sibling engine preset can point at its own.
-	PGIsReady string
-	PollWait  time.Duration
+	PollWait      time.Duration
+
+	// The engine the control plane launched this VM as, delivered by cloud-init,
+	// and the engine the image itself bakes. Two independent statements of the
+	// same fact: the agent refuses to bootstrap when they disagree.
+	Engine      string
+	EngineFile  string
+	BakedEngine string
 
 	// Where the command handlers reach the engine: the client binaries, the
 	// datadir the parameter file is installed into, the socket they connect
 	// over, the OS user they drop to, and the service the stop goes through.
-	PGBin         string
-	PGData        string
+	EngineBinDir  string
+	EngineDataDir string
 	SocketDir     string
-	PGUser        string
+	EngineUser    string
 	RCService     string
 	EngineService string
+	// Where the engine records the pid its probe checks for liveness, for an
+	// engine whose own client cannot tell a server still recovering from one that
+	// is not running at all.
+	EnginePidFile      string
+	EngineProbeTimeout time.Duration
 
 	// Where the data volume is mounted, and the two kernel surfaces a storage
 	// grow resolves its device from. Overridable so a test can point them at
@@ -75,19 +81,21 @@ func loadConfig(envFile string) config {
 		GatewayCA:            get("RDS_GATEWAY_CA"),
 		Region:               get("RDS_REGION"),
 		DBInstanceIdentifier: get("RDS_DB_INSTANCE_IDENTIFIER"),
+		Engine:               get("RDS_ENGINE"),
+		EngineFile:           get("RDS_ENGINE_FILE"),
 		EngineVersion:        get("RDS_ENGINE_VERSION"),
 		HandoffDir:           get("RDS_HANDOFF_DIR"),
 		EngineHost:           get("RDS_ENGINE_HOST"),
-		PGIsReady:            get("RDS_PG_ISREADY"),
-		PGBin:                get("RDS_PG_BIN"),
-		PGData:               get("RDS_PGDATA"),
+		EngineBinDir:         get("RDS_ENGINE_BIN"),
+		EngineDataDir:        get("RDS_ENGINE_DATA"),
 		SocketDir:            get("RDS_SOCKET_DIR"),
-		PGUser:               get("RDS_PG_USER"),
+		EngineUser:           get("RDS_ENGINE_USER"),
 		RCService:            get("RDS_RC_SERVICE"),
 		EngineService:        get("RDS_ENGINE_SERVICE"),
+		EnginePidFile:        get("RDS_ENGINE_PIDFILE"),
 		DataMount:            get("RDS_DATA_MOUNT"),
-		EnginePort:           defaultEnginePort,
 		PollWait:             defaultPollWait,
+		EngineProbeTimeout:   defaultEngineProbeTimeout,
 	}
 	if cfg.GatewayCA == "" {
 		cfg.GatewayCA = defaultGatewayCA
@@ -98,32 +106,20 @@ func loadConfig(envFile string) config {
 	if cfg.EngineHost == "" {
 		cfg.EngineHost = defaultEngineHost
 	}
-	if cfg.PGIsReady == "" {
-		cfg.PGIsReady = defaultPGIsReady
-	}
-	if cfg.PGBin == "" {
-		cfg.PGBin = defaultPGBin
-	}
-	if cfg.PGData == "" {
-		cfg.PGData = defaultPGData
-	}
-	if cfg.SocketDir == "" {
-		cfg.SocketDir = defaultSocketDir
-	}
-	if cfg.PGUser == "" {
-		cfg.PGUser = defaultPGUser
+	if cfg.EngineFile == "" {
+		cfg.EngineFile = defaultEngineFile
 	}
 	if cfg.RCService == "" {
 		cfg.RCService = defaultRCService
 	}
-	if cfg.EngineService == "" {
-		cfg.EngineService = defaultEngineSvcName
-	}
+	// The image's own stamp rather than the delivered engine: an agent that took
+	// its layout from the payload could run one engine's implementation against
+	// another engine's datadir, which is the failure the stamp exists to prevent.
+	cfg.BakedEngine = readBakedEngine(cfg.EngineFile)
+	cfg.applyLayout(engineLayouts[cfg.BakedEngine])
+
 	// rds-datadir reads the same RDS_DATA_MOUNT, so the two agree on where the
 	// volume landed without either asserting it to the other.
-	if cfg.DataMount == "" {
-		cfg.DataMount = defaultDataMount
-	}
 	cfg.MountsFile = defaultMountsFile
 	cfg.SysBlock = defaultSysBlock
 	// The authoritative port comes from the bootstrap config; this is only what
@@ -138,5 +134,50 @@ func loadConfig(envFile string) config {
 			cfg.PollWait = d
 		}
 	}
+	if v := get("RDS_ENGINE_PROBE_TIMEOUT"); v != "" {
+		if d, err := time.ParseDuration(v); err == nil && d > 0 {
+			cfg.EngineProbeTimeout = d
+		}
+	}
 	return cfg
 }
+
+// Fills in whatever the delivered configuration left unset. An override always
+// wins, so a test can point any of these at a fixture. An unrecognised engine
+// leaves the layout empty, and New refuses rather than guessing at one.
+func (c *config) applyLayout(layout engineLayout) {
+	if c.EngineBinDir == "" {
+		c.EngineBinDir = layout.binDir
+	}
+	if c.EngineDataDir == "" {
+		c.EngineDataDir = layout.dataDir
+	}
+	if c.SocketDir == "" {
+		c.SocketDir = layout.socketDir
+	}
+	if c.EngineUser == "" {
+		c.EngineUser = layout.osUser
+	}
+	if c.EngineService == "" {
+		c.EngineService = layout.service
+	}
+	if c.EnginePidFile == "" {
+		c.EnginePidFile = layout.pidFile
+	}
+	if c.DataMount == "" {
+		c.DataMount = layout.dataMount
+	}
+	if c.EnginePort == 0 {
+		c.EnginePort = layout.port
+	}
+}
+
+// The engine the image bakes. An absent or unreadable stamp reads as empty,
+// which New refuses: an agent that guessed would defeat the stamp.
+func readBakedEngine(path string) string {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return ""
+	}
+	return strings.ToLower(strings.TrimSpace(string(raw)))
+}

cmd/rds-agent/config_test.go

@@ -3,21 +3,34 @@ package main
 import (
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 	"time"
 )
 
+// Stamps the engine a real image bakes and loads the configuration against it,
+// which is what selects the guest layout the agent's defaults come from.
+func testLoadConfig(t *testing.T, engine string) config {
+	t.Helper()
+	stamp := filepath.Join(t.TempDir(), "engine")
+	if err := os.WriteFile(stamp, []byte(engine+"\n"), 0o444); err != nil {
+		t.Fatalf("write the engine stamp: %v", err)
+	}
+	t.Setenv("RDS_ENGINE_FILE", stamp)
+	return loadConfig(filepath.Join(t.TempDir(), "absent.env"))
+}
+
 func TestLoadConfig_Defaults(t *testing.T) {
-	cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env"))
+	cfg := testLoadConfig(t, enginePostgres)
 
 	if cfg.GatewayCA != defaultGatewayCA || cfg.HandoffDir != defaultHandoffDir {
 		t.Errorf("CA/handoff = %q/%q, want the built-in defaults", cfg.GatewayCA, cfg.HandoffDir)
 	}
-	if cfg.EngineHost != defaultEngineHost || cfg.EnginePort != defaultEnginePort {
-		t.Errorf("engine endpoint = %s:%d, want the local default", cfg.EngineHost, cfg.EnginePort)
+	if cfg.EngineHost != defaultEngineHost || cfg.PollWait != defaultPollWait {
+		t.Errorf("engine host/pollWait = %q/%v, want the built-in defaults", cfg.EngineHost, cfg.PollWait)
 	}
-	if cfg.PGIsReady != defaultPGIsReady || cfg.PollWait != defaultPollWait {
-		t.Errorf("probe/pollWait = %q/%v, want the built-in defaults", cfg.PGIsReady, cfg.PollWait)
+	if cfg.EngineProbeTimeout != defaultEngineProbeTimeout {
+		t.Errorf("engine probe timeout = %v, want %v", cfg.EngineProbeTimeout, defaultEngineProbeTimeout)
 	}
 	// Identity is optional: the gateway resolves it from the caller's
 	// credentials, so an agent with no configured identifier is normal.
@@ -26,11 +39,94 @@ func TestLoadConfig_Defaults(t *testing.T) {
 	}
 }
 
+// The guest layout is the image's to state, not the control plane's: everything
+// the agent reaches the engine through comes from the stamp it was built with.
+func TestLoadConfig_TakesTheLayoutFromTheBakedEngine(t *testing.T) {
+	cfg := testLoadConfig(t, enginePostgres)
+
+	if cfg.BakedEngine != enginePostgres {
+		t.Fatalf("baked engine = %q, want %q", cfg.BakedEngine, enginePostgres)
+	}
+	layout := engineLayouts[enginePostgres]
+	if cfg.EngineBinDir != layout.binDir || cfg.EngineDataDir != layout.dataDir {
+		t.Errorf("bin/data = %q/%q, want the PostgreSQL layout", cfg.EngineBinDir, cfg.EngineDataDir)
+	}
+	if cfg.EngineUser != layout.osUser || cfg.EngineService != layout.service {
+		t.Errorf("user/service = %q/%q, want the PostgreSQL layout", cfg.EngineUser, cfg.EngineService)
+	}
+	if cfg.SocketDir != layout.socketDir || cfg.DataMount != layout.dataMount {
+		t.Errorf("socket/mount = %q/%q, want the PostgreSQL layout", cfg.SocketDir, cfg.DataMount)
+	}
+	if cfg.EnginePort != layout.port {
+		t.Errorf("port = %d, want the PostgreSQL default %d", cfg.EnginePort, layout.port)
+	}
+}
+
+// An image with no stamp cannot say which engine it is, and an agent that
+// guessed would be the failure the stamp exists to prevent.
+func TestNewEngine_RefusesAnImageWithNoEngineStamp(t *testing.T) {
+	t.Setenv("RDS_ENGINE_FILE", filepath.Join(t.TempDir(), "absent-engine"))
+	cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env"))
+
+	if _, err := newProbe(cfg, staticProbe(0)); err == nil {
+		t.Error("newProbe built a probe for an image carrying no engine stamp")
+	}
+	if _, err := newEngine(cfg, nil, nil, nil); err == nil {
+		t.Error("newEngine built an engine for an image carrying no engine stamp")
+	}
+}
+
+func TestNewEngine_RefusesAnEngineItDoesNotImplement(t *testing.T) {
+	cfg := testLoadConfig(t, "oracle")
+
+	_, err := newEngine(cfg, nil, nil, nil)
+	if err == nil {
+		t.Fatal("newEngine built an engine this agent does not implement")
+	}
+	if !strings.Contains(err.Error(), "oracle") {
+		t.Errorf("error = %v, want it to name the engine the image bakes", err)
+	}
+}
+
+// The implementation is the agent's, but the definition it validates names and
+// classifies parameters against is the control plane's, which now offers MariaDB.
+func TestNewEngine_BuildsMariaDBForAMariaDBImage(t *testing.T) {
+	cfg := testLoadConfig(t, engineMariaDB)
+	probe, err := newProbe(cfg, staticProbe(0))
+	if err != nil {
+		t.Fatalf("newProbe: %v", err)
+	}
+
+	built, err := newEngine(cfg, nil, nil, probe)
+	if err != nil {
+		t.Fatalf("newEngine: %v", err)
+	}
+	if _, ok := built.(*mariadbEngine); !ok {
+		t.Errorf("newEngine built %T for a MariaDB image", built)
+	}
+}
+
+// The probe reads the pidfile Alpine's packaged service starts mariadbd with,
+// because nothing in the generated configuration can move it: the service passes
+// --pid-file on the command line, which beats an option file.
+func TestLoadConfig_TakesTheMariaDBPidfileFromTheLayout(t *testing.T) {
+	cfg := testLoadConfig(t, engineMariaDB)
+
+	if cfg.EnginePidFile != engineLayouts[engineMariaDB].pidFile {
+		t.Errorf("pidfile = %q, want the MariaDB layout's", cfg.EnginePidFile)
+	}
+	if cfg.EnginePidFile == "" {
+		t.Error("an engine whose probe needs a pidfile was given none")
+	}
+}
+
 func TestLoadConfig_ReadsEnvFile(t *testing.T) {
+	t.Setenv("RDS_ENGINE_FILE", filepath.Join(t.TempDir(), "absent-engine"))
 	path := filepath.Join(t.TempDir(), "agent.env")
 	body := "RDS_GATEWAY_URL=https://gw.internal:9999\n" +
 		"RDS_REGION=ap-southeast-2\n" +
 		"RDS_DB_INSTANCE_IDENTIFIER=db-1\n" +
+		"RDS_ENGINE=postgres\n" +
 		"RDS_ENGINE_VERSION=18.1\n" +
 		"RDS_ENGINE_PORT=6543\n" +
 		"RDS_POLL_WAIT=5s\n"
@@ -45,19 +141,22 @@ func TestLoadConfig_ReadsEnvFile(t *testing.T) {
 	if cfg.DBInstanceIdentifier != "db-1" || cfg.EngineVersion != "18.1" {
 		t.Errorf("identity = %q/%q, want db-1/18.1", cfg.DBInstanceIdentifier, cfg.EngineVersion)
 	}
+	if cfg.Engine != enginePostgres {
+		t.Errorf("delivered engine = %q, want the one cloud-init carried", cfg.Engine)
+	}
 	if cfg.EnginePort != 6543 || cfg.PollWait != 5*time.Second {
 		t.Errorf("port/pollWait = %d/%v, want 6543/5s", cfg.EnginePort, cfg.PollWait)
 	}
 }
 
-// An unparseable override falls back to the default rather than leaving the
-// agent with a zero port or a zero-length long poll.
+// An unparseable override falls back to the layout's own default rather than
+// leaving the agent with a zero port or a zero-length long poll.
 func TestLoadConfig_IgnoresUnusableOverrides(t *testing.T) {
 	t.Setenv("RDS_ENGINE_PORT", "not-a-port")
 	t.Setenv("RDS_POLL_WAIT", "forever")
 
-	cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env"))
-	if cfg.EnginePort != defaultEnginePort || cfg.PollWait != defaultPollWait {
+	cfg := testLoadConfig(t, enginePostgres)
+	if cfg.EnginePort != engineLayouts[enginePostgres].port || cfg.PollWait != defaultPollWait {
 		t.Errorf("port/pollWait = %d/%v, want the defaults", cfg.EnginePort, cfg.PollWait)
 	}
 }

cmd/rds-agent/defaultparams_test.go

@@ -0,0 +1,69 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// The include the image build validates against a real mariadbd. It is checked
+// in rather than generated during the build because the build runs inside the
+// libguestfs appliance, where there is no Go toolchain to resolve the catalog.
+const mariadbDefaultParametersFixture = "../../scripts/images/rds-mariadb/default-parameters.cnf"
+
+// Byte-for-byte what rds-init installs on a smallest-class instance that
+// overrides nothing. Regenerate with RDS_UPDATE_FIXTURES=1 go test ./cmd/rds-agent/.
+//
+// Keeping it in step matters because setup.sh feeds exactly this file to
+// mariadbd: a catalog that grew a name the server refuses at startup would
+// otherwise reach a customer as an instance that never boots.
+func TestMariaDBDefaultParametersFixture_MatchesTheCatalog(t *testing.T) {
+	engine, err := handlers_rds.LookupEngine("mariadb")
+	if err != nil {
+		t.Fatalf("LookupEngine: %v", err)
+	}
+	class := handlers_rds.SmallestInstanceClass()
+	params, err := engine.ResolveEffectiveParameters(class, nil)
+	if err != nil {
+		t.Fatalf("ResolveEffectiveParameters(%s): %v", class, err)
+	}
+	body, err := renderParameters("mariadb", params)
+	if err != nil {
+		t.Fatalf("renderParameters: %v", err)
+	}
+	want := mariadbParametersHeader + body
+
+	path := filepath.FromSlash(mariadbDefaultParametersFixture)
+	if os.Getenv("RDS_UPDATE_FIXTURES") == "1" {
+		if err := os.WriteFile(path, []byte(want), 0o644); err != nil {
+			t.Fatalf("update %s: %v", path, err)
+		}
+		return
+	}
+
+	got, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatalf("read %s: %v", path, err)
+	}
+	if string(got) != want {
+		t.Errorf("%s is stale for class %s.\nRegenerate with RDS_UPDATE_FIXTURES=1 go test ./cmd/rds-agent/\n\ngot:\n%s\nwant:\n%s",
+			path, class, got, want)
+	}
+}
+
+// The one thing the fixture exists to prove cannot regress without the image
+// build catching it, asserted here too so a plain preflight names it.
+func TestMariaDBDefaultParametersFixture_CarriesNoSetOnlyName(t *testing.T) {
+	got, err := os.ReadFile(filepath.FromSlash(mariadbDefaultParametersFixture))
+	if err != nil {
+		t.Fatalf("read the fixture: %v", err)
+	}
+	for _, name := range []string{"\ntime_zone ", "\ntx_isolation ", "\ninnodb_buffer_pool_instances "} {
+		if strings.Contains(string(got), name) {
+			t.Errorf("the default include carries %q, which mariadbd refuses at startup", name)
+		}
+	}
+}

cmd/rds-agent/engine.go

@@ -3,25 +3,21 @@ package main
 import (
 	"bytes"
 	"context"
-	"errors"
 	"fmt"
-	"os"
 	"os/exec"
 	"os/user"
 	"path/filepath"
 	"strconv"
 	"strings"
-	"sync"
-	"sync/atomic"
 	"syscall"
 	"time"
 
 	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
 )
 
-// The in-guest half of the engine seam. Everything PostgreSQL-shaped that a
-// control-plane command needs lives behind this, so the command registry above
-// stays engine-agnostic.
+// The in-guest half of the engine seam. Everything engine-shaped that a
+// control-plane command needs lives behind this, so the command registry stays
+// engine-agnostic.
 type engineOps interface {
 	// Rotates the master role's password live. Never persisted anywhere in the
 	// guest: it exists only for the length of this call.
@@ -49,13 +45,119 @@ type parameterRecovery interface {
 	Restart(ctx context.Context) error
 }
 
-// Records the parameter file only after the probe observes the postmaster
-// serving it. This is separate from apply, where static values are installed
-// but not adopted until a later restart.
+// Records the parameter file only after the probe observes the engine serving
+// it. This is separate from apply, where static values are installed but not
+// adopted until a later restart.
 type servingParameterRecorder interface {
 	RecordServingParameters(ctx context.Context) error
 }
 
+// Everything one engine implementation owns. The narrow interfaces above are
+// what the heartbeat, the commander and the rollback guard each hold; this is
+// only what the agent has to construct.
+type engine interface {
+	engineOps
+	parameterRecovery
+	servingParameterRecorder
+	// Follows the port the control plane assigned, once the bootstrap fetch
+	// lands. Called from a different goroutine than the commands that read it.
+	setPort(port int)
+}
+
+// The guest layout an engine preset lays down, keyed by the engine the image
+// bakes. Every default that belongs to one engine lives here, so nothing
+// outside an implementation names its paths, service, port or probe.
+type engineLayout struct {
+	binDir    string
+	dataDir   string
+	socketDir string
+	osUser    string
+	service   string
+	dataMount string
+	port      int
+	// Where the engine records the pid the probe checks for liveness. Empty for
+	// an engine whose probe answers without one.
+	pidFile string
+	// The engine's own liveness signal, which is the only part of engineProbe
+	// that is not shared.
+	newProbe func(cfg config, run probeRunner) *engineProbe
+	// The implementation every control-plane directive is served by. It fails
+	// when the control plane this agent was built against does not offer the
+	// engine the image bakes, rather than serving one it has no definition for.
+	newEngine func(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) (engine, error)
+}
+
+// The names the control plane knows these implementations by, which are also
+// what setup.sh stamps into each image.
+const (
+	enginePostgres = "postgres"
+	engineMariaDB  = "mariadb"
+)
+
+var engineLayouts = map[string]engineLayout{
+	enginePostgres: {
+		binDir:    "/usr/libexec/postgresql18",
+		dataDir:   "/var/lib/postgresql/18/data",
+		socketDir: "/run/postgresql",
+		osUser:    "postgres",
+		service:   "postgresql",
+		dataMount: "/var/lib/postgresql",
+		port:      5432,
+		newProbe:  newPostgresProbe,
+		newEngine: func(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) (engine, error) {
+			return newPostgresEngine(cfg, run, startSess, probe), nil
+		},
+	},
+	engineMariaDB: {
+		binDir:    "/usr/bin",
+		dataDir:   "/var/lib/mysql/data",
+		socketDir: "/run/mysqld",
+		osUser:    "mysql",
+		service:   "mariadb",
+		dataMount: "/var/lib/mysql",
+		port:      3306,
+		// Alpine's packaged service starts mysqld_safe with
+		// --pid-file=/run/mysqld/$RC_SVCNAME.pid, and a command-line option beats
+		// the configuration file, so this is the path rather than anything the
+		// platform drop-in could name. setup.sh asserts it against the packaged
+		// init script at build time.
+		pidFile:   "/run/mysqld/mariadb.pid",
+		newProbe:  newMariaDBProbe,
+		newEngine: newMariaDBEngineFromCatalog,
+	},
+}
+
+// Builds the health probe the image's engine supplies.
+func newProbe(cfg config, run probeRunner) (*engineProbe, error) {
+	layout, err := layoutFor(cfg)
+	if err != nil {
+		return nil, err
+	}
+	return layout.newProbe(cfg, run), nil
+}
+
+// Builds the implementation the image bakes. Nothing here consults the control
+// plane for which engine to be: the agent runs the engine it is made of, and the
+// delivered engine is only ever checked against it.
+func newEngine(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) (engine, error) {
+	layout, err := layoutFor(cfg)
+	if err != nil {
+		return nil, err
+	}
+	return layout.newEngine(cfg, run, startSess, probe)
+}
+
+func layoutFor(cfg config) (engineLayout, error) {
+	if cfg.BakedEngine == "" {
+		return engineLayout{}, fmt.Errorf("this image carries no engine stamp at %s", cfg.EngineFile)
+	}
+	layout, ok := engineLayouts[cfg.BakedEngine]
+	if !ok {
+		return engineLayout{}, fmt.Errorf("this image bakes engine %q, which this agent does not implement", cfg.BakedEngine)
+	}
+	return layout, nil
+}
+
 // One child process. Env replaces the agent's own environment rather than
 // extending it, so a secret placed here reaches only the process that needs it.
 type command struct {
@@ -65,10 +167,14 @@ type command struct {
 	Stdin string
 	// The OS user to drop to before exec. Empty runs as the agent itself.
 	User string
+	// The statement that makes this client print the session sentinel. Read only
+	// when the command is started as a held session, whose reader has to know
+	// when the engine has finished each script it was fed.
+	SentinelStatement string
 }
 
-// Returns stdout. stderr is folded into the error, because psql reports the
-// actual SQL failure there while exiting with a bare status.
+// Returns stdout. stderr is folded into the error, because both clients report
+// the actual SQL failure there while exiting with a bare status.
 type commandRunner func(ctx context.Context, c command) (string, error)
 
 func execCommandRunner(ctx context.Context, c command) (string, error) {
@@ -113,384 +219,16 @@ func lookupCredential(username string) (*syscall.Credential, error) {
 	return &syscall.Credential{Uid: uint32(uid), Gid: uint32(gid)}, nil
 }
 
-// The engine is reached over its unix socket under peer authentication, so the
-// agent drops to the postgres OS user rather than holding a password of its own.
-type postgresEngine struct {
-	run       commandRunner
-	startSess sessionRunner
-	psql      string
-	rcService string
-	service   string
-	pgData    string
-	socketDir string
-	osUser    string
-	probe     *engineProbe
-	// Set from the bootstrap config, on a different goroutine than the commands
-	// that read it.
-	port atomic.Int64
-
-	// Serializes parameter installs, serving snapshots and rollback restores so
-	// none can copy or replace an intermediate configuration.
-	paramMu       sync.Mutex
-	repairTimeout time.Duration
-	repairPoll    time.Duration
-
-	// The backup mode currently held, if any. Guarded because the expiry timer
-	// releases it on its own goroutine.
-	mu   sync.Mutex
-	held *quiesceHold
-}
-
-var (
-	_ engineOps                = (*postgresEngine)(nil)
-	_ parameterRecovery        = (*postgresEngine)(nil)
-	_ servingParameterRecorder = (*postgresEngine)(nil)
-)
-
-const (
-	parameterRepairTimeout = 90 * time.Second
-	parameterRepairPoll    = time.Second
-)
-
-func newPostgresEngine(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) *postgresEngine {
-	e := &postgresEngine{
-		run:           run,
-		startSess:     startSess,
-		psql:          filepath.Join(cfg.PGBin, "psql"),
-		rcService:     cfg.RCService,
-		service:       cfg.EngineService,
-		pgData:        cfg.PGData,
-		socketDir:     cfg.SocketDir,
-		osUser:        cfg.PGUser,
-		probe:         probe,
-		repairTimeout: parameterRepairTimeout,
-		repairPoll:    parameterRepairPoll,
-	}
-	e.port.Store(int64(cfg.EnginePort))
-	return e
-}
-
-func (e *postgresEngine) setPort(port int) {
-	e.port.Store(int64(port))
-}
-
-// The role name and the password ride the environment and are re-quoted by
-// psql, so neither reaches a shell word or an argv another process can read.
-func (e *postgresEngine) SetPassword(ctx context.Context, username, password string) error {
-	if username == "" || password == "" {
-		return fmt.Errorf("set-password requires both %s and %s",
-			handlers_rds.CommandParamMasterUsername, handlers_rds.CommandParamMasterUserPassword)
-	}
-	// The statement below runs as the cluster superuser over the socket under
-	// peer auth, so a reserved name in the command payload would hand the
-	// customer the bootstrap superuser rather than rotate their own role.
-	if err := handlers_rds.EnginePostgres().ValidateUsernameNotReserved(username); err != nil {
-		return fmt.Errorf("refusing to set the password of a role the engine reserves: %w", err)
-	}
-	// psql interpolates the password into the ALTER ROLE before the server sees
-	// it, and these three are what would write it to the log — the last on any
-	// failure at its own default. SUSET, so the parameter group cannot win.
-	const sql = `SET log_statement = 'none';
-SET log_min_duration_statement = -1;
-SET log_min_error_statement = 'panic';
-\getenv master RDS_MASTER_USERNAME
-\getenv password RDS_MASTER_PASSWORD
-ALTER ROLE :"master" WITH LOGIN PASSWORD :'password';
-`
-	_, err := e.psqlRun(ctx, sql,
-		"RDS_MASTER_USERNAME="+username,
-		"RDS_MASTER_PASSWORD="+password,
-	)
-	if err != nil {
-		// psql echoes the failing statement, which here would carry the new
-		// password back to the control plane and into the event ring.
-		return fmt.Errorf("apply the master password: %s", redact(err.Error(), password))
-	}
-	return nil
-}
-
-// The include the resolved parameter set is rendered to, and the copy of the
-// last one the engine accepted. Both live beside the data rather than in /etc,
-// matching rds-init: a class change boots a fresh root volume, which would
-// otherwise revert them when a fresh root volume is used.
-const (
-	parametersFileName = "10-rds-parameters.conf"
-	// Deliberately not a .conf name: include_dir globs *.conf, so the rollback
-	// copy must not be read as a second set of settings.
-	lastGoodFileName = "10-rds-parameters.last-good"
-)
-
-// Installs the resolved set, validates it with the engine's own config parser
-// before the engine ever adopts it, and reloads. A value the engine refuses is
-// rolled back here rather than left on the data volume, where it would survive
-// every VM replace and turn the next restart into a boot loop.
-func (e *postgresEngine) ApplyParameters(ctx context.Context, params []handlers_rds.Parameter) ([]string, error) {
-	e.paramMu.Lock()
-	defer e.paramMu.Unlock()
-
-	initialState, _ := e.probe.state(ctx)
-	if initialState == engineRecovering {
-		return nil, errors.New("apply parameters while the engine is still starting or recovering")
-	}
-	if initialState == engineServing {
-		// A command can arrive before the first heartbeat. Seed the configuration
-		// currently being served before replacing its file in that window.
-		if err := e.recordServingParametersLocked(ctx); err != nil {
-			return nil, fmt.Errorf("record the parameters serving before the apply: %w", err)
-		}
-	}
-
-	// The check has to run against the file in place, because the engine parses
-	// the datadir's own include_dir. The window that leaves is closed from both
-	// ends: the rollback below, and the last-known-good restore the agent runs at
-	// boot when the engine does not come up.
-	_, restore, err := e.installParameters(params)
-	if err != nil {
-		return nil, err
-	}
-	if err := e.checkConfig(ctx); err != nil {
-		return nil, errors.Join(fmt.Errorf("the engine rejected the parameter set: %w", err), restore())
-	}
-
-	state, _ := e.probe.state(ctx)
-	switch state {
-	case engineAbsent:
-		return e.restartOnRepairSetLocked(ctx)
-	case engineRecovering:
-		return nil, errors.Join(errors.New("the engine entered startup or recovery during the parameter apply"), restore())
-	}
-
-	if _, err := e.psqlRun(ctx, "SELECT pg_reload_conf();\n"); err != nil {
-		// The engine may have gone down after the first probe. In that case start
-		// it on the new, parser-checked repair set rather than restoring the set
-		// that already left it unable to serve.
-		if state, _ := e.probe.state(ctx); state == engineAbsent {
-			return e.restartOnRepairSetLocked(ctx)
-		}
-		return nil, errors.Join(fmt.Errorf("reload the engine configuration: %w", err), restore())
-	}
-	return e.pendingRestartParameters(ctx)
-}
-
-func (e *postgresEngine) restartOnRepairSetLocked(ctx context.Context) ([]string, error) {
-	repairCtx, cancel := context.WithTimeout(ctx, e.repairTimeout)
-	defer cancel()
-
-	if err := e.Restart(repairCtx); err != nil {
-		return nil, fmt.Errorf("start the engine on the repaired parameter set: %w", err)
-	}
-
-	timer := time.NewTimer(0)
-	defer timer.Stop()
-	lastMessage := "engine did not respond"
-	for {
-		select {
-		case <-repairCtx.Done():
-			if ctx.Err() != nil {
-				return nil, fmt.Errorf("wait for the engine on the repaired parameter set: %w", ctx.Err())
-			}
-			return nil, fmt.Errorf("wait for the engine on the repaired parameter set: %s: %w", lastMessage, repairCtx.Err())
-		case <-timer.C:
-		}
-
-		state, message := e.probe.state(repairCtx)
-		if state == engineServing {
-			pending, err := e.pendingRestartParameters(repairCtx)
-			if err == nil {
-				return pending, nil
-			}
-			lastMessage = err.Error()
-		} else if message != "" {
-			lastMessage = message
-		}
-		timer.Reset(e.repairPoll)
-	}
-}
-
-func (e *postgresEngine) pendingRestartParameters(ctx context.Context) ([]string, error) {
-	out, err := e.psqlRun(ctx, "SELECT name FROM pg_settings WHERE pending_restart ORDER BY name;\n")
-	if err != nil {
-		return nil, fmt.Errorf("read the settings pending a restart: %w", err)
-	}
-	var pending []string
-	for line := range strings.SplitSeq(out, "\n") {
-		if name := strings.TrimSpace(line); name != "" {
-			pending = append(pending, name)
-		}
-	}
-	return pending, nil
-}
-
-func (e *postgresEngine) parametersPath() string {
-	return filepath.Join(e.pgData, "conf.d", parametersFileName)
-}
-
-func (e *postgresEngine) lastGoodPath() string {
-	return filepath.Join(e.pgData, "conf.d", lastGoodFileName)
-}
-
-// Written as root and handed to the engine's user, at the same path and mode
-// rds-init installs it at, so a later boot overwrites rather than shadows it.
-// Returns the content that was there and a func putting it back, so the caller
-// can undo the install without re-rendering anything.
-func (e *postgresEngine) installParameters(params []handlers_rds.Parameter) (previous []byte, restore func() error, err error) {
-	path := e.parametersPath()
-	// A missing file is the first-ever apply, and its rollback is a removal.
-	previous, readErr := os.ReadFile(path)
-	if readErr != nil && !os.IsNotExist(readErr) {
-		return nil, nil, fmt.Errorf("read the installed parameters at %s: %w", path, readErr)
-	}
-	existed := readErr == nil
-
-	if err := e.writeEngineFile(path, []byte(renderParameters(params))); err != nil {
-		return nil, nil, err
-	}
-	return previous, func() error {
-		if !existed {
-			if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
-				return fmt.Errorf("withdraw the rejected parameters at %s: %w", path, err)
-			}
-			return nil
-		}
-		return e.writeEngineFile(path, previous)
-	}, nil
-}
-
-// Snapshots the include only when the running postmaster has no settings still
-// pending a restart. The parameter mutex keeps an apply from replacing the file
-// between that check and the copy.
-func (e *postgresEngine) RecordServingParameters(ctx context.Context) error {
-	e.paramMu.Lock()
-	defer e.paramMu.Unlock()
-	return e.recordServingParametersLocked(ctx)
-}
-
-func (e *postgresEngine) recordServingParametersLocked(ctx context.Context) error {
-	if _, err := os.Stat(e.parametersPath()); err != nil {
-		if os.IsNotExist(err) {
-			return nil
-		}
-		return fmt.Errorf("inspect the serving parameters: %w", err)
-	}
-	pending, err := e.pendingRestartParameters(ctx)
-	if err != nil {
-		return err
-	}
-	if len(pending) > 0 {
-		return nil
-	}
-
-	content, err := os.ReadFile(e.parametersPath())
-	if err != nil {
-		return fmt.Errorf("read the serving parameters: %w", err)
-	}
-	lastGood, err := os.ReadFile(e.lastGoodPath())
-	if err == nil && bytes.Equal(content, lastGood) {
-		return nil
-	}
-	if err != nil && !os.IsNotExist(err) {
-		return fmt.Errorf("read the last known good parameters: %w", err)
-	}
-	return e.writeEngineFile(e.lastGoodPath(), content)
-}
-
-// Puts the last set the engine accepted back in place, for a restart that failed
-// after a parameter change. The probe is checked again under the parameter lock
-// so a repair that just brought the engine back cannot be reversed.
-func (e *postgresEngine) RestoreLastKnownGoodParameters(ctx context.Context) (bool, error) {
-	e.paramMu.Lock()
-	defer e.paramMu.Unlock()
-
-	if state, _ := e.probe.state(ctx); state != engineAbsent {
-		return false, nil
-	}
-	lastGood, err := os.ReadFile(e.lastGoodPath())
-	if err != nil {
-		if os.IsNotExist(err) {
-			return false, nil
-		}
-		return false, fmt.Errorf("read the last known good parameters: %w", err)
-	}
-	current, err := os.ReadFile(e.parametersPath())
-	if err != nil && !os.IsNotExist(err) {
-		return false, fmt.Errorf("read the installed parameters: %w", err)
-	}
-	if bytes.Equal(current, lastGood) {
-		return false, nil
-	}
-	if err := e.writeEngineFile(e.parametersPath(), lastGood); err != nil {
-		return false, err
-	}
-	return true, nil
-}
-
-// Atomic: a temp file in the same directory, renamed over the target. The temp
-// name deliberately does not end in .conf, so a crash between write and rename
-// cannot leave the engine reading a half-written include.
-func (e *postgresEngine) writeEngineFile(path string, content []byte) error {
-	tmp := path + ".new"
-	if err := os.WriteFile(tmp, content, 0o600); err != nil {
-		return fmt.Errorf("write %s: %w", tmp, err)
-	}
-	if err := e.chownToEngine(tmp); err != nil {
-		os.Remove(tmp)
-		return err
-	}
-	if err := os.Rename(tmp, path); err != nil {
-		os.Remove(tmp)
-		return fmt.Errorf("install %s: %w", path, err)
-	}
-	return nil
-}
-
-// The engine's own parser, run offline against the datadir. Reading one setting
-// back is enough: postgres parses postgresql.conf and every include first, and
-// exits non-zero naming the file and line of an unknown parameter or a value
-// outside its range.
-func (e *postgresEngine) checkConfig(ctx context.Context) error {
-	if _, err := e.run(ctx, command{
-		Name: filepath.Join(filepath.Dir(e.psql), "postgres"),
-		Args: []string{"-D", e.pgData, "-C", "shared_buffers"},
-		Env:  []string{"PATH=" + defaultGuestPath},
-		User: e.osUser,
-	}); err != nil {
-		return err
-	}
-	return nil
-}
-
-// The engine reads its own config, so a root-owned file would be unreadable to
-// it. A guest with no such user is a broken image, not something to work around.
-func (e *postgresEngine) chownToEngine(path string) error {
-	credential, err := lookupCredential(e.osUser)
-	if err != nil {
-		return err
-	}
-	if err := os.Chown(path, int(credential.Uid), int(credential.Gid)); err != nil {
-		return fmt.Errorf("hand %s to %s: %w", path, e.osUser, err)
-	}
-	return nil
-}
-
-// Through the service manager rather than pg_ctl, so the supervisor records the
-// engine as stopped and does not restart it underneath a VM that is going down.
-func (e *postgresEngine) Stop(ctx context.Context) error {
-	return e.serviceAction(ctx, "stop")
-}
-
-// Only the parameter rollback calls this: a restart the control plane wants goes
-// through RebootDBInstance, which cycles the VM.
-func (e *postgresEngine) Restart(ctx context.Context) error {
-	return e.serviceAction(ctx, "restart")
-}
-
-func (e *postgresEngine) serviceAction(ctx context.Context, action string) error {
-	if _, err := e.run(ctx, command{
-		Name: e.rcService,
-		Args: []string{e.service, action},
+// Through the service manager rather than the engine's own control tool, so the
+// supervisor records the state and does not restart the engine underneath a VM
+// that is going down.
+func serviceAction(ctx context.Context, run commandRunner, rcService, service, action string) error {
+	if _, err := run(ctx, command{
+		Name: rcService,
+		Args: []string{service, action},
 		Env:  []string{"PATH=" + defaultGuestPath},
 	}); err != nil {
-		return fmt.Errorf("%s the %s service: %w", action, e.service, err)
+		return fmt.Errorf("%s the %s service: %w", action, service, err)
 	}
 	return nil
 }
@@ -498,32 +236,6 @@ func (e *postgresEngine) serviceAction(ctx context.Context, action string) error
 // A minimal environment for a child that is not carrying secrets.
 const defaultGuestPath = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
 
-// Feeds sql on stdin rather than as an argument, so a statement is never
-// visible in the process table.
-func (e *postgresEngine) psqlRun(ctx context.Context, sql string, env ...string) (string, error) {
-	return e.run(ctx, command{
-		Name:  e.psql,
-		Args:  e.psqlArgs(),
-		Env:   append([]string{"PATH=" + defaultGuestPath}, env...),
-		Stdin: sql,
-		User:  e.osUser,
-	})
-}
-
-// Reading the script from stdin is what lets one invocation serve both a
-// one-shot run and a session held open across several statements.
-func (e *postgresEngine) psqlArgs() []string {
-	return []string{
-		"--no-psqlrc", "--quiet", "--no-align", "--tuples-only",
-		"-v", "ON_ERROR_STOP=1",
-		"-h", e.socketDir,
-		"-p", strconv.FormatInt(e.port.Load(), 10),
-		"-U", e.osUser,
-		"-d", "postgres",
-		"-f", "-",
-	}
-}
-
 func redact(s, secret string) string {
 	if secret == "" {
 		return s

cmd/rds-agent/engine_contract_test.go

@@ -0,0 +1,329 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"os"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// A probe the test drives directly, in place of whichever binaries the engine's
+// own state function would have run. Each implementation's state function is
+// covered on its own; what these cases exercise is what the engine does with the
+// answer.
+func drivenProbe(state *engineState) *engineProbe {
+	return newEngineProbe(0, func(context.Context, int64) (engineState, string) {
+		return *state, "driven by the test"
+	})
+}
+
+// One engine behind the interfaces the heartbeat, the commander and the rollback
+// guard hold, plus the few facts a shared case needs to say what it means.
+type engineHarness struct {
+	engine  engine
+	runner  *recordingRunner
+	params  parameterStore
+	service string
+	// The state the probe reports, moved by the test rather than by the engine.
+	state *engineState
+	// Points the next quiesce at a session the test can inspect, and returns the
+	// commands those sessions were opened with.
+	useSession func(*fakeSession) *[]command
+	// A setting this engine's catalog really carries, so a shared case installs
+	// something the implementation would genuinely be handed.
+	parameter handlers_rds.Parameter
+	// What the engine's own include directive globs. Neither the rollback copy
+	// nor the serving copy may end in it.
+	includeSuffix string
+}
+
+func engineHarnesses() []struct {
+	name  string
+	build func(*testing.T) *engineHarness
+} {
+	return []struct {
+		name  string
+		build func(*testing.T) *engineHarness
+	}{
+		{name: enginePostgres, build: buildPostgresHarness},
+		{name: engineMariaDB, build: buildMariaDBHarness},
+	}
+}
+
+func forEachEngine(t *testing.T, run func(*testing.T, *engineHarness)) {
+	t.Helper()
+	for _, fixture := range engineHarnesses() {
+		t.Run(fixture.name, func(t *testing.T) { run(t, fixture.build(t)) })
+	}
+}
+
+func buildPostgresHarness(t *testing.T) *engineHarness {
+	t.Helper()
+	runner := &recordingRunner{}
+	e := newTestEngine(t, runner.run)
+	state := engineServing
+	e.probe = drivenProbe(&state)
+
+	return &engineHarness{
+		engine: e, runner: runner, params: e.params, service: e.service, state: &state,
+		useSession: func(session *fakeSession) *[]command {
+			started := &[]command{}
+			e.startSess = func(_ context.Context, c command) (engineSession, error) {
+				*started = append(*started, c)
+				return session, nil
+			}
+			return started
+		},
+		parameter:     handlers_rds.Parameter{Name: "work_mem", Value: "4096"},
+		includeSuffix: ".conf",
+	}
+}
+
+func buildMariaDBHarness(t *testing.T) *engineHarness {
+	t.Helper()
+	runner := &recordingRunner{}
+	e := newTestMariaDBEngine(t, runner.run)
+	state := engineServing
+	e.probe = drivenProbe(&state)
+
+	return &engineHarness{
+		engine: e, runner: runner, params: e.params, service: e.service, state: &state,
+		useSession: func(session *fakeSession) *[]command {
+			started := &[]command{}
+			e.startSess = func(_ context.Context, c command) (engineSession, error) {
+				*started = append(*started, c)
+				return session, nil
+			}
+			return started
+		},
+		parameter:     handlers_rds.Parameter{Name: "max_connections", Value: "200"},
+		includeSuffix: ".cnf",
+	}
+}
+
+// The parameters land where rds-init installs them, so the next boot overwrites
+// the file rather than shadowing it with a second copy.
+func TestEngineContract_ApplyParametersInstallsWhereRdsInitDoes(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		if _, err := h.engine.ApplyParameters(t.Context(), []handlers_rds.Parameter{h.parameter}); err != nil {
+			t.Fatalf("ApplyParameters: %v", err)
+		}
+
+		installed, err := os.ReadFile(h.params.installedPath())
+		if err != nil {
+			t.Fatalf("read the installed parameters: %v", err)
+		}
+		if !strings.Contains(string(installed), h.parameter.Name+" = '"+h.parameter.Value+"'") {
+			t.Errorf("installed parameters = %q, want the resolved value", installed)
+		}
+		if !strings.HasSuffix(h.params.installedPath(), h.includeSuffix) {
+			t.Errorf("the installed set at %s is not a file the engine includes", h.params.installedPath())
+		}
+		// Comparison material, not a second set of settings: an include glob that
+		// read either of these would apply a withdrawn configuration.
+		copies := []string{h.params.lastGoodPath()}
+		if h.params.serving != "" {
+			copies = append(copies, h.params.servingPath())
+		}
+		for _, path := range copies {
+			if strings.HasSuffix(path, h.includeSuffix) {
+				t.Errorf("the copy at %s would be included by the engine", path)
+			}
+		}
+	})
+}
+
+// An engine still starting or replaying has not finished adopting the set it is
+// on, so replacing that set underneath it would be installing over a moving
+// target.
+func TestEngineContract_ApplyParametersRefusesWhileTheEngineIsRecovering(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		*h.state = engineRecovering
+
+		if _, err := h.engine.ApplyParameters(t.Context(), []handlers_rds.Parameter{h.parameter}); err == nil {
+			t.Fatal("ApplyParameters succeeded against an engine that was still coming up")
+		}
+		if _, err := os.Stat(h.params.installedPath()); !os.IsNotExist(err) {
+			t.Errorf("the refused set was installed anyway (stat err = %v)", err)
+		}
+	})
+}
+
+// The boot-time half of the parameter safety net: the last set the engine
+// accepted goes back only when the engine is genuinely down, and only once.
+func TestEngineContract_RestoreLastKnownGoodOnlyRollsBackADownEngine(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		accepted := h.parameter
+		if _, err := h.engine.ApplyParameters(t.Context(), []handlers_rds.Parameter{accepted}); err != nil {
+			t.Fatalf("ApplyParameters: %v", err)
+		}
+		if err := h.engine.RecordServingParameters(t.Context()); err != nil {
+			t.Fatalf("RecordServingParameters: %v", err)
+		}
+		if _, err := h.engine.ApplyParameters(t.Context(),
+			[]handlers_rds.Parameter{{Name: accepted.Name, Value: accepted.Value + "0"}}); err != nil {
+			t.Fatalf("second ApplyParameters: %v", err)
+		}
+
+		// A guard whose deadline expired while an apply brought the engine back must
+		// not reverse that repair.
+		restored, err := h.engine.RestoreLastKnownGoodParameters(t.Context())
+		if err != nil {
+			t.Fatalf("RestoreLastKnownGoodParameters against a serving engine: %v", err)
+		}
+		if restored {
+			t.Fatal("rolled the parameters back while the engine was serving")
+		}
+
+		*h.state = engineAbsent
+		restored, err = h.engine.RestoreLastKnownGoodParameters(t.Context())
+		if err != nil {
+			t.Fatalf("RestoreLastKnownGoodParameters: %v", err)
+		}
+		if !restored {
+			t.Fatal("did not roll back to a set that differs from the last accepted one")
+		}
+		installed, err := os.ReadFile(h.params.installedPath())
+		if err != nil {
+			t.Fatalf("read the installed parameters: %v", err)
+		}
+		if !strings.Contains(string(installed), accepted.Name+" = '"+accepted.Value+"'") {
+			t.Errorf("installed = %q, want the last accepted set", installed)
+		}
+
+		// Repeating a rollback would only churn a cluster that is failing for
+		// another reason.
+		restored, err = h.engine.RestoreLastKnownGoodParameters(t.Context())
+		if err != nil {
+			t.Fatalf("RestoreLastKnownGoodParameters: %v", err)
+		}
+		if restored {
+			t.Error("rolled back again when the installed set already matched")
+		}
+	})
+}
+
+// Through the service manager rather than the engine's own control tool, so the
+// supervisor records the engine as stopped and does not restart it underneath a
+// VM that is going down.
+func TestEngineContract_StopGoesThroughTheServiceManager(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		if err := h.engine.Stop(t.Context()); err != nil {
+			t.Fatalf("Stop: %v", err)
+		}
+		if len(h.runner.calls) != 1 {
+			t.Fatalf("ran %d commands, want the service stop", len(h.runner.calls))
+		}
+		call := h.runner.calls[0]
+		if !slices.Equal(call.Args, []string{h.service, "stop"}) {
+			t.Errorf("stop ran %s %v, want the service manager", call.Name, call.Args)
+		}
+	})
+}
+
+// One backup at a time, released by Unquiesce or by its own deadline, whichever
+// comes first. The deadline is what keeps a control plane that dies mid-snapshot
+// from leaving the engine held.
+func TestEngineContract_QuiesceHoldsUntilReleased(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		session := &fakeSession{}
+		started := h.useSession(session)
+
+		if err := h.engine.Quiesce(t.Context(), "orders-db-pre-upgrade", time.Minute); err != nil {
+			t.Fatalf("Quiesce: %v", err)
+		}
+		if len(*started) != 1 {
+			t.Fatalf("started %d sessions, want 1", len(*started))
+		}
+		if session.closes() != 0 {
+			t.Fatal("the session was closed while the backup was meant to be held")
+		}
+
+		err := h.engine.Quiesce(t.Context(), "second", time.Minute)
+		if err == nil {
+			t.Fatal("a second quiesce was accepted while the first was held")
+		}
+		if !strings.Contains(err.Error(), "orders-db-pre-upgrade") {
+			t.Errorf("error %q does not name the backup already holding the engine", err)
+		}
+		if len(*started) != 1 {
+			t.Errorf("started %d sessions, want the second refused before opening one", len(*started))
+		}
+
+		if err := h.engine.Unquiesce(t.Context()); err != nil {
+			t.Fatalf("Unquiesce: %v", err)
+		}
+		if session.closes() != 1 {
+			t.Errorf("closed %d times, want the session ended exactly once", session.closes())
+		}
+	})
+}
+
+// A hold that expired mid-snapshot means the snapshot was not taken against a
+// held checkpoint, so the release reports it rather than succeeding silently.
+func TestEngineContract_QuiesceReleasesItselfOnItsDeadline(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		session := &fakeSession{}
+		h.useSession(session)
+
+		if err := h.engine.Quiesce(t.Context(), "orders-db-pre-upgrade", 10*time.Millisecond); err != nil {
+			t.Fatalf("Quiesce: %v", err)
+		}
+		deadline := time.Now().Add(2 * time.Second)
+		for session.closes() == 0 && time.Now().Before(deadline) {
+			time.Sleep(time.Millisecond)
+		}
+		if session.closes() != 1 {
+			t.Fatal("the backup session outlived its deadline")
+		}
+
+		err := h.engine.Unquiesce(t.Context())
+		if err == nil {
+			t.Fatal("Unquiesce succeeded after the hold expired")
+		}
+		if !strings.Contains(err.Error(), "expired") {
+			t.Errorf("error %q does not say the hold had expired", err)
+		}
+	})
+}
+
+// A failed start leaves nothing held, so a retry is not refused by a hold that
+// never took effect.
+func TestEngineContract_QuiesceEndsTheSessionWhenTheBackupWillNotStart(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		session := &fakeSession{execErr: errors.New("the engine refused the backup")}
+		h.useSession(session)
+
+		if err := h.engine.Quiesce(t.Context(), "orders-db-pre-upgrade", time.Minute); err == nil {
+			t.Fatal("Quiesce succeeded against an engine that refused the backup")
+		}
+		if session.closes() != 1 {
+			t.Errorf("closed %d times, want the failed session ended", session.closes())
+		}
+		if err := h.engine.Unquiesce(t.Context()); err == nil {
+			t.Error("a failed quiesce left a hold behind")
+		}
+	})
+}
+
+// Every command the control plane can issue is served by both implementations,
+// so a directive cannot be answered as unsupported on one engine and honoured on
+// the other.
+func TestEngineContract_ServesEveryRegisteredCommand(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		registry := newCommandRegistry(h.engine, &fakeStorage{})
+		for _, name := range []string{
+			handlers_rds.CommandSetPassword, handlers_rds.CommandApplyParams, handlers_rds.CommandStopEngine,
+			handlers_rds.CommandGrowFilesystem, handlers_rds.CommandQuiesce, handlers_rds.CommandUnquiesce,
+		} {
+			if _, ok := registry[name]; !ok {
+				t.Errorf("command %q has no handler", name)
+			}
+		}
+	})
+}

cmd/rds-agent/engine_mariadb.go

@@ -0,0 +1,638 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"slices"
+	"strconv"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// The control plane's own statements about this engine, resolved once at
+// startup. Held as functions rather than as the Engine value so nothing here
+// re-derives a rule the API already owns.
+type controlPlaneRules struct {
+	// The whole master-username rule rather than only the reserved set. MariaDB's
+	// client has no identifier quoting, so the name is interpolated into SQL by
+	// this process and every part of the rule is load-bearing here.
+	validateUsername func(username string) error
+	// Whether a setting takes effect only at a restart. Already customer-facing
+	// and authoritative, since the API refuses ApplyMethod=immediate on one.
+	isStatic func(name string) bool
+	// The catalog name behind a spelling read back out of an option file. Only a
+	// setting whose startup spelling differs from the customer's moves.
+	catalogName func(optionFileName string) string
+}
+
+func controlPlaneRulesFrom(meta handlers_rds.Engine) controlPlaneRules {
+	// Built once per engine: everything read back out of a generated file has to
+	// return to the catalog's namespace before it is classified or reported, or a
+	// startup spelling would read as an unknown name.
+	catalogNames := map[string]string{}
+	for _, name := range meta.CatalogParameterNames() {
+		if optionFileName := meta.OptionFileName(name); optionFileName != name {
+			catalogNames[optionFileName] = name
+		}
+	}
+
+	return controlPlaneRules{
+		validateUsername: meta.ValidateMasterUsername,
+		isStatic: func(name string) bool {
+			spec, ok := meta.LookupParameter(name)
+			// A name the catalog does not carry cannot be shown to have been adopted
+			// without a restart, and is never issued as a live SET GLOBAL either.
+			return !ok || spec.ApplyType == handlers_rds.ApplyTypeStatic
+		},
+		catalogName: func(optionFileName string) string {
+			if name, ok := catalogNames[optionFileName]; ok {
+				return name
+			}
+			return optionFileName
+		},
+	}
+}
+
+// The engine is reached over its unix socket as root, which the datadir's
+// unix_socket plugin authenticates from the connecting process' own uid — the
+// direct analogue of PostgreSQL's peer auth, and the reason the agent holds no
+// password of its own.
+type mariadbEngine struct {
+	quiesceState
+
+	rules     controlPlaneRules
+	run       commandRunner
+	startSess sessionRunner
+	client    string
+	admin     string
+	socket    string
+	rcService string
+	service   string
+	probe     *engineProbe
+
+	params parameterStore
+	// Serializes parameter installs, serving snapshots and rollback restores so
+	// none can copy or replace an intermediate configuration.
+	paramMu       sync.Mutex
+	repairTimeout time.Duration
+	repairPoll    time.Duration
+}
+
+var _ engine = (*mariadbEngine)(nil)
+
+const (
+	mariadbClientBinary = "mariadb"
+	mariadbAdminBinary  = "mariadb-admin"
+	// The account mariadb-install-db creates for the unix_socket plugin, which
+	// rds-init keeps for the platform and never hands to the customer.
+	mariadbSuperuser                  = "root"
+	mariadbProbeConnectTimeoutSeconds = 3
+	// Named by the platform drop-in rds-init writes, so both halves reach the
+	// same socket without either asserting it to the other.
+	mariadbSocketFile = "mysqld.sock"
+	// The customer reaches the instance over its customer ENI, so the master is
+	// created and rotated on the wildcard host rather than on localhost.
+	mariadbMasterHost = "%"
+)
+
+// The generated drop-ins on the data volume. The rollback and serving copies
+// deliberately do not end in .cnf: !includedir reads *.cnf, and neither is a
+// second set of settings.
+const (
+	mariadbParametersFile = "10-rds-parameters.cnf"
+	mariadbLastGoodFile   = "10-rds-parameters.last-good"
+	mariadbServingFile    = "10-rds-parameters.serving"
+	// MariaDB treats a setting before any group as a fatal parsing error, so
+	// every generated file carries the header rds-init writes.
+	mariadbParametersHeader = "[mysqld]\n"
+)
+
+// The layout's factory. The rules come from the control plane's own definition
+// of this engine, so a build whose control plane does not offer MariaDB refuses
+// to run this implementation rather than inventing a definition for it.
+func newMariaDBEngineFromCatalog(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) (engine, error) {
+	meta, err := handlers_rds.LookupEngine(engineMariaDB)
+	if err != nil {
+		return nil, fmt.Errorf("this image bakes %s, which this build's control plane does not offer: %w", engineMariaDB, err)
+	}
+	return newMariaDBEngine(cfg, controlPlaneRulesFrom(meta), run, startSess, probe), nil
+}
+
+func newMariaDBEngine(cfg config, rules controlPlaneRules, run commandRunner, startSess sessionRunner, probe *engineProbe) *mariadbEngine {
+	return &mariadbEngine{
+		rules:     rules,
+		run:       run,
+		startSess: startSess,
+		client:    filepath.Join(cfg.EngineBinDir, mariadbClientBinary),
+		admin:     filepath.Join(cfg.EngineBinDir, mariadbAdminBinary),
+		socket:    mariadbSocketPath(cfg),
+		rcService: cfg.RCService,
+		service:   cfg.EngineService,
+		probe:     probe,
+		params: parameterStore{
+			// The mount point rather than the datadir one level inside it: the
+			// include directory has to outlive the sweep a failed bootstrap runs.
+			dir:       filepath.Join(cfg.DataMount, "conf.d"),
+			installed: mariadbParametersFile,
+			lastGood:  mariadbLastGoodFile,
+			serving:   mariadbServingFile,
+			header:    mariadbParametersHeader,
+			osUser:    cfg.EngineUser,
+			engine:    engineMariaDB,
+		},
+		repairTimeout: parameterRepairTimeout,
+		repairPoll:    parameterRepairPoll,
+	}
+}
+
+func mariadbSocketPath(cfg config) string {
+	return filepath.Join(cfg.SocketDir, mariadbSocketFile)
+}
+
+// Nothing this implementation runs needs the assigned port: every connection it
+// makes is over the engine's unix socket, and the platform drop-in rds-init
+// writes is what puts the server on the port. The probe follows it separately.
+func (e *mariadbEngine) setPort(int) {}
+
+// Reports whether a pid is a live process. EPERM is a process this agent may not
+// signal rather than one that is gone, which matters because a stale pidfile
+// read as "alive" is far less dangerous than a live engine read as "absent".
+type processLivenessFn func(pid int) bool
+
+func processAlive(pid int) bool {
+	if pid <= 0 {
+		return false
+	}
+	err := syscall.Kill(pid, 0)
+	return err == nil || errors.Is(err, syscall.EPERM)
+}
+
+func newMariaDBProbe(cfg config, run probeRunner) *engineProbe {
+	return newEngineProbe(cfg.EnginePort, mariadbProbeState(cfg, run, processAlive))
+}
+
+// Three stages, because MariaDB has no single signal that separates an engine
+// still coming up from one that is not there at all: during InnoDB crash
+// recovery mariadbd opens neither its socket nor its port, so a ping fails
+// exactly as it would against nothing.
+//
+// Losing that distinction would break the rollback guard, which treats a
+// recovering engine as a reason to reset its deadline. An instance killed hard
+// mid-write can spend minutes replaying its redo log; read as absent, the guard
+// would roll the parameter file back and restart the server mid-recovery,
+// discarding the work — a rollback that exists to break a boot loop would create
+// one, on an instance whose parameters were never at fault.
+//
+// The port is not consulted. Both stages below reach the server over its unix
+// socket, whose path does not move with the port.
+func mariadbProbeState(cfg config, run probeRunner, alive processLivenessFn) probeStateFn {
+	pidFile, socket := cfg.EnginePidFile, mariadbSocketPath(cfg)
+	admin, client := filepath.Join(cfg.EngineBinDir, mariadbAdminBinary), filepath.Join(cfg.EngineBinDir, mariadbClientBinary)
+	connect := []string{
+		"--no-defaults", "--protocol=socket", "--socket=" + socket,
+		"--user=" + mariadbSuperuser,
+		fmt.Sprintf("--connect-timeout=%d", mariadbProbeConnectTimeoutSeconds),
+	}
+	probeTimeout := cfg.EngineProbeTimeout
+	if probeTimeout <= 0 {
+		probeTimeout = defaultEngineProbeTimeout
+	}
+	runBounded := func(ctx context.Context, name string, args ...string) (int, error) {
+		probeCtx, cancel := context.WithTimeout(ctx, probeTimeout)
+		defer cancel()
+		return run(probeCtx, name, args...)
+	}
+
+	return func(ctx context.Context, _ int64) (engineState, string) {
+		pid, err := readPidFile(pidFile)
+		switch {
+		case os.IsNotExist(err):
+			return engineAbsent, fmt.Sprintf("the engine has written no pidfile at %s", pidFile)
+		case err != nil:
+			return engineAbsent, fmt.Sprintf("the engine pidfile %s could not be read: %v", pidFile, err)
+		case !alive(pid):
+			return engineAbsent, fmt.Sprintf("the engine pidfile %s names pid %d, which is not running", pidFile, pid)
+		}
+
+		// The process is up from here on, so every remaining failure is an engine
+		// that is not serving yet rather than one that is gone — including a probe
+		// binary that will not run at all. Reporting absent against a live server
+		// would have the rollback guard restart one that is making progress.
+		switch code, err := runBounded(ctx, admin, append(slices.Clone(connect), "ping")...); {
+		case err != nil:
+			return engineRecovering, fmt.Sprintf("engine probe could not run: %v", err)
+		case code != 0:
+			return engineRecovering, "engine is not answering on its socket yet (startup or crash recovery)"
+		}
+
+		// ping answers successfully even on ER_ACCESS_DENIED, so on its own it
+		// certifies a server that may be unable to execute anything at all. The
+		// statement is a literal in the argv rather than on stdin because it
+		// carries nothing secret and a probe reads no result back.
+		query := append(slices.Clone(connect), "--batch", "--skip-column-names", "--execute=SELECT 1")
+		switch code, err := runBounded(ctx, client, query...); {
+		case err != nil:
+			return engineRecovering, fmt.Sprintf("engine probe could not run: %v", err)
+		case code != 0:
+			return engineRecovering, "engine answered its socket but would not execute a statement"
+		}
+		return engineServing, ""
+	}
+}
+
+func readPidFile(path string) (int, error) {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return 0, err
+	}
+	pid, err := strconv.Atoi(strings.TrimSpace(string(raw)))
+	if err != nil {
+		return 0, fmt.Errorf("%s does not hold a pid: %w", path, err)
+	}
+	return pid, nil
+}
+
+func (e *mariadbEngine) Stop(ctx context.Context) error {
+	return serviceAction(ctx, e.run, e.rcService, e.service, "stop")
+}
+
+// Only the parameter rollback and a failed apply call this: a restart the
+// control plane wants goes through RebootDBInstance, which cycles the VM.
+//
+// The serving copy is refreshed first, because the server is about to start on
+// whatever is installed. That is exactly what rds-init does on every boot, and
+// it is what keeps the two equal by construction whenever the engine starts.
+func (e *mariadbEngine) Restart(ctx context.Context) error {
+	if err := e.recordServingCopy(); err != nil {
+		return err
+	}
+	return serviceAction(ctx, e.run, e.rcService, e.service, "restart")
+}
+
+// The installed set copied to the name the pending-restart comparison reads. A
+// live SET GLOBAL apply deliberately does not touch it: that value was adopted
+// without a restart, which is what "not pending" means.
+func (e *mariadbEngine) recordServingCopy() error {
+	content, err := os.ReadFile(e.params.installedPath())
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return fmt.Errorf("read the installed parameters: %w", err)
+	}
+	return e.params.write(e.params.servingPath(), content)
+}
+
+// A value safe to interpolate into a single-quoted SQL literal. The quote is
+// doubled, which holds in every sql_mode, and the backslash is doubled, which is
+// why every statement built here pins a mode with backslash escapes on.
+func sqlLiteral(value string) string {
+	return strings.ReplaceAll(strings.ReplaceAll(value, `\`, `\\`), `'`, `''`)
+}
+
+// Turns off the two logs that would otherwise copy a statement's text, and pins
+// the mode the escaping above assumes. All three are session-scoped: a rotation
+// must not leave the customer's own general log switched off behind it.
+const mariadbSessionGuard = `SET SESSION sql_log_off = 1;
+SET SESSION slow_query_log = 0;
+SET SESSION sql_mode = 'NO_ENGINE_SUBSTITUTION';
+`
+
+// Rotates the master's password. The statement is built here rather than by the
+// client, which has neither psql's identifier interpolation nor its parameter
+// quoting, and it rides stdin so it is never visible in the process table.
+func (e *mariadbEngine) SetPassword(ctx context.Context, username, password string) error {
+	if username == "" || password == "" {
+		return fmt.Errorf("set-password requires both %s and %s",
+			handlers_rds.CommandParamMasterUsername, handlers_rds.CommandParamMasterUserPassword)
+	}
+	// This runs as the server's own superuser over the socket, so a reserved name
+	// in the command payload would rotate the platform's account rather than the
+	// customer's. The rest of the rule matters just as much here, because the name
+	// is interpolated into the statement below rather than quoted by the client.
+	if err := e.rules.validateUsername(username); err != nil {
+		return fmt.Errorf("refusing to set the password of role %q: %w", username, err)
+	}
+
+	escaped := sqlLiteral(password)
+	sql := fmt.Sprintf("%sALTER USER '%s'@'%s' IDENTIFIED BY '%s';\n",
+		mariadbSessionGuard, username, mariadbMasterHost, escaped)
+	if _, err := e.clientRun(ctx, sql); err != nil {
+		// The client echoes the failing statement, which carries the escaped form
+		// of the password rather than the raw one — so redacting only the raw one
+		// would leak exactly the passwords that needed escaping.
+		return fmt.Errorf("apply the master password: %s", redact(redact(err.Error(), password), escaped))
+	}
+	return nil
+}
+
+// Installs the resolved set and applies the half of it a running server can
+// adopt. MariaDB re-reads no configuration file while it is up, so SET GLOBAL is
+// the only way an immediate apply reaches it; the static half waits in the
+// installed file for the next start, and is what comes back as pending.
+//
+// There is no offline parser to check the set with first — MariaDB ships no
+// equivalent of postgres -C — so the safety net for a static value the server
+// will not start on is the boot-time rollback to the last accepted set.
+func (e *mariadbEngine) ApplyParameters(ctx context.Context, params []handlers_rds.Parameter) ([]string, error) {
+	e.paramMu.Lock()
+	defer e.paramMu.Unlock()
+
+	initialState, _ := e.probe.state(ctx)
+	if initialState == engineRecovering {
+		return nil, errors.New("apply parameters while the engine is still starting or recovering")
+	}
+	if initialState == engineServing {
+		// A command can arrive before the first heartbeat. Seed the configuration
+		// currently being served before replacing its file in that window.
+		if err := e.recordServingParametersLocked(ctx); err != nil {
+			return nil, fmt.Errorf("record the parameters serving before the apply: %w", err)
+		}
+	}
+
+	restore, err := e.params.install(params)
+	if err != nil {
+		return nil, err
+	}
+
+	state, _ := e.probe.state(ctx)
+	switch state {
+	case engineAbsent:
+		return e.restartOnRepairSetLocked(ctx)
+	case engineRecovering:
+		return nil, errors.Join(errors.New("the engine entered startup or recovery during the parameter apply"), restore())
+	}
+
+	if err := e.applyDynamicParameters(ctx, params); err != nil {
+		// The engine may have gone down after the first probe. In that case start it
+		// on the newly installed set rather than restoring one that already left it
+		// unable to serve.
+		if state, _ := e.probe.state(ctx); state == engineAbsent {
+			return e.restartOnRepairSetLocked(ctx)
+		}
+		return nil, errors.Join(err, restore())
+	}
+	return e.pendingRestartParameters(ctx)
+}
+
+// The dynamic half of the set, in one invocation that stops at the first refusal
+// so the server's own message names the setting it would not take.
+//
+// Only names the catalog carries are emitted, which is also what keeps the
+// statement safe to build: a name is an identifier here, and the classification
+// that selects it comes from the same table the API validated the request
+// against. MariaDB cannot set several globals atomically, so a value set before
+// a refusal stays live until the next restart, when the restored file wins.
+func (e *mariadbEngine) applyDynamicParameters(ctx context.Context, params []handlers_rds.Parameter) error {
+	var b strings.Builder
+	b.WriteString("SET SESSION sql_mode = 'NO_ENGINE_SUBSTITUTION';\n")
+	applied := 0
+	for _, p := range params {
+		if p.Name == "" || e.rules.isStatic(p.Name) {
+			continue
+		}
+		fmt.Fprintf(&b, "SET GLOBAL %s = '%s';\n", p.Name, sqlLiteral(p.Value))
+		applied++
+	}
+	if applied == 0 {
+		return nil
+	}
+	if _, err := e.clientRun(ctx, b.String()); err != nil {
+		return fmt.Errorf("apply the dynamic parameters: %w", err)
+	}
+	return nil
+}
+
+func (e *mariadbEngine) restartOnRepairSetLocked(ctx context.Context) ([]string, error) {
+	return awaitRepairedEngine(ctx, e.probe, e.Restart, e.pendingRestartParameters, e.repairTimeout, e.repairPoll)
+}
+
+// MariaDB has no pending_restart. Nothing in information_schema reports a
+// setting the server has stored but not adopted, so the answer is computed from
+// the files instead: the catalog-static keys whose value in the installed
+// drop-in differs from the one the server actually started on.
+//
+// Comparing live values against the desired set was rejected. mysqld silently
+// rewrites values at startup — rounding the buffer pool and log file sizes, and
+// downgrading max_connections when it cannot obtain enough file descriptors —
+// and each rewrite would read as a permanent pending restart: un-clearable
+// pending-reboot state, permanent drift, and a last-known-good copy frozen for
+// good, silently disabling a recovery mechanism only exercised during an
+// incident.
+func (e *mariadbEngine) pendingRestartParameters(context.Context) ([]string, error) {
+	installed, err := readOptionFile(e.params.installedPath())
+	if err != nil {
+		if os.IsNotExist(err) {
+			// Nothing is installed, so nothing is waiting to be adopted.
+			return nil, nil
+		}
+		return nil, fmt.Errorf("read the installed parameters: %w", err)
+	}
+	serving, err := readOptionFile(e.params.servingPath())
+	if err != nil && !os.IsNotExist(err) {
+		return nil, fmt.Errorf("read the parameters the engine started on: %w", err)
+	}
+	// Back to the catalog's names before anything is classified or reported: the
+	// files are written in the engine's startup spellings, and the answer is a
+	// customer-facing list the API refuses ApplyMethod=immediate against.
+	installed = e.catalogKeyed(installed)
+	serving = e.catalogKeyed(serving)
+
+	// An absent serving copy is a set the engine has not started on: rds-init
+	// writes the two together on every boot, so the whole static half counts as
+	// pending rather than being promoted on the strength of a missing file.
+	var pending []string
+	for name, value := range installed {
+		if !e.rules.isStatic(name) {
+			continue
+		}
+		if served, ok := serving[name]; !ok || served != value {
+			pending = append(pending, name)
+		}
+	}
+	// A static setting the group stopped naming reverts to its default at the next
+	// start, which the running server has not adopted either.
+	for name := range serving {
+		if _, ok := installed[name]; !ok && e.rules.isStatic(name) {
+			pending = append(pending, name)
+		}
+	}
+	slices.Sort(pending)
+	return pending, nil
+}
+
+// Snapshots the installed set as the rollback target, but only when nothing in
+// it is still waiting for a restart. The parameter mutex keeps an apply from
+// replacing the file between that check and the copy.
+func (e *mariadbEngine) RecordServingParameters(ctx context.Context) error {
+	e.paramMu.Lock()
+	defer e.paramMu.Unlock()
+	return e.recordServingParametersLocked(ctx)
+}
+
+func (e *mariadbEngine) recordServingParametersLocked(ctx context.Context) error {
+	return recordLastGood(ctx, e.params, e.pendingRestartParameters)
+}
+
+// Puts the last set the engine accepted back in place, for a start that failed
+// after a parameter change.
+func (e *mariadbEngine) RestoreLastKnownGoodParameters(ctx context.Context) (bool, error) {
+	e.paramMu.Lock()
+	defer e.paramMu.Unlock()
+	return restoreLastGood(ctx, e.params, e.probe)
+}
+
+// The same settings under the names the catalog and the API know them by, since
+// a file is written in the engine's startup spellings and those are not always
+// the customer's.
+func (e *mariadbEngine) catalogKeyed(values map[string]string) map[string]string {
+	out := make(map[string]string, len(values))
+	for name, value := range values {
+		out[e.rules.catalogName(name)] = value
+	}
+	return out
+}
+
+// The subset of MariaDB's option-file syntax the generated drop-ins are written
+// in: a group header, comments, and one `name = value` per line. Nothing else
+// reaches these two files, because only rds-init and this agent write them.
+func readOptionFile(path string) (map[string]string, error) {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	values := make(map[string]string)
+	for line := range strings.SplitSeq(string(raw), "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" || strings.HasPrefix(line, "#") || strings.HasPrefix(line, ";") || strings.HasPrefix(line, "[") {
+			continue
+		}
+		// A bare option such as skip-log-bin carries no value; it is still a setting
+		// whose presence or absence differs.
+		name, value, _ := strings.Cut(line, "=")
+		values[normaliseOptionName(name)] = strings.Trim(strings.TrimSpace(value), `"'`)
+	}
+	return values, nil
+}
+
+// MariaDB reads - and _ as the same character in an option name, so the two
+// spellings of one setting must not compare as two settings.
+func normaliseOptionName(name string) string {
+	return strings.ReplaceAll(strings.TrimSpace(name), "-", "_")
+}
+
+// How long any one BACKUP STAGE waits for the metadata locks it needs. Well
+// under the control plane's own quiesce timeout, so a stage that cannot take its
+// lock fails and is reported rather than being abandoned mid-wait with the
+// request still queued in front of live traffic.
+const mariadbQuiesceLockWait = 20 * time.Second
+
+// MariaDB's own backup API rather than FLUSH TABLES WITH READ LOCK. FTWRL turns
+// a snapshot into a write outage: the whole database is read-only until it is
+// released, so a control plane that died after issuing quiesce would leave the
+// customer read-only for the full hold, on a schedule and unattended. Its
+// acquisition phase is unbounded too, with writes queueing behind a pending lock
+// request. BACKUP STAGE blocks commits rather than all writes and its flush
+// stage covers Aria and MyISAM, so it yields a better consistency point while
+// being far less disruptive — and it is what mariadb-backup itself uses.
+func (e *mariadbEngine) Quiesce(ctx context.Context, label string, hold time.Duration) error {
+	if err := validateQuiesceRequest(label, hold); err != nil {
+		return err
+	}
+
+	// Held across the whole start, so a second quiesce waits and then finds the
+	// first one's hold rather than opening a concurrent backup alongside it.
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	if e.held != nil {
+		return fmt.Errorf("the engine is already quiesced for backup %s", e.held.label)
+	}
+
+	// Deliberately not the command's context: the session has to outlive the call
+	// that started it, and its own deadline is what bounds it instead.
+	session, err := e.startSess(context.WithoutCancel(ctx), command{
+		Name:              e.client,
+		Args:              e.clientArgs(),
+		Env:               []string{"PATH=" + defaultGuestPath},
+		SentinelStatement: "SELECT '" + sessionSentinel + "';\n",
+	})
+	if err != nil {
+		return fmt.Errorf("open a backup session: %w", err)
+	}
+
+	// The stages run in order and on one connection: MariaDB releases the whole
+	// hold with the session that took it, which is what bounds a control plane
+	// that dies mid-snapshot. Fed one at a time so a stage that cannot take its
+	// locks is named rather than reported as a line number.
+	stages := []string{
+		fmt.Sprintf("SET SESSION lock_wait_timeout = %d;\n", int(mariadbQuiesceLockWait.Seconds())),
+		"BACKUP STAGE START;\n",
+		"BACKUP STAGE FLUSH;\n",
+		"BACKUP STAGE BLOCK_DDL;\n",
+		"BACKUP STAGE BLOCK_COMMIT;\n",
+	}
+	for _, sql := range stages {
+		if err := session.Exec(ctx, sql); err != nil {
+			if closeErr := session.Close(); closeErr != nil {
+				slog.Warn("rds-agent: closing a failed backup session", "err", closeErr)
+			}
+			return fmt.Errorf("put the engine into backup mode at %q: %w", strings.TrimSpace(sql), err)
+		}
+	}
+
+	e.beginHoldLocked(label, session, hold)
+	return nil
+}
+
+// Ends the backup cleanly. A missing hold is an error rather than a silent
+// success: it means the deadline fired first, so the snapshot the control plane
+// just took was not taken against a held stage.
+func (e *mariadbEngine) Unquiesce(ctx context.Context) error {
+	held := e.takeHold()
+	if held == nil {
+		return errors.New("the engine is not quiesced; the backup hold had already expired")
+	}
+
+	// The release has to run on the session that took the stages, and the session
+	// is closed either way — the engine ends an unreleased backup with it.
+	execErr := held.session.Exec(ctx, "BACKUP STAGE END;\n")
+	closeErr := held.session.Close()
+	if execErr != nil {
+		return fmt.Errorf("take the engine out of backup mode: %w", errors.Join(execErr, closeErr))
+	}
+	if closeErr != nil {
+		return fmt.Errorf("close the backup session: %w", closeErr)
+	}
+	slog.Info("rds-agent: engine released from backup mode", "label", held.label)
+	return nil
+}
+
+// Feeds sql on stdin rather than as an argument, so a statement is never visible
+// in the process table.
+func (e *mariadbEngine) clientRun(ctx context.Context, sql string) (string, error) {
+	return e.run(ctx, command{
+		Name:  e.client,
+		Args:  e.clientArgs(),
+		Env:   []string{"PATH=" + defaultGuestPath},
+		Stdin: sql,
+	})
+}
+
+// --no-defaults so no option file can move the connection this agent makes, and
+// --unbuffered so a held session's output reaches the reader as each statement
+// finishes rather than when a pipe buffer fills. No User: the agent is root, and
+// root is what the datadir's unix_socket plugin authenticates.
+func (e *mariadbEngine) clientArgs() []string {
+	return []string{
+		"--no-defaults", "--batch", "--skip-column-names", "--unbuffered",
+		"--protocol=socket", "--socket=" + e.socket, "--user=" + mariadbSuperuser,
+	}
+}

cmd/rds-agent/engine_mariadb_test.go

@@ -0,0 +1,796 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"os/user"
+	"path/filepath"
+	"slices"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// Stands in for the control plane's own definition of the engine. It mirrors the
+// production wiring exactly, including that a name the catalog does not carry
+// counts as static: a rule the fake inverted would hide the failure it exists to
+// prevent.
+func testMariaDBRules() controlPlaneRules {
+	applyTypes := map[string]string{
+		"innodb_buffer_pool_size": handlers_rds.ApplyTypeStatic,
+		"innodb_log_file_size":    handlers_rds.ApplyTypeStatic,
+		"max_connections":         handlers_rds.ApplyTypeDynamic,
+		"long_query_time":         handlers_rds.ApplyTypeDynamic,
+		"time_zone":               handlers_rds.ApplyTypeDynamic,
+	}
+	reserved := []string{"root", "mysql", "mariadb.sys", "rdsadmin", "public"}
+
+	return controlPlaneRules{
+		validateUsername: func(username string) error {
+			lower := strings.ToLower(username)
+			if slices.Contains(reserved, lower) || strings.HasPrefix(lower, "mysql.") || strings.HasPrefix(lower, "mariadb.") {
+				return fmt.Errorf("MasterUsername %q is reserved by mariadb", username)
+			}
+			for _, r := range username {
+				if !isTestIdentifierRune(r) {
+					return fmt.Errorf("MasterUsername may contain only letters, digits and underscores")
+				}
+			}
+			return nil
+		},
+		isStatic: func(name string) bool {
+			applyType, ok := applyTypes[name]
+			return !ok || applyType == handlers_rds.ApplyTypeStatic
+		},
+		catalogName: func(optionFileName string) string {
+			if optionFileName == "default_time_zone" {
+				return "time_zone"
+			}
+			return optionFileName
+		},
+	}
+}
+
+func isTestIdentifierRune(r rune) bool {
+	return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_'
+}
+
+func newTestMariaDBEngine(t *testing.T, run commandRunner) *mariadbEngine {
+	t.Helper()
+	cfg := testLoadConfig(t, engineMariaDB)
+	cfg.DataMount = t.TempDir()
+	if err := os.MkdirAll(filepath.Join(cfg.DataMount, "conf.d"), 0o750); err != nil {
+		t.Fatalf("create conf.d: %v", err)
+	}
+	// The guest's mysql user does not exist here, and the chown of the installed
+	// parameter file has to resolve to something real.
+	current, err := user.Current()
+	if err != nil {
+		t.Fatalf("resolve the current user: %v", err)
+	}
+	cfg.EngineUser = current.Username
+
+	state := engineServing
+	return newMariaDBEngine(cfg, testMariaDBRules(), run, nil, drivenProbe(&state))
+}
+
+// The wiring the guest takes its classification from, checked against a catalog
+// that is registered: what a spec says about a parameter is what the guest must
+// see, and an unlisted name must read as static rather than as dynamic.
+func TestControlPlaneRulesFrom_TakesTheClassificationFromTheCatalog(t *testing.T) {
+	rules := controlPlaneRulesFrom(postgresEngineMeta)
+
+	for name, want := range map[string]bool{"shared_buffers": true, "work_mem": false} {
+		spec, ok := postgresEngineMeta.LookupParameter(name)
+		if !ok {
+			t.Fatalf("the catalog no longer carries %s", name)
+		}
+		if (spec.ApplyType == handlers_rds.ApplyTypeStatic) != want {
+			t.Fatalf("%s is %s in the catalog, so this test is asserting the wrong way round", name, spec.ApplyType)
+		}
+		if got := rules.isStatic(name); got != want {
+			t.Errorf("isStatic(%s) = %v, want %v", name, got, want)
+		}
+	}
+	if !rules.isStatic("not_a_parameter_any_catalog_carries") {
+		t.Error("a name the catalog does not carry read as dynamic, so it would be issued as a live SET GLOBAL")
+	}
+	if rules.validateUsername("postgres") == nil {
+		t.Error("the username rule does not refuse a reserved role")
+	}
+}
+
+// The client has no parameter quoting of its own, so the password is built into
+// the statement here. It must reach the process on stdin and nowhere else.
+func TestMariaDBEngine_SetPasswordKeepsTheSecretOutOfArgvAndEnv(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+
+	if err := engine.SetPassword(context.Background(), "mulgamaster", "n3w-pw"); err != nil {
+		t.Fatalf("SetPassword: %v", err)
+	}
+	if len(runner.calls) != 1 {
+		t.Fatalf("ran %d commands, want 1", len(runner.calls))
+	}
+	call := runner.calls[0]
+	for _, arg := range call.Args {
+		if strings.Contains(arg, "n3w-pw") {
+			t.Fatalf("argv %v carries the password", call.Args)
+		}
+	}
+	for _, env := range call.Env {
+		if strings.Contains(env, "n3w-pw") {
+			t.Fatalf("env %v carries the password, which the client would not read anyway", call.Env)
+		}
+	}
+	if !strings.Contains(call.Stdin, "ALTER USER 'mulgamaster'@'%' IDENTIFIED BY 'n3w-pw';") {
+		t.Errorf("SQL %q does not rotate the master on the wildcard host", call.Stdin)
+	}
+}
+
+// ValidateMasterUserPassword permits both of these, and the client offers no
+// equivalent of psql's parameter quoting, so the escaping here is what keeps a
+// password from ending the literal it is inside.
+func TestMariaDBEngine_SetPasswordEscapesQuotesAndBackslashes(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+
+	if err := engine.SetPassword(context.Background(), "mulgamaster", `a'b\c`); err != nil {
+		t.Fatalf("SetPassword: %v", err)
+	}
+	sql := runner.calls[0].Stdin
+	if !strings.Contains(sql, `IDENTIFIED BY 'a''b\\c';`) {
+		t.Errorf("SQL %q does not escape both the quote and the backslash", sql)
+	}
+	// The escaping assumes a mode where backslash escapes are on, so the statement
+	// cannot inherit whatever sql_mode the customer's parameter group set.
+	if !strings.Contains(sql, "SET SESSION sql_mode = 'NO_ENGINE_SUBSTITUTION';") {
+		t.Errorf("SQL %q does not pin the mode its escaping assumes", sql)
+	}
+}
+
+// The password is interpolated into the statement, so a parameter group that
+// turned the general or slow log on would copy it into a log file every snapshot
+// then carries. Both are silenced for this session only: a rotation must not
+// leave the customer's own logging switched off behind it.
+func TestMariaDBEngine_SetPasswordSilencesStatementLoggingForItsSessionOnly(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+
+	if err := engine.SetPassword(context.Background(), "mulgamaster", "n3w-pw"); err != nil {
+		t.Fatalf("SetPassword: %v", err)
+	}
+	sql := runner.calls[0].Stdin
+	alter := strings.Index(sql, "ALTER USER")
+	if alter < 0 {
+		t.Fatalf("SQL %q does not alter the user", sql)
+	}
+	for _, guard := range []string{"SET SESSION sql_log_off = 1;", "SET SESSION slow_query_log = 0;"} {
+		at := strings.Index(sql, guard)
+		if at < 0 {
+			t.Errorf("SQL %q is missing %q", sql, guard)
+			continue
+		}
+		if at > alter {
+			t.Errorf("SQL %q sets %q only after the statement has already been logged", sql, guard)
+		}
+	}
+	if strings.Contains(sql, "SET GLOBAL general_log") || strings.Contains(sql, "SET GLOBAL slow_query_log") {
+		t.Errorf("SQL %q turns logging off globally, which outlives the rotation", sql)
+	}
+}
+
+// The client echoes the statement it failed on, which carries the escaped form
+// of the password rather than the raw one — so redacting only the raw one would
+// leak exactly the passwords that needed escaping.
+func TestMariaDBEngine_SetPasswordRedactsBothFormsOfTheSecret(t *testing.T) {
+	secret := `a'b\c`
+	runner := &recordingRunner{err: errors.New(`ERROR 1064 near "IDENTIFIED BY 'a''b\\c'"`)}
+	engine := newTestMariaDBEngine(t, runner.run)
+
+	err := engine.SetPassword(context.Background(), "mulgamaster", secret)
+	if err == nil {
+		t.Fatal("SetPassword succeeded against a failing client")
+	}
+	if strings.Contains(err.Error(), secret) || strings.Contains(err.Error(), sqlLiteral(secret)) {
+		t.Errorf("error %q leaks the password", err)
+	}
+	if !strings.Contains(err.Error(), "[REDACTED]") {
+		t.Errorf("error %q does not mark the redaction", err)
+	}
+}
+
+// The apply runs as the server's own superuser over the socket, and the name is
+// interpolated into the statement rather than quoted by the client, so both
+// halves of the rule are refused before any SQL is built.
+func TestMariaDBEngine_SetPasswordRefusesNamesTheEngineWillNotTake(t *testing.T) {
+	for _, username := range []string{"root", "mysql", "mariadb.sys", "rdsadmin", "PUBLIC", "mysql.infoschema", `ma'ster`} {
+		t.Run(username, func(t *testing.T) {
+			runner := &recordingRunner{}
+			engine := newTestMariaDBEngine(t, runner.run)
+
+			if err := engine.SetPassword(context.Background(), username, "n3w-pw"); err == nil {
+				t.Fatalf("SetPassword accepted %q", username)
+			}
+			if len(runner.calls) != 0 {
+				t.Errorf("ran %d commands for a refused name, want 0", len(runner.calls))
+			}
+		})
+	}
+}
+
+func TestMariaDBEngine_SetPasswordRejectsAnIncompleteCommand(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+
+	if err := engine.SetPassword(context.Background(), "mulgamaster", ""); err == nil {
+		t.Error("SetPassword accepted an empty password")
+	}
+	if len(runner.calls) != 0 {
+		t.Errorf("ran %d commands for an incomplete request, want 0", len(runner.calls))
+	}
+}
+
+// A running server re-reads no configuration file, so the dynamic half has to
+// reach it as SET GLOBAL. The static half is installed and waits for a restart,
+// which is what comes back as pending.
+func TestMariaDBEngine_ApplyParametersSetsOnlyTheDynamicHalfLive(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+
+	pending, err := engine.ApplyParameters(context.Background(), []handlers_rds.Parameter{
+		{Name: "max_connections", Value: "200"},
+		{Name: "innodb_buffer_pool_size", Value: "1073741824"},
+	})
+	if err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+
+	installed, err := os.ReadFile(engine.params.installedPath())
+	if err != nil {
+		t.Fatalf("read the installed parameters: %v", err)
+	}
+	if !strings.HasPrefix(string(installed), "[mysqld]\n") {
+		t.Errorf("installed parameters = %q, want the group header rds-init writes", installed)
+	}
+	for _, want := range []string{"max_connections = '200'", "innodb_buffer_pool_size = '1073741824'"} {
+		if !strings.Contains(string(installed), want) {
+			t.Errorf("installed parameters = %q, want %q", installed, want)
+		}
+	}
+
+	if len(runner.calls) != 1 {
+		t.Fatalf("ran %d commands, want the one live apply", len(runner.calls))
+	}
+	sql := runner.calls[0].Stdin
+	if !strings.Contains(sql, "SET GLOBAL max_connections = '200';") {
+		t.Errorf("SQL %q does not set the dynamic value live", sql)
+	}
+	if strings.Contains(sql, "innodb_buffer_pool_size") {
+		t.Errorf("SQL %q sets a static value live, which the server would refuse", sql)
+	}
+	if !slices.Equal(pending, []string{"innodb_buffer_pool_size"}) {
+		t.Errorf("pending = %v, want the static setting the engine has not adopted", pending)
+	}
+}
+
+// A set with nothing dynamic in it must not open a connection at all: the
+// installed file is the whole of the change.
+func TestMariaDBEngine_ApplyParametersRunsNoStatementForAStaticOnlySet(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+
+	if _, err := engine.ApplyParameters(context.Background(),
+		[]handlers_rds.Parameter{{Name: "innodb_log_file_size", Value: "268435456"}}); err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+	if len(runner.calls) != 0 {
+		t.Errorf("ran %d commands for a static-only set, want 0", len(runner.calls))
+	}
+}
+
+// A value the server refuses leaves the installed file back where it was, so it
+// does not sit on the data volume turning the next restart into a boot loop.
+func TestMariaDBEngine_ApplyParametersRollsBackAValueTheServerRefuses(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+
+	if _, err := engine.ApplyParameters(context.Background(),
+		[]handlers_rds.Parameter{{Name: "max_connections", Value: "200"}}); err != nil {
+		t.Fatalf("first ApplyParameters: %v", err)
+	}
+
+	runner.err = errors.New("ERROR 1231 (42000): Variable 'max_connections' can't be set to the value of '0'")
+	if _, err := engine.ApplyParameters(context.Background(),
+		[]handlers_rds.Parameter{{Name: "max_connections", Value: "0"}}); err == nil {
+		t.Fatal("ApplyParameters succeeded against a value the server refused")
+	}
+
+	installed, err := os.ReadFile(engine.params.installedPath())
+	if err != nil {
+		t.Fatalf("read the installed parameters: %v", err)
+	}
+	if !strings.Contains(string(installed), "max_connections = '200'") {
+		t.Errorf("installed parameters = %q, want the previous accepted set restored", installed)
+	}
+}
+
+// The whole of the pending-restart answer, since MariaDB has none of its own:
+// the installed drop-in against the one the server actually started on, over the
+// static keys only.
+func TestMariaDBEngine_PendingRestartComparesTheInstalledSetAgainstTheServingCopy(t *testing.T) {
+	engine := newTestMariaDBEngine(t, (&recordingRunner{}).run)
+	write := func(path, body string) {
+		t.Helper()
+		if err := os.WriteFile(path, []byte("[mysqld]\n"+body), 0o600); err != nil {
+			t.Fatalf("write %s: %v", path, err)
+		}
+	}
+	pending := func() []string {
+		t.Helper()
+		names, err := engine.pendingRestartParameters(t.Context())
+		if err != nil {
+			t.Fatalf("pendingRestartParameters: %v", err)
+		}
+		return names
+	}
+
+	// A fresh boot: rds-init wrote the two together, so nothing is waiting.
+	write(engine.params.installedPath(), "innodb_buffer_pool_size = '1073741824'\nmax_connections = '100'\n")
+	write(engine.params.servingPath(), "innodb_buffer_pool_size = '1073741824'\nmax_connections = '100'\n")
+	if names := pending(); len(names) != 0 {
+		t.Errorf("pending = %v, want nothing on a set the engine started on", names)
+	}
+
+	// A dynamic value the server took live is not pending: it was adopted without
+	// a restart, which is exactly what the serving copy not moving records.
+	write(engine.params.installedPath(), "innodb_buffer_pool_size = '1073741824'\nmax_connections = '200'\n")
+	if names := pending(); len(names) != 0 {
+		t.Errorf("pending = %v, want nothing for a value adopted live", names)
+	}
+
+	// A static value is.
+	write(engine.params.installedPath(), "innodb_buffer_pool_size = '2147483648'\nmax_connections = '200'\n")
+	if names := pending(); !slices.Equal(names, []string{"innodb_buffer_pool_size"}) {
+		t.Errorf("pending = %v, want the static setting", names)
+	}
+
+	// So is a static setting the group stopped naming, which reverts to its
+	// default at the next start.
+	write(engine.params.installedPath(), "max_connections = '200'\n")
+	if names := pending(); !slices.Equal(names, []string{"innodb_buffer_pool_size"}) {
+		t.Errorf("pending = %v, want the withdrawn static setting", names)
+	}
+
+	// MariaDB reads - and _ as the same character, so one setting spelled two ways
+	// is not two settings.
+	write(engine.params.installedPath(), "innodb-buffer-pool-size = '1073741824'\nmax_connections = '200'\n")
+	if names := pending(); len(names) != 0 {
+		t.Errorf("pending = %v, want the two spellings of one setting to compare equal", names)
+	}
+}
+
+// rds-init writes the serving copy beside the installed one on every boot, so a
+// set with no copy beside it is one the engine has not started on. Promoting it
+// would point the rollback at a configuration that has never served.
+func TestMariaDBEngine_PendingRestartTreatsAMissingServingCopyAsUnadopted(t *testing.T) {
+	engine := newTestMariaDBEngine(t, (&recordingRunner{}).run)
+	if err := os.WriteFile(engine.params.installedPath(),
+		[]byte("[mysqld]\ninnodb_buffer_pool_size = '1073741824'\nmax_connections = '100'\n"), 0o600); err != nil {
+		t.Fatalf("write installed parameters: %v", err)
+	}
+
+	names, err := engine.pendingRestartParameters(t.Context())
+	if err != nil {
+		t.Fatalf("pendingRestartParameters: %v", err)
+	}
+	if !slices.Equal(names, []string{"innodb_buffer_pool_size"}) {
+		t.Errorf("pending = %v, want the static half of a set the engine never started on", names)
+	}
+}
+
+// The restart the agent performs is the only thing that moves the serving copy,
+// which is what makes a fresh start clear whatever was pending.
+func TestMariaDBEngine_RestartRefreshesTheServingCopy(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+	installed := []byte("[mysqld]\ninnodb_buffer_pool_size = '2147483648'\n")
+	if err := os.WriteFile(engine.params.installedPath(), installed, 0o600); err != nil {
+		t.Fatalf("write installed parameters: %v", err)
+	}
+	if err := os.WriteFile(engine.params.servingPath(), []byte("[mysqld]\ninnodb_buffer_pool_size = '1073741824'\n"), 0o600); err != nil {
+		t.Fatalf("write serving parameters: %v", err)
+	}
+
+	if err := engine.Restart(context.Background()); err != nil {
+		t.Fatalf("Restart: %v", err)
+	}
+
+	serving, err := os.ReadFile(engine.params.servingPath())
+	if err != nil {
+		t.Fatalf("read the serving copy: %v", err)
+	}
+	if string(serving) != string(installed) {
+		t.Errorf("serving copy = %q, want the set the engine was just started on", serving)
+	}
+	names, err := engine.pendingRestartParameters(t.Context())
+	if err != nil {
+		t.Fatalf("pendingRestartParameters: %v", err)
+	}
+	if len(names) != 0 {
+		t.Errorf("pending = %v, want a restart to have cleared it", names)
+	}
+	// The copy the comparison reads must never be parsed as a second set of
+	// settings by the engine's own include glob.
+	if strings.HasSuffix(engine.params.servingPath(), ".cnf") {
+		t.Errorf("the serving copy at %s would be included by the engine", engine.params.servingPath())
+	}
+}
+
+// The rollback target advances only once a restart has proved the installed
+// static values can serve, not when a dynamic apply merely set them live.
+func TestMariaDBEngine_LastKnownGoodWaitsForARestartToAdoptAStaticSet(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+
+	if _, err := engine.ApplyParameters(t.Context(),
+		[]handlers_rds.Parameter{{Name: "innodb_buffer_pool_size", Value: "1073741824"}}); err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+	if err := engine.RecordServingParameters(t.Context()); err != nil {
+		t.Fatalf("RecordServingParameters: %v", err)
+	}
+	if _, err := os.Stat(engine.params.lastGoodPath()); !os.IsNotExist(err) {
+		t.Errorf("a set the engine has not restarted onto became the rollback target (stat err = %v)", err)
+	}
+
+	if err := engine.Restart(t.Context()); err != nil {
+		t.Fatalf("Restart: %v", err)
+	}
+	if err := engine.RecordServingParameters(t.Context()); err != nil {
+		t.Fatalf("RecordServingParameters after the restart: %v", err)
+	}
+	lastGood, err := os.ReadFile(engine.params.lastGoodPath())
+	if err != nil {
+		t.Fatalf("read the last known good parameters: %v", err)
+	}
+	if !strings.Contains(string(lastGood), "innodb_buffer_pool_size = '1073741824'") {
+		t.Errorf("last known good = %q, want the set the engine restarted onto", lastGood)
+	}
+}
+
+// A dynamic apply is adopted without a restart, so it may advance the rollback
+// target on the next healthy beat rather than waiting for one.
+func TestMariaDBEngine_LastKnownGoodAdvancesOnADynamicApply(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+	if err := os.WriteFile(engine.params.servingPath(), []byte("[mysqld]\n"), 0o600); err != nil {
+		t.Fatalf("write serving parameters: %v", err)
+	}
+
+	if _, err := engine.ApplyParameters(t.Context(),
+		[]handlers_rds.Parameter{{Name: "max_connections", Value: "200"}}); err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+	if err := engine.RecordServingParameters(t.Context()); err != nil {
+		t.Fatalf("RecordServingParameters: %v", err)
+	}
+
+	lastGood, err := os.ReadFile(engine.params.lastGoodPath())
+	if err != nil {
+		t.Fatalf("read the last known good parameters: %v", err)
+	}
+	if !strings.Contains(string(lastGood), "max_connections = '200'") {
+		t.Errorf("last known good = %q, want the set the engine is already running", lastGood)
+	}
+}
+
+// A probe that reports what the test hands it, in place of whichever binaries
+// the engine's own state function would have run.
+func mariadbTestProbeConfig(t *testing.T) config {
+	t.Helper()
+	cfg := testLoadConfig(t, engineMariaDB)
+	cfg.EnginePidFile = filepath.Join(t.TempDir(), "mariadb.pid")
+	return cfg
+}
+
+func writePid(t *testing.T, path string, pid int) {
+	t.Helper()
+	if err := os.WriteFile(path, []byte(strconv.Itoa(pid)+"\n"), 0o644); err != nil {
+		t.Fatalf("write pidfile: %v", err)
+	}
+}
+
+// The three stages, and what each one separates. During InnoDB crash recovery
+// mariadbd opens neither its socket nor its port, so nothing the client can ask
+// distinguishes a server replaying its redo log from no server at all.
+func TestMariaDBProbe_SeparatesRecoveringFromAbsent(t *testing.T) {
+	tests := []struct {
+		name     string
+		writePid bool
+		alive    bool
+		pingCode int
+		pingErr  error
+		execCode int
+		want     engineState
+	}{
+		{name: "no pidfile at all", want: engineAbsent},
+		{name: "pidfile names a dead process", writePid: true, want: engineAbsent},
+		{name: "process alive but socket silent", writePid: true, alive: true, pingCode: 1, want: engineRecovering},
+		{name: "process alive but probe cannot run", writePid: true, alive: true, pingErr: errors.New("no such file"), want: engineRecovering},
+		{name: "answers ping but executes nothing", writePid: true, alive: true, execCode: 1, want: engineRecovering},
+		{name: "serving", writePid: true, alive: true, want: engineServing},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfg := mariadbTestProbeConfig(t)
+			if tt.writePid {
+				writePid(t, cfg.EnginePidFile, 4242)
+			}
+			run := func(_ context.Context, name string, args ...string) (int, error) {
+				if strings.HasSuffix(name, mariadbAdminBinary) {
+					return tt.pingCode, tt.pingErr
+				}
+				if !slices.Contains(args, "--execute=SELECT 1") {
+					t.Errorf("the second stage ran %s %v, want a statement the server has to execute", name, args)
+				}
+				return tt.execCode, nil
+			}
+			stateFn := mariadbProbeState(cfg, run, func(int) bool { return tt.alive })
+
+			got, message := stateFn(context.Background(), int64(cfg.EnginePort))
+			if got != tt.want {
+				t.Errorf("state = %v, want %v (%s)", got, tt.want, message)
+			}
+			if got != engineServing && message == "" {
+				t.Error("a non-serving result carried no message explaining it")
+			}
+		})
+	}
+}
+
+// The failure the guard exists for still behaves correctly: a static value
+// mariadbd will not accept makes it exit during startup, so the pidfile names a
+// process that is gone and the deadline runs.
+func TestMariaDBProbe_BoundsAStalledClient(t *testing.T) {
+	cfg := mariadbTestProbeConfig(t)
+	cfg.EngineProbeTimeout = 20 * time.Millisecond
+	writePid(t, cfg.EnginePidFile, 4242)
+
+	started := time.Now()
+	state, message := mariadbProbeState(cfg,
+		func(ctx context.Context, _ string, args ...string) (int, error) {
+			if !slices.Contains(args, "--connect-timeout=3") {
+				t.Errorf("probe args = %v, want a client connection timeout", args)
+			}
+			<-ctx.Done()
+			return -1, ctx.Err()
+		},
+		func(int) bool { return true })(t.Context(), int64(cfg.EnginePort))
+
+	if state != engineRecovering {
+		t.Errorf("state = %v, want recovering after a live engine's probe timed out", state)
+	}
+	if !strings.Contains(message, context.DeadlineExceeded.Error()) {
+		t.Errorf("message = %q, want the probe deadline", message)
+	}
+	if elapsed := time.Since(started); elapsed > time.Second {
+		t.Errorf("probe took %v, want the configured deadline to bound it", elapsed)
+	}
+}
+
+func TestMariaDBProbe_MapsToTheHealthTheControlPlaneReads(t *testing.T) {
+	cfg := mariadbTestProbeConfig(t)
+	writePid(t, cfg.EnginePidFile, 4242)
+	alive := true
+	serving := true
+	probe := newEngineProbe(cfg.EnginePort, mariadbProbeState(cfg,
+		func(context.Context, string, ...string) (int, error) {
+			if serving {
+				return 0, nil
+			}
+			return 1, nil
+		},
+		func(int) bool { return alive }))
+
+	if got, _ := probe.Check(context.Background()); got != handlers_rds.EngineHealthHealthy {
+		t.Fatalf("health = %q, want healthy", got)
+	}
+	serving = false
+	if got, _ := probe.Check(context.Background()); got != handlers_rds.EngineHealthStarting {
+		t.Errorf("health while recovering = %q, want starting", got)
+	}
+	alive = false
+	if got, _ := probe.Check(context.Background()); got != handlers_rds.EngineHealthUnhealthy {
+		t.Errorf("health after the engine went away = %q, want unhealthy", got)
+	}
+}
+
+// An instance killed hard mid-write can spend minutes replaying its redo log. If
+// that read as absent, the guard would roll the parameter file back and restart
+// the server mid-recovery — a rollback that exists to break a boot loop would
+// create one, on an instance whose parameters were never at fault.
+func TestMariaDBProbe_LiveButUnreachableEngineResetsTheRollbackDeadline(t *testing.T) {
+	cfg := mariadbTestProbeConfig(t)
+	writePid(t, cfg.EnginePidFile, 4242)
+	probe := newEngineProbe(cfg.EnginePort, mariadbProbeState(cfg,
+		func(context.Context, string, ...string) (int, error) { return 1, nil },
+		func(int) bool { return true }))
+
+	recovery := &fakeRecovery{restored: true}
+	guard := newTestGuard(recovery, probe)
+	// Bounded so the deadline-resetting loop cannot run forever if it stops
+	// resetting; the assertion is that the rollback was never reached.
+	ctx, cancel := context.WithTimeout(t.Context(), 200*time.Millisecond)
+	defer cancel()
+	guard.Run(ctx)
+
+	if recovery.restarts != 0 {
+		t.Errorf("restarts = %d, want none while the engine is still recovering", recovery.restarts)
+	}
+}
+
+// MariaDB releases a BACKUP STAGE with the connection that took it, so the whole
+// sequence has to run on one held session — which is also what bounds a control
+// plane that dies mid-snapshot.
+func TestMariaDBEngine_QuiesceTakesTheBackupStagesOnAHeldSession(t *testing.T) {
+	session := &fakeSession{}
+	engine := newTestMariaDBEngine(t, (&recordingRunner{}).run)
+	var started []command
+	engine.startSess = func(_ context.Context, c command) (engineSession, error) {
+		started = append(started, c)
+		return session, nil
+	}
+
+	if err := engine.Quiesce(context.Background(), "orders-db-pre-upgrade", time.Minute); err != nil {
+		t.Fatalf("Quiesce: %v", err)
+	}
+
+	if len(started) != 1 {
+		t.Fatalf("started %d sessions, want 1", len(started))
+	}
+	if !slices.Contains(started[0].Args, "--unbuffered") {
+		t.Errorf("session argv %v does not flush after each statement, so the sentinel could sit in a pipe buffer", started[0].Args)
+	}
+	statements := session.statements()
+	want := []string{
+		"SET SESSION lock_wait_timeout = 20;",
+		"BACKUP STAGE START;",
+		"BACKUP STAGE FLUSH;",
+		"BACKUP STAGE BLOCK_DDL;",
+		"BACKUP STAGE BLOCK_COMMIT;",
+	}
+	if len(statements) != len(want) {
+		t.Fatalf("statements = %q, want the bounded stage sequence %q", statements, want)
+	}
+	for i, sql := range want {
+		if strings.TrimSpace(statements[i]) != sql {
+			t.Errorf("statement %d = %q, want %q", i, statements[i], sql)
+		}
+	}
+	// FLUSH TABLES WITH READ LOCK would make the whole database read-only for the
+	// length of the hold rather than blocking commits.
+	if strings.Contains(strings.Join(statements, " "), "FLUSH TABLES") {
+		t.Error("the quiesce took a read lock over the whole database")
+	}
+	if session.closes() != 0 {
+		t.Error("the session was closed while the backup was meant to be held")
+	}
+}
+
+// The stage bound has to leave the control plane's own quiesce timeout room for
+// the whole sequence, or a stage abandoned mid-wait leaves its lock request
+// queued in front of live traffic.
+func TestMariaDBEngine_QuiesceLockWaitFitsInsideTheControlPlaneTimeout(t *testing.T) {
+	const stages = 4
+	if stages*mariadbQuiesceLockWait >= 2*time.Minute {
+		t.Errorf("%d stages of %s can outlast the control plane's quiesce timeout", stages, mariadbQuiesceLockWait)
+	}
+}
+
+func TestMariaDBEngine_UnquiesceEndsTheBackupAndTheSession(t *testing.T) {
+	session := &fakeSession{}
+	engine := newTestMariaDBEngine(t, (&recordingRunner{}).run)
+	engine.startSess = func(context.Context, command) (engineSession, error) { return session, nil }
+	if err := engine.Quiesce(context.Background(), "orders-db-pre-upgrade", time.Minute); err != nil {
+		t.Fatalf("Quiesce: %v", err)
+	}
+
+	if err := engine.Unquiesce(context.Background()); err != nil {
+		t.Fatalf("Unquiesce: %v", err)
+	}
+
+	statements := session.statements()
+	if last := strings.TrimSpace(statements[len(statements)-1]); last != "BACKUP STAGE END;" {
+		t.Errorf("last statement = %q, want the stage released", last)
+	}
+	if session.closes() != 1 {
+		t.Errorf("closed %d times, want the session ended exactly once", session.closes())
+	}
+}
+
+// The two halves of a time_zone apply use different names: SET GLOBAL takes the
+// customer's, the option file the server's startup spelling. Getting either
+// backwards is a statement the server refuses or a boot it refuses.
+func TestMariaDBEngine_TimeZoneAppliesLiveByItsCustomerFacingName(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+
+	if _, err := engine.ApplyParameters(context.Background(), []handlers_rds.Parameter{
+		{Name: "time_zone", Value: "+10:00"},
+	}); err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+
+	if len(runner.calls) != 1 {
+		t.Fatalf("ran %d commands, want the one live apply", len(runner.calls))
+	}
+	sql := runner.calls[0].Stdin
+	if !strings.Contains(sql, "SET GLOBAL time_zone = '+10:00';") {
+		t.Errorf("SQL %q does not set time_zone under the name MariaDB accepts for SET", sql)
+	}
+	if strings.Contains(sql, "default_time_zone") {
+		t.Errorf("SQL %q names the startup spelling, which is not a settable variable", sql)
+	}
+
+	installed, err := os.ReadFile(engine.params.installedPath())
+	if err != nil {
+		t.Fatalf("read the installed parameters: %v", err)
+	}
+	if !strings.Contains(string(installed), "default_time_zone = '+10:00'") {
+		t.Errorf("installed parameters = %q, want the startup spelling", installed)
+	}
+}
+
+// A dynamic setting applied live deliberately leaves the serving copy alone, so
+// the two files differ by design. Read back under the startup spelling the
+// catalog does not carry, that difference would classify as static and report a
+// pending restart that nothing could ever clear.
+func TestMariaDBEngine_ALiveTimeZoneChangeLeavesNothingPendingRestart(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+
+	if err := os.WriteFile(engine.params.servingPath(),
+		[]byte("[mysqld]\ndefault_time_zone = 'SYSTEM'\n"), 0o600); err != nil {
+		t.Fatalf("seed the serving copy: %v", err)
+	}
+
+	pending, err := engine.ApplyParameters(context.Background(), []handlers_rds.Parameter{
+		{Name: "time_zone", Value: "+10:00"},
+	})
+	if err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+	if len(pending) != 0 {
+		t.Errorf("pending = %v, want nothing: time_zone is dynamic and was applied live", pending)
+	}
+}
+
+// The list is customer-facing, so a static setting still has to be reported
+// under the name the customer set rather than the engine's startup spelling.
+func TestMariaDBEngine_PendingRestartReportsCatalogNames(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+
+	if err := os.WriteFile(engine.params.servingPath(),
+		[]byte("[mysqld]\ninnodb_buffer_pool_size = '536870912'\ndefault_time_zone = 'SYSTEM'\n"), 0o600); err != nil {
+		t.Fatalf("seed the serving copy: %v", err)
+	}
+
+	pending, err := engine.ApplyParameters(context.Background(), []handlers_rds.Parameter{
+		{Name: "innodb_buffer_pool_size", Value: "1073741824"},
+		{Name: "time_zone", Value: "SYSTEM"},
+	})
+	if err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+	if !slices.Equal(pending, []string{"innodb_buffer_pool_size"}) {
+		t.Errorf("pending = %v, want only the static setting under its catalog name", pending)
+	}
+}

cmd/rds-agent/engine_postgres.go

@@ -0,0 +1,380 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// The engine is reached over its unix socket under peer authentication, so the
+// agent drops to the postgres OS user rather than holding a password of its own.
+type postgresEngine struct {
+	quiesceState
+
+	// The control plane's own metadata for this engine, resolved once at
+	// startup: the live password apply runs as the cluster superuser, so it
+	// re-checks the role name against the same reserved set the API validates.
+	meta      handlers_rds.Engine
+	run       commandRunner
+	startSess sessionRunner
+	psql      string
+	rcService string
+	service   string
+	pgData    string
+	socketDir string
+	osUser    string
+	probe     *engineProbe
+	// Set from the bootstrap config, on a different goroutine than the commands
+	// that read it.
+	port atomic.Int64
+
+	params parameterStore
+	// Serializes parameter installs, serving snapshots and rollback restores so
+	// none can copy or replace an intermediate configuration.
+	paramMu       sync.Mutex
+	repairTimeout time.Duration
+	repairPoll    time.Duration
+}
+
+var _ engine = (*postgresEngine)(nil)
+
+// pg_isready is what gives PostgreSQL three states for free: exit 0 is serving,
+// exit 1 is a postmaster up and rejecting connections, and anything else is an
+// engine that is not there at all. Resolved on PATH, where the client package
+// puts it.
+const postgresProbeBinary = "pg_isready"
+
+func newPostgresProbe(cfg config, run probeRunner) *engineProbe {
+	return newEngineProbe(cfg.EnginePort, postgresProbeState(cfg.EngineHost, run))
+}
+
+func postgresProbeState(host string, run probeRunner) probeStateFn {
+	return func(ctx context.Context, port int64) (engineState, string) {
+		portArg := strconv.FormatInt(port, 10)
+		code, err := run(ctx, postgresProbeBinary, "-h", host, "-p", portArg, "-q")
+		switch {
+		case err != nil:
+			// A missing binary or broken image. Reporting healthy on the strength of
+			// nothing would hide it, so this reads as absent like an engine that did
+			// not answer.
+			return engineAbsent, fmt.Sprintf("engine probe could not run: %v", err)
+		case code == 0:
+			return engineServing, ""
+		case code == 1:
+			return engineRecovering, "engine is rejecting connections (startup or recovery)"
+		default:
+			return engineAbsent, fmt.Sprintf("engine did not respond on %s:%s", host, portArg)
+		}
+	}
+}
+
+// Resolved once, under the name the control plane knows this implementation by.
+// A lookup failure is a mismatch between the agent and the control plane it was
+// built against, so it fails at startup rather than at the first rotation.
+var postgresEngineMeta = mustLookupEngine(enginePostgres)
+
+func mustLookupEngine(name string) handlers_rds.Engine {
+	engine, err := handlers_rds.LookupEngine(name)
+	if err != nil {
+		panic("rds-agent: " + err.Error())
+	}
+	return engine
+}
+
+// The include the resolved parameter set is rendered to, and the copy of the
+// last one the engine accepted. Both live beside the data rather than in /etc,
+// matching rds-init: a class change boots a fresh root volume, which would
+// otherwise revert them.
+const (
+	postgresParametersFile = "10-rds-parameters.conf"
+	// Deliberately not a .conf name: include_dir globs *.conf, so the rollback
+	// copy must not be read as a second set of settings.
+	postgresLastGoodFile = "10-rds-parameters.last-good"
+)
+
+func newPostgresEngine(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) *postgresEngine {
+	e := &postgresEngine{
+		meta:      postgresEngineMeta,
+		run:       run,
+		startSess: startSess,
+		psql:      filepath.Join(cfg.EngineBinDir, "psql"),
+		rcService: cfg.RCService,
+		service:   cfg.EngineService,
+		pgData:    cfg.EngineDataDir,
+		socketDir: cfg.SocketDir,
+		osUser:    cfg.EngineUser,
+		probe:     probe,
+		params: parameterStore{
+			dir:       filepath.Join(cfg.EngineDataDir, "conf.d"),
+			installed: postgresParametersFile,
+			lastGood:  postgresLastGoodFile,
+			osUser:    cfg.EngineUser,
+			engine:    enginePostgres,
+		},
+		repairTimeout: parameterRepairTimeout,
+		repairPoll:    parameterRepairPoll,
+	}
+	e.port.Store(int64(cfg.EnginePort))
+	return e
+}
+
+func (e *postgresEngine) setPort(port int) {
+	e.port.Store(int64(port))
+}
+
+// The role name and the password ride the environment and are re-quoted by
+// psql, so neither reaches a shell word or an argv another process can read.
+func (e *postgresEngine) SetPassword(ctx context.Context, username, password string) error {
+	if username == "" || password == "" {
+		return fmt.Errorf("set-password requires both %s and %s",
+			handlers_rds.CommandParamMasterUsername, handlers_rds.CommandParamMasterUserPassword)
+	}
+	// The statement below runs as the cluster superuser over the socket under
+	// peer auth, so a reserved name in the command payload would hand the
+	// customer the bootstrap superuser rather than rotate their own role.
+	if err := e.meta.ValidateUsernameNotReserved(username); err != nil {
+		return fmt.Errorf("refusing to set the password of a role the engine reserves: %w", err)
+	}
+	// psql interpolates the password into the ALTER ROLE before the server sees
+	// it, and these three are what would write it to the log — the last on any
+	// failure at its own default. SUSET, so the parameter group cannot win.
+	const sql = `SET log_statement = 'none';
+SET log_min_duration_statement = -1;
+SET log_min_error_statement = 'panic';
+\getenv master RDS_MASTER_USERNAME
+\getenv password RDS_MASTER_PASSWORD
+ALTER ROLE :"master" WITH LOGIN PASSWORD :'password';
+`
+	_, err := e.psqlRun(ctx, sql,
+		"RDS_MASTER_USERNAME="+username,
+		"RDS_MASTER_PASSWORD="+password,
+	)
+	if err != nil {
+		// psql echoes the failing statement, which here would carry the new
+		// password back to the control plane and into the event ring.
+		return fmt.Errorf("apply the master password: %s", redact(err.Error(), password))
+	}
+	return nil
+}
+
+// Installs the resolved set, validates it with the engine's own config parser
+// before the engine ever adopts it, and reloads. A value the engine refuses is
+// rolled back here rather than left on the data volume, where it would survive
+// every VM replace and turn the next restart into a boot loop.
+func (e *postgresEngine) ApplyParameters(ctx context.Context, params []handlers_rds.Parameter) ([]string, error) {
+	e.paramMu.Lock()
+	defer e.paramMu.Unlock()
+
+	initialState, _ := e.probe.state(ctx)
+	if initialState == engineRecovering {
+		return nil, errors.New("apply parameters while the engine is still starting or recovering")
+	}
+	if initialState == engineServing {
+		// A command can arrive before the first heartbeat. Seed the configuration
+		// currently being served before replacing its file in that window.
+		if err := e.recordServingParametersLocked(ctx); err != nil {
+			return nil, fmt.Errorf("record the parameters serving before the apply: %w", err)
+		}
+	}
+
+	// The check has to run against the file in place, because the engine parses
+	// the datadir's own include_dir. The window that leaves is closed from both
+	// ends: the rollback below, and the last-known-good restore the agent runs at
+	// boot when the engine does not come up.
+	restore, err := e.params.install(params)
+	if err != nil {
+		return nil, err
+	}
+	if err := e.checkConfig(ctx); err != nil {
+		return nil, errors.Join(fmt.Errorf("the engine rejected the parameter set: %w", err), restore())
+	}
+
+	state, _ := e.probe.state(ctx)
+	switch state {
+	case engineAbsent:
+		return e.restartOnRepairSetLocked(ctx)
+	case engineRecovering:
+		return nil, errors.Join(errors.New("the engine entered startup or recovery during the parameter apply"), restore())
+	}
+
+	if _, err := e.psqlRun(ctx, "SELECT pg_reload_conf();\n"); err != nil {
+		// The engine may have gone down after the first probe. In that case start
+		// it on the new, parser-checked repair set rather than restoring the set
+		// that already left it unable to serve.
+		if state, _ := e.probe.state(ctx); state == engineAbsent {
+			return e.restartOnRepairSetLocked(ctx)
+		}
+		return nil, errors.Join(fmt.Errorf("reload the engine configuration: %w", err), restore())
+	}
+	return e.pendingRestartParameters(ctx)
+}
+
+func (e *postgresEngine) restartOnRepairSetLocked(ctx context.Context) ([]string, error) {
+	return awaitRepairedEngine(ctx, e.probe, e.Restart, e.pendingRestartParameters, e.repairTimeout, e.repairPoll)
+}
+
+// The postmaster's own answer: a static setting it has stored but not adopted is
+// what it reports here, so nothing in the guest has to classify one.
+func (e *postgresEngine) pendingRestartParameters(ctx context.Context) ([]string, error) {
+	out, err := e.psqlRun(ctx, "SELECT name FROM pg_settings WHERE pending_restart ORDER BY name;\n")
+	if err != nil {
+		return nil, fmt.Errorf("read the settings pending a restart: %w", err)
+	}
+	var pending []string
+	for line := range strings.SplitSeq(out, "\n") {
+		if name := strings.TrimSpace(line); name != "" {
+			pending = append(pending, name)
+		}
+	}
+	return pending, nil
+}
+
+// Snapshots the include only when the running postmaster has no settings still
+// pending a restart. The parameter mutex keeps an apply from replacing the file
+// between that check and the copy.
+func (e *postgresEngine) RecordServingParameters(ctx context.Context) error {
+	e.paramMu.Lock()
+	defer e.paramMu.Unlock()
+	return e.recordServingParametersLocked(ctx)
+}
+
+func (e *postgresEngine) recordServingParametersLocked(ctx context.Context) error {
+	return recordLastGood(ctx, e.params, e.pendingRestartParameters)
+}
+
+// Puts the last set the engine accepted back in place, for a restart that failed
+// after a parameter change.
+func (e *postgresEngine) RestoreLastKnownGoodParameters(ctx context.Context) (bool, error) {
+	e.paramMu.Lock()
+	defer e.paramMu.Unlock()
+	return restoreLastGood(ctx, e.params, e.probe)
+}
+
+// The engine's own parser, run offline against the datadir. Reading one setting
+// back is enough: postgres parses postgresql.conf and every include first, and
+// exits non-zero naming the file and line of an unknown parameter or a value
+// outside its range.
+func (e *postgresEngine) checkConfig(ctx context.Context) error {
+	if _, err := e.run(ctx, command{
+		Name: filepath.Join(filepath.Dir(e.psql), "postgres"),
+		Args: []string{"-D", e.pgData, "-C", "shared_buffers"},
+		Env:  []string{"PATH=" + defaultGuestPath},
+		User: e.osUser,
+	}); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (e *postgresEngine) Stop(ctx context.Context) error {
+	return serviceAction(ctx, e.run, e.rcService, e.service, "stop")
+}
+
+// Only the parameter rollback calls this: a restart the control plane wants goes
+// through RebootDBInstance, which cycles the VM.
+func (e *postgresEngine) Restart(ctx context.Context) error {
+	return serviceAction(ctx, e.run, e.rcService, e.service, "restart")
+}
+
+// Feeds sql on stdin rather than as an argument, so a statement is never
+// visible in the process table.
+func (e *postgresEngine) psqlRun(ctx context.Context, sql string, env ...string) (string, error) {
+	return e.run(ctx, command{
+		Name:  e.psql,
+		Args:  e.psqlArgs(),
+		Env:   append([]string{"PATH=" + defaultGuestPath}, env...),
+		Stdin: sql,
+		User:  e.osUser,
+	})
+}
+
+// Reading the script from stdin is what lets one invocation serve both a
+// one-shot run and a session held open across several statements.
+func (e *postgresEngine) psqlArgs() []string {
+	return []string{
+		"--no-psqlrc", "--quiet", "--no-align", "--tuples-only",
+		"-v", "ON_ERROR_STOP=1",
+		"-h", e.socketDir,
+		"-p", strconv.FormatInt(e.port.Load(), 10),
+		"-U", e.osUser,
+		"-d", "postgres",
+		"-f", "-",
+	}
+}
+
+// Puts the engine into backup mode: the datadir is checkpointed and the engine
+// stops writing over the pages a snapshot is about to read. The hold is released
+// by Unquiesce, or by its own deadline, whichever comes first.
+func (e *postgresEngine) Quiesce(ctx context.Context, label string, hold time.Duration) error {
+	if err := validateQuiesceRequest(label, hold); err != nil {
+		return err
+	}
+
+	// Held across the whole start, so a second quiesce waits and then finds the
+	// first one's hold rather than opening a concurrent backup alongside it.
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	if e.held != nil {
+		return fmt.Errorf("the engine is already quiesced for backup %s", e.held.label)
+	}
+
+	// Deliberately not the command's context: the session has to outlive the call
+	// that started it, and its own deadline is what bounds it instead.
+	session, err := e.startSess(context.WithoutCancel(ctx), command{
+		Name:              e.psql,
+		Args:              e.psqlArgs(),
+		Env:               []string{"PATH=" + defaultGuestPath, "RDS_BACKUP_LABEL=" + label},
+		User:              e.osUser,
+		SentinelStatement: `\echo ` + sessionSentinel + "\n",
+	})
+	if err != nil {
+		return fmt.Errorf("open a backup session: %w", err)
+	}
+
+	// fast forces an immediate checkpoint rather than spreading it over the
+	// checkpoint interval, which would hold the snapshot open for minutes.
+	const sql = `\getenv label RDS_BACKUP_LABEL
+SELECT pg_backup_start(:'label', fast => true);
+`
+	if err := session.Exec(ctx, sql); err != nil {
+		if closeErr := session.Close(); closeErr != nil {
+			slog.Warn("rds-agent: closing a failed backup session", "err", closeErr)
+		}
+		return fmt.Errorf("put the engine into backup mode: %w", err)
+	}
+
+	e.beginHoldLocked(label, session, hold)
+	return nil
+}
+
+// Ends the backup cleanly. A missing hold is an error rather than a silent
+// success: it means the deadline fired first, so the snapshot the control plane
+// just took was not taken against a held checkpoint.
+func (e *postgresEngine) Unquiesce(ctx context.Context) error {
+	held := e.takeHold()
+	if held == nil {
+		return errors.New("the engine is not quiesced; the backup hold had already expired")
+	}
+
+	// The stop has to run on the session that started the backup, and the session
+	// is closed either way — the engine aborts an unstopped backup with it.
+	execErr := held.session.Exec(ctx, "SELECT pg_backup_stop();\n")
+	closeErr := held.session.Close()
+	if execErr != nil {
+		return fmt.Errorf("take the engine out of backup mode: %w", errors.Join(execErr, closeErr))
+	}
+	if closeErr != nil {
+		return fmt.Errorf("close the backup session: %w", closeErr)
+	}
+	slog.Info("rds-agent: engine released from backup mode", "label", held.label)
+	return nil
+}

cmd/rds-agent/engine_test.go

@@ -141,9 +141,9 @@ func (r *recordingRunner) run(_ context.Context, c command) (string, error) {
 
 func newTestEngine(t *testing.T, run commandRunner) *postgresEngine {
 	t.Helper()
-	cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env"))
-	cfg.PGData = t.TempDir()
-	if err := os.MkdirAll(filepath.Join(cfg.PGData, "conf.d"), 0o700); err != nil {
+	cfg := testLoadConfig(t, enginePostgres)
+	cfg.EngineDataDir = t.TempDir()
+	if err := os.MkdirAll(filepath.Join(cfg.EngineDataDir, "conf.d"), 0o700); err != nil {
 		t.Fatalf("create conf.d: %v", err)
 	}
 	// The guest's postgres user does not exist here, and the chown of the
@@ -152,8 +152,8 @@ func newTestEngine(t *testing.T, run commandRunner) *postgresEngine {
 	if err != nil {
 		t.Fatalf("resolve the current user: %v", err)
 	}
-	cfg.PGUser = current.Username
-	return newPostgresEngine(cfg, run, nil, newEngineProbe(cfg, staticProbe(0)))
+	cfg.EngineUser = current.Username
+	return newPostgresEngine(cfg, run, nil, newPostgresProbe(cfg, staticProbe(0)))
 }
 
 // The password reaches psql through the environment and is re-quoted there,
@@ -341,7 +341,7 @@ func TestPostgresEngine_ApplyParametersRestartsOnARepairSetWhileEngineIsDown(t *
 	codes := []int{2, 2, 0}
 	probeCall := 0
 	cfg := testProbeConfig()
-	engine.probe = newEngineProbe(cfg, func(context.Context, string, ...string) (int, error) {
+	engine.probe = newPostgresProbe(cfg, func(context.Context, string, ...string) (int, error) {
 		code := codes[min(probeCall, len(codes)-1)]
 		probeCall++
 		return code, nil
@@ -372,7 +372,7 @@ func TestPostgresEngine_ApplyParametersRestartsOnARepairSetWhileEngineIsDown(t *
 	if !strings.Contains(runner.calls[2].Stdin, "pending_restart") {
 		t.Errorf("third call = %+v, want the pending-restart read", runner.calls[2])
 	}
-	installed, err := os.ReadFile(engine.parametersPath())
+	installed, err := os.ReadFile(engine.params.installedPath())
 	if err != nil {
 		t.Fatalf("read installed parameters: %v", err)
 	}
@@ -392,7 +392,7 @@ func TestPostgresEngine_ApplyParametersKeepsRepairSetWhenRestartTimesOut(t *test
 	if err == nil || !strings.Contains(err.Error(), "wait for the engine") {
 		t.Fatalf("ApplyParameters error = %v, want the repair wait failure", err)
 	}
-	installed, readErr := os.ReadFile(engine.parametersPath())
+	installed, readErr := os.ReadFile(engine.params.installedPath())
 	if readErr != nil {
 		t.Fatalf("read installed parameters: %v", readErr)
 	}
@@ -400,24 +400,3 @@ func TestPostgresEngine_ApplyParametersKeepsRepairSetWhenRestartTimesOut(t *test
 		t.Errorf("installed parameters = %q, want the checked repair set retained", installed)
 	}
 }
-
-// Through the service manager, so the supervisor records the engine as stopped
-// and does not restart it underneath a VM that is going down.
-func TestPostgresEngine_StopGoesThroughTheServiceManager(t *testing.T) {
-	runner := &recordingRunner{}
-	engine := newTestEngine(t, runner.run)
-
-	if err := engine.Stop(context.Background()); err != nil {
-		t.Fatalf("Stop: %v", err)
-	}
-	if len(runner.calls) != 1 {
-		t.Fatalf("ran %d commands, want 1", len(runner.calls))
-	}
-	call := runner.calls[0]
-	if filepath.Base(call.Name) != "rc-service" {
-		t.Errorf("ran %q, want rc-service", call.Name)
-	}
-	if len(call.Args) != 2 || call.Args[0] != "postgresql" || call.Args[1] != "stop" {
-		t.Errorf("args = %v, want [postgresql stop]", call.Args)
-	}
-}

cmd/rds-agent/health.go

@@ -3,9 +3,7 @@ package main
 import (
 	"context"
 	"errors"
-	"fmt"
 	"os/exec"
-	"strconv"
 	"sync/atomic"
 
 	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
@@ -20,6 +18,9 @@ func execProbeRunner(ctx context.Context, name string, args ...string) (int, err
 	if err == nil {
 		return 0, nil
 	}
+	if ctxErr := ctx.Err(); ctxErr != nil {
+		return -1, ctxErr
+	}
 	var exitErr *exec.ExitError
 	if errors.As(err, &exitErr) {
 		return exitErr.ExitCode(), nil
@@ -27,12 +28,16 @@ func execProbeRunner(ctx context.Context, name string, args ...string) (int, err
 	return -1, err
 }
 
-// Shells out to pg_isready rather than dialling the port, since only a startup
-// exchange separates an engine that is serving from one still in recovery.
+// What the engine reports about itself, which is the only part of the probe
+// that differs between engines. The port is passed in rather than read from the
+// engine, so the atomic below stays the one place it is followed from.
+type probeStateFn func(ctx context.Context, port int64) (engineState, string)
+
+// Engine-neutral: the latch, the port, the degraded mapping and the translation
+// to a reported health are shared, and only the state function is supplied by
+// the engine the image bakes.
 type engineProbe struct {
-	run    probeRunner
-	binary string
-	host   string
+	stateFn probeStateFn
 	// Set from the bootstrap config once it lands, from a different goroutine
 	// than the heartbeat that reads it.
 	port atomic.Int64
@@ -41,9 +46,9 @@ type engineProbe struct {
 	seenHealthy bool
 }
 
-func newEngineProbe(cfg config, run probeRunner) *engineProbe {
-	p := &engineProbe{run: run, binary: cfg.PGIsReady, host: cfg.EngineHost}
-	p.port.Store(int64(cfg.EnginePort))
+func newEngineProbe(port int, stateFn probeStateFn) *engineProbe {
+	p := &engineProbe{stateFn: stateFn}
+	p.port.Store(int64(port))
 	return p
 }
 
@@ -60,7 +65,7 @@ type engineState int
 const (
 	// Not answering at all, or the probe could not run.
 	engineAbsent engineState = iota
-	// The postmaster is up and rejecting connections: startup or crash recovery.
+	// The process is up but not yet serving: startup or crash recovery.
 	engineRecovering
 	engineServing
 )
@@ -83,21 +88,7 @@ func (p *engineProbe) Check(ctx context.Context) (handlers_rds.EngineHealth, str
 // The raw probe result. Safe to call from a goroutine other than the
 // heartbeat's: it touches no latched state.
 func (p *engineProbe) state(ctx context.Context) (engineState, string) {
-	port := strconv.FormatInt(p.port.Load(), 10)
-	code, err := p.run(ctx, p.binary, "-h", p.host, "-p", port, "-q")
-	switch {
-	case err != nil:
-		// A missing binary or broken image. Reporting healthy on the strength of
-		// nothing would hide it, so this reads as absent like an engine that did
-		// not answer.
-		return engineAbsent, fmt.Sprintf("engine probe could not run: %v", err)
-	case code == 0:
-		return engineServing, ""
-	case code == 1:
-		return engineRecovering, "engine is rejecting connections (startup or recovery)"
-	default:
-		return engineAbsent, fmt.Sprintf("engine did not respond on %s:%s", p.host, port)
-	}
+	return p.stateFn(ctx, p.port.Load())
 }
 
 // A non-answering engine: starting until it has answered once, unhealthy after.

cmd/rds-agent/health_test.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"strings"
 	"testing"
+	"time"
 
 	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
 )
@@ -28,7 +29,7 @@ func TestEngineProbe_MapsExitCodesToHealth(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			probe := newEngineProbe(testProbeConfig(), func(context.Context, string, ...string) (int, error) {
+			probe := newPostgresProbe(testProbeConfig(), func(context.Context, string, ...string) (int, error) {
 				return tt.code, tt.runErr
 			})
 			probe.seenHealthy = tt.seenHealthy
@@ -51,7 +52,7 @@ func TestEngineProbe_MapsExitCodesToHealth(t *testing.T) {
 // has to flip the first time the engine answers.
 func TestEngineProbe_LatchesAfterFirstHealthy(t *testing.T) {
 	code := 0
-	probe := newEngineProbe(testProbeConfig(), func(context.Context, string, ...string) (int, error) {
+	probe := newPostgresProbe(testProbeConfig(), func(context.Context, string, ...string) (int, error) {
 		return code, nil
 	})
 
@@ -68,7 +69,7 @@ func TestEngineProbe_LatchesAfterFirstHealthy(t *testing.T) {
 // must follow it rather than keep asking the default.
 func TestEngineProbe_ProbesTheAssignedPort(t *testing.T) {
 	var gotArgs []string
-	probe := newEngineProbe(testProbeConfig(), func(_ context.Context, _ string, args ...string) (int, error) {
+	probe := newPostgresProbe(testProbeConfig(), func(_ context.Context, _ string, args ...string) (int, error) {
 		gotArgs = args
 		return 0, nil
 	})
@@ -81,6 +82,19 @@ func TestEngineProbe_ProbesTheAssignedPort(t *testing.T) {
 	}
 }
 
-func testProbeConfig() config {
-	return config{EngineHost: defaultEngineHost, EnginePort: defaultEnginePort, PGIsReady: defaultPGIsReady}
+func TestExecProbeRunner_PreservesTheProbeDeadline(t *testing.T) {
+	ctx, cancel := context.WithTimeout(t.Context(), 20*time.Millisecond)
+	defer cancel()
+
+	code, err := execProbeRunner(ctx, "sh", "-c", "while :; do :; done")
+	if code != -1 {
+		t.Errorf("exit code = %d, want -1 for a timed-out probe", code)
+	}
+	if !errors.Is(err, context.DeadlineExceeded) {
+		t.Errorf("error = %v, want the context deadline", err)
+	}
+}
+
+func testProbeConfig() config {
+	return config{EngineHost: defaultEngineHost, EnginePort: engineLayouts[enginePostgres].port}
 }

cmd/rds-agent/heartbeat_test.go

@@ -17,7 +17,7 @@ func TestHeartbeater_FirstServingProbeSeedsLastKnownGood(t *testing.T) {
 	runner := &recordingRunner{}
 	engine := newTestEngine(t, runner.run)
 	content := []byte("work_mem = '4096'\n")
-	if err := os.WriteFile(engine.parametersPath(), content, 0o600); err != nil {
+	if err := os.WriteFile(engine.params.installedPath(), content, 0o600); err != nil {
 		t.Fatalf("write installed parameters: %v", err)
 	}
 
@@ -25,7 +25,7 @@ func TestHeartbeater_FirstServingProbeSeedsLastKnownGood(t *testing.T) {
 	h := newHeartbeater(cp, engine.probe, engine, 0)
 	h.beat(context.Background())
 
-	lastGood, err := os.ReadFile(engine.lastGoodPath())
+	lastGood, err := os.ReadFile(engine.params.lastGoodPath())
 	if err != nil {
 		t.Fatalf("read last known good: %v", err)
 	}
@@ -39,7 +39,7 @@ func TestHeartbeater_FirstServingProbeSeedsLastKnownGood(t *testing.T) {
 }
 
 func TestHeartbeater_BoundsBootstrapFailure(t *testing.T) {
-	h := newHeartbeater(newFakeControlPlane(), newEngineProbe(testProbeConfig(), staticProbe(2)), nil, 0)
+	h := newHeartbeater(newFakeControlPlane(), newPostgresProbe(testProbeConfig(), staticProbe(2)), nil, 0)
 	h.setBootstrapFailure("bootstrap fetch", errors.New(strings.Repeat("界", 1000)))
 
 	failure := h.bootstrapFailure.Load()
@@ -60,7 +60,7 @@ func TestHeartbeater_BoundsBootstrapFailure(t *testing.T) {
 func TestHeartbeater_ChecksServingParametersOnEveryHealthyProbe(t *testing.T) {
 	code := 0
 	cfg := testProbeConfig()
-	probe := newEngineProbe(cfg, func(context.Context, string, ...string) (int, error) {
+	probe := newPostgresProbe(cfg, func(context.Context, string, ...string) (int, error) {
 		return code, nil
 	})
 	recorder := &countingServingRecorder{}

cmd/rds-agent/parameters.go

@@ -0,0 +1,223 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"time"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// The generated parameter files and what it takes to install one. They live on
+// the data volume rather than under /etc, at the same paths rds-init writes them
+// to: a class change is a VM replacement, so a set under /etc would be discarded
+// with the boot volume and the instance would silently revert to catalog
+// defaults while the API still reported the customer's values.
+type parameterStore struct {
+	dir string
+	// The include the resolved set is rendered to.
+	installed string
+	// The copy of the last set the engine accepted, which a boot loop is rolled
+	// back to. Deliberately not a name the engine's own include glob reads, so it
+	// is never parsed as a second set of settings.
+	lastGood string
+	// What the engine actually started on, for an engine that cannot be asked
+	// which of its settings are still pending a restart. Empty for one that can.
+	serving string
+	// Prepended to every file written here, for a configuration format that has
+	// no setting outside a group.
+	header string
+	// The engine parses its own configuration, so a root-owned file would be
+	// unreadable to it.
+	osUser string
+	// Names the engine whose option-file spellings these files are written in.
+	engine string
+}
+
+func (s parameterStore) installedPath() string { return filepath.Join(s.dir, s.installed) }
+func (s parameterStore) lastGoodPath() string  { return filepath.Join(s.dir, s.lastGood) }
+func (s parameterStore) servingPath() string   { return filepath.Join(s.dir, s.serving) }
+
+// Byte-for-byte what rds-init writes for the same set, so the copies compared
+// against it are comparing values rather than formatting.
+func (s parameterStore) render(params []handlers_rds.Parameter) ([]byte, error) {
+	body, err := renderParameters(s.engine, params)
+	if err != nil {
+		return nil, err
+	}
+	return []byte(s.header + body), nil
+}
+
+// Installs the resolved set at the path rds-init uses, so a later boot
+// overwrites rather than shadows it. Returns a func putting back whatever was
+// there, so the caller can undo the install without re-rendering anything.
+func (s parameterStore) install(params []handlers_rds.Parameter) (restore func() error, err error) {
+	path := s.installedPath()
+	// A missing file is the first-ever apply, and its rollback is a removal.
+	previous, readErr := os.ReadFile(path)
+	if readErr != nil && !os.IsNotExist(readErr) {
+		return nil, fmt.Errorf("read the installed parameters at %s: %w", path, readErr)
+	}
+	existed := readErr == nil
+
+	rendered, err := s.render(params)
+	if err != nil {
+		return nil, err
+	}
+	if err := s.write(path, rendered); err != nil {
+		return nil, err
+	}
+	return func() error {
+		if !existed {
+			if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
+				return fmt.Errorf("withdraw the rejected parameters at %s: %w", path, err)
+			}
+			return nil
+		}
+		return s.write(path, previous)
+	}, nil
+}
+
+// Atomic: a temp file in the same directory, renamed over the target. The temp
+// name deliberately does not end in the engine's include suffix, so a crash
+// between write and rename cannot leave the engine reading a half-written file.
+func (s parameterStore) write(path string, content []byte) error {
+	tmp := path + ".new"
+	if err := os.WriteFile(tmp, content, 0o600); err != nil {
+		return fmt.Errorf("write %s: %w", tmp, err)
+	}
+	if err := s.chownToEngine(tmp); err != nil {
+		os.Remove(tmp)
+		return err
+	}
+	if err := os.Rename(tmp, path); err != nil {
+		os.Remove(tmp)
+		return fmt.Errorf("install %s: %w", path, err)
+	}
+	return nil
+}
+
+// A guest with no engine user is a broken image, not something to work around.
+func (s parameterStore) chownToEngine(path string) error {
+	credential, err := lookupCredential(s.osUser)
+	if err != nil {
+		return err
+	}
+	if err := os.Chown(path, int(credential.Uid), int(credential.Gid)); err != nil {
+		return fmt.Errorf("hand %s to %s: %w", path, s.osUser, err)
+	}
+	return nil
+}
+
+// Snapshots the installed set as the rollback target, but only once the engine
+// is running all of it. A set with a setting still pending a restart has not
+// been shown to serve, and promoting it would leave the rollback pointing at the
+// configuration that is about to fail.
+func recordLastGood(ctx context.Context, store parameterStore, pending pendingRestartFn) error {
+	if _, err := os.Stat(store.installedPath()); err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return fmt.Errorf("inspect the serving parameters: %w", err)
+	}
+	names, err := pending(ctx)
+	if err != nil {
+		return err
+	}
+	if len(names) > 0 {
+		return nil
+	}
+
+	content, err := os.ReadFile(store.installedPath())
+	if err != nil {
+		return fmt.Errorf("read the serving parameters: %w", err)
+	}
+	lastGood, err := os.ReadFile(store.lastGoodPath())
+	if err == nil && bytes.Equal(content, lastGood) {
+		return nil
+	}
+	if err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("read the last known good parameters: %w", err)
+	}
+	return store.write(store.lastGoodPath(), content)
+}
+
+// Puts the last set the engine accepted back in place, for a restart that failed
+// after a parameter change. The probe is checked again under the caller's
+// parameter lock, so a repair that just brought the engine back is not reversed.
+func restoreLastGood(ctx context.Context, store parameterStore, probe *engineProbe) (bool, error) {
+	if state, _ := probe.state(ctx); state != engineAbsent {
+		return false, nil
+	}
+	lastGood, err := os.ReadFile(store.lastGoodPath())
+	if err != nil {
+		if os.IsNotExist(err) {
+			return false, nil
+		}
+		return false, fmt.Errorf("read the last known good parameters: %w", err)
+	}
+	current, err := os.ReadFile(store.installedPath())
+	if err != nil && !os.IsNotExist(err) {
+		return false, fmt.Errorf("read the installed parameters: %w", err)
+	}
+	if bytes.Equal(current, lastGood) {
+		return false, nil
+	}
+	if err := store.write(store.installedPath(), lastGood); err != nil {
+		return false, err
+	}
+	return true, nil
+}
+
+// The settings the engine has accepted but will not honour until it restarts.
+// Each implementation answers it its own way, and both the apply and the serving
+// snapshot are gated on the same answer.
+type pendingRestartFn func(ctx context.Context) ([]string, error)
+
+const (
+	parameterRepairTimeout = 90 * time.Second
+	parameterRepairPoll    = time.Second
+)
+
+// Starts a down engine on the set that was just installed and waits for it to
+// serve. Reached when the engine went away during an apply: the installed set is
+// the newer of the two, so starting on it beats restoring one that already left
+// the engine unable to serve.
+func awaitRepairedEngine(ctx context.Context, probe *engineProbe, restart func(context.Context) error,
+	pending pendingRestartFn, timeout, poll time.Duration) ([]string, error) {
+	repairCtx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	if err := restart(repairCtx); err != nil {
+		return nil, fmt.Errorf("start the engine on the repaired parameter set: %w", err)
+	}
+
+	timer := time.NewTimer(0)
+	defer timer.Stop()
+	lastMessage := "engine did not respond"
+	for {
+		select {
+		case <-repairCtx.Done():
+			if ctx.Err() != nil {
+				return nil, fmt.Errorf("wait for the engine on the repaired parameter set: %w", ctx.Err())
+			}
+			return nil, fmt.Errorf("wait for the engine on the repaired parameter set: %s: %w", lastMessage, repairCtx.Err())
+		case <-timer.C:
+		}
+
+		state, message := probe.state(repairCtx)
+		if state == engineServing {
+			names, err := pending(repairCtx)
+			if err == nil {
+				return names, nil
+			}
+			lastMessage = err.Error()
+		} else if message != "" {
+			lastMessage = message
+		}
+		timer.Reset(poll)
+	}
+}

cmd/rds-agent/paramrecovery_test.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"errors"
 	"os"
-	"path/filepath"
 	"slices"
 	"strings"
 	"testing"
@@ -50,7 +49,7 @@ func TestPostgresEngine_ApplyParametersRollsBackAValueTheEngineRejects(t *testin
 		t.Fatal("ApplyParameters succeeded against a config the engine rejected")
 	}
 
-	installed, err := os.ReadFile(engine.parametersPath())
+	installed, err := os.ReadFile(engine.params.installedPath())
 	if err != nil {
 		t.Fatalf("read the installed parameters: %v", err)
 	}
@@ -69,7 +68,7 @@ func TestPostgresEngine_ApplyParametersWithdrawsTheFirstRejectedSet(t *testing.T
 		[]handlers_rds.Parameter{{Name: "work_mem", Value: "0"}}); err == nil {
 		t.Fatal("ApplyParameters succeeded against a config the engine rejected")
 	}
-	if _, err := os.Stat(engine.parametersPath()); !os.IsNotExist(err) {
+	if _, err := os.Stat(engine.params.installedPath()); !os.IsNotExist(err) {
 		t.Errorf("the rejected include is still installed (stat err = %v)", err)
 	}
 }
@@ -97,7 +96,7 @@ func TestPostgresEngine_ApplyBeforeFirstHeartbeatSeedsLastKnownGood(t *testing.T
 	runner := &pendingAfterReloadRunner{}
 	engine := newTestEngine(t, runner.run)
 	first := []byte("shared_buffers = '32768'\n")
-	if err := os.WriteFile(engine.parametersPath(), first, 0o600); err != nil {
+	if err := os.WriteFile(engine.params.installedPath(), first, 0o600); err != nil {
 		t.Fatalf("write serving parameter set: %v", err)
 	}
 
@@ -108,7 +107,7 @@ func TestPostgresEngine_ApplyBeforeFirstHeartbeatSeedsLastKnownGood(t *testing.T
 	if !slices.Equal(pending, []string{"shared_buffers"}) {
 		t.Errorf("pending = %v, want shared_buffers", pending)
 	}
-	lastGood, err := os.ReadFile(engine.lastGoodPath())
+	lastGood, err := os.ReadFile(engine.params.lastGoodPath())
 	if err != nil {
 		t.Fatalf("read last known good: %v", err)
 	}
@@ -153,10 +152,10 @@ func TestPostgresEngine_ServingSnapshotWaitsForParameterApply(t *testing.T) {
 	}
 	engine := newTestEngine(t, runner.run)
 	first := []byte("shared_buffers = '32768'\n")
-	if err := os.WriteFile(engine.parametersPath(), first, 0o600); err != nil {
+	if err := os.WriteFile(engine.params.installedPath(), first, 0o600); err != nil {
 		t.Fatalf("write serving parameter set: %v", err)
 	}
-	if err := os.WriteFile(engine.lastGoodPath(), first, 0o600); err != nil {
+	if err := os.WriteFile(engine.params.lastGoodPath(), first, 0o600); err != nil {
 		t.Fatalf("write last known good: %v", err)
 	}
 
@@ -192,7 +191,7 @@ func TestPostgresEngine_ServingSnapshotWaitsForParameterApply(t *testing.T) {
 		t.Error("serving snapshot completed while the parameter apply held its transaction")
 	}
 
-	lastGood, err := os.ReadFile(engine.lastGoodPath())
+	lastGood, err := os.ReadFile(engine.params.lastGoodPath())
 	if err != nil {
 		t.Fatalf("read last known good: %v", err)
 	}
@@ -216,7 +215,7 @@ func TestPostgresEngine_LastKnownGoodTracksTheServingSet(t *testing.T) {
 	}
 	assertLastGood := func(value string) {
 		t.Helper()
-		lastGood, err := os.ReadFile(engine.lastGoodPath())
+		lastGood, err := os.ReadFile(engine.params.lastGoodPath())
 		if err != nil {
 			t.Fatalf("read the last known good parameters: %v", err)
 		}
@@ -240,8 +239,8 @@ func TestPostgresEngine_LastKnownGoodTracksTheServingSet(t *testing.T) {
 	apply("16384")
 	assertLastGood("8192")
 
-	if strings.HasSuffix(engine.lastGoodPath(), ".conf") {
-		t.Errorf("the rollback copy at %s would be included by the engine", engine.lastGoodPath())
+	if strings.HasSuffix(engine.params.lastGoodPath(), ".conf") {
+		t.Errorf("the rollback copy at %s would be included by the engine", engine.params.lastGoodPath())
 	}
 }
 
@@ -249,7 +248,7 @@ func TestPostgresEngine_RecordServingParametersSkipsPendingRestart(t *testing.T)
 	runner := &recordingRunner{}
 	engine := newTestEngine(t, runner.run)
 	first := []byte("shared_buffers = '32768'\n")
-	if err := os.WriteFile(engine.parametersPath(), first, 0o600); err != nil {
+	if err := os.WriteFile(engine.params.installedPath(), first, 0o600); err != nil {
 		t.Fatalf("write first parameter set: %v", err)
 	}
 	if err := engine.RecordServingParameters(t.Context()); err != nil {
@@ -257,14 +256,14 @@ func TestPostgresEngine_RecordServingParametersSkipsPendingRestart(t *testing.T)
 	}
 
 	second := []byte("shared_buffers = '65536'\n")
-	if err := os.WriteFile(engine.parametersPath(), second, 0o600); err != nil {
+	if err := os.WriteFile(engine.params.installedPath(), second, 0o600); err != nil {
 		t.Fatalf("write pending parameter set: %v", err)
 	}
 	runner.out = "shared_buffers\n"
 	if err := engine.RecordServingParameters(t.Context()); err != nil {
 		t.Fatalf("record with a pending restart: %v", err)
 	}
-	lastGood, err := os.ReadFile(engine.lastGoodPath())
+	lastGood, err := os.ReadFile(engine.params.lastGoodPath())
 	if err != nil {
 		t.Fatalf("read last known good: %v", err)
 	}
@@ -278,7 +277,7 @@ func TestPostgresEngine_RecordServingParametersSkipsPendingRestart(t *testing.T)
 	if err := engine.RecordServingParameters(t.Context()); err != nil {
 		t.Fatalf("record the restarted parameter set: %v", err)
 	}
-	lastGood, err = os.ReadFile(engine.lastGoodPath())
+	lastGood, err = os.ReadFile(engine.params.lastGoodPath())
 	if err != nil {
 		t.Fatalf("read advanced last known good: %v", err)
 	}
@@ -330,7 +329,7 @@ func TestPostgresEngine_RestoreLastKnownGoodParameters(t *testing.T) {
 	if !restored {
 		t.Fatal("did not restore a set that differs from the last known good")
 	}
-	installed, err := os.ReadFile(engine.parametersPath())
+	installed, err := os.ReadFile(engine.params.installedPath())
 	if err != nil {
 		t.Fatalf("read the installed parameters: %v", err)
 	}
@@ -369,8 +368,8 @@ func (f *fakeRecovery) Restart(context.Context) error {
 // A probe whose result the test drives directly, standing in for pg_isready.
 func stubProbe(t *testing.T, code int, err error) *engineProbe {
 	t.Helper()
-	cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env"))
-	return newEngineProbe(cfg, func(context.Context, string, ...string) (int, error) {
+	cfg := testLoadConfig(t, enginePostgres)
+	return newPostgresProbe(cfg, func(context.Context, string, ...string) (int, error) {
 		return code, err
 	})
 }

cmd/rds-agent/quiesce.go

@@ -15,10 +15,11 @@ import (
 	"time"
 )
 
-// PostgreSQL's non-exclusive backup API ties the backup to the session that
-// started it: when that session ends, for any reason, the backup is aborted.
-// That is what makes the hold self-expiring — a control plane that dies mid
-// snapshot cannot leave the engine in backup mode forever.
+// Both engines tie a backup hold to the session that took it: PostgreSQL aborts
+// a non-exclusive backup when its session ends, and MariaDB releases a BACKUP
+// STAGE with the connection that started it. That is what makes the hold
+// self-expiring — a control plane that dies mid snapshot cannot leave the engine
+// in backup mode forever.
 type engineSession interface {
 	// Runs sql and waits for the engine to finish it.
 	Exec(ctx context.Context, sql string) error
@@ -31,7 +32,8 @@ type engineSession interface {
 type sessionRunner func(ctx context.Context, c command) (engineSession, error)
 
 // Written after every statement so the reader knows the engine is done with it.
-// The statements this carries return LSNs, so it cannot appear in a result.
+// The statements this carries return LSNs and backup stages, so it cannot appear
+// in a result.
 const sessionSentinel = "--rds-agent-ready--"
 
 // The backup mode currently held, and the timer that ends it regardless of what
@@ -42,98 +44,65 @@ type quiesceHold struct {
 	expiry  *time.Timer
 }
 
-// Puts the engine into backup mode: the datadir is checkpointed and the engine
-// stops writing over the pages a snapshot is about to read. The hold is released
-// by Unquiesce, or by its own deadline, whichever comes first.
-func (e *postgresEngine) Quiesce(ctx context.Context, label string, hold time.Duration) error {
+// The hold bookkeeping, which is the same for both engines: only the statements
+// that take and release it differ, not the single-hold rule or the deadline that
+// releases it whatever happens to the caller.
+type quiesceState struct {
+	// Guarded because the expiry timer releases the hold on its own goroutine.
+	mu   sync.Mutex
+	held *quiesceHold
+}
+
+// The two things every quiesce needs, checked before a session is opened so a
+// malformed command costs the engine nothing.
+func validateQuiesceRequest(label string, hold time.Duration) error {
 	if label == "" {
 		return errors.New("quiesce requires a backup label")
 	}
 	if hold <= 0 {
 		return errors.New("quiesce requires a positive hold deadline")
 	}
+	return nil
+}
 
-	// Held across the whole start, so a second quiesce waits and then finds the
-	// first one's hold rather than opening a concurrent backup alongside it.
-	e.mu.Lock()
-	defer e.mu.Unlock()
-	if e.held != nil {
-		return fmt.Errorf("the engine is already quiesced for backup %s", e.held.label)
-	}
-
-	// Deliberately not the command's context: the session has to outlive the call
-	// that started it, and its own deadline is what bounds it instead.
-	session, err := e.startSess(context.WithoutCancel(ctx), command{
-		Name:  e.psql,
-		Args:  e.psqlArgs(),
-		Env:   []string{"PATH=" + defaultGuestPath, "RDS_BACKUP_LABEL=" + label},
-		Stdin: "",
-		User:  e.osUser,
-	})
-	if err != nil {
-		return fmt.Errorf("open a backup session: %w", err)
-	}
-
-	// fast forces an immediate checkpoint rather than spreading it over the
-	// checkpoint interval, which would hold the snapshot open for minutes.
-	const sql = `\getenv label RDS_BACKUP_LABEL
-SELECT pg_backup_start(:'label', fast => true);
-`
-	if err := session.Exec(ctx, sql); err != nil {
-		if closeErr := session.Close(); closeErr != nil {
-			slog.Warn("rds-agent: closing a failed backup session", "err", closeErr)
-		}
-		return fmt.Errorf("put the engine into backup mode: %w", err)
-	}
-
-	e.held = &quiesceHold{
+// Records the hold and arms its deadline. Called with mu held, so a second
+// quiesce waits and then finds this hold rather than opening a concurrent
+// backup alongside it.
+func (q *quiesceState) beginHoldLocked(label string, session engineSession, hold time.Duration) {
+	q.held = &quiesceHold{
 		label:   label,
 		session: session,
-		expiry:  time.AfterFunc(hold, func() { e.expireQuiesce(label) }),
+		expiry:  time.AfterFunc(hold, func() { q.expire(label) }),
 	}
 	slog.Info("rds-agent: engine quiesced for backup", "label", label, "hold", hold)
-	return nil
 }
 
-// Ends the backup cleanly. A missing hold is an error rather than a silent
-// success: it means the deadline fired first, so the snapshot the control plane
-// just took was not taken against a held checkpoint.
-func (e *postgresEngine) Unquiesce(ctx context.Context) error {
-	e.mu.Lock()
-	held := e.held
-	e.held = nil
-	e.mu.Unlock()
+// Takes the hold off the engine, if one is still there, and disarms its
+// deadline. The caller ends the session, because the statement that releases the
+// backup has to run on it first.
+func (q *quiesceState) takeHold() *quiesceHold {
+	q.mu.Lock()
+	defer q.mu.Unlock()
 
-	if held == nil {
-		return errors.New("the engine is not quiesced; the backup hold had already expired")
+	held := q.held
+	q.held = nil
+	if held != nil {
+		held.expiry.Stop()
 	}
-	held.expiry.Stop()
-
-	// The stop has to run on the session that started the backup, and the session
-	// is closed either way — the engine aborts an unstopped backup with it.
-	execErr := held.session.Exec(ctx, "SELECT pg_backup_stop();\n")
-	closeErr := held.session.Close()
-	if execErr != nil {
-		return fmt.Errorf("take the engine out of backup mode: %w", errors.Join(execErr, closeErr))
-	}
-	if closeErr != nil {
-		return fmt.Errorf("close the backup session: %w", closeErr)
-	}
-	slog.Info("rds-agent: engine released from backup mode", "label", held.label)
-	return nil
+	return held
 }
 
-// The deadline. Ending the session is enough — the engine aborts the backup it
-// was holding — so this never has to reach the engine itself.
-func (e *postgresEngine) expireQuiesce(label string) {
-	e.mu.Lock()
-	held := e.held
+// The deadline. Ending the session is enough — both engines release the hold
+// with it — so this never has to reach the engine itself.
+func (q *quiesceState) expire(label string) {
+	q.mu.Lock()
+	held := q.held
 	if held == nil || held.label != label {
-		e.mu.Unlock()
+		q.mu.Unlock()
 		return
 	}
-	e.held = nil
-	e.mu.Unlock()
+	q.held = nil
+	q.mu.Unlock()
 
 	slog.Warn("rds-agent: backup hold expired; releasing the engine", "label", label)
 	if err := held.session.Close(); err != nil {
@@ -141,21 +110,27 @@ func (e *postgresEngine) expireQuiesce(label string) {
 	}
 }
 
-// A psql child with its stdin held open, so statements can be fed to it one at a
-// time and the engine keeps seeing a single session.
-type psqlSession struct {
+// A client child with its stdin held open, so statements can be fed to it one at
+// a time and the engine keeps seeing a single session.
+type clientSession struct {
 	cmd    *exec.Cmd
 	stdin  io.WriteCloser
 	stdout *bufio.Reader
 	stderr *bytes.Buffer
+	// The statement that makes this client print the sentinel, which is the only
+	// part of a held session that differs between engines.
+	sentinel string
 
 	mu     sync.Mutex
 	closed bool
 }
 
-var _ engineSession = (*psqlSession)(nil)
+var _ engineSession = (*clientSession)(nil)
 
 func execSessionRunner(ctx context.Context, c command) (engineSession, error) {
+	if c.SentinelStatement == "" {
+		return nil, fmt.Errorf("%s was started as a session with no sentinel statement", c.Name)
+	}
 	cmd := exec.CommandContext(ctx, c.Name, c.Args...)
 	cmd.Env = c.Env
 	if c.User != "" {
@@ -180,24 +155,30 @@ func execSessionRunner(ctx context.Context, c command) (engineSession, error) {
 	if err := cmd.Start(); err != nil {
 		return nil, fmt.Errorf("start %s: %w", c.Name, err)
 	}
-	return &psqlSession{cmd: cmd, stdin: stdin, stdout: bufio.NewReader(stdout), stderr: &stderr}, nil
+	return &clientSession{
+		cmd:      cmd,
+		stdin:    stdin,
+		stdout:   bufio.NewReader(stdout),
+		stderr:   &stderr,
+		sentinel: c.SentinelStatement,
+	}, nil
 }
 
-// Feeds sql and waits for the sentinel that follows it. ON_ERROR_STOP makes psql
-// exit on a failed statement, so a sentinel that never arrives is the failure
-// itself and the engine's own message is on stderr.
-func (s *psqlSession) Exec(ctx context.Context, sql string) error {
+// Feeds sql and waits for the sentinel that follows it. Both clients are run in
+// a mode that exits on a failed statement, so a sentinel that never arrives is
+// the failure itself and the engine's own message is on stderr.
+func (s *clientSession) Exec(ctx context.Context, sql string) error {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	if s.closed {
 		return errors.New("the backup session is closed")
 	}
-	if _, err := io.WriteString(s.stdin, sql+"\\echo "+sessionSentinel+"\n"); err != nil {
+	if _, err := io.WriteString(s.stdin, sql+s.sentinel); err != nil {
 		return fmt.Errorf("%w: %s", err, strings.TrimSpace(s.stderr.String()))
 	}
 
-	// Read on its own goroutine so a psql that never answers costs the caller its
-	// deadline rather than blocking it forever.
+	// Read on its own goroutine so a client that never answers costs the caller
+	// its deadline rather than blocking it forever.
 	done := make(chan error, 1)
 	go func() { done <- s.awaitSentinel() }()
 	select {
@@ -208,7 +189,7 @@ func (s *psqlSession) Exec(ctx context.Context, sql string) error {
 	}
 }
 
-func (s *psqlSession) awaitSentinel() error {
+func (s *clientSession) awaitSentinel() error {
 	for {
 		line, err := s.stdout.ReadString('\n')
 		if strings.TrimSpace(line) == sessionSentinel {
@@ -226,7 +207,7 @@ func (s *psqlSession) awaitSentinel() error {
 // Closing stdin ends the script, and the wait reaps the child. A session that
 // will not end is killed, because leaving it is what would keep the engine in
 // backup mode.
-func (s *psqlSession) Close() error {
+func (s *clientSession) Close() error {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	if s.closed {

cmd/rds-agent/quiesce_test.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"context"
-	"errors"
 	"slices"
 	"strings"
 	"sync"
@@ -113,81 +112,6 @@ func TestPostgresEngine_UnquiesceStopsTheBackupAndEndsTheSession(t *testing.T) {
 	}
 }
 
-// A hold that expired mid-snapshot means the snapshot was not taken against a
-// held checkpoint, so the release reports it rather than succeeding silently.
-func TestPostgresEngine_UnquiesceWithoutAHoldIsAnError(t *testing.T) {
-	engine, _ := newQuiesceEngine(t, &fakeSession{})
-
-	err := engine.Unquiesce(context.Background())
-	if err == nil {
-		t.Fatal("Unquiesce succeeded with no hold to release")
-	}
-	if !strings.Contains(err.Error(), "expired") {
-		t.Errorf("error %q does not say the hold had expired", err)
-	}
-}
-
-// The engine aborts the backup when the session ends, which is what makes the
-// hold self-expiring: a control plane that dies cannot leave it held forever.
-func TestPostgresEngine_QuiesceReleasesItselfOnItsDeadline(t *testing.T) {
-	session := &fakeSession{}
-	engine, _ := newQuiesceEngine(t, session)
-
-	if err := engine.Quiesce(context.Background(), "orders-db-pre-upgrade", 10*time.Millisecond); err != nil {
-		t.Fatalf("Quiesce: %v", err)
-	}
-	deadline := time.Now().Add(2 * time.Second)
-	for session.closes() == 0 && time.Now().Before(deadline) {
-		time.Sleep(time.Millisecond)
-	}
-
-	if session.closes() != 1 {
-		t.Fatal("the backup session outlived its deadline")
-	}
-	// The hold is gone with it, so a later release is honest about that.
-	if err := engine.Unquiesce(context.Background()); err == nil {
-		t.Error("Unquiesce succeeded after the hold expired")
-	}
-}
-
-// One backup at a time: a second start would open a concurrent backup alongside
-// the first rather than extending it.
-func TestPostgresEngine_QuiesceRefusesASecondHold(t *testing.T) {
-	session := &fakeSession{}
-	engine, started := newQuiesceEngine(t, session)
-	if err := engine.Quiesce(context.Background(), "first", time.Minute); err != nil {
-		t.Fatalf("Quiesce: %v", err)
-	}
-
-	err := engine.Quiesce(context.Background(), "second", time.Minute)
-	if err == nil {
-		t.Fatal("a second quiesce was accepted while the first was held")
-	}
-	if !strings.Contains(err.Error(), "first") {
-		t.Errorf("error %q does not name the backup already holding the engine", err)
-	}
-	if len(*started) != 1 {
-		t.Errorf("started %d sessions, want the second refused before opening one", len(*started))
-	}
-}
-
-// A failed start leaves nothing held, so a retry is not refused by a hold that
-// never took effect.
-func TestPostgresEngine_QuiesceEndsTheSessionWhenTheBackupWillNotStart(t *testing.T) {
-	session := &fakeSession{execErr: errors.New(`ERROR:  a backup is already in progress`)}
-	engine, _ := newQuiesceEngine(t, session)
-
-	if err := engine.Quiesce(context.Background(), "orders-db-pre-upgrade", time.Minute); err == nil {
-		t.Fatal("Quiesce succeeded against an engine that refused the backup")
-	}
-	if session.closes() != 1 {
-		t.Errorf("closed %d times, want the failed session ended", session.closes())
-	}
-	if engine.held != nil {
-		t.Error("a failed quiesce left a hold behind")
-	}
-}
-
 func TestCommandRegistry_QuiesceCarriesTheLabelAndDeadline(t *testing.T) {
 	engine := &fakeEngine{}
 	reply := newCommander(nil, newCommandRegistry(engine, &fakeStorage{}), 0).execute(context.Background(),

cmd/rds-agent/storage.go

@@ -28,8 +28,6 @@ const (
 	defaultXFSGrowfs  = "xfs_growfs"
 	defaultMountsFile = "/proc/mounts"
 	defaultSysBlock   = "/sys/class/block"
-	// Where rds-datadir mounts the data volume; the datadir sits one level in.
-	defaultDataMount = "/var/lib/postgresql"
 	// growpart says this when the partition already fills the disk, which a
 	// resumed grow is the normal way to reach.
 	growpartNoChange = "NOCHANGE"

cmd/rds-agent/storage_test.go

@@ -61,12 +61,12 @@ func newTestStorage(t *testing.T, device, fstype string, run commandRunner) *gue
 	mounts := filepath.Join(dir, "mounts")
 	content := "proc /proc proc rw 0 0\n"
 	if device != "" {
-		content += device + " " + defaultDataMount + " " + fstype + " rw,relatime 0 0\n"
+		content += device + " " + testDataMount + " " + fstype + " rw,relatime 0 0\n"
 	}
 	if err := os.WriteFile(mounts, []byte(content), 0o600); err != nil {
 		t.Fatalf("write the mount table: %v", err)
 	}
-	cfg := config{DataMount: defaultDataMount, MountsFile: mounts, SysBlock: filepath.Join(dir, "block")}
+	cfg := config{DataMount: testDataMount, MountsFile: mounts, SysBlock: filepath.Join(dir, "block")}
 	if err := os.MkdirAll(cfg.SysBlock, 0o755); err != nil {
 		t.Fatalf("create the sysfs tree: %v", err)
 	}
@@ -93,6 +93,9 @@ func seedPartition(t *testing.T, g *guestStorage, disk, name, number string) {
 
 // The data volume is formatted whole, so the usual grow is resize2fs against
 // the device with no partition to push out first.
+// Where rds-datadir mounts the data volume in the PostgreSQL preset.
+const testDataMount = "/var/lib/postgresql"
+
 func TestGrowFilesystem_ResizesAnExt4DeviceOnline(t *testing.T) {
 	log := &runLog{}
 	g := newTestStorage(t, "/dev/vdb", "ext4", log.run)
@@ -108,7 +111,7 @@ func TestGrowFilesystem_ResizesAnExt4DeviceOnline(t *testing.T) {
 	if args := log.calls[0].Args; len(args) != 1 || args[0] != "/dev/vdb" {
 		t.Errorf("resize2fs args = %v, want the device", args)
 	}
-	if !strings.Contains(message, "/dev/vdb") || !strings.Contains(message, defaultDataMount) {
+	if !strings.Contains(message, "/dev/vdb") || !strings.Contains(message, testDataMount) {
 		t.Errorf("reply message = %q, want the device and mount it grew", message)
 	}
 }
@@ -125,7 +128,7 @@ func TestGrowFilesystem_GrowsXFSByMountPoint(t *testing.T) {
 	if got := log.names(); len(got) != 1 || got[0] != defaultXFSGrowfs {
 		t.Fatalf("ran %v, want just %s", got, defaultXFSGrowfs)
 	}
-	if args := log.calls[0].Args; len(args) != 1 || args[0] != defaultDataMount {
+	if args := log.calls[0].Args; len(args) != 1 || args[0] != testDataMount {
 		t.Errorf("xfs_growfs args = %v, want the mount point", args)
 	}
 }

cmd/spinifex/cmd/admin.go

@@ -1524,10 +1524,8 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	// is node-local, so it is derived here from this node's own config dirs.
 	northstarConfigPath := filepath.Join(dirs.Northstar, "northstar.toml")
 
-	// Parse multi-node predastore configuration. A single-node install is host
-	// 1, the only [[host]] its predastore.toml declares; the multi-node path
-	// below replaces this with the host matching this machine.
-	predastoreHostID := 1
+	// Parse multi-node predastore configuration (legacy flag-based approach for single-node)
+	var predastoreHostID int
 	if predastoreNodesStr != "" {
 		ips := strings.Split(predastoreNodesStr, ",")
 		if len(ips) < 2 {
@@ -1844,10 +1842,8 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 	// is node-local, so it is derived here from this node's own config dirs.
 	northstarConfigPath := filepath.Join(dirs.Northstar, "northstar.toml")
 
-	// Generate multi-node predastore config. A single-node install is host 1,
-	// the only [[host]] its predastore.toml declares; the multi-node path below
-	// replaces this with the host matching this machine.
-	predastoreHostID := 1
+	// Generate multi-node predastore config
+	var predastoreHostID int
 	hasPredastoreConfig := len(predastoreNodes) >= 2
 	if hasPredastoreConfig {
 		predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, accessKey, secretKey, region, natsToken, configDir, spxRoot, bindIP, compactionInterval, northstarCreds)
@@ -1862,14 +1858,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 			os.Exit(1)
 		}
 
-		// One machine is one predastore host, so the node list entry matching
-		// this bind IP is this node's host ID. Without one the service has no
-		// [[host]] to run and refuses to start.
 		predastoreHostID = admin.FindNodeIDByIP(predastoreNodes, bindIP)
-		if predastoreHostID == 0 {
-			fmt.Fprintf(os.Stderr, "❌ Error: bind IP %s not found in the predastore node list\n", bindIP)
-			os.Exit(1)
-		}
 		fmt.Printf("✅ Created: multi-node predastore.toml (host ID: %d)\n", predastoreHostID)
 	}
 
@@ -2403,10 +2392,8 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 
 	northstarCreds, northstarConfigPath := northstarFromFormation(creds, dirs)
 
-	// Generate multi-node predastore config. A single-node install is host 1,
-	// the only [[host]] its predastore.toml declares; the multi-node path below
-	// replaces this with the host matching this machine.
-	predastoreHostID := 1
+	// Generate multi-node predastore config
+	var predastoreHostID int
 	hasPredastoreConfig := len(predastoreNodes) >= 2
 
 	if hasPredastoreConfig {

cmd/spinifex/cmd/admin_test.go

@@ -768,7 +768,8 @@ func predastoreMultinodeNodes() []admin.PredastoreNodeConfig {
 type predastoreBucketAuth struct {
 	Buckets []struct {
 		Name string `toml:"name"`
-	} `toml:"bucket"`
+		Type string `toml:"type"`
+	} `toml:"buckets"`
 	Auth []struct {
 		AccessKeyID     string `toml:"access_key_id"`
 		SecretAccessKey string `toml:"secret_access_key"`
@@ -820,10 +821,16 @@ func TestPredastoreMultinodeTemplate_NorthstarProvisioned(t *testing.T) {
 	var cfg predastoreBucketAuth
 	require.NoError(t, toml.Unmarshal([]byte(content), &cfg))
 
-	// The zone bucket is declared once for the whole cluster: every host's gate
-	// reads the same config, so a resolver on any node can serve it.
 	assert.True(t, cfg.hasBucket(admin.NorthstarBucketName), "northstar zone bucket not rendered")
 
+	// The zone bucket must be readable from every node's local predastore, so
+	// a resolver on any node can serve it.
+	for _, b := range cfg.Buckets {
+		if b.Name == admin.NorthstarBucketName {
+			assert.Equal(t, "distributed", b.Type)
+		}
+	}
+
 	// BootstrapBaseZone writes zone files with the system key, not the
 	// resolver's. Without this grant, seeding fails on every node.
 	systemActions, ok := cfg.policyFor("AK", admin.NorthstarBucketName)
@@ -1048,15 +1055,15 @@ func TestSpinifexTomlTemplate_PeerNorthstarStanzaIsPathOnly(t *testing.T) {
 }
 
 // Unset knob must leave production config byte-identical: no [compaction] block
-// and the original trailing bytes unchanged — the [iam] table both templates
-// end on, and the newline after it.
+// and the original trailing bytes unchanged (single-node has no trailing
+// newline, multinode keeps one).
 func TestPredastoreTemplates_UnsetCompactionEmitsNoBlock(t *testing.T) {
 	single := renderSingleNodePredastore(t, admin.ConfigSettings{
 		Region: "ap-southeast-2", BindIP: "0.0.0.0", ConfigDir: "/cfg",
 		AccessKey: "AK", SecretKey: "SK", NatsToken: "tok",
 	})
 	assert.NotContains(t, single, "[compaction]")
-	assert.True(t, strings.HasSuffix(single, "access_keys_bucket = \"spinifex-iam-access-keys\"\n"),
+	assert.True(t, strings.HasSuffix(single, `access_keys_bucket = "spinifex-iam-access-keys"`),
 		"unset single-node tail changed: %q", single[len(single)-40:])
 
 	multi, err := admin.GenerateMultiNodePredastoreConfig(

cmd/spinifex/cmd/admin_upgrade.go

@@ -8,6 +8,10 @@ import (
 
 	"github.com/mulgadc/spinifex/spinifex/migrate"
 
+	// Config migrations that need more than the framework register themselves
+	// here rather than in migrate, which every handler imports.
+	_ "github.com/mulgadc/spinifex/spinifex/migrate/predastoretopology"
+
 	"github.com/spf13/cobra"
 )
 

cmd/spinifex/cmd/admin_upgrade_test.go

@@ -13,21 +13,17 @@ import (
 // TestUpgradeSeesEveryConfigTarget pins the registrations `spx admin upgrade`
 // depends on. A migration whose package is no longer linked here would leave
 // the command reporting nothing pending on an install that needs migrating.
-//
-// spinifex.toml is the only config target: predastore.toml is not migrated at
-// all, so a predastore config on disk must contribute neither a version nor a
-// pending step.
 func TestUpgradeSeesEveryConfigTarget(t *testing.T) {
 	configDir := t.TempDir()
 
-	// A target must exist on disk to be reported at all.
+	// Both targets must exist on disk to be reported at all.
 	require.NoError(t, os.MkdirAll(filepath.Join(configDir, "predastore"), 0750))
 	require.NoError(t, os.WriteFile(filepath.Join(configDir, "spinifex.toml"), []byte("version = \"3\"\n"), 0640))
-	require.NoError(t, os.WriteFile(filepath.Join(configDir, "predastore", "predastore.toml"), []byte("version = 1\n"), 0640))
+	require.NoError(t, os.WriteFile(filepath.Join(configDir, "predastore", "predastore.toml"), []byte("version = \"4\"\n"), 0640))
 
 	versions, err := migrate.DefaultRegistry.ConfigVersions(configDir)
 	require.NoError(t, err)
-	assert.Equal(t, map[string]int{"spinifex.toml": 3}, versions)
+	assert.Equal(t, map[string]int{"spinifex.toml": 3, "predastore.toml": 4}, versions)
 
 	pending, err := migrate.DefaultRegistry.PendingConfig(configDir)
 	require.NoError(t, err)
@@ -36,5 +32,8 @@ func TestUpgradeSeesEveryConfigTarget(t *testing.T) {
 	for _, p := range pending {
 		targets[p.Target] = [2]int{p.FromVersion, p.ToVersion}
 	}
-	assert.Equal(t, map[string][2]int{"spinifex.toml": {3, 4}}, targets)
+	assert.Equal(t, map[string][2]int{
+		"spinifex.toml":   {3, 4},
+		"predastore.toml": {4, 5},
+	}, targets)
 }

cmd/spinifex/cmd/predastore_bind_test.go

@@ -58,11 +58,11 @@ host_id = 3
 	assert.Equal(t, 3, bind.HostID)
 }
 
-// TestDerivePredastoreBind_HostIDAbsentDerivesZero covers a spinifex.toml
-// that omits host_id: the derivation substitutes nothing, so the zero it
-// returns names no [[host]] and the start command rejects it rather than
-// guessing which host of the topology this process runs.
-func TestDerivePredastoreBind_HostIDAbsentDerivesZero(t *testing.T) {
+// TestDerivePredastoreBind_HostIDAbsentRunsWholeTopology covers the
+// single-node rule: those spinifex.toml templates omit host_id entirely,
+// pinning every node of the topology to one host, which only a
+// whole-topology process can serve.
+func TestDerivePredastoreBind_HostIDAbsentRunsWholeTopology(t *testing.T) {
 	resetGlobalViper(t)
 
 	path := writeSpinifexToml(t, `

cmd/spinifex/cmd/predastore_planes_test.go

@@ -1,136 +0,0 @@
-package cmd
-
-import (
-	"os"
-	"path/filepath"
-	"testing"
-
-	pds "github.com/mulgadc/predastore"
-	"github.com/mulgadc/spinifex/spinifex/admin"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// loadRenderedPredastore parses a rendered template with predastore's own
-// loader. Asserting on the TOML text would only prove the template renders;
-// this proves the cluster it describes is one predastore will start.
-func loadRenderedPredastore(t *testing.T, content string) *pds.Config {
-	t.Helper()
-	path := filepath.Join(t.TempDir(), "predastore.toml")
-	require.NoError(t, os.WriteFile(path, []byte(content), 0o600))
-	cfg, err := pds.LoadConfig(path)
-	require.NoError(t, err, "the installer must not write a config predastore rejects")
-	return cfg
-}
-
-// singleNodeSettings is what a real install renders with: the paths predastore
-// validates are absolute, which the shared fixture leaves empty because the
-// tests around it only read the rendered text.
-func singleNodeSettings() admin.ConfigSettings {
-	settings := northstarSettings()
-	settings.ConfigDir = "/etc/spinifex"
-	settings.PredastoreDataDir = admin.PredastoreDataDir("/var/lib/spinifex")
-	return settings
-}
-
-func gateNode(t *testing.T, host pds.HostConfig) pds.NodeConfig {
-	t.Helper()
-	for _, n := range host.Nodes {
-		if n.Role == pds.RoleGate {
-			return n
-		}
-	}
-	t.Fatalf("host %d runs no gate", host.ID)
-	return pds.NodeConfig{}
-}
-
-// A single machine holds every shard of an object whatever the erasure code
-// is, so parity there costs a split, an encode and a second write for a copy
-// the same disk failure takes with it. Redundancy belongs to the pool under
-// the data directory.
-func TestPredastoreSingleNodeTemplateUsesNoParity(t *testing.T) {
-	t.Parallel()
-
-	cfg := loadRenderedPredastore(t, renderTemplate(t, predastoreTomlTemplate, singleNodeSettings()))
-
-	assert.Equal(t, 1, cfg.RS.Data, "a single node splits nothing")
-	assert.Equal(t, 0, cfg.RS.Parity, "a single node has nowhere to put parity")
-}
-
-// A single node is one server: one of each role. A second blob node would only
-// hash-partition the keys across another index on the same drive, and a second
-// meta node would add an fsync to every commit for a quorum sharing the disk it
-// exists to survive.
-func TestPredastoreSingleNodeTemplateRunsOneOfEachRole(t *testing.T) {
-	t.Parallel()
-
-	cfg := loadRenderedPredastore(t, renderTemplate(t, predastoreTomlTemplate, singleNodeSettings()))
-
-	require.Len(t, cfg.Hosts, 1, "a single node is one host")
-
-	byRole := make(map[pds.Role]int)
-	for _, n := range cfg.Hosts[0].Nodes {
-		byRole[n.Role]++
-	}
-	assert.Equal(t, map[pds.Role]int{pds.RoleGate: 1, pds.RoleBlob: 1, pds.RoleMeta: 1}, byRole)
-}
-
-// S3 is a public service and replication is not. The gate binds every
-// interface; raft and blob traffic bind the host's own address and stay there.
-func TestPredastoreSingleNodeTemplateSplitsThePlanes(t *testing.T) {
-	t.Parallel()
-
-	cfg := loadRenderedPredastore(t, renderTemplate(t, predastoreTomlTemplate, singleNodeSettings()))
-
-	require.Len(t, cfg.Hosts, 1)
-	assert.Equal(t, "0.0.0.0", gateNode(t, cfg.Hosts[0]).BindAddr, "S3 must answer on every interface")
-	assert.NotEqual(t, "0.0.0.0", cfg.Hosts[0].BindAddr, "the cluster plane must not be a wildcard")
-}
-
-func TestPredastoreMultiNodeTemplateSplitsThePlanes(t *testing.T) {
-	t.Parallel()
-
-	content, err := admin.GenerateMultiNodePredastoreConfig(
-		predastoreMultiNodeTemplate, predastoreMultinodeNodes(), "AK", "SK", "ap-southeast-2", "tok",
-		"/cfg", "/var/lib/spinifex", "10.0.0.1", 0, admin.NorthstarCredentials{},
-	)
-	require.NoError(t, err)
-
-	cfg := loadRenderedPredastore(t, content)
-
-	require.Len(t, cfg.Hosts, 3)
-	for _, host := range cfg.Hosts {
-		// Every machine's replication plane is the address its peers dial, so
-		// a second interface on any of them carries no cluster traffic.
-		assert.Equal(t, host.Addr, host.BindAddr, "host %d binds the cluster plane off its dial address", host.ID)
-		assert.Equal(t, "0.0.0.0", gateNode(t, host).BindAddr, "host %d must serve public S3", host.ID)
-	}
-}
-
-// The bind_addr under [[host.node]] is the gate's alone: no other role has a
-// listener to point anywhere, and predastore rejects one that tries.
-func TestPredastoreTemplatesBindOnlyTheGate(t *testing.T) {
-	t.Parallel()
-
-	multinode, err := admin.GenerateMultiNodePredastoreConfig(
-		predastoreMultiNodeTemplate, predastoreMultinodeNodes(), "AK", "SK", "ap-southeast-2", "tok",
-		"/cfg", "/var/lib/spinifex", "10.0.0.1", 0, admin.NorthstarCredentials{},
-	)
-	require.NoError(t, err)
-
-	for name, content := range map[string]string{
-		"single-node": renderTemplate(t, predastoreTomlTemplate, singleNodeSettings()),
-		"multi-node":  multinode,
-	} {
-		t.Run(name, func(t *testing.T) {
-			t.Parallel()
-			for _, host := range loadRenderedPredastore(t, content).Hosts {
-				for _, n := range host.Nodes {
-					if n.Role != pds.RoleGate {
-						assert.Empty(t, n.BindAddr, "node %d has role %q and must bind nothing of its own", n.ID, n.Role)
-					}
-				}
-			}
-		})
-	}
-}

cmd/spinifex/cmd/service.go

@@ -104,9 +104,9 @@ type predastoreBind struct {
 // normalization for callers that DIAL predastore, not for the address
 // predastore itself binds to.
 //
-// host_id resolves to 0 when spinifex.toml omits the key. That names no
-// [[host]] of the predastore topology, so the start command rejects it rather
-// than substituting one.
+// host_id defaults to 0 when spinifex.toml omits the key, which runs the whole
+// predastore topology in this one process — the single-node deployment. Only a
+// multi-node config names a host (>= 1), selecting just that host's nodes.
 func derivePredastoreBind(clusterConfig *config.ClusterConfig) (predastoreBind, error) {
 	node := clusterConfig.Node
 	bindKey := "nodes." + node + ".predastore.host"
@@ -138,6 +138,7 @@ var predastoreStartCmd = &cobra.Command{
 		port := viper.GetInt("predastore-port")
 		host := viper.GetString("predastore-host")
 		basePath := viper.GetString("predastore-base-path")
+		debug := viper.GetBool("predastore-debug")
 		hostID := viper.GetInt("predastore-host-id")
 
 		// Derive bind host/port/host-id from spinifex.toml when its path is
@@ -165,10 +166,9 @@ var predastoreStartCmd = &cobra.Command{
 			}
 		}
 
-		// Required, no default: the host id selects the [[host]] of the
-		// predastore topology this process runs, and nothing can guess it.
-		if hostID <= 0 {
-			fmt.Println("Host ID is not set (--host-id, SPINIFEX_PREDASTORE_HOST_ID or host_id in spinifex.toml)")
+		// Required, no default
+		if basePath == "" {
+			fmt.Println("Base path is not set")
 			return
 		}
 
@@ -200,19 +200,26 @@ var predastoreStartCmd = &cobra.Command{
 			return
 		}
 
-		defer initTelemetry("predastore", false)()
+		pprofEnabled := viper.GetBool("pprof")
+		pprofOutput := viper.GetString("pprof-output")
+
+		defer initTelemetry("predastore", debug)()
 
 		service, err := service.New("predastore", &predastore.Config{
 			Port:       port,
 			Host:       host,
 			BasePath:   basePath,
 			ConfigPath: configPath,
+			Debug:      debug,
 			TlsCert:    tlsCert,
 			TlsKey:     tlsKey,
 
 			EncryptionKeyFile: encryptionKeyFile,
 
 			HostID: hostID,
+
+			PprofEnabled:    pprofEnabled,
+			PprofOutputPath: pprofOutput,
 		})
 
 		if err != nil {
@@ -235,11 +242,7 @@ var predastoreStopCmd = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 		fmt.Println("Stopping predastore service...")
 
-		// Only the pid directory matters here: Stop finds the running process
-		// through it, and start wrote the file there.
-		service, err := service.New("predastore", &predastore.Config{
-			BasePath: viper.GetString("predastore-base-path"),
-		})
+		service, err := service.New("predastore", &predastore.Config{})
 
 		if err != nil {
 			fmt.Println("Error stopping predastore service:", err)
@@ -1153,7 +1156,7 @@ var qmpCollectorStatusCmd = &cobra.Command{
 // init()); pflag panics on a redefined flag.
 func registerPredastoreNamespacedFlags() {
 	predastoreCmd.PersistentFlags().String("host", "0.0.0.0", "Predastore (S3) host")
-	predastoreCmd.PersistentFlags().String("base-path", "", "Directory holding the predastore service pid file")
+	predastoreCmd.PersistentFlags().String("base-path", "", "Predastore (S3) base path")
 	predastoreCmd.PersistentFlags().String("config-path", "", "Predastore (S3) config path")
 	bindPredastoreNamespacedEnv()
 }
@@ -1172,13 +1175,16 @@ func bindPredastoreNamespacedEnv() {
 	viper.BindPFlag("predastore-config-path", predastoreCmd.PersistentFlags().Lookup("config-path"))
 }
 
-// bindPredastoreCollisionEnv namespaces predastore's port, tls-cert, tls-key,
-// encryption-key-file and host-id keys, which nats and awsgw also bind bare.
-// Each derived env name now matches its own BindEnv target.
+// bindPredastoreCollisionEnv namespaces predastore's port, debug, tls-cert,
+// tls-key, encryption-key-file and host-id keys, which nats and awsgw also
+// bind bare. Each derived env name now matches its own BindEnv target.
 func bindPredastoreCollisionEnv() {
 	viper.BindEnv("predastore-port", "SPINIFEX_PREDASTORE_PORT")
 	viper.BindPFlag("predastore-port", predastoreCmd.PersistentFlags().Lookup("port"))
 
+	viper.BindEnv("predastore-debug", "SPINIFEX_PREDASTORE_DEBUG")
+	viper.BindPFlag("predastore-debug", predastoreCmd.PersistentFlags().Lookup("debug"))
+
 	viper.BindEnv("predastore-tls-cert", "SPINIFEX_PREDASTORE_TLS_CERT")
 	viper.BindPFlag("predastore-tls-cert", predastoreCmd.PersistentFlags().Lookup("tls-cert"))
 
@@ -1253,10 +1259,13 @@ func init() {
 	// Predastore Port
 	predastoreCmd.PersistentFlags().Int("port", 8443, "Predastore (S3) port")
 
-	// Predastore host, config-path (namespaced viper keys —
+	// Predastore host, base-path, config-path (namespaced viper keys —
 	// see registerPredastoreNamespacedFlags doc comment)
 	registerPredastoreNamespacedFlags()
 
+	// Predastore Debug
+	predastoreCmd.PersistentFlags().Bool("debug", false, "Predastore (S3) debug")
+
 	// Predastore TLS Cert
 	predastoreCmd.PersistentFlags().String("tls-cert", "", "Predastore (S3) TLS certificate")
 
@@ -1267,15 +1276,25 @@ func init() {
 	predastoreCmd.PersistentFlags().String("encryption-key-file", "", "Path to this node's 32-byte AES-256 master key file (required)")
 
 	// Predastore host ID: which [[host]] of the predastore topology this
-	// process runs. Required, and it must name a host the predastore config
-	// declares; the default 0 names none. Set it via spinifex.toml's host_id,
-	// SPINIFEX_PREDASTORE_HOST_ID or --host-id.
-	predastoreCmd.PersistentFlags().Int("host-id", 0, "Predastore cluster host ID, naming a [[host]] in the predastore config (required)")
+	// process is. Default 0 runs every node of the topology in this process,
+	// which is the single-node deployment. Multi-node deployments set the
+	// host's real ID (>= 1) via SPINIFEX_PREDASTORE_HOST_ID or --host-id.
+	predastoreCmd.PersistentFlags().Int("host-id", 0, "Predastore cluster host ID (0 = run the whole topology in this process, >= 1 = this host only)")
 
-	// Namespaced viper keys for port/tls-cert/tls-key/encryption-key-file/host-id
+	// Namespaced viper keys for port/debug/tls-cert/tls-key/encryption-key-file/host-id
 	// (see bindPredastoreCollisionEnv doc comment)
 	bindPredastoreCollisionEnv()
 
+	// Predastore CPU Profiling
+	predastoreCmd.PersistentFlags().Bool("pprof", false, "Enable CPU profiling (also via PPROF_ENABLED=1)")
+	viper.BindEnv("pprof", "PPROF_ENABLED")
+	viper.BindPFlag("pprof", predastoreCmd.PersistentFlags().Lookup("pprof"))
+
+	// Predastore CPU Profile Output Path
+	predastoreCmd.PersistentFlags().String("pprof-output", "/tmp/predastore-cpu.prof", "CPU profile output path")
+	viper.BindEnv("pprof-output", "PPROF_OUTPUT")
+	viper.BindPFlag("pprof-output", predastoreCmd.PersistentFlags().Lookup("pprof-output"))
+
 	predastoreCmd.AddCommand(predastoreStartCmd)
 	predastoreCmd.AddCommand(predastoreStopCmd)
 	predastoreCmd.AddCommand(predastoreStatusCmd)

cmd/spinifex/cmd/service_bind_collision_test.go

@@ -21,18 +21,19 @@ func bindAllServiceCollisionEnv(t *testing.T) {
 	bindAwsgwCollisionEnv()
 }
 
-// TestSubcommandDebugKeysResolveIndependently proves nats and awsgw each
-// resolve their own SPINIFEX_<SERVICE>_DEBUG value instead of sharing one
-// last-registration-wins "debug" viper key. Predastore has no debug flag: it
-// logs at whatever level the predastore library is built to.
+// TestSubcommandDebugKeysResolveIndependently proves predastore, nats and
+// awsgw each resolve their own SPINIFEX_<SERVICE>_DEBUG value instead of
+// sharing one last-registration-wins "debug" viper key.
 func TestSubcommandDebugKeysResolveIndependently(t *testing.T) {
 	t.Cleanup(func() { viper.Reset() })
 	viper.Reset()
 	bindAllServiceCollisionEnv(t)
 
+	t.Setenv("SPINIFEX_PREDASTORE_DEBUG", "true")
 	t.Setenv("SPINIFEX_NATS_DEBUG", "false")
 	t.Setenv("SPINIFEX_AWSGW_DEBUG", "true")
 
+	assert.True(t, viper.GetBool("predastore-debug"))
 	assert.False(t, viper.GetBool("nats-debug"))
 	assert.True(t, viper.GetBool("awsgw-debug"))
 }
@@ -98,6 +99,7 @@ func TestGenericSpinifexEnvDoesNotShadowNamespacedKeys(t *testing.T) {
 	t.Setenv("SPINIFEX_HOST", "255.255.255.255")
 	t.Setenv("SPINIFEX_PORT", "1")
 
+	t.Setenv("SPINIFEX_PREDASTORE_DEBUG", "false")
 	t.Setenv("SPINIFEX_NATS_DEBUG", "false")
 	t.Setenv("SPINIFEX_AWSGW_DEBUG", "false")
 	t.Setenv("SPINIFEX_NATS_HOST", "10.0.0.9")
@@ -105,6 +107,7 @@ func TestGenericSpinifexEnvDoesNotShadowNamespacedKeys(t *testing.T) {
 	t.Setenv("SPINIFEX_PREDASTORE_PORT", "8443")
 	t.Setenv("SPINIFEX_NATS_PORT", "4222")
 
+	assert.False(t, viper.GetBool("predastore-debug"), "generic SPINIFEX_DEBUG must not shadow predastore-debug")
 	assert.False(t, viper.GetBool("nats-debug"), "generic SPINIFEX_DEBUG must not shadow nats-debug")
 	assert.False(t, viper.GetBool("awsgw-debug"), "generic SPINIFEX_DEBUG must not shadow awsgw-debug")
 	assert.Equal(t, "10.0.0.9", viper.GetString("nats-service-host"), "generic SPINIFEX_HOST must not shadow nats-service-host")

cmd/spinifex/cmd/templates/predastore-multinode.toml

@@ -1,10 +1,13 @@
 # Predastore S3-compatible storage configuration file in TOML format.
 # Generated for multi-node cluster
-version = 1
+version = "5"
 region = "{{.Region}}"
 
-# Reed-Solomon erasure code. data + parity must not exceed the number of blob
-# nodes in the cluster, and each machine contributes exactly one.
+# The gateway listen address comes from the daemon CLI options: a top-level
+# host key would collide with the [[host]] topology tables below.
+
+debug = false
+
 [rs]
 {{- if eq (len .Nodes) 2}}
 data = 1
@@ -14,57 +17,46 @@ data = 2
 parity = 1
 {{- end}}
 
-# One host per machine: a predastore process owning a data directory, a TLS
-# identity and the nodes pinned to it. addr is what the other hosts dial, so it
-# carries no port and names one machine.
-#
-# bind_addr is the cluster plane: raft consensus and blob shard traffic listen
-# there and nowhere else. It is the address peers dial rather than a wildcard,
-# so on a multi-homed machine replication never reaches the public interface.
-# The gate below binds the public plane separately.
-#
-# Each machine runs a gate for the S3 API, a blob node for erasure-coded object
-# shards and a meta node for the Raft quorum over global state. Ports are
-# unique within a host, so every machine uses the same three.
-{{- range $host := .Nodes}}
+# One host per machine: a predastore process owning a socket and a data
+# directory. bind_addr binds every interface; public_addr is what the other
+# hosts dial.
+{{- range .Nodes}}
 
 [[host]]
-id = {{$host.ID}}
-bind_addr = "{{$host.Host}}"
-addr = "{{$host.Host}}"
-data_dir = "{{$.PredastoreDataDir}}"
-tls_cert = "{{$.ConfigDir}}/server.pem"
-tls_key = "{{$.ConfigDir}}/server.key"
-encryption_key = "{{$.ConfigDir}}/predastore/encryption.key"
-{{- range $.ClusterNodes}}
-{{- if eq .HostID $host.ID}}
-
-[[host.node]]
 id = {{.ID}}
-role = "{{.Role}}"
-port = {{.Port}}
-{{- if eq .Role "gate"}}
-# S3 is the public plane and answers on every interface; the host's bind_addr
-# above keeps raft and blob traffic off them. Only a gate may set this.
-bind_addr = "0.0.0.0"
-{{- end}}
-{{- end}}
-{{- end}}
+bind_addr = "0.0.0.0:6660"
+public_addr = "{{.Host}}:6660"
+data_dir = "{{$.PredastoreDataDir}}"
 {{- end}}
 
-[[bucket]]
+# Nodes are roles pinned to a host: each machine carries one shard-storage
+# node for erasure-coded object shards and one state replica for the Raft
+# quorum over global state.
+{{- range .ClusterNodes}}
+
+[[node]]
+id = {{.ID}}
+host_id = {{.HostID}}
+role = "{{.Role}}"
+{{- end}}
+
+[[buckets]]
 name = "predastore"
 region = "{{.Region}}"
+type = "distributed"
 public = false
+encryption = ""
 account_id = "000000000000"
 
 {{- if .NorthstarAccessKey}}
 # Northstar DNS zone storage. Read-only for the northstar service; the control
 # plane writes zone files with the system credentials.
-[[bucket]]
+[[buckets]]
 name = "{{.NorthstarBucket}}"
 region = "{{.Region}}"
+type = "distributed"
 public = false
+encryption = ""
 account_id = "000000000000"
 {{- end}}
 
@@ -109,6 +101,7 @@ burst = 200
 [iam]
 nats_url = "nats://{{if eq .BindIP "0.0.0.0"}}localhost{{else}}{{.BindIP}}{{end}}:4222"
 nats_token = "{{.NatsToken}}"
+nats_ca_cert = "{{.ConfigDir}}/ca.pem"
 master_key_path = "{{.ConfigDir}}/master.key"
 access_keys_bucket = "spinifex-iam-access-keys"
 {{- if .CompactionIntervalSeconds}}

cmd/spinifex/cmd/templates/predastore.toml

@@ -1,77 +1,86 @@
 # Predastore S3-compatible storage configuration file in TOML format.
-# Single node: one host running every node in one process.
-version = 1
+# MULTI NODE used for testing
+version = "5"
 region = "{{.Region}}"
 
-# Reed-Solomon erasure code. data + parity must not exceed the number of blob
-# nodes in the cluster.
-#
-# RS(1,0) is one shard on one node: no split, no parity encode and no fan-out.
-# Every shard of a wider code would land on this one machine anyway, so parity
-# buys nothing a second copy of the same disk failure would not take with it.
-# Redundancy here is the pool's job — ZFS raidz or a mirror under the data
-# directory. Multi-node installs get a real erasure code from the multi-node
-# template: RS(2,1) minimum, RS(3,2) recommended.
-[rs]
-data = 1
-parity = 0
+# The gateway listen address comes from the daemon CLI options: a top-level
+# host key would collide with the [[host]] topology table below.
 
-# A host is one predastore process: it owns a data directory, a TLS identity
-# and the nodes pinned to it. Everything is colocated here, so the nodes talk
-# over the in-process pipe and only the gate binds a socket.
-#
-# bind_addr is the cluster plane — raft and blob traffic. A single-host install
-# opens no socket for it at all, but it is set to the loopback address rather
-# than left wide, so adding a second host later cannot quietly publish
-# replication to every interface.
+# Enable verbose debugging
+debug = false
+
+# Optionally configure base-path if full directory path not defined for "fs" buckets
+# base_path = "/home/predastore/data/"
+
+# Specify reed solomon encoding - 
+# RS(2,1), 2 data, 1 parity. Minimum
+# RS(3,2), 3 data, 2 parity. Recommended for production
+[rs]
+data = 2
+parity = 1
+
+# A host is one predastore process: it owns a socket and a data directory,
+# and runs the nodes pinned to it. A single-node deployment declares one host
+# and colocates every node in it, so all intra-cluster traffic goes over the
+# in-process pipe and no socket is bound. bind_addr/public_addr are unused
+# while everything is colocated; they only matter once nodes are split across
+# processes.
 [[host]]
 id = 1
-bind_addr = "127.0.0.1"
-addr = "127.0.0.1"
+bind_addr = "127.0.0.1:6660"
+public_addr = "127.0.0.1:6660"
 data_dir = "{{.PredastoreDataDir}}"
-tls_cert = "{{.ConfigDir}}/server.pem"
-tls_key = "{{.ConfigDir}}/server.key"
-encryption_key = "{{.ConfigDir}}/predastore/encryption.key"
 
-# The gate serves the S3 API. Without one the host answers no S3 request.
-# S3 is the public plane, so it binds every interface while the host's
-# bind_addr above keeps everything else private. Only a gate may set this.
-[[host.node]]
+# Shard nodes: the distributed hash ring places object shards across these.
+# RS(2,1) needs 3.
+[[node]]
 id = 1
-role = "gate"
-port = 8443
-bind_addr = "0.0.0.0"
+host_id = 1
+role = "shard-storage"
 
-# One blob node holds every object. Extra blob nodes would only hash-partition
-# the keys across more segment files and more badger indexes on the same drive,
-# which costs memory and compaction work and buys no redundancy. Colocated
-# nodes bind no socket, but each still needs a port unique within its host.
-[[host.node]]
+[[node]]
 id = 2
-role = "blob"
-port = 6660
+host_id = 1
+role = "shard-storage"
 
-# One meta node is the whole Raft quorum. Replicas here would share the disk
-# they are meant to survive, so each commit would pay their fsyncs for a quorum
-# that fails together.
-[[host.node]]
+[[node]]
 id = 3
-role = "meta"
-port = 7660
+host_id = 1
+role = "shard-storage"
 
-[[bucket]]
+# State replicas: the Raft quorum over global state (buckets, object index).
+[[node]]
+id = 4
+host_id = 1
+role = "state-replica"
+
+[[node]]
+id = 5
+host_id = 1
+role = "state-replica"
+
+[[node]]
+id = 6
+host_id = 1
+role = "state-replica"
+
+[[buckets]]
 name = "predastore"
 region = "{{.Region}}"
+type = "distributed"
 public = false
+encryption = ""
 account_id = "000000000000"
 
 {{- if .NorthstarAccessKey}}
 # Northstar DNS zone storage. Read-only for the northstar service; the control
 # plane writes zone files with the system credentials.
-[[bucket]]
+[[buckets]]
 name = "{{.NorthstarBucket}}"
 region = "{{.Region}}"
+type = "distributed"
 public = false
+encryption = ""
 account_id = "000000000000"
 {{- end}}
 
@@ -116,10 +125,11 @@ burst = 200
 [iam]
 nats_url = "nats://{{if eq .BindIP "0.0.0.0"}}localhost{{else}}{{.BindIP}}{{end}}:4222"
 nats_token = "{{.NatsToken}}"
+nats_ca_cert = "{{.ConfigDir}}/ca.pem"
 master_key_path = "{{.ConfigDir}}/master.key"
 access_keys_bucket = "spinifex-iam-access-keys"
 {{- if .CompactionIntervalSeconds}}
 
 [compaction]
 interval_seconds = {{.CompactionIntervalSeconds}}
-{{- end}}
+{{- end}}

cmd/spinifex/cmd/templates/spinifex.toml

@@ -127,9 +127,10 @@ bridge_mode = "{{.BridgeMode}}"
 
 [nodes.{{.Node}}.predastore]
 # Wildcard rather than the bind IP: S3 is a public service and has to answer on
-# the wan plane. This settles the gate node's own bind address and port, and
-# nothing else: raft and blob traffic bind [[host]].bind_addr from
-# predastore.toml, which is the private plane and stays there.
+# the wan plane, and predastore-start.sh scrapes this line into
+# SPINIFEX_PREDASTORE_HOST, which outranks the host set in predastore.toml. Only
+# the S3 API moves — the [[db]] and [[nodes]] peer addresses in
+# predastore-multinode.toml stay on the bind address.
 #
 # This doubles as the local dial target, and 0.0.0.0 is not one: config loading
 # rewrites it to 127.0.0.1 for this node, and admin.DialTarget does the same for

cmd/spinifex/cmd/viperblock_region_test.go

@@ -36,7 +36,7 @@ func TestPredastoreTemplateBucketsUseConfiguredRegion(t *testing.T) {
 		Buckets []struct {
 			Name   string `toml:"name"`
 			Region string `toml:"region"`
-		} `toml:"bucket"`
+		} `toml:"buckets"`
 	}
 	require.NoError(t, toml.Unmarshal([]byte(content), &parsed))
 

docs/COMMANDS.md

@@ -10,7 +10,7 @@ Service lifecycle commands for starting, stopping, and checking status of all Sp
 
 | Command | Flags | Description |
 |---------|-------|-------------|
-| `spx service predastore start` | `--port` (default: 8443, the host's S3 gate port), `--host` (default: 0.0.0.0, the address the host's sockets bind), `--base-path` (directory holding the service pid file), `--config-path` (required), `--tls-cert` (required), `--tls-key` (required), `--encryption-key-file` (required), `--host-id` (required; names a `[[host]]` in the predastore config) | Creates predastore service instance with S3-compatible storage backend → starts service, serving that host's nodes and the S3 gate among them until the process is signalled. When `--config`/`SPINIFEX_CONFIG_PATH` points at a cluster spinifex.toml, `--host`/`--port`/`--host-id` default to that node's `[nodes.<node>.predastore]` section instead of the flag defaults above; an explicit flag or `SPINIFEX_PREDASTORE_*` env var still overrides it. `host_id` absent from spinifex.toml resolves to 0, which names no host and is rejected. |
+| `spx service predastore start` | `--port` (default: 8443), `--host` (default: 0.0.0.0), `--base-path`, `--config-path`, `--debug`, `--tls-cert`, `--tls-key`, `--encryption-key-file`, `--host-id` (default: 0 = run the whole topology in this process), `--pprof`, `--pprof-output` | Creates predastore service instance with S3-compatible storage backend → starts service. When `--config`/`SPINIFEX_CONFIG_PATH` points at a cluster spinifex.toml, `--host`/`--port`/`--host-id` default to that node's `[nodes.<node>.predastore]` section instead of the flag defaults above; an explicit flag or `SPINIFEX_PREDASTORE_*` env var still overrides it. `host_id` absent from spinifex.toml resolves to 0, running the whole topology in this process. |
 | `spx service predastore stop` | — | Stops the predastore service |
 | `spx service predastore status` | — | Reports predastore service status |
 | `spx service viperblock start` | `--s3-host` (default: 0.0.0.0:8443), `--s3-bucket` (default: predastore), `--s3-region` (default: ap-southeast-2), `--plugin-path` (auto-detected via `nbdkit --dump-config plugindir`; typically `/usr/lib/x86_64-linux-gnu/nbdkit/plugins/nbdkit-viperblock-plugin.so` on amd64, overridable via `SPINIFEX_VIPERBLOCK_PLUGIN_PATH` in `/etc/spinifex/systemd.env`) | Loads cluster config → connects to NATS and Predastore → starts viperblock block storage service with NBD plugin |
@@ -845,16 +845,35 @@ Spinifex both stores externally-issued certificates (`import-certificate`) and i
 
 ---
 
-## RDS (PostgreSQL)
+## RDS (PostgreSQL and MariaDB)
 
-Each DB instance is one dedicated system-owned VM running the engine directly, launched from the `spinifex-rds-postgres` AMI, tagged `spinifex:managed-by=rds` and therefore hidden from the customer's EC2 API. The engine is reached over a customer-account ENI injected into a subnet of the DB subnet group, so **the endpoint is private — reachable from inside the VPC only**, from any subnet of it rather than the endpoint ENI's own. `Endpoint.Address` is `{db-instance-identifier}.{account-id}.{region}.rds.{base-domain}` where northstar is configured, and the endpoint ENI's private IP where it is not; the IP is stable across VM replacement either way. Default port 5432.
+Each DB instance is one dedicated system-owned VM running the engine directly, launched from the `spinifex-rds-postgres` or `spinifex-rds-mariadb` AMI, tagged `spinifex:managed-by=rds` and therefore hidden from the customer's EC2 API. The engine is reached over a customer-account ENI injected into a subnet of the DB subnet group, so **the endpoint is private — reachable from inside the VPC only**, from any subnet of it rather than the endpoint ENI's own. `Endpoint.Address` is `{db-instance-identifier}.{account-id}.{region}.rds.{base-domain}` where northstar is configured, and the endpoint ENI's private IP where it is not; the IP is stable across VM replacement either way.
 
-- **Engine:** `postgres` 18 only — `EngineVersion` accepts `18` or `18.x`, and there is no in-place upgrade.
+Everything below applies to both engines unless it names one. `Engine` is fixed at create — there is no in-place engine change, no cross-engine snapshot restore and no migration between the two.
+
+| | `postgres` | `mariadb` |
+|---|---|---|
+| Version | 18 | 11.8 |
+| `EngineVersion` accepted | `18`, or omitted | `11.8`, or omitted |
+| Default port | 5432 | 3306 |
+| Parameter-group family | `postgres18` | `mariadb11.8` |
+| Implicit default group | `default.postgres18` | `default.mariadb11.8` |
+| `DBName` maximum | 63 characters | 64 characters |
+| `MasterUsername` maximum | 63 characters | 80 characters |
+| Reserved usernames | `postgres`, `rdsadmin`, `rds_superuser`, and the `pg_` prefix | `root`, `mysql`, `mariadb.sys`, `rdsadmin`, `PUBLIC`, and the `mysql.` and `mariadb.` prefixes |
+| Catalog size | 51 parameters | 54 parameters |
+
+**`mysql` is not an accepted engine and is not an alias for `mariadb`.** MariaDB is offered under its own AWS engine name, exactly as AWS RDS offers it. Naming `mysql` fails with `InvalidParameterValue`; a client — including Terraform's `aws_db_instance` — must set `engine = "mariadb"`. Aliasing would report an engine and a version the instance is not running, and the discrepancy would propagate into `DescribeDBInstances`, parameter-group families and snapshot metadata.
+
+- **Engine version:** pinned per engine, and an `EngineVersion` naming anything but the pin is rejected. A minor version such as `18.4` or `11.8.8` is rejected as well: the AMI does not promise any particular minor, so accepting one would be a promise the platform cannot keep. There is no in-place upgrade.
 - **Instance classes:** `db.t3.{micro,small,medium,large}` and `db.m5.{large,xlarge}` — a naming facade over the platform's EC2 sizing table. Any other class is rejected at create.
 - **Storage:** gp3 only, 20–65536 GiB, always encrypted with the cluster key. Grow-only, and a grow is **stop/start with downtime** — the volume cannot be resized while attached.
-- **TLS:** offered, not enforced. The engine serves a per-instance certificate signed by the cluster CA (the `ca.pem` baked into AMIs) carrying both the ENI IP and the DNS name in its SAN set, so `sslmode=verify-full` works by name or by address. A deployment with no cluster CA starts the engine without TLS.
-- **Master user:** administrative but **not a PostgreSQL superuser**, as on AWS. It gets `CREATEDB` and `CREATEROLE`, owns the initial database, and holds `rds_superuser` (`pg_monitor`, `pg_signal_backend`, `pg_checkpoint`) with `ADMIN OPTION`, so it can grant that set onward. It cannot `COPY ... FROM/TO PROGRAM`, touch server-side files, run `ALTER SYSTEM`, or install **untrusted** extensions — those need the cluster superuser, which no customer credential is. Trusted extensions install normally in a database it owns. `postgres`, `rdsadmin` and `rds_superuser` are rejected as `--master-username`.
-- **Backups:** daily COW snapshots of the data volume inside `PreferredBackupWindow`. Retention defaults to 7 days and caps at 7; `0` disables automated backups. No point-in-time recovery.
+- **`DBName`:** optional; an omitted name creates no initial database. When supplied it must begin with a letter and hold only letters, digits and underscores, within the per-engine limit above. The rule is narrower than either engine's own, because the guest interpolates the name into a `CREATE DATABASE`.
+- **TLS:** offered, not enforced, on both engines. The engine serves a per-instance certificate signed by the cluster CA (the `ca.pem` baked into AMIs) carrying both the ENI IP and the DNS name in its SAN set, so PostgreSQL's `sslmode=verify-full` verifies by name or by address, and MariaDB's `--ssl-ca` plus `--ssl-verify-server-cert` verifies against the endpoint name. A deployment with no cluster CA starts the engine without TLS.
+- **PostgreSQL master user:** administrative but **not a PostgreSQL superuser**, as on AWS. It gets `CREATEDB` and `CREATEROLE`, owns the initial database, and holds `rds_superuser` (`pg_monitor`, `pg_signal_backend`, `pg_checkpoint`) with `ADMIN OPTION`, so it can grant that set onward. It cannot `COPY ... FROM/TO PROGRAM`, touch server-side files, run `ALTER SYSTEM`, or install **untrusted** extensions — those need the cluster superuser, which no customer credential is. Trusted extensions install normally in a database it owns.
+- **MariaDB master user:** created as `'<name>'@'%'` with global `RELOAD`, `PROCESS`, `SHOW DATABASES`, `REPLICATION SLAVE` and `REPLICATION CLIENT`, while application DML, DDL, routines, events, triggers and delegation are granted only on customer databases. MariaDB has no deny or partial-revoke mechanism, so granting those object privileges `ON *.*` would also grant them on `mysql.global_priv` and let the master acquire `SUPER`, `FILE` or `SHUTDOWN`. The initial `DBName` is granted automatically. Create another customer database with `CALL _spinifex_rds.create_database('name')`; plain `CREATE DATABASE`, arbitrary account administration and access to `mysql.*` are intentionally unavailable on the stock MariaDB build.
+- **MariaDB storage engine:** `default_storage_engine` is platform-owned and pinned to **InnoDB**, and is absent from the parameter catalog — a deliberate divergence from AWS, which exposes it as modifiable. Only InnoDB keeps a redo log, so only InnoDB is covered by the snapshot guarantee. An explicit `ENGINE=Aria` or `ENGINE=MyISAM` on `CREATE TABLE` still works — the limitation is documented rather than prevented — but such a table has no redo log, so a snapshot taken without a successful quiesce can leave it inconsistent with no way back. Use InnoDB, as AWS also recommends.
+- **Backups:** daily COW snapshots of the data volume inside `PreferredBackupWindow`. Retention defaults to 7 days and caps at 7; `0` disables automated backups. No point-in-time recovery, and MariaDB's binary log is off.
 - **Availability:** single-AZ. Engine crashes restart in-guest and VM crashes on a live host restart via `ec2-health-restart`, but an instance whose host is lost is reported `failed` with a reason in `StatusInfos` and needs operator recovery.
 
 Statuses: `creating`, `available`, `modifying`, `backing-up`, `rebooting`, `stopping`, `stopped`, `starting`, `deleting`, `failed`. (`recovering` is defined in the state machine but unreachable until auto-recovery lands.)
@@ -863,7 +882,7 @@ Statuses: `creating`, `available`, `modifying`, `backing-up`, `rebooting`, `stop
 
 | Command | Implemented Flags | Missing Flags | Status |
 |---------|-------------------|---------------|--------|
-| `create-db-instance` | `--db-instance-identifier`, `--engine` (postgres), `--engine-version` (18), `--db-instance-class`, `--allocated-storage` (20–65536 GiB), `--storage-type` (gp3), `--master-username`, `--master-user-password`, `--db-name`, `--port` (1150–65535), `--db-subnet-group-name` (unnamed → the account's default VPC subnet), `--vpc-security-group-ids`, `--db-parameter-group-name`, `--backup-retention-period` (0–7, default 7), `--preferred-backup-window`, `--preferred-maintenance-window` (unnamed → assigned from the identifier), `--deletion-protection`, `--tags`, `--storage-encrypted` (true only) | `--auto-minor-version-upgrade`, `--copy-tags-to-snapshot`, Performance Insights / Enhanced Monitoring flags (all accepted, no-op); see "Rejected Parameters" for those that fail loudly | **DONE** |
+| `create-db-instance` | `--db-instance-identifier`, `--engine` (postgres, mariadb), `--engine-version` (the engine's pin, or omitted), `--db-instance-class`, `--allocated-storage` (20–65536 GiB), `--storage-type` (gp3), `--master-username`, `--master-user-password`, `--db-name` (letters, digits and underscores, beginning with a letter, max 63 on postgres and 64 on mariadb), `--port` (1150–65535), `--db-subnet-group-name` (unnamed → the account's default VPC subnet), `--vpc-security-group-ids`, `--db-parameter-group-name`, `--backup-retention-period` (0–7, default 7), `--preferred-backup-window`, `--preferred-maintenance-window` (unnamed → assigned from the identifier), `--deletion-protection`, `--tags`, `--storage-encrypted` (true only) | `--auto-minor-version-upgrade`, `--copy-tags-to-snapshot`, Performance Insights / Enhanced Monitoring flags (all accepted, no-op); see "Rejected Parameters" for those that fail loudly | **DONE** |
 | `describe-db-instances` | `--db-instance-identifier` | `--filters`, `--max-records`, `--marker` (parsed, not applied) | **DONE** |
 | `modify-db-instance` | `--db-instance-identifier`, `--master-user-password`, `--allocated-storage` (grow only, stop/start), `--db-instance-class` (VM replace), `--db-parameter-group-name`, `--vpc-security-group-ids`, `--deletion-protection`, `--backup-retention-period`, `--preferred-backup-window`, `--preferred-maintenance-window`, `--apply-immediately` | `--new-db-instance-identifier`, `--engine-version`, `--db-port-number`, `--db-subnet-group-name`, `--max-allocated-storage`, `--ca-certificate-identifier` (all rejected, not ignored) | **DONE** |
 | `delete-db-instance` | `--db-instance-identifier`, `--skip-final-snapshot`, `--final-db-snapshot-identifier` (exactly one is required) | `--delete-automated-backups` (no-op — automated snapshots are always purged with the instance) | **DONE** — `DeletionProtection` blocks the call; a final snapshot **retains** the data volume until that snapshot is deleted |
@@ -875,7 +894,7 @@ Statuses: `creating`, `available`, `modifying`, `backing-up`, `rebooting`, `stop
 
 | Command | Implemented Flags | Missing Flags | Status |
 |---------|-------------------|---------------|--------|
-| `create-db-snapshot` | `--db-snapshot-identifier`, `--db-instance-identifier`, `--tags` | — | **DONE** — the agent quiesces the engine first; a failed quiesce falls back to a crash-consistent snapshot and writes a `DescribeEvents` warning |
+| `create-db-snapshot` | `--db-snapshot-identifier`, `--db-instance-identifier`, `--tags` | — | **DONE** — the agent quiesces the engine first (`pg_backup_start` on postgres, `BACKUP STAGE` on mariadb); a failed quiesce falls back to a crash-consistent snapshot and writes a `DescribeEvents` warning naming what that engine does and does not recover |
 | `describe-db-snapshots` | `--db-snapshot-identifier`, `--db-instance-identifier`, `--snapshot-type` (manual, automated) | `--filters`, `--dbi-resource-id`, `--include-shared`, `--include-public` (rejected), `--max-records`, `--marker` | **DONE** |
 | `delete-db-snapshot` | `--db-snapshot-identifier` | — | **DONE** — refused while a restored volume still references the snapshot; the last snapshot released reclaims a retained data volume |
 | `restore-db-instance-from-db-snapshot` | `--db-instance-identifier`, `--db-snapshot-identifier`, `--db-instance-class`, `--allocated-storage` (≥ the snapshot's), `--storage-type` (gp3), `--port`, `--db-subnet-group-name`, `--vpc-security-group-ids`, `--db-parameter-group-name`, `--deletion-protection`, `--tags`, `--engine` (must match the snapshot) | `--db-name` (rejected when it differs from the snapshot's), plus the create rejections | **DONE** — unnamed fields are inherited from the snapshot; the master password comes from the restored datadir |
@@ -903,15 +922,17 @@ Statuses: `creating`, `available`, `modifying`, `backing-up`, `rebooting`, `stop
 
 | Command | Implemented Flags | Missing Flags | Status |
 |---------|-------------------|---------------|--------|
-| `create-db-parameter-group` | `--db-parameter-group-name`, `--db-parameter-group-family` (postgres18), `--description`, `--tags` | — | **DONE** — names beginning with `default.` are reserved; the implicit group is `default.postgres18` |
-| `describe-db-parameter-groups` | `--db-parameter-group-name` | `--filters`, `--max-records`, `--marker` | **DONE** — the implicit default group is always listed |
+| `create-db-parameter-group` | `--db-parameter-group-name`, `--db-parameter-group-family` (`postgres18` or `mariadb11.8`), `--description`, `--tags` | — | **DONE** — names beginning with `default.` are reserved. **`--db-parameter-group-family` defaults to `postgres18` when omitted**, so a MariaDB group must name its family explicitly or the mismatch surfaces one call later, at `create-db-instance` |
+| `describe-db-parameter-groups` | `--db-parameter-group-name` | `--filters`, `--max-records`, `--marker` | **DONE** — every engine's implicit default group is always listed, so both `default.postgres18` and `default.mariadb11.8` appear in every account |
 | `modify-db-parameter-group` | `--db-parameter-group-name`, `--parameters` (ParameterName, ParameterValue, ApplyMethod) | — | **DONE** — the whole batch is validated before anything is written and propagated to attached instances; `immediate` on a static parameter is rejected, as AWS does |
-| `describe-db-parameters` | `--db-parameter-group-name`, `--source` (user, engine-default) | `--filters`, `--max-records`, `--marker` | **DONE** — 51-parameter PostgreSQL 18 catalog; memory defaults are computed per instance class and reported as literals |
+| `describe-db-parameters` | `--db-parameter-group-name`, `--source` (user, engine-default) | `--filters`, `--max-records`, `--marker` | **DONE** — the catalog of the group's own family: 51 parameters on `postgres18`, 54 on `mariadb11.8`. Memory defaults are computed per instance class and reported as literals |
 | `delete-db-parameter-group` | `--db-parameter-group-name` | — | **DONE** — refused for a default group and while any instance references it |
 | `reset-db-parameter-group` | — | all | **NOT STARTED** (`InvalidAction`) |
 
 A group takes effect on an instance when `modify-db-instance --db-parameter-group-name` attaches it — immediately with `--apply-immediately`, otherwise at the next maintenance window. Later group edits propagate to every attached instance. Dynamic parameters are written into the engine's config and reloaded live; static ones are recorded `pending-reboot` and applied by `reboot-db-instance`.
 
+**A group's family must match the instance's engine.** A `postgres18` group cannot be attached to a MariaDB instance and a `mariadb11.8` group cannot be attached to a PostgreSQL one, on create, on modify, on a deferred apply or on a snapshot restore; the mismatch is refused with `InvalidParameterCombination`. The catalogs are per-family too, so `modify-db-parameter-group` will not store one engine's parameter name into the other's group. Each catalog is a curated, validated subset rather than the engine's full variable set, and the platform-owned settings — the port, the datadir, the socket, the bind address, the `ssl_*` family, and on MariaDB `secure_file_priv`, `log_bin`, `default_storage_engine` and the InnoDB buffer-pool chunk settings — are absent from it rather than present-and-unmodifiable, because they are not the customer's to set.
+
 ### RDS — Tags
 
 | Command | Implemented Flags | Missing Flags | Status |
@@ -942,7 +963,8 @@ Policy: a parameter whose omission would create a false safety, security or avai
 | `DBSecurityGroups` | EC2-Classic security groups — use `VpcSecurityGroupIds` |
 | `DBClusterIdentifier`, `DBClusterSnapshotIdentifier` | Clustered engines are not offered |
 | `EnableCloudwatchLogsExports` | Log export is not implemented |
-| `EngineVersion` other than 18, `Engine` on modify | No in-place engine or version change |
+| `EngineVersion` other than the engine's pin, `Engine` on modify | No in-place engine or version change |
+| `Engine=mysql` (and Aurora engines) | Oracle MySQL is not offered; `mariadb` is a distinct engine, not an alias for it |
 | `NewDBInstanceIdentifier` | The identifier is the DNS label and the KV key |
 | `DBPortNumber`, `DBSubnetGroupName` on modify | Both would move the endpoint |
 | `MaxAllocatedStorage` | Storage autoscaling is not implemented |
@@ -961,14 +983,14 @@ Recognised actions below return `OperationNotSupported`, so a client sees "not o
 
 | Feature | Actions | Priority | Status |
 |---------|---------|----------|--------|
-| MySQL engine | — (second AMI preset + `Engine` implementation) | High | **NOT STARTED** |
-| Point-in-time recovery (WAL archiving to predastore) | `restore-db-instance-to-point-in-time` | High | **NOT STARTED** |
+| Oracle MySQL engine | — | Low | **NOT PLANNED** — MariaDB is offered instead, under its own `mariadb` engine name; see the note above on why it is not aliased onto `mysql` |
+| Point-in-time recovery (WAL archiving to predastore on postgres, binlog on mariadb) | `restore-db-instance-to-point-in-time` | High | **NOT STARTED** |
 | Auto-recovery from node loss (failure is detected, not repaired) | — | High | **NOT STARTED** |
 | Read replicas | `create-db-instance-read-replica`, `promote-read-replica` | Medium | **NOT STARTED** |
 | Aurora / DB clusters | `create-db-cluster`, `modify-db-cluster`, `delete-db-cluster`, `describe-db-clusters`, `failover-db-cluster` | Low | **NOT STARTED** |
 | Option groups | `create-option-group`, `modify-option-group`, `delete-option-group`, `describe-option-groups` | Low | **NOT STARTED** |
 | Engine-version / orderable-option data sources | `describe-db-engine-versions`, `describe-orderable-db-instance-options` | Low | **NOT STARTED** (`InvalidAction`) — not required by `aws_db_instance`; pin the class and version literally in Terraform |
-| Multi-AZ standby, online (no-downtime) storage grow, storage autoscaling, IAM database auth, Performance Insights, Enhanced Monitoring, log exports, per-tenant private DNS zones, enforced TLS | — | — | **NOT STARTED** |
+| Multi-AZ standby, online (no-downtime) storage grow, storage autoscaling, IAM database auth, Performance Insights, Enhanced Monitoring, log exports, per-tenant private DNS zones, enforced TLS, cross-engine snapshot restore, in-place migration between the two engines | — | — | **NOT STARTED** |
 
 IAM: `AmazonRDSFullAccess` and `AmazonRDSReadOnlyAccess` are available as managed policies. They grant `rds:` verb prefixes rather than `rds:*`, because `rds:*` would also appear to grant the internal agent actions the gateway reserves for a DB VM's own role.
 

docs/DESIGN.md

@@ -238,22 +238,6 @@ Object storage used for:
 - SSH public key storage (`/bucket/ec2/{key-name}.pub`)
 - Volume metadata and configuration
 
-Spinifex runs Predastore in-process: `spx service predastore start` loads `/etc/spinifex/predastore/predastore.toml`, selects the `[[host]]` naming this machine, and serves it until the process is signalled. The topology is written by `spx admin init` / `join` from the servers that form the cluster.
-
-One `[[host]]` is one process. The nodes pinned to it under `[[host.node]]` carry the roles:
-
-| Role | Port | Responsibility |
-|------|------|----------------|
-| `gate` | TCP 8443 | The S3 API. Holds no cluster state; dials the meta and blob nodes to serve a request. |
-| `blob` | UDP 6660 | Erasure-coded object shards, encrypted at rest under this host's master key. |
-| `meta` | UDP 7660 | Raft replica of global state — buckets and the object index. |
-
-Ports are unique within a host but repeat across the cluster, so a multi-node cluster puts one node of each role on every machine and every machine uses the same three ports. Nodes sharing a host talk over an in-process pipe and bind no socket; nodes on other hosts are reached over QUIC, authenticated by the cluster CA.
-
-A single-node install is the same arrangement with one host: seven nodes — a gate, three blob and three meta — colocated in one process, needing seven distinct ports (8443, 6660–6662, 7660–7662) to stay unique within the host even though only the gate opens a listener. It has no code path of its own.
-
-The daemon reports the topology on `spinifex.storage.config` by parsing the config file. It does not connect to Predastore to do so, and no Predastore node exposes a status endpoint.
-
 ## Configuration
 
 Cluster configuration (`spinifex/config/config.go`):

docs/admin/update/README.md

@@ -32,9 +32,6 @@ Updating Spinifex is the same command used to install it. The installer detects
 
 For operators who want to review migrations before they are applied, a manual upgrade path is also supported.
 
-> [!WARNING]
-> **Object storage is exempt when upgrading from v1.15.0 or earlier to v1.16.0.** Predastore's configuration schema and on-disk layout changed with the object storage cutover in v1.16.0, and no migration converts an installation from before it. `spx admin upgrade` will not report anything pending for `predastore.toml`, and updating the binary over such an installation leaves Predastore unable to start. Those clusters have to be re-initialised from scratch, which discards their stored objects — export anything you need first. Upgrades between v1.16.0 and later releases are unaffected.
-
 ## Instructions
 
 ## Step 1. Re-run the Installer

docs/aws-model-operation-coverage.md

@@ -12,7 +12,7 @@ Implemented means a modelled operation is registered to a real handler. Stub and
 | ecs | 2014-11-13 | 56 | 39 | 31 | 5 | 0 | 20 | 3 |
 | elasticloadbalancingv2 | 2015-12-01 | 46 | 37 | 33 | 0 | 0 | 13 | 4 |
 | iam | 2010-05-08 | 159 | 75 | 75 | 0 | 0 | 84 | 0 |
-| rds | 2014-10-31 | 162 | 41 | 24 | 0 | 12 | 126 | 5 |
+| rds | 2014-10-31 | 162 | 40 | 24 | 0 | 12 | 126 | 4 |
 | s3 | 2006-03-01 | 99 | — | — | — | — | — | — |
 | sts | 2011-06-15 | 8 | 4 | 4 | 0 | 0 | 4 | 0 |
 
@@ -527,9 +527,9 @@ None.
 
 </details>
 
-<details><summary>Registered outside the pinned model (5)</summary>
+<details><summary>Registered outside the pinned model (4)</summary>
 
-`AcknowledgeDBBootstrap`, `GetDBBootstrapConfig`, `PollDBCommands`, `RegisterDBInstance`, `SubmitDBStateChange`
+`GetDBBootstrapConfig`, `PollDBCommands`, `RegisterDBInstance`, `SubmitDBStateChange`
 
 
 </details>

docs/identity/iam-groups/README.md

@@ -1,6 +1,6 @@
 ---
-title: "AWS IAM Groups for Shared User Permissions"
-description: "Organise AWS IAM users into groups, manage membership, and attach managed or inline policies once for the whole team instead of granting them user by user."
+title: "IAM Groups"
+description: "Organise IAM users into groups and manage permissions once for the whole team."
 category: "Identity"
 tags:
   - iam
@@ -15,9 +15,9 @@ resources:
     url: "https://docs.aws.amazon.com/IAM/latest/UserGuide/id_groups.html"
 ---
 
-# AWS IAM Groups for Shared User Permissions
+# IAM Groups
 
-> Organise AWS IAM users into groups, manage membership, and attach managed or inline policies once for the whole team instead of granting them user by user.
+> Organise IAM users into groups and manage permissions once for the whole team.
 
 ## Table of Contents
 
@@ -33,11 +33,11 @@ resources:
 
 ## Overview
 
-A group is a collection of [IAM users](/docs/iam-users-and-policies). Policies attached to a group apply to every member, so instead of attaching the same policy to each developer individually, you attach it once to a `developers` group and manage membership.
+A group is a collection of IAM users. Policies attached to a group apply to every member, so instead of attaching the same policy to each developer individually, you attach it once to a `developers` group and manage membership.
 
 Members inherit both the group's **attached managed policies** and its **inline policies**, combined with any policies on the user itself. Policy evaluation is the same as everywhere else: an explicit `Deny` in any applicable policy wins, and anything not allowed is denied.
 
-Groups cannot be nested, and a group is not a principal — it cannot sign requests or own access keys. The identity you assume to get credentials is an [IAM role](/docs/iam-roles-and-instance-profiles), not a group. Like all IAM resources, groups are scoped to the account that creates them.
+Groups cannot be nested, and a group is not a principal — it cannot sign requests or own access keys. Like all IAM resources, groups are scoped to the account that creates them.
 
 ## Prerequisites
 

docs/identity/iam-roles-and-instance-profiles/README.md

@@ -1,6 +1,6 @@
 ---
-title: "AWS IAM Roles and Instance Profiles for EC2"
-description: "Create AWS IAM roles with trust policies, wrap them in instance profiles, and launch EC2 instances that receive credentials with no static access keys at all."
+title: "IAM Roles and Instance Profiles"
+description: "Create IAM roles with trust policies, wrap them in instance profiles, and grant EC2 instances credentials without static keys."
 category: "Identity"
 tags:
   - iam
@@ -18,9 +18,9 @@ resources:
     url: "https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2_instance-profiles.html"
 ---
 
-# AWS IAM Roles and Instance Profiles for EC2
+# IAM Roles and Instance Profiles
 
-> Create AWS IAM roles with trust policies, wrap them in instance profiles, and launch EC2 instances that receive credentials with no static access keys at all.
+> Create IAM roles with trust policies, wrap them in instance profiles, and grant EC2 instances credentials without static keys.
 
 ## Table of Contents
 
@@ -39,7 +39,7 @@ resources:
 
 ## Overview
 
-A role is an IAM identity with permissions but no long-lived credentials. Unlike an [IAM group](/docs/iam-groups), which only bundles users together, a role is a principal in its own right: instead of an access key pair it has a **trust policy** declaring who may assume it, and whoever assumes the role receives short-lived, auto-rotating STS credentials.
+A role is an IAM identity with permissions but no long-lived credentials. Instead of an access key pair, a role has a **trust policy** declaring who may assume it; whoever assumes the role receives short-lived, auto-rotating STS credentials.
 
 An **instance profile** is the container that binds a role to an EC2 instance. An instance launched with a profile gets the role's credentials delivered through [IMDS](/docs/imds) — no static keys baked into the image, no `~/.aws/credentials` inside the guest.
 

docs/identity/iam-users-and-policies/README.md

@@ -1,6 +1,6 @@
 ---
-title: "AWS IAM Users, Policies, and Access Keys"
-description: "Create AWS IAM users, issue and rotate access keys, and write and attach JSON policies that control exactly what each user can do in a Spinifex account."
+title: "IAM Users and Policies"
+description: "Create IAM users, manage access keys, and control permissions with policies."
 category: "Identity"
 tags:
   - iam
@@ -17,9 +17,9 @@ resources:
     url: "https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements.html"
 ---
 
-# AWS IAM Users, Policies, and Access Keys
+# IAM Users and Policies
 
-> Create AWS IAM users, issue and rotate access keys, and write and attach JSON policies that control exactly what each user can do in a Spinifex account.
+> Create IAM users, manage access keys, and control permissions with policies.
 
 ## Table of Contents
 
@@ -119,7 +119,7 @@ aws iam delete-user --user-name alice
 
 ## Access Keys
 
-Access keys are how users authenticate with the AWS CLI. Each user can have up to **2 access keys** at a time, allowing key rotation without downtime. For credentials that expire on their own instead of a long-lived key pair, see [STS and Temporary Credentials](/docs/sts). Workloads running inside an EC2 instance should not carry a user's access key at all — they pick up role credentials automatically through [IMDS](/docs/imds).
+Access keys are how users authenticate with the AWS CLI. Each user can have up to **2 access keys** at a time, allowing key rotation without downtime.
 
 ### Create an Access Key
 

docs/identity/imds/README.md

@@ -1,6 +1,6 @@
 ---
-title: "AWS IMDS: Instance Metadata Service (IMDSv2)"
-description: "Query AWS instance metadata, read user data, and fetch short-lived IAM role credentials from inside a guest VM using IMDSv2 session tokens on Spinifex."
+title: "IMDS (Instance Metadata Service)"
+description: "Access instance metadata, user data, and IAM role credentials from inside a guest VM with IMDSv2."
 category: "Identity"
 tags:
   - imds
@@ -18,9 +18,9 @@ resources:
     url: "https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html"
 ---
 
-# AWS IMDS: Instance Metadata Service (IMDSv2)
+# Instance Metadata Service (IMDS)
 
-> Query AWS instance metadata, read user data, and fetch short-lived IAM role credentials from inside a guest VM using IMDSv2 session tokens on Spinifex.
+> Access instance metadata, user data, and IAM role credentials from inside a guest VM with IMDSv2.
 
 ## Table of Contents
 
@@ -153,7 +153,7 @@ Returns the decoded user data, or 404 if the instance was launched without any.
 
 ## IAM Role Credentials
 
-Instances launched with an IAM instance profile get short-lived, auto-rotating credentials through IMDS — no static keys baked into the image. Creating roles and instance profiles is covered in [IAM Roles and Instance Profiles](/docs/iam-roles-and-instance-profiles). These are the same temporary credentials [STS](/docs/sts) issues, delivered to the guest automatically instead of through an explicit `assume-role` call.
+Instances launched with an IAM instance profile get short-lived, auto-rotating credentials through IMDS — no static keys baked into the image. Creating roles and instance profiles is covered in [IAM Roles and Instance Profiles](/docs/iam-roles-and-instance-profiles).
 
 ```bash
 # Discover the role name

docs/identity/sts/README.md

@@ -1,6 +1,6 @@
 ---
-title: "AWS STS Temporary Credentials and AssumeRole"
-description: "Use AWS STS to issue temporary credentials: assume an IAM role, request a session token, or federate Kubernetes workloads with OIDC web identity on Spinifex."
+title: "STS and Temporary Credentials"
+description: "Mint short-lived credentials with STS — assume roles, issue session tokens, and federate Kubernetes workloads with OIDC."
 category: "Identity"
 tags:
   - sts
@@ -18,9 +18,9 @@ resources:
     url: "https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html"
 ---
 
-# AWS STS Temporary Credentials and AssumeRole
+# STS and Temporary Credentials
 
-> Use AWS STS to issue temporary credentials: assume an IAM role, request a session token, or federate Kubernetes workloads with OIDC web identity on Spinifex.
+> Mint short-lived credentials with STS — assume roles, issue session tokens, and federate Kubernetes workloads with OIDC.
 
 ## Table of Contents
 
@@ -43,7 +43,7 @@ STS (Security Token Service) issues **temporary credentials**: an `ASIA`-prefixe
 Spinifex supports the three main ways to obtain them:
 
 - **`assume-role`** — exchange your IAM user credentials for a [role's](/docs/iam-roles-and-instance-profiles) permissions, gated by the role's trust policy.
-- **`get-session-token`** — get a time-boxed copy of your own [IAM user's](/docs/iam-users-and-policies) permissions.
+- **`get-session-token`** — get a time-boxed copy of your own user's permissions.
 - **`assume-role-with-web-identity`** — exchange an OIDC ID token (a Kubernetes ServiceAccount token) for role credentials, with no IAM credentials at all.
 
 EC2 instances get role credentials a fourth way, automatically through [IMDS](/docs/imds) — no STS call needed in the guest.

docs/install/install-multi-node/README.md

@@ -26,7 +26,6 @@ resources:
 ## Table of Contents
 
 - [Overview](#overview)
-- [Object Storage Layout](#object-storage-layout)
 - [Prerequisites](#prerequisites)
 - [Instructions](#instructions)
 - [Converting ISO-Installed Nodes](#converting-iso-installed-nodes)
@@ -54,28 +53,6 @@ Servers beyond the third join as compute nodes and do not run a database, which
 - UDP port 6081 open between hosts (Geneve tunnels)
 - TCP ports 4222, 4248, 6641, 6642 open between hosts (NATS, OVN)
 - TCP ports 6643, 6644 open between database servers (OVN database clustering)
-- TCP port 8443 and UDP ports 6660, 7660 open between hosts (Predastore S3, object shards, metadata consensus)
-
-Predastore uses the same three ports on every server, so the surface does not widen as the cluster grows. See [Object Storage Layout](#object-storage-layout) below.
-
-## Object Storage Layout
-
-Predastore is configured for the whole cluster in `/etc/spinifex/predastore/predastore.toml`. Each server is one `[[host]]` — a single Predastore process owning that machine's data directory and TLS identity — carrying three nodes under `[[host.node]]`:
-
-| Role | Port | Purpose |
-|------|------|---------|
-| `gate` | TCP 8443 | Serves the S3 API. Every server runs one, so any of them answers an S3 request. |
-| `blob` | UDP 6660 | Holds erasure-coded object shards. One per machine. |
-| `meta` | UDP 7660 | Member of the Raft quorum over global state — buckets and the object index. |
-
-Ports have to be unique within a host but not across the cluster, so every machine uses the same three. Blob and meta traffic between hosts runs over QUIC, authenticated by the cluster CA; nodes on the same machine talk over an in-process pipe and open no socket, which is why a single-server install listens on 8443 alone.
-
-Reed-Solomon parameters are chosen from the cluster size, since each machine contributes exactly one blob node: two servers get `RS(1,1)`, three or more get `RS(2,1)`. `RS(2,1)` survives the loss of any one server's shards.
-
-You do not configure any of this by hand. `spx admin init` and `spx admin join` build the topology from the servers that actually form the cluster in Step 4, and each machine gets the same file with its own host ID recorded in `spinifex.toml`.
-
-> [!WARNING]
-> **There is no upgrade path onto this layout.** A cluster installed before the object storage cutover cannot be migrated — the on-disk directory structure and the configuration schema both changed, and no migration rewrites either. Such a cluster has to be re-initialised from scratch, which discards its stored objects. Export anything you need first.
 
 ## Prerequisites
 

docs/reference-architectures/RTX-Pro-6000-EKS/README.md

@@ -137,19 +137,19 @@ lspci -d 10de: -nn
 
 **4. Attach Predastore storage**
 
-The X14 has four NVMe drives: two 1.5 TB SSDs (one carries the OS) and two ~880 GB SSDs. The OS occupies its own dedicated NVMe; the remaining three drives are pre-formatted and already mounted at `/mnt/nvme-1`, `/mnt/nvme-2`, and `/mnt/nvme-3`. Predastore spreads object shards across these three drives — one blob node per physical drive, with Reed–Solomon redundancy so a single drive failure is recoverable.
+The X14 has four NVMe drives: two 1.5 TB SSDs (one carries the OS) and two ~880 GB SSDs. The OS occupies its own dedicated NVMe; the remaining three drives are pre-formatted and already mounted at `/mnt/nvme-1`, `/mnt/nvme-2`, and `/mnt/nvme-3`. Predastore is distributed across these three drives — one storage node per physical drive, with Reed–Solomon redundancy so a single drive failure is recoverable.
 
-A single-node install is one Predastore host running seven nodes, each with a directory named for its node ID under `/var/lib/spinifex/predastore/cluster`. The three blob nodes are `node-2`, `node-3` and `node-4`, so `nvme-$i` takes `node-$((i + 1))`. The gate keeps no data at all, and the three meta nodes hold only the Raft log for buckets and the object index, which is small enough to leave on the OS drive.
-
-Relocate the blob node directories onto the mounted drives:
+Relocate the Predastore data directories onto the mounted drives:
 
 ```bash
 sudo systemctl stop spinifex.target
 
 for i in 1 2 3; do
-  node="node-$((i + 1))"
-  sudo mv /var/lib/spinifex/predastore/cluster/$node /mnt/nvme-$i/$node
-  sudo ln -s /mnt/nvme-$i/$node /var/lib/spinifex/predastore/cluster/$node
+  sudo mkdir -p /mnt/nvme-$i/nodes /mnt/nvme-$i/db
+  sudo mv /var/lib/spinifex/predastore/distributed/nodes/node-$i /mnt/nvme-$i/nodes/node-$i
+  sudo mv /var/lib/spinifex/predastore/distributed/db/node-$i    /mnt/nvme-$i/db/node-$i
+  sudo ln -s /mnt/nvme-$i/nodes/node-$i /var/lib/spinifex/predastore/distributed/nodes/node-$i
+  sudo ln -s /mnt/nvme-$i/db/node-$i    /var/lib/spinifex/predastore/distributed/db/node-$i
 done
 
 sudo systemctl start spinifex.target

docs/reference-architectures/SMCI-MIG-orchestration/README.md

@@ -138,9 +138,7 @@ sudo systemctl status spinifex.target
 
 ### 4. Attach Predastore storage
 
-Spinifex spreads Predastore's object shards across multiple blob nodes using Reed–Solomon encoding for data redundancy. This chassis has four NVMe drives — one for the OS, three dedicated to data — so we back each blob node with its own physical drive, giving us fault-tolerant storage on a single machine — conceptually similar to RAID 5, where data and parity are spread across drives so a single drive failure is recoverable.
-
-A single-node install is one Predastore host running seven nodes, each with a directory named for its node ID under `/var/lib/spinifex/predastore/cluster`. The three blob nodes are `node-2`, `node-3` and `node-4`, and they are the ones worth their own drive: the gate keeps no data, and the three meta nodes hold only the Raft log for buckets and the object index.
+Spinifex distributes the Predastore object storage volume across multiple nodes using Reed–Solomon encoding for data redundancy. This chassis has four NVMe drives — one for the OS, three dedicated to data — so we back each Predastore storage node with its own physical drive, giving us fault-tolerant distributed storage on a single machine — conceptually similar to RAID 5, where data and parity are spread across drives so a single drive failure is recoverable.
 
 Confirm drive assignments with `lsblk` before proceeding, as device names vary between systems.
 
@@ -150,17 +148,20 @@ lsblk   # identify the OS drive and the three data drives
 # Stop services so Predastore isn't writing while we relocate its data directories
 sudo systemctl stop spinifex.target
 
-# --- Repeat the block below for each data drive (node-2/nvme-1, node-3/nvme-2, node-4/nvme-3) ---
+# --- Repeat the block below for each data drive (node-1/nvme-1, node-2/nvme-2, node-3/nvme-3) ---
 
 # Mount the physical drive at a stable path
 sudo mkdir -p /mnt/nvme-1
 sudo mount /dev/nvme1n1 /mnt/nvme-1
+sudo mkdir -p /mnt/nvme-1/nodes /mnt/nvme-1/db
 
-# Move the blob node's data off the OS drive onto the physical NVMe
-sudo mv /var/lib/spinifex/predastore/cluster/node-2 /mnt/nvme-1/node-2
+# Move Predastore's node data and metadata off the OS drive onto the physical NVMe
+sudo mv /var/lib/spinifex/predastore/distributed/nodes/node-1 /mnt/nvme-1/nodes/node-1
+sudo mv /var/lib/spinifex/predastore/distributed/db/node-1    /mnt/nvme-1/db/node-1
 
-# Symlink the original path back so Predastore finds its data unchanged
-sudo ln -s /mnt/nvme-1/node-2 /var/lib/spinifex/predastore/cluster/node-2
+# Symlink the original paths back so Predastore finds its data unchanged
+sudo ln -s /mnt/nvme-1/nodes/node-1 /var/lib/spinifex/predastore/distributed/nodes/node-1
+sudo ln -s /mnt/nvme-1/db/node-1    /var/lib/spinifex/predastore/distributed/db/node-1
 
 # Persist the mount across reboots
 echo "/dev/nvme1n1  /mnt/nvme-1  auto  defaults  0  2" | sudo tee -a /etc/fstab

docs/security/malware-protection/README.md

@@ -80,7 +80,7 @@ All Spinifex nodes must include the following in their malware protection scope.
 
 | Path | Contents |
 |------|----------|
-| `/etc/spinifex/` | Cluster config directory: `spinifex.toml`, `awsgw.toml`, `nats.conf`, and `predastore/predastore.toml`. |
+| `/etc/spinifex/` | Cluster config directory: `spinifex.toml`, `predastore.toml`, `nats.conf`. |
 | `/etc/spinifex/ca.pem`, `/etc/spinifex/ca.key` | Cluster root CA certificate and private key (leader only). |
 | `/etc/spinifex/server.pem`, `/etc/spinifex/server.key` | Per-node TLS certificate and key. |
 | `/etc/spinifex/master.key` | Master encryption key that wraps per-volume DEKs and IAM secrets. **Highest-value artifact on the node.** |

docs/security/network-connections/README.md

@@ -85,9 +85,9 @@ The inventory in [§1](#1-inbound-listeners)–[§2](#2-outbound-connections) sa
 | 4432 | Formation server | HTTPS | External (bootstrap only) | Cluster join coordination; active only while a join token is valid. See *Formation port lifecycle* below. | Short-lived bearer token + TLS¹ |
 | 4222 | spinifex-nats (client) | NATS + TLS | Cluster | Internal service bus for EC2/EBS/VPC/S3 handlers | Token + mutual TLS (cluster CA) |
 | 4248 | spinifex-nats (cluster) | NATS + TLS | Cluster | Inter-node NATS federation | Token + mutual TLS (cluster CA) |
-| 8443 | spinifex-predastore (gate) | HTTPS | Cluster | S3-compatible object storage (AMIs, snapshots, user objects) | AWS SigV4 + TLS |
-| 6660 | predastore (blob node) | QUIC / UDP | Cluster | Erasure-coded object shard transport between hosts. Multi-node clusters only — see *Predastore ports* below. | TLS 1.3, server certificate verified against the cluster CA |
-| 7660 | predastore (meta node) | QUIC / UDP | Cluster | Raft consensus over global state — buckets and the object index — between hosts. Multi-node clusters only. | TLS 1.3, server certificate verified against the cluster CA |
+| 8443 | spinifex-predastore | HTTPS | Cluster | S3-compatible object storage (AMIs, snapshots, user objects) | AWS SigV4 + TLS |
+| 6660–6662 | predastore (Raft) | TCP | Cluster | Metadata consensus (3 nodes) | Cluster network only |
+| 9991–9993 | predastore (data shards) | TCP | Cluster | Erasure-coded data shard transport | Cluster network only |
 | 6641 | OVN Northbound DB (client) | OVSDB/TCP | Cluster | Logical network topology consumed by vpcd | Cluster network only; TLS planned |
 | 6642 | OVN Southbound DB (client) | OVSDB/TCP | Cluster | Chassis / port / MAC binding state | Cluster network only; TLS planned |
 | 6643 | OVN Northbound DB (RAFT) | OVSDB/TCP | Cluster | NB database RAFT replication between the 3 quorum nodes | Cluster network only; TLS planned |
@@ -97,10 +97,6 @@ The inventory in [§1](#1-inbound-listeners)–[§2](#2-outbound-connections) sa
 
 ¹ **Formation port lifecycle.** 4432 opens during `spx admin init` / `spx admin join` while a bootstrap token is outstanding and closes once the cluster is formed (token TTL default 30 min, `--token-ttl`). The server presents an ephemeral self-signed cert that pre-dates trust bootstrap, so the joining node does not verify the certificate chain for this single dial. Authenticity rests on the operator supplying the leader address out-of-band plus possession of the bearer token. Document in the security plan so reviewers do not flag 4432 as a persistent open port.
 
-**Predastore ports.** A Predastore cluster is described in `/etc/spinifex/predastore/predastore.toml` as a set of `[[host]]` blocks — one per machine, each running a single process — with the nodes pinned to it declared under `[[host.node]]`. There are three roles: a `gate` serving the S3 API, a `blob` node holding erasure-coded object shards, and a `meta` node in the Raft quorum over global state. Ports must be unique within a host but are not unique across the cluster, so **every machine uses the same three ports** — 8443, 6660 and 7660. It is three fixed ports per machine, not a range, and adding machines does not widen it.
-
-Nodes on the same host talk over an in-process pipe and bind no socket at all, so a single-node install opens only 8443; 6660 and 7660 appear only once a second host exists. The gate is dialled by S3 clients but never by peer nodes, so it binds no QUIC socket of its own — it takes an ephemeral UDP port to dial out with.
-
 **Development-only listeners.** When `dev_networking=true`, QEMU opens arbitrary host TCP ports for SSH port-forwarding into guest VMs. Production installs (the `/etc/spinifex` layout) do not enable this; it must not appear on compliance nodes.
 
 ## 2. Outbound Connections
@@ -118,7 +114,7 @@ Spinifex nodes initiate a small, fixed set of outbound connections.
 | `https://d2yp8ipz5jfqcw.cloudfront.net` | Alpine image for managed HAProxy load-balancer | HTTPS | TLS + checksum verification |
 | `https://install.mulgadc.com/install` | One-shot install telemetry POST on `spx admin init` / `join`. | HTTPS | TLS |
 
-**To peer nodes (cluster-internal):** NATS federation (4248), Predastore S3 (8443), OVN NB/SB (6641/6642) — see [§3](#3-cross-node-internal-connections) for encryption and verification of each. The daemon also polls local NATS monitoring at `127.0.0.1:8222/varz` (loopback HTTP). It opens no connection to Predastore for status: the storage topology it reports comes from reading `predastore.toml`, and no Predastore node serves a status endpoint.
+**To peer nodes (cluster-internal):** NATS federation (4248), Predastore S3 (8443), OVN NB/SB (6641/6642) — see [§3](#3-cross-node-internal-connections) for encryption and verification of each. The daemon also probes local Predastore Raft status at `<bind IP>:6660/status` (TLS, cluster CA) and local NATS monitoring at `127.0.0.1:8222/varz` (loopback HTTP).
 
 **Update checks and metadata.** Spinifex does not check for updates and does not consume a cloud metadata service (`169.254.169.254` is served *by* the cluster to guest VMs). Node software updates come from the operator's OS package channel. The install-telemetry endpoint above is the only vendor-operated destination contacted by a node; closed-egress deployments should disable it and record the opt-out in the security plan.
 
@@ -131,14 +127,14 @@ Control-plane and data-plane traffic between Spinifex nodes, for completeness an
 | Connection | Port(s) | Encryption / Auth | Notes |
 |-----------|---------|-------------------|-------|
 | NATS cluster routes | 4248 | Mutual TLS + cluster token | Full mesh between NATS servers |
-| Predastore S3 (gate) | TCP 8443 | TLS + AWS SigV4 | Cross-node object reads/writes |
-| Predastore blob | UDP 6660 | QUIC with TLS 1.3; server certificate verified against the cluster CA | Erasure-coded object shards. Same port on every machine. |
-| Predastore meta | UDP 7660 | QUIC with TLS 1.3; server certificate verified against the cluster CA | Raft consensus over buckets and the object index. Same port on every machine. |
+| Predastore S3 | 8443 | TLS + AWS SigV4 | Cross-node object reads/writes |
+| Predastore Raft | 6660–6662 | Cluster network only | Metadata consensus |
+| Predastore shards | 9991–9993 | Cluster network only | Erasure-coded data shards |
 | OVN NB/SB (client) | 6641 / 6642 | Cluster network only (TLS planned) | Network control plane; vpcd and ovn-controller dial the quorum |
 | OVN NB/SB (RAFT) | 6643 / 6644 | Cluster network only (TLS planned) | NB/SB database replication across the 3 quorum nodes |
 | OVN tunnels (Geneve) | UDP 6081 | None | Tenant traffic overlay between chassis, inside the cluster subnet |
 
-Nodes **must** sit on a network segment that is not routed to tenant/guest VMs or to the internet. The Predastore blob and meta transports and the OVN DBs are cluster-internal and must not be reachable from anywhere else.
+Nodes **must** sit on a network segment that is not routed to tenant/guest VMs or to the internet. Predastore Raft/shards and OVN DBs are cluster-internal and must not be reachable from anywhere else.
 
 ## 4. Limiting Controls
 
@@ -149,8 +145,8 @@ Default external surface is three listeners — **9999** (AWS API), **3000** (UI
 tcp dport { 22, 9999, 3000 } accept
 
 # Cluster-only: replace 10.0.1.0/24 with your cluster CIDR
-ip saddr 10.0.1.0/24 tcp dport { 4222, 4248, 8443, 6641-6644 } accept
-ip saddr 10.0.1.0/24 udp dport { 6081, 6660, 7660 } accept
+ip saddr 10.0.1.0/24 tcp dport { 4222, 4248, 8443, 6641-6644, 6660-6662, 9991-9993 } accept
+ip saddr 10.0.1.0/24 udp dport 6081 accept
 
 # Default deny
 tcp dport 0-65535 drop
@@ -167,7 +163,7 @@ Every listener and outbound destination is controlled by one of these files. Cha
 |------|------|----------|
 | `/etc/spinifex/spinifex.toml` | `nodes.<node>.{awsgw,nats,predastore,daemon}.host`, `nodes.<node>.vpcd.ovn_{nb,sb}_addr`, `nodes.<node>.daemon.dev_networking` | Per-service bind addresses/ports; dev-mode QEMU port forwarding. |
 | `/etc/spinifex/nats.conf` | `listen`, `cluster.listen`, `cluster.routes`, `http`, `tls`, `cluster.authorization` | NATS client/cluster/monitoring listeners, peer routes, TLS, cluster token. |
-| `/etc/spinifex/predastore/predastore.toml` | `[[host]].bind_addr`, `[[host]].addr`, `[[host]].tls_cert`, `[[host]].tls_key`, `[[host.node]].role`, `[[host.node]].port` | Predastore host and node layout: `bind_addr` is the address the host's sockets bind, `addr` is the address peer hosts dial it on, and both carry no port — the nodes pinned to the host supply their own. The service's `--host`/`--port` override the bind address and the gate's S3 port. |
+| `/etc/spinifex/predastore.toml` | `[[host]].bind_addr`, `[[host]].public_addr`, `[[node]].role`, `tls.*` | Predastore intra-cluster listener (one socket per host, carrying every node pinned to it), node placement, TLS certs. The S3 listener itself is set by the service's `--host`/`--port`. |
 | OVN packages (`ovn-central`, `ovn-host`) | `ovn-nb-db`, `ovn-sb-db` (via `ovs-vsctl set open_vswitch …`) | OVN DB bind addresses. |
 | Spinifex UI service | Built-in defaults: `host = "0.0.0.0"`, `port = 3000`. No `spinifex.toml` block today. | UI listener. |
 | `spx admin init` / `spx admin join` | `--port`, `--token-ttl`, `--no-telemetry` (or `SPX_NO_TELEMETRY=1`) | Formation port, token TTL, telemetry opt-out. |

docs/terraform-workbooks/rds-quickstart/README.md

@@ -1,5 +1,5 @@
 ---
-title: "RDS Quickstart"
+title: "RDS Quickstart (PostgreSQL)"
 description: "Stand up a managed PostgreSQL database on Spinifex with Terraform — a VPC, a DB subnet group, a parameter group, an aws_db_instance, and a client VM inside the VPC that connects to it with psql."
 category: "Terraform Workbooks"
 tags:
@@ -16,7 +16,7 @@ sections:
   - troubleshooting
 resources:
   - title: "RDS CLI Reference"
-    url: "https://github.com/mulgadc/spinifex/blob/main/docs/COMMANDS.md#rds-postgresql"
+    url: "https://github.com/mulgadc/spinifex/blob/main/docs/COMMANDS.md#rds-postgresql-and-mariadb"
   - title: "Terraform AWS Provider — aws_db_instance"
     url: "https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/db_instance"
   - title: "Spinifex Repository"
@@ -25,10 +25,12 @@ resources:
     url: "https://opentofu.org/"
 ---
 
-# Terraform: RDS Quickstart
+# Terraform: RDS Quickstart (PostgreSQL)
 
 > The smallest end-to-end RDS example on Spinifex: a VPC with a public client subnet and a private DB subnet, a DB subnet group, a parameter group, a PostgreSQL DB instance, and a client VM with `psql` already pointed at the endpoint.
 
+This workbook is **PostgreSQL throughout** — the engine, the `postgres18` parameter group, port `5432`, the client package and the `~/.pgpass` credential are all PostgreSQL's. Spinifex also serves MariaDB 11.8 under `engine = "mariadb"`; running this workbook against it means changing `engine`, `engine_version`, the parameter-group `family` to `mariadb11.8`, the security-group port to `3306` and the client to `mariadb-client`. See the [RDS command reference](https://github.com/mulgadc/spinifex/blob/main/docs/COMMANDS.md#rds-postgresql-and-mariadb) for both engines' pins, families and limits. Note that `mysql` is not an accepted engine and is not an alias for `mariadb`.
+
 ## Overview
 
 A Spinifex DB instance runs in a VM you never see. What lands in your VPC is a single **endpoint ENI** in one of the DB subnet group's subnets, so the endpoint is **always private** — there is no `publicly_accessible` mode, and a request for one is rejected. That is why this workbook builds a client VM: it is the only place from which the database can be reached.
@@ -127,7 +129,7 @@ psql "sslmode=verify-full sslrootcert=/home/ubuntu/ca.pem" \
 | `spinifex_endpoint` | `https://127.0.0.1:9999` | Gateway as seen from the host running Terraform. |
 | `instance_type` | `t3.small` | EC2 type for the **client** VM. |
 | `db_instance_class` | `db.t3.micro` | DB class. One of the curated `db.*` subset. |
-| `engine_version` | `18` | PostgreSQL major. `18` is the only version served. |
+| `engine_version` | `18` | PostgreSQL major. `18` is the only version served, and a minor such as `18.4` is rejected. |
 | `db_name` | `appdb` | Database created at bootstrap. |
 | `db_username` | `appuser` | Master user. `postgres`, `rdsadmin`, `rds_superuser` and `pg_*` are reserved. |
 | `db_password` | `QuickstartS3cret1` | Master password — override it. No `/`, `"`, `@` or spaces. |

go.mod

@@ -21,9 +21,9 @@ require (
 	github.com/insomniacslk/dhcp v0.0.0-20260407060928-11b94ed970f2
 	github.com/klauspost/cpuid/v2 v2.4.0
 	github.com/miekg/dns v1.1.72
-	github.com/mulgadc/northstar v1.16.0
-	github.com/mulgadc/predastore v1.16.0
-	github.com/mulgadc/viperblock v1.16.0
+	github.com/mulgadc/northstar v1.15.1-0.20260810233215-04fade67fb2c
+	github.com/mulgadc/predastore v1.15.0
+	github.com/mulgadc/viperblock v1.15.0
 	github.com/nats-io/nats-server/v2 v2.14.4
 	github.com/nats-io/nats.go v1.52.0
 	github.com/opencontainers/runtime-spec v1.3.0

go.sum

@@ -805,6 +805,7 @@ github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYF
 github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
 github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w=
 github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE=
+github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc=
 github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE=
@@ -864,6 +865,7 @@ github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbV
 github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
 github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8=
 github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
@@ -1319,12 +1321,14 @@ github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELU
 github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
 github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
 github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
-github.com/mulgadc/northstar v1.16.0 h1:uX+A7PUwC51MRK6Q8dsEx40U2ZL0S7qz2x7lPTaAykc=
-github.com/mulgadc/northstar v1.16.0/go.mod h1:1tRJZAKsXvsnMaAHAt5za+sDsRpkaoDGH0s8LST2Xyg=
-github.com/mulgadc/predastore v1.16.0 h1:czQBOcl9NiB/V1A3kD5zgOsPePh2FHK94rAE3yHy9eE=
-github.com/mulgadc/predastore v1.16.0/go.mod h1:ph+eASnkMogVZ0cMiO/6aUw1S1+Gl3+NnVtlqCNdBo8=
-github.com/mulgadc/viperblock v1.16.0 h1:S1kT8fQ+Gbmj0I0+CCLXKwf6dMS7GpDZck86iGvLR+A=
-github.com/mulgadc/viperblock v1.16.0/go.mod h1:pyacEYyZvoxYhZbw5wMpYTW2DnDGpLs9Jackgh/CNgk=
+github.com/mulgadc/northstar v1.15.0 h1:DbJE/w3FLk19PNwgaC+Rnu/gP0E5GNMK9EGwR4TPqqU=
+github.com/mulgadc/northstar v1.15.0/go.mod h1:QbP2ydcCbGHidIdp61O2HOcJANFgGetS5zw60mbjxfQ=
+github.com/mulgadc/northstar v1.15.1-0.20260810233215-04fade67fb2c h1:Ia73hhrUvRRV7iEdCczHMhzrXtOJlQRotMN7rcMGFuA=
+github.com/mulgadc/northstar v1.15.1-0.20260810233215-04fade67fb2c/go.mod h1:1tRJZAKsXvsnMaAHAt5za+sDsRpkaoDGH0s8LST2Xyg=
+github.com/mulgadc/predastore v1.15.0 h1:IGQX8bbuCrwU5mC4JyjxgdL8UXGfTA1Cu9g9Kuu8Irw=
+github.com/mulgadc/predastore v1.15.0/go.mod h1:vjad49Pql/NtQLGfAC43SSDBcPLD6ijbqMbddbt0vA4=
+github.com/mulgadc/viperblock v1.15.0 h1:nZ8d8q8lSgMucd43GYgvwpmnCWXN379z6H2MxUHGo9g=
+github.com/mulgadc/viperblock v1.15.0/go.mod h1:/pkP/liHmG+yzojZ3+xSvAcqMHb/zfiEQVsHeK9EPYM=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@@ -1627,6 +1631,8 @@ go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
 go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
 go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
 go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
+go.opentelemetry.io/contrib/bridges/otelslog v0.19.0 h1:5RgvxieNq9tS3ewrV1vnODvbHPfKUIJcYtF9Cvz+6aQ=
+go.opentelemetry.io/contrib/bridges/otelslog v0.19.0/go.mod h1:iTBIdNwx/xmUhfgJs6+84S4dIK059811cO1eUBjKcHY=
 go.opentelemetry.io/contrib/bridges/otelslog v0.20.0 h1:oEl2Pw/i4OQwhAuda2pAHFAcOMivA+Xa+iTccBfab/g=
 go.opentelemetry.io/contrib/bridges/otelslog v0.20.0/go.mod h1:yMSQaiiq5dpfrSJCYLBcqFeJkFFI67seT4ngvx6jfVo=
 go.opentelemetry.io/contrib/detectors/gcp v1.28.0/go.mod h1:9BIqH22qyHWAiZxQh0whuJygro59z+nbMVuc7ciiGug=
@@ -1664,8 +1670,12 @@ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0/go.mod h1:
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 h1:LMuyCAyfalSjDyjdC65nK6N0zoTT63+E/u95X0JovZI=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0/go.mod h1:085m8qbm4hgc8rZWGDEa4vmyyo2c3nPxUslYUKUIU04=
+go.opentelemetry.io/contrib/instrumentation/runtime v0.69.0 h1:MtkMsuRo3zEXTTMALfyrszwCDZTkB6wolyPjbwFAdq0=
+go.opentelemetry.io/contrib/instrumentation/runtime v0.69.0/go.mod h1:FYTxnpsm+UPD0erZNq20GvnM8T2YQHiHtT2vokdpoac=
 go.opentelemetry.io/contrib/instrumentation/runtime v0.70.0 h1:1+WLVYezXA9tkuVzKQri8zgB1cEIVYKUSoYIRjsBiMU=
 go.opentelemetry.io/contrib/instrumentation/runtime v0.70.0/go.mod h1:rbAXUUXqQDMxpSnmof4VtcZ+7YpZQEtjXSCIfdvR0Go=
 go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo=
@@ -1689,15 +1699,24 @@ go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1Nkl
 go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g=
 go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc=
 go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0=
+go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU=
 go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc=
 go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU=
 go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ=
+go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.20.0 h1:rydZ9sxbcFdm/oWrVyfLTjHIygMgv0bEeMd+3B/BvoM=
+go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.20.0/go.mod h1:earQ25dooT0Hhspq59DZ8YCC50jWfOlFEeWoxy/P444=
 go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.21.0 h1:WseeVYf5dJZTsyPiyW5L14k5qsSibqXAMTSiFEDiWr0=
 go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.21.0/go.mod h1:SiLZnQS6Qk2eCpvr2CH/XMAOa64TWGXxEZJZCpD2Lmc=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.44.0 h1:SUplec5dp06reu1zaXmOXdvqH398taqrDXqUl99jxSc=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.44.0/go.mod h1:ho2g4N+ane+swq5I/VBkKWnRDY4kUINH3FuqyZqX/Ug=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.45.0 h1:klTViGcsvLCd1xN3rZzfZ12NslC/OimbmR+k+A006RI=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.45.0/go.mod h1:jRsK04CWmXuY8A0O+wMpSf+t90RHZ53o5Qmxn2PQPfk=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 h1:4YsVu3B8+3qtWYYrsUYgn0OG78pN0rnNPRGX4SbokQI=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0/go.mod h1:+wnlSn0mD1ADVMe3v9Z/WIaiz6q6gL2J/ejaAmdmv80=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 h1:QRefszxJmfPdjXUUm3j6iDzY03mTPXMjqErFqQ67vUg=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0/go.mod h1:Tiz03lTBVBrm7eWZBOidzEaYaJa8tjwGUGv6d8mlTyk=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0 h1:qazEJlUOQzhCpzQpFETGby7EdqjI1wsd0W+6Gg1SCTU=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0/go.mod h1:fOD2Yefuxixkx3ahVNf0O/PERb6r4OlbxfATVnYvzCo=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0 h1:fG5MCxGz8+2VtrN/WgqSpJFctVz24gpxj8CxkKmc8Ww=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0/go.mod h1:BmAYTn+3ysbRe+IU2msxmf5Rx3g6DHvex+tWI3LdhYI=
 go.opentelemetry.io/otel/exporters/prometheus v0.57.0/go.mod h1:QpFWz1QxqevfjwzYdbMb4Y1NnlJvqSGwyuU0B4iuc9c=
@@ -1705,6 +1724,8 @@ go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.29.0/go.mod h1:BLbf7zb
 go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.35.0/go.mod h1:U2R3XyVPzn0WX7wOIypPuptulsMcPDPs/oiSVOMVnHY=
 go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0/go.mod h1:dowW6UsM9MKbJq5JTz2AMVp3/5iW5I/TStsk8S+CfHw=
 go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.42.0/go.mod h1:so9ounLcuoRDu033MW/E0AD4hhUjVqswrMF5FoZlBcw=
+go.opentelemetry.io/otel/log v0.20.0 h1:/5i0vuHxCLWUfChWG41K9wkM0jafruPw9NU1/RCJirs=
+go.opentelemetry.io/otel/log v0.20.0/go.mod h1:wOcMcjsZpG8x7Bak7IhSi/lg8wscV2C1VdrKCLPlt0E=
 go.opentelemetry.io/otel/log v0.21.0 h1:SLsVDGmtyBrdw8/a2Z0bOIxou/+bN4z56GebH7T0LvA=
 go.opentelemetry.io/otel/log v0.21.0/go.mod h1:iReetQrZL9Wyg84cCkOoCmqDHS5RCFfyxC7J+r8fn8g=
 go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU=
@@ -1728,12 +1749,13 @@ go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0
 go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc=
 go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI=
 go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY=
+go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc=
 go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo=
 go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M=
 go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s=
+go.opentelemetry.io/otel/metric/x v0.66.0 h1:YkCrx1zLOChi9ZcZ6euupOcsgzbVlec7D/xoEU1+cTA=
 go.opentelemetry.io/otel/metric/x v0.66.0/go.mod h1:d1+BDj9t96do0/1LoU1ayfCv79ZgNE41qbhBvnMOBZk=
 go.opentelemetry.io/otel/metric/x v0.67.0 h1:PcicCNZFkZ4bXfSooXdo3WN7RBOVOtjVdo1wD358Uns=
-go.opentelemetry.io/otel/metric/x v0.67.0/go.mod h1:FBjCWZe6wgcqxcMtjdGiClDKXb2YxxXii0CXftE4QtI=
 go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw=
 go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc=
 go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E=
@@ -1754,13 +1776,17 @@ go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwm
 go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE=
 go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWmkHuGbayMts=
 go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg=
+go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58=
 go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0=
 go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw=
 go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA=
+go.opentelemetry.io/otel/sdk/log v0.20.0 h1:vM3xI7TQgKPiSghe6urZtAkyFY7SodrSpC83CffDFuY=
+go.opentelemetry.io/otel/sdk/log v0.20.0/go.mod h1:Knej2nmsTUzN79T2eeXdRsjjPcoxoq2pUyUHz9TFyyU=
 go.opentelemetry.io/otel/sdk/log v0.21.0 h1:QsE7XSR0ktQdKmRKGnR+f1ObGF32WG+7MER/P9KgmYc=
 go.opentelemetry.io/otel/sdk/log v0.21.0/go.mod h1:m9mApjCoD2/1QuKCAptjv+BrG9WKOvQLVdNx+iBldTo=
+go.opentelemetry.io/otel/sdk/log/logtest v0.20.0 h1:OqdRZ1guyzamK3M6LlRsmGqRrjkHWw6WZOKKli5ELpg=
+go.opentelemetry.io/otel/sdk/log/logtest v0.20.0/go.mod h1:PuMIlm7zAt7c3z8zfOI5ox4iT1Z87We+PF6YoINux/M=
 go.opentelemetry.io/otel/sdk/log/logtest v0.21.0 h1:X+JBBgKlswCGYsmgL0CnoUUtlE//VB345c84jYAYkdQ=
-go.opentelemetry.io/otel/sdk/log/logtest v0.21.0/go.mod h1:HD1575K8e6sIFBBDd5tZB3t9DlMytWXq9FuR+Y4rfjE=
 go.opentelemetry.io/otel/sdk/metric v1.28.0/go.mod h1:cWPjykihLAPvXKi4iZc1dpER3Jdq2Z0YLse3moQUCpg=
 go.opentelemetry.io/otel/sdk/metric v1.29.0/go.mod h1:6zZLdCl2fkauYoZIOn/soQIDSWFmNSRcICarHfuhNJQ=
 go.opentelemetry.io/otel/sdk/metric v1.30.0/go.mod h1:waS6P3YqFNzeP01kuo/MBBYqaoBJl7efRQHOaydhy1Y=
@@ -1775,6 +1801,7 @@ go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5w
 go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg=
 go.opentelemetry.io/otel/sdk/metric v1.42.0/go.mod h1:Ua6AAlDKdZ7tdvaQKfSmnFTdHx37+J4ba8MwVCYM5hc=
 go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A=
+go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI=
 go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA=
 go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o=
 go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA=
@@ -1799,6 +1826,7 @@ go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4Etq
 go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA=
 go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc=
 go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0=
+go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk=
 go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE=
 go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag=
 go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc=
@@ -1809,6 +1837,8 @@ go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v8
 go.opentelemetry.io/proto/otlp v1.7.0/go.mod h1:fSKjH6YJ7HDlwzltzyMj036AJ3ejJLCgCSHGj4efDDo=
 go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE=
 go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4=
+go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g=
+go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk=
 go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk=
 go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
@@ -2122,6 +2152,8 @@ golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
 golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs=
 golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ=
 golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww=
+golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
+golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
 golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
 golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -2836,6 +2868,7 @@ google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9/go.
 google.golang.org/genproto/googleapis/api v0.0.0-20260511170946-3700d4141b60/go.mod h1:7yoXV7RIh5gblj/xVYoogxAWvA9wUeVbpsK/M694l00=
 google.golang.org/genproto/googleapis/api v0.0.0-20260519071638-aa98bba5eb94/go.mod h1:1dCETSCY2YKZNXQE3h4fun3TYwF5p8jejRKZgfWAgAY=
 google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY=
+google.golang.org/genproto/googleapis/api v0.0.0-20260618152121-87f3d3e198d3 h1:ctPmKL12ZsoKAlmPUsoW70zEDiYF+/H6aLieXxgAU0k=
 google.golang.org/genproto/googleapis/api v0.0.0-20260618152121-87f3d3e198d3/go.mod h1:Z4WJ5pJOYWFWcHEQUelD5QaZDknIQkpIL/+fyJOT9+A=
 google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d h1:FarXi840EJWSHYTN3ERkADbPWjl307+FGrA22KAVjjc=
 google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d/go.mod h1:K/+WGbmBY7aNW1HDw1fJnKYo10i0DkAX6pows00dLig=
@@ -3018,6 +3051,7 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20260519071638-aa98bba5eb94/go.
 google.golang.org/genproto/googleapis/rpc v0.0.0-20260523011958-0a33c5d7ca68/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20260610212136-7ab31c22f7ad/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260618152121-87f3d3e198d3 h1:phvBWCAQMGN1945mp5fjCXP6jEF0+a0+4TjokS4sxNY=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20260618152121-87f3d3e198d3/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d h1:IL4hdHzcUv2l/gcg98/Rj3FbtE6axwqslOW8SW0C+S0=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
@@ -3114,6 +3148,8 @@ google.golang.org/grpc v1.79.2/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhH
 google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
 google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4=
 google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I=
+google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE=
+google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA=
 google.golang.org/grpc v1.83.0 h1:JeNZEKJFbQxArAMl+hiytHauacDNqJUllNfmIMmpqnQ=
 google.golang.org/grpc v1.83.0/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ=
 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=

scripts/README.md

@@ -55,6 +55,7 @@ Each subdirectory holds a `manifest.conf`, init scripts, and a `setup.sh` that `
 | `eks-node/` | The unified Alpine EKS node AMI (K3s server *or* agent, selected at first boot). `setup.sh` installs the pinned, SHA-verified K3s binary and the role init scripts. |
 | `eks-node-gpu/` | The Ubuntu EKS GPU worker AMI. Agent role only — GPU nodes never run the control plane — plus the NVIDIA GPU stack. |
 | `rds-postgres/` | The `spinifex-rds-postgres` AMI. `setup.sh` sets exec bits on the OpenRC services and creates the root-only directory cloud-init drops the agent env file and gateway CA into. |
+| `rds-mariadb/` | The `spinifex-rds-mariadb` AMI. Same layout as `rds-postgres/`, plus the `/etc/my.cnf.d` include drop-in and the `conf.d/mariadb` dependency that keeps the packaged service off an unbootstrapped datadir. |
 | `ubuntu-gpu-{amd,nvidia}-setup.sh` | Chroot setup for GPU-capable Ubuntu guest images: ROCm CLI tools and AMD firmware, or the DKMS-built headless NVIDIA driver. Guest-side only — host VFIO setup is `spx admin gpu setup`. |
 
 ### `images/eks-node/` — in-guest helpers

scripts/images/common/rds-datadir

@@ -1,10 +1,14 @@
 #!/bin/sh
 # rds-datadir — select, authorize, prepare, and mount the exact data volume
-# assigned by the control plane before rds-init touches the PostgreSQL datadir.
+# assigned by the control plane before rds-init touches the engine's datadir.
+#
+# Shared by every RDS image preset. The mount point and the OS user that owns it
+# are the one engine-specific pair here, and both are supplied by the preset's
+# own service rather than defaulted, so no image can inherit another's layout.
 set -eu
 
-DATA_MOUNT="${RDS_DATA_MOUNT:-/var/lib/postgresql}"
-PG_USER="${RDS_PG_USER:-postgres}"
+DATA_MOUNT="${RDS_DATA_MOUNT:-}"
+ENGINE_USER="${RDS_ENGINE_USER:-}"
 SYS_BLOCK="${RDS_SYS_BLOCK:-/sys/block}"
 DEV_DIR="${RDS_DEV_DIR:-/dev}"
 MOUNTS_FILE="${RDS_MOUNTS_FILE:-/proc/mounts}"
@@ -103,6 +107,14 @@ consume_format_authorization() {
 }
 
 main() {
+    # An unset layout is a broken image, not something to guess at: mounting the
+    # customer's volume where the engine is not looking would read as an empty
+    # datadir and bootstrap a second, empty database beside their data.
+    [ -n "${DATA_MOUNT}" ] ||
+        fail "RDS_DATA_MOUNT is not set; refusing to guess where this engine's datadir belongs"
+    [ -n "${ENGINE_USER}" ] ||
+        fail "RDS_ENGINE_USER is not set; refusing to guess which user owns this engine's datadir"
+
     load_handoff
 
     command -v blkid >/dev/null 2>&1 ||
@@ -184,8 +196,8 @@ main() {
 
     mount -t "${_fstype}" "${_dev}" "${DATA_MOUNT}" ||
         fail "could not mount ${_dev} at ${DATA_MOUNT}"
-    chown "${PG_USER}:${PG_USER}" "${DATA_MOUNT}" ||
-        fail "could not set ${PG_USER} ownership on ${DATA_MOUNT}"
+    chown "${ENGINE_USER}:${ENGINE_USER}" "${DATA_MOUNT}" ||
+        fail "could not set ${ENGINE_USER} ownership on ${DATA_MOUNT}"
     chmod 0750 "${DATA_MOUNT}"
 
     log "${DATA_MOUNT} ready on ${_dev}"

scripts/images/common/rds-datadir_initd_test.sh

@@ -0,0 +1,63 @@
+#!/bin/sh
+set -eu
+
+# rds-datadir is shared by every engine preset and refuses to guess a layout, so
+# each preset's service is the only statement of where its volume mounts and who
+# owns it. A preset that stopped exporting them would fail closed at boot; a
+# preset that exported another engine's would mount the customer's volume where
+# its engine is not looking, which reads as an empty datadir.
+
+SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
+IMAGES=$(CDPATH='' cd -- "${SCRIPT_DIR}/.." && pwd)
+WORK=$(mktemp -d)
+trap 'rm -rf "${WORK}"' EXIT
+
+FAILS=0
+fail() { echo "FAIL: $*" >&2; FAILS=$((FAILS + 1)); }
+pass() { echo "ok: $*"; }
+
+# Sourced rather than executed: start_pre is what OpenRC runs, and the values it
+# exports are what reach the mount script.
+layout() {
+    (
+        # shellcheck disable=SC1090
+        . "${IMAGES}/$1/rds-datadir.initd"
+        # Read by the sourced start_pre, which shellcheck cannot see.
+        export AGENT_ENV="$2"
+        start_pre
+        printf '%s %s\n' "${RDS_DATA_MOUNT:-}" "${RDS_ENGINE_USER:-}"
+    )
+}
+
+check_preset() {
+    _preset=$1
+    _mount=$2
+    _user=$3
+
+    _got=$(layout "${_preset}" "${WORK}/absent.env")
+    if [ "${_got}" = "${_mount} ${_user}" ]; then
+        pass "${_preset}: exports ${_mount} owned by ${_user}"
+    else
+        fail "${_preset}: exports '${_got}', want '${_mount} ${_user}'"
+    fi
+
+    # The control plane can move the mount point without a rebuild, so a
+    # delivered value has to win over the image's own default.
+    printf "RDS_DATA_MOUNT=%s\n" "${WORK}/delivered" > "${WORK}/agent.env"
+    _got=$(layout "${_preset}" "${WORK}/agent.env")
+    if [ "${_got}" = "${WORK}/delivered ${_user}" ]; then
+        pass "${_preset}: a delivered mount point wins"
+    else
+        fail "${_preset}: delivered mount point ignored, got '${_got}'"
+    fi
+}
+
+check_preset rds-postgres /var/lib/postgresql postgres
+check_preset rds-mariadb /var/lib/mysql mysql
+
+if [ "${FAILS}" -eq 0 ]; then
+    echo "rds-datadir.initd: all presets state their layout"
+    exit 0
+fi
+echo "FAILED: ${FAILS} case(s)"
+exit 1

scripts/images/common/rds-datadir_test.sh

@@ -56,6 +56,7 @@ fail() { echo "FAIL: $*"; FAILS=$((FAILS + 1)); }
 pass() { echo "ok: $*"; }
 
 DATA_MOUNT="${WORK}/data"
+ENGINE_USER="dbengine"
 SYS_BLOCK="${WORK}/sys/block"
 DEV_DIR="${WORK}/dev"
 MOUNTS="${WORK}/mounts"
@@ -91,6 +92,8 @@ EOF
 }
 
 reset_state() {
+    DATA_MOUNT="${WORK}/data"
+    ENGINE_USER="dbengine"
     rm -rf "${WORK}/sys" "${DEV_DIR}" "${DATA_MOUNT}" "$(dirname "${HANDOFF}")"
     mkdir -p "${SYS_BLOCK}" "${DEV_DIR}" "${DATA_MOUNT}"
     printf '/dev/vda1 / ext4 rw,relatime 0 0\n' > "${MOUNTS}"
@@ -104,6 +107,7 @@ reset_state() {
 
 run() {
     env RDS_DATA_MOUNT="${DATA_MOUNT}" \
+        RDS_ENGINE_USER="${ENGINE_USER}" \
         RDS_SYS_BLOCK="${SYS_BLOCK}" \
         RDS_DEV_DIR="${DEV_DIR}" \
         RDS_MOUNTS_FILE="${MOUNTS}" \
@@ -152,6 +156,9 @@ if run_ok "authorized blank exact volume"; then
     grep -q "mkfs.ext4 .*${DEV_DIR}/vdc" "${MKFS_CALLS}" && pass "authorized: exact disk formatted" || fail "authorized: exact disk not formatted"
     grep -q -- '-m 0' "${MKFS_CALLS}" && pass "authorized: no reserved blocks" || fail "authorized: reserved blocks left on"
     grep -q "mount -t ext4 ${DEV_DIR}/vdc ${DATA_MOUNT}" "${MOUNT_CALLS}" && pass "authorized: mounted" || fail "authorized: not mounted"
+    # The engine reads its own datadir, so the mount is handed to the preset's
+    # user rather than to whichever engine this script was first written for.
+    grep -q "chown ${ENGINE_USER}:${ENGINE_USER} ${DATA_MOUNT}" "${CHOWN_CALLS}" && pass "authorized: handed to the engine user" || fail "authorized: not handed to ${ENGINE_USER}"
     grep -q '^RDS_MASTER_PASSWORD=' "${HANDOFF}" && pass "authorized: bootstrap material preserved" || fail "authorized: bootstrap material removed"
     if grep -q '^RDS_FORMAT_AUTHORIZED=' "${HANDOFF}"; then fail "authorized: grant retained"; else pass "authorized: grant consumed"; fi
 fi
@@ -309,6 +316,24 @@ printf '%s %s ext4 rw 0 0\n' "${DEV_DIR}/wrong" "${DATA_MOUNT}" >> "${MOUNTS}"
 run_fails "already mounted wrong device" "not expected device"
 nothing_formatted "already mounted wrong device"
 
+# An image whose service states no layout must not fall back to another engine's
+# paths: a volume mounted where the engine is not looking reads as an empty
+# datadir, and the bootstrap that follows serves an empty database beside the
+# customer's data. Both refusals come before the handoff is even read.
+reset_state
+add_disk vdb volexpected
+write_handoff vol-expected volexpected 1 true
+DATA_MOUNT=""
+run_fails "no mount point" "RDS_DATA_MOUNT is not set"
+nothing_formatted "no mount point"
+
+reset_state
+add_disk vdb volexpected
+write_handoff vol-expected volexpected 1 true
+ENGINE_USER=""
+run_fails "no engine user" "RDS_ENGINE_USER is not set"
+nothing_formatted "no engine user"
+
 if [ "${FAILS}" -eq 0 ]; then
     echo "PASS: all rds-datadir cases"
     exit 0

scripts/images/rds-mariadb/default-parameters.cnf

@@ -0,0 +1,56 @@
+[mysqld]
+# Resolved parameter group, written by rds-agent.
+autocommit = 'on'
+character_set_server = 'utf8mb4'
+collation_server = 'utf8mb4_general_ci'
+connect_timeout = '10'
+event_scheduler = 'off'
+general_log = 'off'
+group_concat_max_len = '1048576'
+idle_transaction_timeout = '0'
+innodb_adaptive_hash_index = 'off'
+innodb_autoinc_lock_mode = '1'
+innodb_buffer_pool_size = '805306368'
+innodb_file_per_table = 'on'
+innodb_flush_log_at_trx_commit = '1'
+innodb_flush_method = 'o_direct'
+innodb_flush_neighbors = '1'
+innodb_io_capacity = '200'
+innodb_io_capacity_max = '2000'
+innodb_lock_wait_timeout = '50'
+innodb_log_file_size = '100663296'
+innodb_max_dirty_pages_pct = '90'
+innodb_purge_threads = '4'
+innodb_read_io_threads = '4'
+innodb_stats_persistent = 'on'
+innodb_strict_mode = 'on'
+innodb_write_io_threads = '4'
+interactive_timeout = '28800'
+join_buffer_size = '262144'
+key_buffer_size = '134217728'
+lock_wait_timeout = '86400'
+log_output = 'file'
+log_queries_not_using_indexes = 'off'
+log_warnings = '2'
+long_query_time = '10'
+max_allowed_packet = '16777216'
+max_connect_errors = '100'
+max_connections = '85'
+max_heap_table_size = '16777216'
+max_prepared_stmt_count = '16382'
+max_statement_time = '0'
+net_read_timeout = '30'
+net_write_timeout = '60'
+performance_schema = 'off'
+read_buffer_size = '131072'
+read_rnd_buffer_size = '262144'
+slow_query_log = 'off'
+sort_buffer_size = '2097152'
+sql_mode = 'STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION'
+table_definition_cache = '400'
+table_open_cache = '2000'
+thread_cache_size = '256'
+default_time_zone = 'SYSTEM'
+tmp_table_size = '16777216'
+transaction_isolation = 'repeatable-read'
+wait_timeout = '28800'

scripts/images/rds-mariadb/manifest.conf

@@ -0,0 +1,59 @@
+IMAGE_NAME="rds-mariadb"
+IMAGE_DESCRIPTION="Spinifex RDS MariaDB instance — Alpine + MariaDB 11.8"
+DISTRO="alpine"
+ALPINE_VERSION="3.24.1"
+
+# Engine binaries and per-boot diagnostics only. The datadir lives on the data
+# volume, so the boot volume stays a disposable COW clone.
+IMAGE_SIZE="2G"
+
+# Alpine's mariadb package name carries no version, unlike postgresql18, so the
+# fuzzy pin is what keeps a rebuild on this series: =~11.8 admits a patch release
+# and refuses a series bump, which would otherwise produce an image tagged
+# engine-version=11.8 running 11.9. mariadb-common carries my.cnf and the charset
+# and error-message data; the client package carries the binaries the agent
+# probes and rotates passwords with.
+# blkid is a hard requirement — without it rds-datadir refuses to run rather
+# than risk formatting a customer's datadir.
+APK_PACKAGES="mariadb=~11.8 mariadb-client=~11.8 mariadb-common=~11.8 mariadb-openrc=~11.8 ca-certificates cloud-utils-growpart e2fsprogs-extra xfsprogs blkid openrc"
+
+# rds-agent is the only built binary (static, FIPS boot like the host build).
+BUILD_COMMANDS="CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -ldflags \"-s -w\" -o bin/rds-agent ./cmd/rds-agent"
+
+INSTALL_BINARIES="bin/rds-agent:/usr/local/bin/rds-agent"
+
+# mulga-mgmt-net is shared with eks-node: this is a three-NIC system VM (system
+# ENI, customer ENI, mgmt0), so the host's fw_cfg netcfg blob needs a consumer in
+# the guest, or mgmt0 never gets its static address and rds-agent cannot reach
+# the gateway, and the customer ENI never comes up so nothing reaches the DB.
+INSTALL_FILES="\
+scripts/images/common/rds-datadir:/usr/local/sbin/rds-datadir \
+scripts/images/rds-mariadb/rds-datadir.initd:/etc/init.d/rds-datadir \
+scripts/images/rds-mariadb/rds-init:/usr/local/sbin/rds-init \
+scripts/images/rds-mariadb/rds-init.initd:/etc/init.d/rds-init \
+scripts/images/rds-mariadb/rds-agent.initd:/etc/init.d/rds-agent \
+scripts/images/common/mulga-mgmt-net.initd:/etc/init.d/mulga-mgmt-net \
+scripts/images/common/mulga-mgmt-net-routes.initd:/etc/init.d/mulga-mgmt-net-routes \
+scripts/images/common/mulga-mgmt-net.sh:/usr/local/sbin/mulga-mgmt-net \
+scripts/images/rds-mariadb/zz-rds-include.cnf:/etc/my.cnf.d/zz-rds-include.cnf \
+scripts/images/rds-mariadb/default-parameters.cnf:/usr/local/share/spinifex-rds/default-parameters.cnf \
+scripts/images/rds-mariadb/mariadb.confd:/etc/conf.d/mariadb"
+
+# Each service's own depend() asserts the ordering; this lists it to match.
+# mulga-mgmt-net is NOT here: setup.sh adds it to the boot runlevel, since a
+# default-runlevel entry would run after cloud-init-local and be too late. Its
+# route-enforcement second pass is the opposite — it has to follow cloud-init, so
+# the default runlevel is exactly where it belongs.
+#
+# The packaged mariadb service only warns about an empty datadir instead of
+# refusing, so it is not ordering that keeps it off an unbootstrapped volume:
+# /etc/conf.d/mariadb makes it need rds-init, which fails closed.
+ENABLE_SERVICES="rds-datadir rds-agent rds-init mariadb mulga-mgmt-net-routes"
+
+SETUP_SCRIPT="scripts/images/rds-mariadb/setup.sh"
+
+AMI_NAME="spinifex-rds-mariadb"
+
+# The contract tag prevents the control plane from issuing a format grant to an
+# older image that still discovers generic vol* devices.
+SYSTEM_TAG="spinifex:managed-by=rds engine=mariadb engine-version=11.8 rds-data-volume-contract=format-auth-v1"

scripts/images/rds-mariadb/mariadb.confd

@@ -0,0 +1,15 @@
+# /etc/conf.d/mariadb — Spinifex RDS overrides for the stock Alpine mariadb
+# service. The package ships no conf.d file of its own, so this adds one rather
+# than replacing one.
+
+# The packaged service's start_pre only prints "Datadir is empty or invalid" and
+# starts anyway, so an rds-init that failed — an unreachable gateway, a data
+# volume that did not attach, a datadir stamped for another engine — would be
+# answered by launching mysqld_safe against whatever is mounted. OpenRC reads
+# rc_need from here, so the engine depends on the bootstrap without the packaged
+# init script being patched.
+rc_need="rds-init"
+
+# Everything else the service resolves from the configuration itself: datadir,
+# socket and port come from the platform drop-in rds-init writes onto the data
+# volume, which the packaged !includedir reads last.

scripts/images/rds-mariadb/rds-agent.initd

@@ -0,0 +1,77 @@
+#!/sbin/openrc-run
+# rds-agent — Spinifex RDS in-guest agent. Reaches the control plane over
+# TLS+SigV4 (never NATS), signing with IMDS instance-role credentials. Static
+# config comes from /etc/spinifex-rds/agent.env, which carries no secrets.
+
+description="Spinifex RDS instance agent"
+command="/usr/local/bin/rds-agent"
+command_background="yes"
+pidfile="/run/rds-agent.pid"
+output_log="/var/log/rds-agent.log"
+error_log="/var/log/rds-agent.log"
+
+start_stop_daemon_args="--wait 2000"
+
+AGENT_ENV="/etc/spinifex-rds/agent.env"
+
+# The agent retries indefinitely, so this only bounds how long the boot waits
+# before letting rds-init fail visibly rather than hanging forever.
+handoff_timeout="${RDS_HANDOFF_TIMEOUT:-120}"
+
+# How much of the agent log to mirror to the console when the handoff times out.
+# Enough to carry a few backed-off retries with their errors, not the whole boot.
+handoff_tail="${RDS_HANDOFF_TAIL:-20}"
+
+depend() {
+    need net
+    after localmount
+    before rds-datadir rds-init mariadb
+}
+
+start_pre() {
+    # An absent file is tolerated, so the agent still starts and logs why it
+    # cannot reach the gateway.
+    if [ -f "${AGENT_ENV}" ]; then
+        # shellcheck disable=SC1090
+        . "${AGENT_ENV}"
+        export RDS_GATEWAY_URL RDS_GATEWAY_CA RDS_REGION RDS_DB_INSTANCE_IDENTIFIER \
+            RDS_ENGINE RDS_ENGINE_VERSION RDS_HANDOFF_DIR RDS_ENGINE_HOST RDS_ENGINE_PORT \
+            RDS_POLL_WAIT RDS_DATA_MOUNT
+    fi
+}
+
+start_post() {
+    # Resolve this after start_pre loads agent.env so custom handoff directories
+    # are shared by the agent, this wait, and rds-init.
+    HANDOFF_ENV="${RDS_HANDOFF_DIR:-/run/spinifex-rds}/bootstrap.env"
+
+    # `before rds-init` only orders the starts; rds-init needs the fetched config
+    # itself. The agent writes bootstrap.env last and renames it into place, so
+    # its appearance means the whole handoff is there.
+    ebegin "Waiting for the RDS bootstrap handoff"
+    _waited=0
+    while [ ! -f "${HANDOFF_ENV}" ]; do
+        if [ "${_waited}" -ge "${handoff_timeout}" ]; then
+            eend 1 "no bootstrap handoff after ${handoff_timeout}s"
+            # Mirror the agent's own tail onto the console. The log lives inside a
+            # guest nothing can reach when this fails — an unreachable gateway is
+            # the usual cause — so the host-side console capture is the only way
+            # the dial error survives, and without it the failure reads as silence.
+            if [ -s "${output_log}" ]; then
+                einfo "last ${handoff_tail} lines of ${output_log}:"
+                tail -n "${handoff_tail}" "${output_log}" 2>/dev/null | while IFS= read -r _line; do
+                    einfo "  [rds-agent] ${_line}"
+                done
+            else
+                einfo "${output_log} is empty — the agent produced no output at all"
+            fi
+            # The agent keeps retrying, so it is left running: rds-init fails on
+            # the missing handoff and the engine stays down, which is what the
+            # control plane observes.
+            return 0
+        fi
+        sleep 1
+        _waited=$((_waited + 1))
+    done
+    eend 0
+}

scripts/images/rds-mariadb/rds-agent_initd_test.sh

@@ -0,0 +1,48 @@
+#!/bin/sh
+set -eu
+
+SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
+CASE=$(mktemp -d)
+trap 'rm -rf "${CASE}"' EXIT
+
+source_time_handoff="${CASE}/source-time"
+configured_handoff="${CASE}/configured"
+mkdir -p "${configured_handoff}"
+touch "${configured_handoff}/bootstrap.env"
+{
+    printf 'RDS_HANDOFF_DIR=%s\n' "${configured_handoff}"
+    printf 'RDS_ENGINE=mariadb\n'
+} >"${CASE}/agent.env"
+
+# Simulate OpenRC sourcing the service before start_pre loads agent.env.
+export RDS_HANDOFF_DIR="${source_time_handoff}"
+# shellcheck source=rds-agent.initd
+. "${SCRIPT_DIR}/rds-agent.initd"
+export AGENT_ENV="${CASE}/agent.env"
+export handoff_timeout=0
+
+EEND_STATUS=
+ebegin() { :; }
+einfo() { :; }
+eend() { EEND_STATUS=$1; }
+
+start_pre
+start_post
+
+if [ "${HANDOFF_ENV}" != "${configured_handoff}/bootstrap.env" ]; then
+    echo "FAIL: handoff path did not use RDS_HANDOFF_DIR from agent.env" >&2
+    exit 1
+fi
+if [ "${EEND_STATUS}" != "0" ]; then
+    echo "FAIL: post-start wait did not find the configured handoff" >&2
+    exit 1
+fi
+# The agent checks this against the engine its own image bakes. Left out of the
+# export list it would read as no assertion at all, so a VM launched as the
+# wrong engine would bootstrap instead of refusing.
+if [ "${RDS_ENGINE:-}" != "mariadb" ]; then
+    echo "FAIL: start_pre did not export RDS_ENGINE from agent.env" >&2
+    exit 1
+fi
+
+echo "rds-agent.initd: all tests passed"

scripts/images/rds-mariadb/rds-datadir.initd

@@ -0,0 +1,35 @@
+#!/sbin/openrc-run
+# rds-datadir — boot oneshot that discovers the hot-plugged data volume, formats
+# it if authorized, and mounts it at the datadir mount point. It runs after the
+# agent has written the authenticated handoff and before anything can use the
+# MariaDB datadir. Soft ordering keeps a failed boot visible on the console.
+
+description="Spinifex RDS data volume mount"
+
+AGENT_ENV="/etc/spinifex-rds/agent.env"
+
+depend() {
+    after localmount rds-agent
+    before rds-init mariadb
+}
+
+start_pre() {
+    # This image's layout, stated here because the mount script is shared by
+    # every engine preset. A delivered RDS_DATA_MOUNT still wins, so the control
+    # plane can move the mount point without rebuilding the image.
+    RDS_DATA_MOUNT="/var/lib/mysql"
+    RDS_ENGINE_USER="mysql"
+    if [ -f "${AGENT_ENV}" ]; then
+        # shellcheck disable=SC1090
+        . "${AGENT_ENV}"
+    fi
+    export RDS_HANDOFF_DIR RDS_DATA_MOUNT RDS_ENGINE_USER
+}
+
+start() {
+    ebegin "Mounting the RDS data volume"
+    # To the console for the same reason rds-init's is: the serial log is the
+    # only view of a boot that failed before the engine came up.
+    /usr/local/sbin/rds-datadir > /dev/console 2>&1
+    eend $?
+}

scripts/images/rds-mariadb/rds-engine-stamp_test.sh

@@ -0,0 +1,62 @@
+#!/bin/sh
+set -eu
+
+# The engine this preset bakes, and the series it pins, are stated in places that
+# have to agree: the stamp setup.sh writes, the manifest tags resolveEngineAMI
+# selects on, the apk pin the packages are installed from, and the engine value
+# the control plane launches instances with. A drift between them launches the
+# wrong image, refuses the right one, or reports a version the VM is not running.
+#
+# The agent's layout table is checked by the postgres preset's copy of this test
+# only, until this engine has a guest implementation to key it by.
+
+SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
+REPO_ROOT=$(CDPATH='' cd -- "${SCRIPT_DIR}/../../.." && pwd)
+ENGINE="mariadb"
+ENGINE_GO="${REPO_ROOT}/spinifex/handlers/rds/engine_mariadb.go"
+
+FAILS=0
+fail() { echo "FAIL: $*" >&2; FAILS=$((FAILS + 1)); }
+
+stamped=$(sed -n "s|^printf '\\(.*\\)\\\\n' > /etc/spinifex-rds/engine\$|\\1|p" "${SCRIPT_DIR}/setup.sh")
+if [ "${stamped}" != "${ENGINE}" ]; then
+    fail "setup.sh stamps engine '${stamped}', want '${ENGINE}'"
+fi
+
+if ! grep -q "engine=${ENGINE} " "${SCRIPT_DIR}/manifest.conf"; then
+    fail "manifest.conf SYSTEM_TAG does not carry engine=${ENGINE}"
+    grep '^SYSTEM_TAG' "${SCRIPT_DIR}/manifest.conf" >&2
+fi
+
+if ! grep -q "Name: *\"${ENGINE}\"" "${ENGINE_GO}"; then
+    fail "the control plane has no engine named ${ENGINE}"
+fi
+
+if ! grep -q "^ENGINE=\"${ENGINE}\"\$" "${SCRIPT_DIR}/rds-init"; then
+    fail "rds-init does not stamp the data volume '${ENGINE}'"
+    grep '^ENGINE=' "${SCRIPT_DIR}/rds-init" >&2
+fi
+
+# The tag is what an instance's recorded EngineVersion resolves an AMI by, so it
+# has to be the same series the control plane pins and the packages install.
+version=$(sed -n 's/^[[:space:]]*MajorVersion: *"\([^"]*\)".*/\1/p' "${ENGINE_GO}")
+if [ -z "${version}" ]; then
+    fail "could not read MajorVersion from ${ENGINE_GO}"
+else
+    if ! grep -q "engine-version=${version} " "${SCRIPT_DIR}/manifest.conf"; then
+        fail "manifest.conf SYSTEM_TAG does not carry engine-version=${version}"
+        grep '^SYSTEM_TAG' "${SCRIPT_DIR}/manifest.conf" >&2
+    fi
+    # Alpine's package name carries no version, so this fuzzy pin is the only
+    # thing standing between a rebuild and a silent series bump.
+    if ! grep -q "APK_PACKAGES=.*${ENGINE}=~${version} " "${SCRIPT_DIR}/manifest.conf"; then
+        fail "manifest.conf does not pin the ${ENGINE} packages to =~${version}"
+        grep '^APK_PACKAGES' "${SCRIPT_DIR}/manifest.conf" >&2
+    fi
+fi
+
+if [ "${FAILS}" -eq 0 ]; then
+    echo "rds-engine-stamp: all tests passed"
+    exit 0
+fi
+exit 1

scripts/images/rds-mariadb/rds-init

@@ -0,0 +1,903 @@
+#!/bin/sh
+# rds-init — MariaDB bootstrap for the spinifex-rds-mariadb AMI.
+#
+# A boot oneshot ordered before the mariadb service. This is the MariaDB half of
+# the engine seam: mariadb-install-db, the master grant, the datadir and the 3306
+# default live here and nowhere else in the platform.
+#
+# Handoff contract, written by rds-agent into a root-owned tmpfs directory:
+#
+#   /run/spinifex-rds/bootstrap.env    0600  KEY=value shell fragment:
+#       RDS_MODE=initialize|attach
+#       RDS_DB_INSTANCE_IDENTIFIER=<the DB instance this VM backs>
+#       RDS_MASTER_USERNAME=<master user name>
+#       RDS_MASTER_PASSWORD=<present only when RDS_MODE=initialize>
+#       RDS_DB_NAME=<initial database; optional, as on AWS>
+#       RDS_PORT=<listen port; defaults to 3306>
+#       RDS_DATA_VOLUME_ID / RDS_DATA_VOLUME_SERIAL=<expected volume identity>
+#       RDS_VM_GENERATION / RDS_FORMAT_AUTHORIZED=<create-only format grant>
+#       RDS_BOOTSTRAP_PENDING / RDS_PAYLOAD_ID=<the control plane is still
+#                                            waiting for this payload to be applied>
+#   /run/spinifex-rds/parameters.conf  0600  resolved parameter-group values, one
+#                                            name = 'value' per line
+#   /run/spinifex-rds/server.crt       0600  serving cert, cluster-CA-signed
+#   /run/spinifex-rds/server.key       0600  its private key
+#
+# Nothing there survives a reboot: the next boot re-fetches and the cert is
+# re-minted rather than persisted.
+#
+# The data volume also carries an engine stamp, written before the datadir is
+# inspected and compared on every boot, so another engine's volume is refused
+# rather than initialised over.
+#
+# rds-init keys off the datadir, never off RDS_MODE. A datadir holding the mysql
+# system database means skip mariadb-install-db and only refresh config, so a
+# replaced VM comes back on its existing data volume without re-bootstrapping.
+# RDS_BOOTSTRAP_PENDING is not an exception: it is a control-plane-resolved
+# precondition compared against on-disk evidence, the same shape as
+# RDS_FORMAT_AUTHORIZED.
+set -eu
+
+# The engine this image bakes, stamped onto the data volume and compared against
+# it on every boot. Held here rather than read from /etc/spinifex-rds/engine:
+# this script is the MariaDB half of the seam, so it is the same constant.
+ENGINE="mariadb"
+
+MARIADB_BIN="${RDS_MARIADB_BIN:-/usr/bin}"
+ENGINE_USER="${RDS_ENGINE_USER:-mysql}"
+
+# Mount point of the data volume, and the datadir one level inside it. The
+# datadir is deliberately not the mount point itself: a failed bootstrap clears
+# it, and the include directory, the engine stamp and the completion receipt all
+# have to outlive that. It also keeps lost+found and the include directory out of
+# a directory whose every subdirectory MariaDB lists as a database.
+DATA_MOUNT="${RDS_DATA_MOUNT:-/var/lib/mysql}"
+DATADIR="${RDS_DATADIR:-${DATA_MOUNT}/data}"
+
+# Read through the image's baked /etc/my.cnf.d/zz-rds-include.cnf. Generated
+# configuration lives here rather than under /etc because a class change is a VM
+# replacement — fresh boot volume, same data volume — and parameters under /etc
+# would be discarded by it while the API still reported the customer's values.
+CONF_DIR="${RDS_CONF_DIR:-${DATA_MOUNT}/conf.d}"
+PARAM_FILE="${CONF_DIR}/10-rds-parameters.cnf"
+# What the server actually started on. The agent computes pending-restart by
+# comparing the installed file against this one, so writing it here — before the
+# engine starts — is what makes the two equal by construction on every boot.
+# Deliberately not a .cnf name: !includedir reads *.cnf and this is comparison
+# material, not a second set of settings.
+SERVING_FILE="${CONF_DIR}/10-rds-parameters.serving"
+PLATFORM_FILE="${CONF_DIR}/90-rds-init.cnf"
+
+# Written between mariadb-install-db and a working master user. It lives on the
+# data volume because no trap survives a crash or a hard kill, and it is what
+# stops the next boot attaching to a datadir whose master user was never created.
+BOOTSTRAP_SENTINEL="${DATADIR}/rds-bootstrap-incomplete"
+
+# Proof that the master user was durably applied, written once the datadir and
+# the user are both done. Deliberately outside DATADIR, which
+# clear_unfinished_datadir rm -rf's.
+RECEIPT_DIR="${RDS_RECEIPT_DIR:-${DATA_MOUNT}/.spinifex-rds/bootstrap}"
+RECEIPT_FILE="${RECEIPT_DIR}/receipt.env"
+
+# Which engine's data this volume holds. Outside DATADIR for the same reason the
+# receipt is, and beside it rather than inside the bootstrap directory: it
+# outlives any one bootstrap and is read before the datadir is ever inspected.
+ENGINE_STAMP="${RDS_ENGINE_STAMP:-${DATA_MOUNT}/.spinifex-rds/engine}"
+
+HANDOFF_DIR="${RDS_HANDOFF_DIR:-/run/spinifex-rds}"
+ENV_FILE="${HANDOFF_DIR}/bootstrap.env"
+
+SOCKET_DIR="${RDS_SOCKET_DIR:-/run/mysqld}"
+SOCKET="${SOCKET_DIR}/mysqld.sock"
+TLS_DIR="${RDS_TLS_DIR:-${SOCKET_DIR}/tls}"
+LOG_DIR="${RDS_LOG_DIR:-/var/log/mysql}"
+
+# The bootstrap server listens on a unix socket in this directory and nowhere
+# else. It is 0700 and mysql-owned, so only mysql and root can reach the short
+# window before the master user exists.
+BOOTSTRAP_DIR="${RDS_BOOTSTRAP_DIR:-/run/rds-init}"
+BOOTSTRAP_SOCKET="${BOOTSTRAP_DIR}/mysqld.sock"
+BOOTSTRAP_PID="${BOOTSTRAP_DIR}/mysqld.pid"
+
+# secure_file_priv names the one directory FILE-privileged statements may read
+# and write. It must be a real directory: the empty string means no restriction
+# at all. Root-owned and empty, on the boot volume, so it stays empty.
+SECURE_FILE_DIR="${RDS_SECURE_FILE_DIR:-/var/lib/mysql-files}"
+
+MOUNTS_FILE="${RDS_MOUNTS_FILE:-/proc/mounts}"
+
+# How long the private bootstrap server gets to open its socket, as probes one
+# BOOTSTRAP_POLL apart. InnoDB creates its redo log and system tablespace on this
+# first start, so it is not instant.
+BOOTSTRAP_PROBES="${RDS_BOOTSTRAP_PROBES:-60}"
+BOOTSTRAP_POLL="${RDS_BOOTSTRAP_POLL:-1}"
+# Failure cleanup gives the private server this long to honor TERM before KILL.
+BOOTSTRAP_STOP_GRACE="${RDS_BOOTSTRAP_STOP_GRACE:-10}"
+
+# The server truncates a longer name, so the database created would not be the
+# one the customer asked for, and MariaDB maps the name onto a directory name.
+MAX_DB_NAME_LEN=64
+MAX_USERNAME_LEN=80
+MANAGEMENT_DB="_spinifex_rds"
+CREATE_DATABASE_PROCEDURE="create_database"
+
+log() { echo "[rds-init] $*"; }
+fail() { echo "[rds-init] ERROR: $*" >&2; exit 1; }
+
+# Between these two, the datadir carries a mysql system database but no usable
+# master user. _server_pid is the private bootstrap server, if one is running.
+_created_datadir=0
+_bootstrapped=0
+_server_pid=""
+
+# Runs from a trap rather than a statement, so neither `set -e` on an unguarded
+# step between mariadb-install-db and a working master user nor a signal can skip
+# it — either would leave a healthy-looking engine with no usable master user,
+# unrepairable.
+clear_unfinished_datadir() {
+    [ "${_created_datadir}" = "1" ] || return 0
+    [ "${_bootstrapped}" = "1" ] && return 0
+    # A server left holding a datadir that is about to be deleted keeps its
+    # socket and pidfile and blocks the mariadb service for the life of the boot.
+    kill_bootstrap_server
+    # The receipt is installed before the traps retire, so a sweep in between
+    # would leave one vouching for a datadir that is about to be deleted. The
+    # agent would acknowledge it and the control plane would destroy the only
+    # copy of the password this instance can ever be initialised with.
+    rm -f -- "${RECEIPT_FILE}"
+    if rm -rf -- "${DATADIR}" && [ ! -e "${DATADIR}" ]; then
+        echo "[rds-init] ERROR: bootstrap did not complete; datadir ${DATADIR} cleared" >&2
+        return 0
+    fi
+    echo "[rds-init] ERROR: bootstrap did not complete and datadir ${DATADIR} could not be cleared; recover it out of band before retrying" >&2
+}
+
+kill_bootstrap_server() {
+    [ -n "${_server_pid}" ] || return 0
+    _pid="${_server_pid}"
+
+    # wait has no timeout in POSIX sh. A watchdog bounds it and escalates, while
+    # this shell remains the one that waits and reaps its own child.
+    kill -TERM "${_pid}" 2>/dev/null || true
+    (
+        sleep "${BOOTSTRAP_STOP_GRACE}"
+        kill -KILL "${_pid}" 2>/dev/null || true
+    ) &
+    _watchdog_pid=$!
+
+    # A signal exit is expected here: cleanup cares that the process is reaped
+    # and cannot touch the datadir, not whether its incomplete startup was clean.
+    wait "${_pid}" 2>/dev/null || true
+    _server_pid=""
+    kill -TERM "${_watchdog_pid}" 2>/dev/null || true
+    wait "${_watchdog_pid}" 2>/dev/null || true
+}
+
+load_handoff() {
+    [ -f "${ENV_FILE}" ] || fail "no bootstrap handoff at ${ENV_FILE} — rds-agent has not delivered the bootstrap config"
+
+    # shellcheck disable=SC1090
+    . "${ENV_FILE}"
+
+    RDS_MODE="${RDS_MODE:-}"
+    RDS_DB_INSTANCE_IDENTIFIER="${RDS_DB_INSTANCE_IDENTIFIER:-}"
+    RDS_MASTER_USERNAME="${RDS_MASTER_USERNAME:-}"
+    RDS_MASTER_PASSWORD="${RDS_MASTER_PASSWORD:-}"
+    RDS_DB_NAME="${RDS_DB_NAME:-}"
+    RDS_PORT="${RDS_PORT:-3306}"
+    RDS_VM_GENERATION="${RDS_VM_GENERATION:-}"
+    # Absent from an agent that predates the receipt protocol, which reads as
+    # "nothing is pending" and leaves every path below unchanged.
+    RDS_BOOTSTRAP_PENDING="${RDS_BOOTSTRAP_PENDING:-0}"
+    RDS_PAYLOAD_ID="${RDS_PAYLOAD_ID:-}"
+
+    [ -n "${RDS_MASTER_USERNAME}" ] || fail "bootstrap config has no RDS_MASTER_USERNAME"
+
+    # The name is interpolated into CREATE USER by the shell — the mariadb client
+    # has no identifier quoting of its own — so the rule the control plane
+    # applies is enforced here rather than trusted.
+    case "${RDS_MASTER_USERNAME}" in
+        [!A-Za-z]* | *[!A-Za-z0-9_]*)
+            fail "RDS_MASTER_USERNAME ${RDS_MASTER_USERNAME} must begin with a letter and hold only letters, digits and underscores; the control plane should have rejected it"
+            ;;
+    esac
+    if [ "${#RDS_MASTER_USERNAME}" -gt "${MAX_USERNAME_LEN}" ]; then
+        fail "RDS_MASTER_USERNAME is longer than the ${MAX_USERNAME_LEN} characters MariaDB allows; the control plane should have rejected it"
+    fi
+
+    # root is the superuser mariadb-install-db creates and rds-init keeps for
+    # itself, mysql is the unix_socket account beside it, mariadb.sys owns the sys
+    # schema, rdsadmin is reserved as on AWS, and PUBLIC is a role name the server
+    # refuses to create a user under. Matched the same way the control plane
+    # matches them: anything it refuses must be refused here too, or the name
+    # reaches mariadb-install-db, fails at CREATE USER and spends the one-shot
+    # password on the way.
+    _master_lower=$(printf '%s' "${RDS_MASTER_USERNAME}" | tr '[:upper:]' '[:lower:]')
+    case "${_master_lower}" in
+        root | mysql | mariadb.sys | rdsadmin | public | mysql.* | mariadb.*)
+            fail "RDS_MASTER_USERNAME ${RDS_MASTER_USERNAME} is reserved by the platform; the control plane should have rejected it"
+            ;;
+    esac
+
+    # The same rule the control plane applies. Here it is the barrier rather than
+    # defence in depth: the mariadb client has no identifier interpolation, so the
+    # name is built into CREATE DATABASE by the shell, and MariaDB maps a database
+    # name onto a directory name — which makes /, \, . and a trailing space
+    # structurally dangerous rather than merely awkward.
+    case "${RDS_DB_NAME}" in
+        '') ;;
+        [!A-Za-z]* | *[!A-Za-z0-9_]*)
+            fail "RDS_DB_NAME ${RDS_DB_NAME} must begin with a letter and hold only letters, digits and underscores; the control plane should have rejected it"
+            ;;
+    esac
+    if [ "${#RDS_DB_NAME}" -gt "${MAX_DB_NAME_LEN}" ]; then
+        fail "RDS_DB_NAME is longer than the ${MAX_DB_NAME_LEN} characters MariaDB allows; the control plane should have rejected it"
+    fi
+
+    case "${RDS_PORT}" in
+        '' | *[!0-9]*) fail "bootstrap config has a non-numeric RDS_PORT: ${RDS_PORT}" ;;
+    esac
+
+    # redact_stream is line-oriented, so a password spanning a line boundary
+    # would match neither half and reach /dev/console in the clear. The control
+    # plane rejects one; refusing here too keeps that an asserted property of
+    # the handoff rather than a silent dependency on another service.
+    case "${RDS_MASTER_PASSWORD}" in
+        *'
+'*) fail "the master password in the bootstrap config spans more than one line; the control plane should have rejected it" ;;
+    esac
+
+    # The form the password takes inside the SQL, computed once beside it. A
+    # statement echoed on failure carries that form and not the raw secret, so
+    # redacting only the raw one would put a trivially unescaped password on the
+    # console for exactly the passwords that needed escaping.
+    RDS_MASTER_PASSWORD_SQL=""
+    if [ -n "${RDS_MASTER_PASSWORD}" ]; then
+        RDS_MASTER_PASSWORD_SQL=$(sql_literal "${RDS_MASTER_PASSWORD}")
+    fi
+
+    log "bootstrap config loaded (mode=${RDS_MODE:-unset}, port=${RDS_PORT})"
+}
+
+# Every replace boots a fresh clone of this AMI, so a datadir off the data volume
+# disappears at the next class change or recovery — silently, as the engine looks
+# fine.
+require_data_volume() {
+    if awk -v mnt="${DATA_MOUNT}" '$2 == mnt { found = 1 } END { exit !found }' "${MOUNTS_FILE}"; then
+        return 0
+    fi
+    [ "${RDS_ALLOW_LOCAL_DATADIR:-0}" = "1" ] ||
+        fail "${DATA_MOUNT} is not a mount point — the data volume is not attached; refusing to initialise on the boot volume"
+    log "warning: ${DATA_MOUNT} is not a mount point; data will not survive a VM replace (RDS_ALLOW_LOCAL_DATADIR=1)"
+}
+
+# True when the data volume holds anything that is not ours. The mount is what
+# is inspected rather than ${DATADIR}: another engine lays its data out under its
+# own name, so a PostgreSQL volume has content here while ${DATADIR} does not
+# exist at all.
+data_volume_has_content() {
+    [ -d "${DATA_MOUNT}" ] || return 1
+
+    # A mount that cannot be read is neither empty nor not: it is a refusal,
+    # because the caller decides from the answer whether to initialise.
+    ls -A -- "${DATA_MOUNT}" >/dev/null 2>&1 ||
+        fail "could not inspect ${DATA_MOUNT}; refusing to decide whether the data volume is ours"
+
+    # Globbed rather than split out of `ls`, so an entry carrying whitespace
+    # counts once. lost+found is the filesystem's, and the metadata and include
+    # directories are ours — none of the three says whose data the volume holds.
+    for _entry in "${DATA_MOUNT}"/* "${DATA_MOUNT}"/.[!.]* "${DATA_MOUNT}"/..?*; do
+        [ -e "${_entry}" ] || [ -L "${_entry}" ] || continue
+        case "${_entry##*/}" in
+            lost+found | .spinifex-rds | "${CONF_DIR##*/}") continue ;;
+        esac
+        return 0
+    done
+    return 1
+}
+
+# Written before the datadir is inspected, so a volume this image goes on to
+# initialise carries the stamp from the first boot onwards. Fatal on failure:
+# an unstamped volume is one the next boot cannot tell from another engine's.
+write_engine_stamp() {
+    _stamp_dir=$(dirname "${ENGINE_STAMP}")
+    install -d -m 0700 -o "${ENGINE_USER}" -g "${ENGINE_USER}" "${_stamp_dir}" ||
+        fail "the datadir engine stamp directory ${_stamp_dir} could not be created; refusing to use a data volume whose engine cannot be recorded"
+
+    _tmp="${ENGINE_STAMP}.new"
+    printf '%s\n' "${ENGINE}" > "${_tmp}" ||
+        fail "the datadir engine stamp ${ENGINE_STAMP} could not be written; refusing to use a data volume whose engine cannot be recorded"
+    chown "${ENGINE_USER}:${ENGINE_USER}" "${_tmp}"
+    chmod 0600 "${_tmp}"
+
+    # Renamed rather than written in place, so a crash leaves either the old
+    # stamp or none — and none is decided on the datadir, where a torn one would
+    # be fatal.
+    sync "${_tmp}" 2>/dev/null || sync
+    mv -f "${_tmp}" "${ENGINE_STAMP}" ||
+        fail "the datadir engine stamp ${ENGINE_STAMP} could not be installed; refusing to use a data volume whose engine cannot be recorded"
+    sync "${_stamp_dir}" 2>/dev/null || sync
+}
+
+# The volume mounts cleanly whichever engine wrote it, and another engine's
+# datadir reads as uninitialised here: rds-init would run mariadb-install-db
+# beside the customer's data and serve an empty database that passes every health
+# probe.
+require_engine_stamp() {
+    if [ -e "${ENGINE_STAMP}" ]; then
+        _stamped=""
+        _stamp_read=0
+        _stamped=$(head -n 1 -- "${ENGINE_STAMP}") && _stamp_read=1
+        [ "${_stamp_read}" = "1" ] ||
+            fail "the datadir engine stamp ${ENGINE_STAMP} exists but could not be read; refusing to attach to a data volume whose engine cannot be established"
+        if [ "${_stamped}" != "${ENGINE}" ]; then
+            fail "the data volume holds a '${_stamped}' datadir but this image runs ${ENGINE}; refusing to touch it. Attach it to a '${_stamped}' instance, or recover the volume out of band."
+        fi
+        log "data volume is stamped ${ENGINE}"
+        return 0
+    fi
+
+    # No MariaDB volume in the field predates the stamp, so unlike PostgreSQL
+    # there is nothing to backfill onto: an unstamped volume with content on it
+    # is by definition not ours, and stamping it would turn the check into a
+    # rubber stamp on exactly the volume it exists to refuse.
+    if data_volume_has_content; then
+        fail "${DATA_MOUNT} holds data but the volume carries no ${ENGINE} engine stamp; refusing to touch it. Recover the volume out of band."
+    fi
+
+    log "data volume carries no engine stamp — recording ${ENGINE}"
+    write_engine_stamp
+}
+
+# Creates the system tables in an empty datadir. Root authenticates through the
+# unix_socket plugin — the direct analogue of PostgreSQL's peer auth, and the
+# reason no bootstrap password is needed.
+initialise_datadir() {
+    # An empty datadir in `attach` mode means the data volume did not come back.
+    # Initialising there would report success and hand the customer an empty
+    # database in place of theirs.
+    if [ "${RDS_MODE}" = "attach" ]; then
+        fail "datadir ${DATADIR} holds no MariaDB system database but the bootstrap config is Mode=attach — the existing data volume is missing; refusing to create an empty database"
+    fi
+    [ -n "${RDS_MASTER_PASSWORD}" ] || fail "Mode=${RDS_MODE:-unset} carries no master password; cannot initialise the datadir"
+
+    install -d -m 0700 -o "${ENGINE_USER}" -g "${ENGINE_USER}" "${DATADIR}"
+
+    # Whether the datadir was empty beforehand gates the failure cleanup below.
+    # A torn write can leave the mysql system database missing over intact user
+    # data, and deleting then would destroy it in response to the one signal that
+    # it is still there.
+    _was_empty=0
+    if _contents=$(ls -A -- "${DATADIR}" 2>/dev/null); then
+        [ -z "${_contents}" ] && _was_empty=1
+    else
+        fail "could not inspect datadir ${DATADIR}; refusing to decide whether failure cleanup is safe"
+    fi
+
+    log "initialising the MariaDB datadir at ${DATADIR}"
+    # --no-defaults: the installed configuration includes the customer's own
+    # parameter file, and a static value this server will not start under would
+    # spend the one-shot master password on a datadir that never gets one. The
+    # real server starts on those parameters, where a rollback exists for them.
+    if ! "${MARIADB_BIN}/mariadb-install-db" --no-defaults \
+        --user="${ENGINE_USER}" --datadir="${DATADIR}" \
+        --auth-root-authentication-method=socket --skip-test-db; then
+        # A half-written datadir carries a mysql directory and would look
+        # initialised forever after, but only what this invocation created may be
+        # cleared.
+        if [ "${_was_empty}" = "1" ]; then
+            if rm -rf -- "${DATADIR}" && [ ! -e "${DATADIR}" ]; then
+                fail "mariadb-install-db failed; datadir cleared"
+            fi
+            fail "mariadb-install-db failed and datadir ${DATADIR} could not be cleared; recover it out of band before retrying"
+        fi
+        fail "mariadb-install-db failed and ${DATADIR} was not empty beforehand — refusing to clear it, since the contents may be a datadir whose system tables were lost; recover the data volume out of band"
+    fi
+
+    # Only a datadir built over an empty directory may be swept later.
+    _created_datadir="${_was_empty}"
+    trap clear_unfinished_datadir EXIT
+    # An EXIT trap does not run on a signal in a POSIX shell, so the same sweep
+    # is armed for the signals a shutdown, an ACPI powerdown or a host-side
+    # force-stop delivers. EXIT is disarmed first so the sweep runs once.
+    trap 'trap - EXIT; clear_unfinished_datadir; exit 1' INT TERM HUP
+
+    # No trap survives a crash or a SIGKILL, so the incomplete state is also
+    # recorded on the volume itself. Cleared once the master user exists.
+    printf '%s\n' \
+        'rds-init created this datadir and did not finish bootstrapping the master user.' \
+        'While this file exists rds-init refuses to start the engine: the datadir carries no' \
+        'usable master user, so it must not be attached to. Recover out of band.' \
+        > "${BOOTSTRAP_SENTINEL}"
+    chown "${ENGINE_USER}:${ENGINE_USER}" "${BOOTSTRAP_SENTINEL}"
+    chmod 0600 "${BOOTSTRAP_SENTINEL}"
+}
+
+# The resolved parameter group, prefixed with the group header. MariaDB parses
+# each file in an include directory on its own and treats a setting before any
+# group as a fatal error, so the header belongs to whoever writes the file rather
+# than to the engine-neutral renderer that produced its body.
+install_parameters() {
+    if [ -f "${HANDOFF_DIR}/parameters.conf" ]; then
+        _tmp="${CONF_DIR}/.10-rds-parameters.cnf.new"
+        { printf '[mysqld]\n'; cat "${HANDOFF_DIR}/parameters.conf"; } > "${_tmp}" ||
+            fail "the resolved parameter group could not be written to ${PARAM_FILE}"
+        chown "${ENGINE_USER}:${ENGINE_USER}" "${_tmp}"
+        chmod 0600 "${_tmp}"
+        mv -f "${_tmp}" "${PARAM_FILE}"
+        record_serving_parameters
+        return 0
+    fi
+    # A handoff without parameters keeps the previous include: dropping it would
+    # silently reset the instance to engine defaults.
+    if [ -f "${PARAM_FILE}" ]; then
+        log "warning: no parameters in the handoff — keeping the parameters already installed"
+        record_serving_parameters
+    fi
+}
+
+# The engine is about to start on whatever is installed, so the two are equal by
+# construction here. The agent refreshes this copy only after a restart it
+# performs itself: a live SET GLOBAL apply deliberately leaves it alone, since
+# that value was adopted without a restart, which is what "not pending" means.
+record_serving_parameters() {
+    install -m 0600 -o "${ENGINE_USER}" -g "${ENGINE_USER}" "${PARAM_FILE}" "${SERVING_FILE}" ||
+        fail "the serving copy of the parameters could not be written to ${SERVING_FILE}"
+}
+
+# The key stays on tmpfs: it is re-minted every fetch, so persisting it would
+# spread a private key into every snapshot.
+install_tls() {
+    if [ -f "${HANDOFF_DIR}/server.crt" ] && [ -f "${HANDOFF_DIR}/server.key" ]; then
+        install -d -m 0700 -o "${ENGINE_USER}" -g "${ENGINE_USER}" "${TLS_DIR}"
+        install -m 0600 -o "${ENGINE_USER}" -g "${ENGINE_USER}" "${HANDOFF_DIR}/server.crt" "${TLS_DIR}/server.crt"
+        install -m 0600 -o "${ENGINE_USER}" -g "${ENGINE_USER}" "${HANDOFF_DIR}/server.key" "${TLS_DIR}/server.key"
+        rm -f "${HANDOFF_DIR}/server.crt" "${HANDOFF_DIR}/server.key"
+        SSL_MODE="on"
+        return 0
+    fi
+    # A re-run after the cert was consumed must not turn TLS off underneath
+    # clients connecting with --ssl-verify-server-cert.
+    if [ -f "${TLS_DIR}/server.crt" ] && [ -f "${TLS_DIR}/server.key" ]; then
+        SSL_MODE="on"
+        return 0
+    fi
+    log "warning: no serving cert in the handoff — the engine will start without TLS"
+    SSL_MODE="off"
+}
+
+# The 90- prefix sorts last within the include directory and MariaDB takes the
+# last occurrence of a setting, so a resolved parameter can never move the
+# endpoint's port, redirect the datadir or turn off TLS.
+#
+# innodb_buffer_pool_chunk_size and innodb_buffer_pool_instances are deliberately
+# absent although they are platform-owned: 11.8 removed the second outright, so
+# naming it is a startup failure rather than an override, and the first is
+# deprecated with no effect. The granularity they used to pin is fixed by the
+# server itself in this series.
+write_platform_config() {
+    _ssl="$1"
+    _tmp="${CONF_DIR}/.90-rds-init.cnf.new"
+    cat > "${_tmp}" <<EOF
+# Managed by rds-init — regenerated on every boot from the bootstrap config.
+# Edits here are lost; instance parameters belong in the parameter group.
+[mysqld]
+port = ${RDS_PORT}
+datadir = ${DATADIR}
+socket = ${SOCKET}
+bind_address = 0.0.0.0
+
+# InnoDB is the only storage engine the snapshot guarantee holds for: it keeps a
+# redo log and recovers from a crash-consistent snapshot, and Aria and MyISAM do
+# not. An explicit ENGINE= clause still works, which keeps the exception
+# deliberate and per-table rather than account-wide and invisible.
+default_storage_engine = InnoDB
+
+# Point-in-time recovery is not offered, so the binary log is off: it would cost
+# write throughput and disk for a capability nothing reads.
+skip-log-bin
+
+# Defence in depth behind the master grant, which withholds FILE: an empty
+# secure_file_priv would mean no restriction at all, so it names a root-owned
+# directory that stays empty.
+secure_file_priv = ${SECURE_FILE_DIR}
+skip-symbolic-links
+EOF
+    if [ "${_ssl}" = "on" ]; then
+        # Offered, not enforced, matching the PostgreSQL image's pg_hba: clients
+        # opt in. require_secure_transport is deliberately absent until enforcement
+        # lands for both engines at once. No ssl_ca, because client certificates
+        # are not required and it would only gate those.
+        cat >> "${_tmp}" <<EOF
+ssl_cert = ${TLS_DIR}/server.crt
+ssl_key = ${TLS_DIR}/server.key
+EOF
+    fi
+    chown "${ENGINE_USER}:${ENGINE_USER}" "${_tmp}"
+    chmod 0600 "${_tmp}"
+    mv -f "${_tmp}" "${PLATFORM_FILE}"
+}
+
+# Everything refreshed on every boot: the resolved parameters and their serving
+# copy, the serving cert and the platform's own settings.
+install_runtime_config() {
+    install -d -m 0750 -o "${ENGINE_USER}" -g "${ENGINE_USER}" "${CONF_DIR}"
+    install -d -m 0755 -o "${ENGINE_USER}" -g "${ENGINE_USER}" "${SOCKET_DIR}"
+    install -d -m 0750 -o "${ENGINE_USER}" -g "${ENGINE_USER}" "${LOG_DIR}"
+    install -d -m 0700 "${SECURE_FILE_DIR}"
+
+    install_parameters
+    install_tls
+    write_platform_config "${SSL_MODE}"
+}
+
+# Brings the server up on a private socket with no TCP listener at all, so the
+# master password can be set before the engine is reachable. --no-defaults for
+# the same reason mariadb-install-db has it, and because the password below is
+# escaped for a known sql_mode rather than the customer's.
+start_bootstrap_server() {
+    install -d -m 0700 -o "${ENGINE_USER}" -g "${ENGINE_USER}" "${BOOTSTRAP_DIR}"
+
+    "${MARIADB_BIN}/mariadbd" --no-defaults \
+        --user="${ENGINE_USER}" --datadir="${DATADIR}" \
+        --skip-networking --socket="${BOOTSTRAP_SOCKET}" \
+        --pid-file="${BOOTSTRAP_PID}" --log-error="${LOG_DIR}/rds-init.log" &
+    _server_pid=$!
+
+    _probes=0
+    while ! bootstrap_admin ping > /dev/null 2>&1; do
+        # The server writes why it gave up to its error log, which is on the boot
+        # volume and readable after the fact; the console only gets the refusal.
+        if ! kill -0 "${_server_pid}" 2>/dev/null; then
+            _server_pid=""
+            fail "the bootstrap server exited before it opened ${BOOTSTRAP_SOCKET}; see ${LOG_DIR}/rds-init.log"
+        fi
+        if [ "${_probes}" -ge "${BOOTSTRAP_PROBES}" ]; then
+            fail "the bootstrap server did not open ${BOOTSTRAP_SOCKET} after ${BOOTSTRAP_PROBES} probes ${BOOTSTRAP_POLL}s apart; see ${LOG_DIR}/rds-init.log"
+        fi
+        sleep "${BOOTSTRAP_POLL}"
+        _probes=$((_probes + 1))
+    done
+}
+
+bootstrap_admin() {
+    "${MARIADB_BIN}/mariadb-admin" --no-defaults \
+        --socket="${BOOTSTRAP_SOCKET}" --user=root "$@"
+}
+
+# Returns the shutdown's status, then the server's own: a caller reading this
+# must be able to tell "it refused to stop" from "it stopped badly", and both
+# from a server still holding the datadir.
+stop_bootstrap_server() {
+    [ -n "${_server_pid}" ] || return 0
+    bootstrap_admin shutdown || return 1
+
+    # `wait` rather than a kill -0 poll: the server is this shell's own child, so
+    # until it is reaped it stays a zombie that kill -0 reports as alive.
+    _wait_rc=0
+    wait "${_server_pid}" || _wait_rc=$?
+    _server_pid=""
+    return "${_wait_rc}"
+}
+
+# Prepended to every bootstrap script. The two logs that would copy the password
+# out of the statement are turned off — global rather than session variables, and
+# safe to set globally because nothing but rds-init can reach this server. The
+# mode is pinned because the password is escaped below for a mode that has
+# NO_BACKSLASH_ESCAPES off.
+BOOTSTRAP_SQL_GUARD="SET GLOBAL general_log = 0;
+SET GLOBAL slow_query_log = 0;
+SET SESSION sql_mode = 'NO_ENGINE_SUBSTITUTION';"
+
+# Replaces every occurrence of the secret with a placeholder. Parameter
+# expansion rather than sed, whose script would carry the secret in an argv any
+# process on the guest can read.
+redact_stream() {
+    _secret="$1"
+    while IFS= read -r _line || [ -n "${_line}" ]; do
+        _out=""
+        while [ -n "${_secret}" ]; do
+            case "${_line}" in
+                *"${_secret}"*)
+                    _out="${_out}${_line%%"${_secret}"*}[REDACTED]"
+                    _line="${_line#*"${_secret}"}"
+                    ;;
+                *) break ;;
+            esac
+        done
+        printf '%s\n' "${_out}${_line}"
+    done
+}
+
+# A value safe to interpolate into a single-quoted SQL literal. The quote is
+# doubled, which holds in every sql_mode, and the backslash is doubled, which is
+# why the guard above pins one where backslash escapes are on. The secret reaches
+# sed on stdin: an argv would be readable by every process in the guest.
+sql_literal() {
+    printf '%s' "$1" | sed -e 's/\\/\\\\/g' -e "s/'/''/g"
+}
+
+# The script rides stdin and never an argv, and both streams are redacted before
+# they reach the caller: the client echoes the statement it failed on, and the
+# initd sends everything here to /dev/console, which the host captures off ttyS0.
+mariadb_bootstrap() {
+    # A pipeline's status is its last command's and POSIX sh has no pipefail, so
+    # the client's own status leaves the subshell through the filesystem. The
+    # guard shifts the client's reported line numbers by the lines it adds.
+    _status_file="${BOOTSTRAP_DIR}/client.status"
+    rm -f "${_status_file}"
+    {
+        _client_rc=0
+        # --batch stops at the first error and exits non-zero, so a failed
+        # statement cannot leave the rest of the script running.
+        { printf '%s\n' "${BOOTSTRAP_SQL_GUARD}"; cat; } |
+            "${MARIADB_BIN}/mariadb" --no-defaults --batch --skip-column-names \
+                --socket="${BOOTSTRAP_SOCKET}" --user=root ||
+            _client_rc=$?
+        printf '%s\n' "${_client_rc}" > "${_status_file}"
+    } 2>&1 | redact_stream "${RDS_MASTER_PASSWORD}" | redact_stream "${RDS_MASTER_PASSWORD_SQL}"
+
+    # Every way of not recovering the client's status is a failed bootstrap
+    # rather than a silent success, and each says so in its own words: blaming
+    # the client for a status file that could not be written would send an
+    # operator looking at a MariaDB that did exactly what it was asked.
+    #
+    # A torn write leaves the file created but empty, so the value is checked
+    # before it is returned: `return ""` is fatal in the guest's ash, and would
+    # kill rds-init here — past the sweep, before the diagnostics.
+    _client_status=1
+    if [ -f "${_status_file}" ]; then
+        _client_status=$(cat "${_status_file}" 2>/dev/null) || _client_status=""
+        case "${_client_status}" in
+            '' | *[!0-9]*)
+                echo "[rds-init] ERROR: the mariadb client ran but its exit status could not be read back from ${_status_file} (got '${_client_status}'); treating the bootstrap as failed" >&2
+                _client_status=1
+                ;;
+        esac
+        rm -f "${_status_file}"
+    else
+        echo "[rds-init] ERROR: the mariadb client ran but its exit status was never recorded at ${_status_file}; treating the bootstrap as failed" >&2
+    fi
+    return "${_client_status}"
+}
+
+# 11.8 creates no anonymous account and --skip-test-db leaves no test database,
+# so on a current image both of these are no-ops. They run anyway: the properties
+# are asserted by the image tests, and an engine or packaging change that
+# reintroduced either would otherwise reach a customer's instance.
+remove_insecure_defaults() {
+    mariadb_bootstrap <<'SQL'
+DELETE FROM mysql.global_priv WHERE User = '';
+DROP DATABASE IF EXISTS test;
+FLUSH PRIVILEGES;
+SQL
+}
+
+# MariaDB has no DENY or partial revocation: an object privilege ON *.* also
+# reaches mysql.global_priv. Application privileges are therefore database-scoped,
+# while this protected definer procedure creates and grants future databases.
+apply_master_user() {
+    log "creating the master user ${RDS_MASTER_USERNAME}"
+    mariadb_bootstrap <<SQL
+CREATE USER IF NOT EXISTS '${RDS_MASTER_USERNAME}'@'%' IDENTIFIED BY '${RDS_MASTER_PASSWORD_SQL}';
+ALTER USER '${RDS_MASTER_USERNAME}'@'%' IDENTIFIED BY '${RDS_MASTER_PASSWORD_SQL}';
+GRANT RELOAD, PROCESS, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT
+  ON *.* TO '${RDS_MASTER_USERNAME}'@'%';
+CREATE DATABASE IF NOT EXISTS \`${MANAGEMENT_DB}\`;
+DROP PROCEDURE IF EXISTS \`${MANAGEMENT_DB}\`.\`${CREATE_DATABASE_PROCEDURE}\`;
+DELIMITER //
+CREATE DEFINER='root'@'localhost' PROCEDURE \`${MANAGEMENT_DB}\`.\`${CREATE_DATABASE_PROCEDURE}\`(
+    IN database_name VARCHAR(64) CHARACTER SET ascii
+)
+SQL SECURITY DEFINER
+BEGIN
+    IF database_name IS NULL
+       OR BINARY database_name NOT REGEXP '^[A-Za-z][A-Za-z0-9_]{0,63}$'
+       OR LOWER(database_name) IN ('mysql', 'information_schema', 'performance_schema', 'sys', '${MANAGEMENT_DB}') THEN
+        SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'database name must be a non-system identifier of at most 64 characters';
+    END IF;
+
+    SET @rds_database = REPLACE(database_name, '\`', '\`\`');
+    SET @rds_statement = CONCAT('CREATE DATABASE IF NOT EXISTS \`', @rds_database, '\`');
+    PREPARE rds_create_database FROM @rds_statement;
+    EXECUTE rds_create_database;
+    DEALLOCATE PREPARE rds_create_database;
+
+    SET @rds_statement = CONCAT(
+        'GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, ',
+        'CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, ',
+        'CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON \`', @rds_database,
+        '\`.* TO ''${RDS_MASTER_USERNAME}''@''%'' WITH GRANT OPTION'
+    );
+    PREPARE rds_grant_database FROM @rds_statement;
+    EXECUTE rds_grant_database;
+    DEALLOCATE PREPARE rds_grant_database;
+END//
+DELIMITER ;
+GRANT EXECUTE ON PROCEDURE \`${MANAGEMENT_DB}\`.\`${CREATE_DATABASE_PROCEDURE}\`
+  TO '${RDS_MASTER_USERNAME}'@'%';
+SQL
+}
+
+# The resolved value of one setting in the installed parameter file, or nothing
+# when the group does not carry it.
+resolved_parameter() {
+    [ -f "${PARAM_FILE}" ] || return 0
+    sed -n "s/^[[:space:]]*$1[[:space:]]*=[[:space:]]*//p" "${PARAM_FILE}" |
+        tail -n 1 | tr -d "\"'[:space:]"
+}
+
+# The bootstrap server runs on --no-defaults, so it is not carrying the character
+# set the real server will start with — and a database keeps the character set it
+# was created under for good. Both values therefore come from the parameter file
+# rather than from the server's compiled default.
+database_charset_clause() {
+    _charset=$(resolved_parameter character_set_server)
+    _collation=$(resolved_parameter collation_server)
+    if [ -z "${_charset}" ]; then
+        _charset="utf8mb4"
+        _collation=""
+    fi
+    # Interpolated into CREATE DATABASE below, and read out of a file rather than
+    # validated as an identifier on the way in.
+    case "${_charset}${_collation}" in
+        *[!A-Za-z0-9_]*)
+            fail "the parameter group's character_set_server or collation_server is not a bare identifier; refusing to interpolate it into CREATE DATABASE"
+            ;;
+    esac
+    if [ -n "${_collation}" ]; then
+        printf 'CHARACTER SET %s COLLATE %s' "${_charset}" "${_collation}"
+        return 0
+    fi
+    printf 'CHARACTER SET %s' "${_charset}"
+}
+
+# As on AWS, an instance created without a DBName gets no initial database. A
+# supplied database is created under its resolved charset before the protected
+# routine applies the same scope future databases receive.
+create_database() {
+    log "creating database ${RDS_DB_NAME}"
+    _charset_clause=$(database_charset_clause)
+    mariadb_bootstrap <<SQL
+CREATE DATABASE IF NOT EXISTS \`${RDS_DB_NAME}\` ${_charset_clause};
+CALL \`${MANAGEMENT_DB}\`.\`${CREATE_DATABASE_PROCEDURE}\`('${RDS_DB_NAME}');
+SQL
+}
+
+# Runs on a freshly initialised datadir only.
+bootstrap_master() {
+    log "starting bootstrap server on ${BOOTSTRAP_DIR}"
+    start_bootstrap_server
+
+    _rc=0
+    remove_insecure_defaults || _rc=$?
+    if [ "${_rc}" -eq 0 ]; then
+        apply_master_user || _rc=$?
+    fi
+    if [ "${_rc}" -eq 0 ] && [ -n "${RDS_DB_NAME}" ]; then
+        create_database || _rc=$?
+    fi
+
+    # `set -e` is suspended for a function called from an AND-OR list, so the
+    # stop status is captured rather than propagated. A failed stop must also not
+    # abort before the bootstrap check below, which names the real cause.
+    _stop_rc=0
+    stop_bootstrap_server || _stop_rc=$?
+
+    # The datadir is swept by the traps, which also cover the steps here that
+    # abort under `set -e` before this check is ever reached.
+    if [ "${_rc}" -ne 0 ]; then
+        fail "bootstrapping the master user/database failed (rc=${_rc})"
+    fi
+
+    # Proof written while the traps are still armed, so a receipt that cannot be
+    # made durable clears the datadir instead of leaving one it cannot vouch for.
+    write_completion_receipt
+
+    # The master user exists from here, so the datadir is complete and nothing
+    # may clear it — including on the stop failure reported below.
+    mark_bootstrap_complete
+
+    # A server still holding the datadir blocks the mariadb service for the life
+    # of the boot, so this is a failure and not a warning.
+    if [ "${_stop_rc}" -ne 0 ]; then
+        fail "the bootstrap server did not stop (rc=${_stop_rc}) and still holds ${DATADIR}; the mariadb service cannot start until it is gone"
+    fi
+}
+
+# The value of one receipt key. Never sourced: the receipt is on the data volume,
+# so a restored volume can carry one written by another instance, and executing
+# it would run whatever that instance's control plane put there.
+receipt_value() {
+    [ -f "${RECEIPT_FILE}" ] || return 0
+    sed -n "s/^$1=//p" "${RECEIPT_FILE}" 2>/dev/null | head -n 1
+}
+
+# Written between the master user and the retirement of the traps, so a receipt
+# that cannot be made durable still clears the datadir rather than leaving a
+# bootstrapped one nobody can prove.
+#
+# Fatal on failure, the mirror image of the sentinel removal below: a healthy
+# engine whose receipt never landed would come up available while the agent
+# blocked forever on a receipt that will never appear, leaving a working database
+# that can never accept a password change or a parameter reload.
+write_completion_receipt() {
+    install -d -m 0700 -o "${ENGINE_USER}" -g "${ENGINE_USER}" "${RECEIPT_DIR}" ||
+        fail "the master user exists but the bootstrap receipt directory ${RECEIPT_DIR} could not be created"
+
+    _tmp="${RECEIPT_FILE}.new"
+    {
+        echo "# Written by rds-init once the master user was applied. Holds no secrets."
+        echo "RDS_RECEIPT_PAYLOAD_ID=${RDS_PAYLOAD_ID}"
+        echo "RDS_RECEIPT_DB_INSTANCE_IDENTIFIER=${RDS_DB_INSTANCE_IDENTIFIER}"
+        echo "RDS_RECEIPT_VM_GENERATION=${RDS_VM_GENERATION}"
+        echo "RDS_RECEIPT_COMPLETED_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+    } > "${_tmp}" ||
+        fail "the master user exists but the bootstrap receipt ${RECEIPT_FILE} could not be written"
+    chown "${ENGINE_USER}:${ENGINE_USER}" "${_tmp}"
+    chmod 0600 "${_tmp}"
+
+    # A receipt still in the page cache is lost by exactly the crash it exists to
+    # survive, so it is on disk before it is renamed into place.
+    sync "${_tmp}" 2>/dev/null || sync
+    mv -f "${_tmp}" "${RECEIPT_FILE}" ||
+        fail "the master user exists but the bootstrap receipt ${RECEIPT_FILE} could not be installed"
+    sync "${RECEIPT_DIR}" 2>/dev/null || sync
+}
+
+# A system database with no sentinel and no receipt is ambiguous from the datadir
+# alone: either an instance that bootstrapped before receipts existed, which must
+# attach, or one that applied the master user and crashed before the receipt was
+# durable, which must not. Only the control plane's view of whether the payload
+# is still staged separates the two, and that is what RDS_BOOTSTRAP_PENDING says.
+require_bootstrap_receipt() {
+    [ "${RDS_BOOTSTRAP_PENDING}" = "1" ] || return 0
+    if [ -n "${RDS_PAYLOAD_ID}" ] &&
+        [ "$(receipt_value RDS_RECEIPT_PAYLOAD_ID)" = "${RDS_PAYLOAD_ID}" ] &&
+        [ "$(receipt_value RDS_RECEIPT_DB_INSTANCE_IDENTIFIER)" = "${RDS_DB_INSTANCE_IDENTIFIER}" ]; then
+        return 0
+    fi
+    fail "the control plane has no record of a completed bootstrap for this datadir; refusing to attach to a datadir whose master user cannot be proven. Recover the volume out of band, or delete and recreate the instance."
+}
+
+# Retires the guards armed at initialisation, in an order where no signal in
+# between can clear a datadir that now carries a usable master user.
+mark_bootstrap_complete() {
+    _bootstrapped=1
+    rm -f "${BOOTSTRAP_SENTINEL}" ||
+        fail "the master user exists but ${BOOTSTRAP_SENTINEL} could not be removed; the next boot will refuse to start until it is deleted"
+    trap - EXIT INT TERM HUP
+}
+
+# Drops the master password from the handoff once applied, so it is not left
+# readable for the life of the boot. The rest is tmpfs and goes at reboot.
+consume_secrets() {
+    [ -f "${ENV_FILE}" ] || return 0
+    _tmp="${ENV_FILE}.new"
+    install -m 0600 /dev/null "${_tmp}"
+    grep -v '^RDS_MASTER_PASSWORD=' "${ENV_FILE}" >> "${_tmp}" || true
+    mv -f "${_tmp}" "${ENV_FILE}"
+}
+
+load_handoff
+require_data_volume
+require_engine_stamp
+
+if [ -d "${DATADIR}/mysql" ]; then
+    # The engine would start cleanly on such a datadir and pass every health
+    # probe with no master user to log in as. The sentinel says the bootstrap did
+    # not finish, not that the volume is empty, so nothing here clears it.
+    if [ -e "${BOOTSTRAP_SENTINEL}" ]; then
+        fail "datadir ${DATADIR} carries ${BOOTSTRAP_SENTINEL}: an earlier bootstrap was interrupted before the master user existed; recover the data volume out of band rather than starting the engine"
+    fi
+    require_bootstrap_receipt
+    log "datadir ${DATADIR} is already initialised — attaching"
+    install_runtime_config
+else
+    initialise_datadir
+    install_runtime_config
+    bootstrap_master
+fi
+
+consume_secrets
+log "bootstrap complete; handing off to the mariadb service"

scripts/images/rds-mariadb/rds-init.initd

@@ -0,0 +1,35 @@
+#!/sbin/openrc-run
+# rds-init — boot oneshot that bootstraps or attaches the MariaDB datadir before
+# the engine starts.
+#
+# `before mariadb` orders the two, and /etc/conf.d/mariadb makes the engine need
+# this oneshot: Alpine's packaged mariadb service only prints a warning when the
+# datadir is empty and starts anyway, so ordering alone would let mysqld_safe
+# launch against an unbootstrapped data volume. `after rds-agent`, not `need`, so
+# a failed fetch fails this oneshot visibly rather than skipping it.
+
+description="Spinifex RDS MariaDB bootstrap"
+
+AGENT_ENV="/etc/spinifex-rds/agent.env"
+
+depend() {
+    need net
+    after rds-agent rds-datadir
+    before mariadb
+}
+
+start_pre() {
+    if [ -f "${AGENT_ENV}" ]; then
+        # shellcheck disable=SC1090
+        . "${AGENT_ENV}"
+        export RDS_HANDOFF_DIR RDS_DATA_MOUNT
+    fi
+}
+
+start() {
+    ebegin "Bootstrapping the RDS MariaDB instance"
+    # The VM has no operator login, so the host-side serial log is the only view
+    # of a bootstrap that failed before the engine came up.
+    /usr/local/sbin/rds-init > /dev/console 2>&1
+    eend $?
+}

scripts/images/rds-mariadb/rds-mycnf_test.sh

@@ -0,0 +1,74 @@
+#!/bin/sh
+set -eu
+
+# The generated parameter files live on the data volume and reach the server
+# through one baked include. Three properties make that work, and each fails
+# silently rather than loudly if it breaks: the include must be read after every
+# packaged drop-in, it must point at the mount rds-datadir uses, and it must be
+# named so MariaDB reads it at all.
+
+SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
+INCLUDE="zz-rds-include.cnf"
+
+FAILS=0
+fail() { echo "FAIL: $*" >&2; FAILS=$((FAILS + 1)); }
+pass() { echo "ok: $*"; }
+
+# MariaDB reads only .cnf and .ini out of an include directory. Any other suffix
+# would leave the customer's parameters silently unread.
+case "${INCLUDE}" in
+    *.cnf) pass "the include is named so MariaDB reads it" ;;
+    *) fail "${INCLUDE} is not a name MariaDB reads out of an include directory" ;;
+esac
+
+if [ ! -f "${SCRIPT_DIR}/${INCLUDE}" ]; then
+    fail "${INCLUDE} is missing from the preset"
+fi
+if ! grep -q "${INCLUDE}:/etc/my.cnf.d/${INCLUDE}" "${SCRIPT_DIR}/manifest.conf"; then
+    fail "manifest.conf does not install ${INCLUDE} into /etc/my.cnf.d"
+fi
+
+# Alpine's own drop-in is the file this has to outsort: MariaDB walks the include
+# directory in byte order and takes the last occurrence of a setting, so a digit
+# prefix would sort ahead of it and let a distribution default beat the platform.
+packaged="mariadb-server.cnf"
+if [ "$(printf '%s\n%s\n' "${INCLUDE}" "${packaged}" | LC_ALL=C sort | tail -n 1)" = "${INCLUDE}" ]; then
+    pass "the include is read after ${packaged}"
+else
+    fail "${packaged} is read after ${INCLUDE}, so packaged defaults win"
+fi
+
+# setup.sh asserts the same property against the real directory at build time.
+grep -q 'LC_ALL=C sort | tail -n 1' "${SCRIPT_DIR}/setup.sh" ||
+    fail "setup.sh does not assert the include is the last drop-in"
+
+include_dir=$(sed -n 's/^!includedir[[:space:]]*//p' "${SCRIPT_DIR}/${INCLUDE}")
+if [ -z "${include_dir}" ]; then
+    fail "${INCLUDE} declares no !includedir"
+else
+    # The generated files survive a VM replacement only by living on the data
+    # volume, so the include has to sit under the mount rds-datadir provides.
+    data_mount=$(sed -n 's/^[[:space:]]*RDS_DATA_MOUNT="\([^"]*\)".*/\1/p' "${SCRIPT_DIR}/rds-datadir.initd")
+    case "${include_dir}" in
+        "${data_mount}"/*) pass "the include target is on the data volume at ${include_dir}" ;;
+        *) fail "${include_dir} is not under the data mount ${data_mount:-<unset>}" ;;
+    esac
+
+    # An !includedir MariaDB cannot open is a fatal defaults-parsing error, which
+    # would take the client down alongside the server on a boot with no volume.
+    grep -q "install -d .* ${include_dir}\$" "${SCRIPT_DIR}/setup.sh" ||
+        fail "setup.sh does not bake ${include_dir} into the image"
+fi
+
+# The packaged drop-in disables TCP. Left in place, the endpoint resolves and the
+# health probe passes while nothing on the customer ENI can ever connect.
+grep -q 's/\^skip-networking\$/' "${SCRIPT_DIR}/setup.sh" ||
+    fail "setup.sh does not disable the packaged skip-networking"
+grep -q "skip\[-_\]networking" "${SCRIPT_DIR}/setup.sh" ||
+    fail "setup.sh does not assert that no configuration file disables TCP"
+
+if [ "${FAILS}" -eq 0 ]; then
+    echo "rds-mycnf: all tests passed"
+    exit 0
+fi
+exit 1

scripts/images/rds-mariadb/rds-openrc_test.sh

@@ -0,0 +1,68 @@
+#!/bin/sh
+set -eu
+
+SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
+WORK=$(mktemp -d)
+trap 'rm -rf "${WORK}"' EXIT
+
+capture_dependencies() {
+    _script=$1
+    (
+        need() { echo "need $*"; }
+        after() { echo "after $*"; }
+        before() { echo "before $*"; }
+        # shellcheck disable=SC1090
+        . "${SCRIPT_DIR}/${_script}"
+        depend
+    )
+}
+
+assert_line() {
+    _file=$1
+    _line=$2
+    grep -Fxq "${_line}" "${_file}" || {
+        echo "FAIL: ${_file} missing dependency: ${_line}" >&2
+        cat "${_file}" >&2
+        exit 1
+    }
+}
+
+capture_dependencies rds-agent.initd > "${WORK}/agent"
+capture_dependencies rds-datadir.initd > "${WORK}/datadir"
+capture_dependencies rds-init.initd > "${WORK}/init"
+
+assert_line "${WORK}/agent" "need net"
+assert_line "${WORK}/agent" "after localmount"
+assert_line "${WORK}/agent" "before rds-datadir rds-init mariadb"
+assert_line "${WORK}/datadir" "after localmount rds-agent"
+assert_line "${WORK}/datadir" "before rds-init mariadb"
+assert_line "${WORK}/init" "need net"
+assert_line "${WORK}/init" "after rds-agent rds-datadir"
+assert_line "${WORK}/init" "before mariadb"
+
+if grep -Fq "before rds-agent" "${WORK}/datadir"; then
+    echo "FAIL: rds-datadir still starts before the bootstrap handoff" >&2
+    exit 1
+fi
+
+# Ordering alone is not enough for this engine: Alpine's mariadb service only
+# warns about an empty datadir and starts anyway, so a failed rds-init would be
+# answered by launching mysqld_safe against an unbootstrapped volume. OpenRC
+# reads the dependency out of the service's conf.d file.
+grep -Fq 'rc_need="rds-init"' "${SCRIPT_DIR}/mariadb.confd" || {
+    echo "FAIL: /etc/conf.d/mariadb does not make the engine need rds-init" >&2
+    exit 1
+}
+grep -Fq "mariadb.confd:/etc/conf.d/mariadb" "${SCRIPT_DIR}/manifest.conf" || {
+    echo "FAIL: manifest.conf does not install mariadb.confd" >&2
+    exit 1
+}
+for svc in rds-init mariadb; do
+    grep -qE "^ENABLE_SERVICES=.*[\" ]${svc}[\" ]" "${SCRIPT_DIR}/manifest.conf" || {
+        echo "FAIL: manifest.conf does not enable ${svc}" >&2
+        grep '^ENABLE_SERVICES' "${SCRIPT_DIR}/manifest.conf" >&2
+        exit 1
+    }
+done
+
+echo "rds-openrc: dependency order tests passed"

scripts/images/rds-mariadb/setup.sh

@@ -0,0 +1,122 @@
+#!/bin/sh
+set -eu
+
+# setup.sh — guest customisation for the spinifex-rds-mariadb AMI. Runs inside
+# the libguestfs appliance under build-system-image.sh, after packages and
+# INSTALL_FILES are placed.
+
+# INSTALL_FILES land 0644; OpenRC requires 0755 on init scripts, and rds-init
+# and rds-datadir are executed directly by their services.
+chmod 0755 /etc/init.d/rds-datadir /etc/init.d/rds-init /etc/init.d/rds-agent \
+    /usr/local/sbin/rds-datadir /usr/local/sbin/rds-init
+chmod 0755 /etc/init.d/mulga-mgmt-net /etc/init.d/mulga-mgmt-net-routes \
+    /usr/local/sbin/mulga-mgmt-net
+
+# mulga-mgmt-net goes in the boot runlevel, not default (where ENABLE_SERVICES
+# lands services). It applies mgmt0's static address, which rds-agent needs to
+# reach the gateway at all, and DHCPs the data NIC so the init-local Ec2 crawl
+# reaches IMDS; a default entry runs after cloud-init-local and is too late.
+rc-update add mulga-mgmt-net boot
+
+# Where cloud-init drops the agent's env file and the gateway CA. Created here
+# so the delivery lands in a root-only directory.
+install -d -m 0700 /etc/spinifex-rds
+
+# The engine this image bakes. rds-agent builds its engine implementation from
+# this file, so running the wrong implementation is structurally impossible
+# rather than merely validated against; a VM launched as another engine is
+# refused before anything touches the datadir.
+printf 'mariadb\n' > /etc/spinifex-rds/engine
+chmod 0444 /etc/spinifex-rds/engine
+
+# Empty in the image: rds-datadir mounts over it at boot. The package already
+# creates it 0750 mysql:mysql; restated so a packaging change cannot loosen it.
+install -d -m 0750 -o mysql -g mysql /var/lib/mysql
+
+# The include target, baked so it exists even when the data volume is not
+# mounted. MariaDB treats an !includedir it cannot open as a fatal error while
+# parsing defaults, which would take down the client alongside the server on
+# exactly the boot an operator needs both to diagnose the missing volume.
+install -d -m 0750 -o mysql -g mysql /var/lib/mysql/conf.d
+
+# The generated configuration must be read last, or a packaged default would win
+# over a platform-owned setting. Sorted in byte order, the way MariaDB itself
+# walks an include directory, so a future package adding a later-sorting file
+# fails the build rather than silently overriding the platform.
+last_dropin=$(ls /etc/my.cnf.d | LC_ALL=C sort | tail -n 1)
+if [ "${last_dropin}" != "zz-rds-include.cnf" ]; then
+    echo "[rds-mariadb-setup] /etc/my.cnf.d/${last_dropin} is read after the RDS include"
+    exit 1
+fi
+
+# Alpine's packaged server drop-in sets skip-networking, which would leave the
+# instance reachable only over its unix socket — the endpoint would resolve, the
+# health probe would pass, and nothing on the customer ENI could ever connect.
+# Disabled at the source rather than countered from a later file, so the image
+# does not depend on override ordering for the endpoint to work at all.
+sed -i 's/^skip-networking$/# skip-networking — disabled: the customer reaches this instance over TCP/' \
+    /etc/my.cnf.d/mariadb-server.cnf
+if grep -rEq '^[[:space:]]*skip[-_]networking' /etc/my.cnf /etc/my.cnf.d; then
+    echo "[rds-mariadb-setup] a packaged configuration file still disables TCP networking"
+    grep -rEn '^[[:space:]]*skip[-_]networking' /etc/my.cnf /etc/my.cnf.d
+    exit 1
+fi
+
+# The resolved defaults checked against this image's own mariadbd. A name the
+# server refuses at startup is not a rejected setting: it aborts the boot with the
+# generated file already on the data volume. cmd/rds-agent keeps this file in step.
+#
+# --verbose --help parses the option files and exits without touching a datadir,
+# so it is the whole check. defaults-extra-file rather than the include directory,
+# which stays empty in the image; it must be the first option mariadbd sees.
+default_params=/usr/local/share/spinifex-rds/default-parameters.cnf
+if [ ! -f "${default_params}" ]; then
+    echo "[rds-mariadb-setup] the catalog's default parameter set was not delivered to ${default_params}"
+    exit 1
+fi
+if ! mariadbd --defaults-extra-file="${default_params}" --user=mysql --verbose --help \
+    >/dev/null 2>/tmp/rds-defaults.err; then
+    echo "[rds-mariadb-setup] mariadbd refuses the catalog's default parameter set"
+    cat /tmp/rds-defaults.err
+    exit 1
+fi
+# Build-time only: nothing at runtime reads it, and an include the server could
+# find twice is a way for a stale copy to shadow the customer's own set.
+rm -f /tmp/rds-defaults.err "${default_params}"
+rmdir /usr/local/share/spinifex-rds 2>/dev/null || true
+
+# The packaged service starts on a warning when the datadir is empty, so the
+# dependency is what keeps it off an unbootstrapped volume. Asserted here, since
+# an image whose engine can come up beside a failed bootstrap is the one failure
+# this preset must not have.
+grep -q '^rc_need="rds-init"' /etc/conf.d/mariadb || {
+    echo "[rds-mariadb-setup] /etc/conf.d/mariadb does not make the engine need rds-init"
+    exit 1
+}
+
+# rds-agent's health probe reads this pid to tell a server replaying its redo log
+# from one that is not running at all. Nothing generated can move the path: the
+# packaged service passes --pid-file on mysqld_safe's command line, which beats
+# any option file. A package that renamed it would leave every instance reporting
+# an engine that is not there, and the rollback guard restarting a healthy one.
+grep -q '^pidfile="/run/mysqld/\$RC_SVCNAME.pid"$' /etc/init.d/mariadb &&
+    grep -q -- '--pid-file=\$pidfile' /etc/init.d/mariadb || {
+    echo "[rds-mariadb-setup] /etc/init.d/mariadb no longer starts the engine with its pidfile at /run/mysqld/mariadb.pid"
+    grep -nE 'pidfile|command_args' /etc/init.d/mariadb
+    exit 1
+}
+
+# Bind /dev/console to the serial port so userspace boot output reaches ttyS0,
+# which the orchestrator captures host-side. Linux makes the last console= the
+# controlling one, and stock Alpine lists tty0 last — reorder so ttyS0 wins.
+sed -i \
+    's|console=ttyS0,115200n8 console=ttyAMA0,115200n8 console=tty0|console=tty0 console=ttyAMA0,115200n8 console=ttyS0,115200n8|' \
+    /etc/update-extlinux.conf /boot/extlinux.conf
+
+# Cut the boot-menu countdown from 10s to ~1s, a fixed tax on every VM start.
+# Both the generator config (seconds) and the rendered output (1/10s) are patched
+# so a regenerate keeps it; a small nonzero keeps the menu interruptible.
+sed -i 's/^timeout=.*/timeout=1/' /etc/update-extlinux.conf
+sed -i 's/^TIMEOUT[[:space:]].*/TIMEOUT 10/' /boot/extlinux.conf
+
+echo "[rds-mariadb-setup] done"

scripts/images/rds-mariadb/zz-rds-include.cnf

@@ -0,0 +1,19 @@
+# /etc/my.cnf.d/zz-rds-include.cnf — Spinifex RDS generated-configuration include.
+#
+# The generated files live on the data volume rather than under /etc. A class
+# change is a VM replacement: a fresh boot volume from this AMI with the same
+# data volume reattached. Parameters under /etc would be discarded by that
+# replacement and the instance would silently revert to catalog defaults while
+# the API still reported the customer's values.
+#
+# zz- so this is the last drop-in the packaged !includedir reads. MariaDB walks
+# an include directory in byte order and takes the last occurrence of a setting,
+# which is what makes a platform-owned value beat a distribution default — and a
+# digit prefix would sort ahead of the packaged mariadb-server.cnf rather than
+# after it. setup.sh asserts the resulting order at build time.
+#
+# The directory is also baked empty into this image. MariaDB treats an
+# !includedir it cannot open as a fatal defaults-parsing error, so a boot with no
+# data volume mounted would otherwise take the client down with the server.
+
+!includedir /var/lib/mysql/conf.d

scripts/images/rds-postgres/manifest.conf

@@ -23,7 +23,7 @@ INSTALL_BINARIES="bin/rds-agent:/usr/local/bin/rds-agent"
 # the guest, or mgmt0 never gets its static address and rds-agent cannot reach
 # the gateway, and the customer ENI never comes up so nothing reaches the DB.
 INSTALL_FILES="\
-scripts/images/rds-postgres/rds-datadir:/usr/local/sbin/rds-datadir \
+scripts/images/common/rds-datadir:/usr/local/sbin/rds-datadir \
 scripts/images/rds-postgres/rds-datadir.initd:/etc/init.d/rds-datadir \
 scripts/images/rds-postgres/rds-init:/usr/local/sbin/rds-init \
 scripts/images/rds-postgres/rds-init.initd:/etc/init.d/rds-init \

scripts/images/rds-postgres/rds-agent.initd

@@ -35,8 +35,8 @@ start_pre() {
         # shellcheck disable=SC1090
         . "${AGENT_ENV}"
         export RDS_GATEWAY_URL RDS_GATEWAY_CA RDS_REGION RDS_DB_INSTANCE_IDENTIFIER \
-            RDS_ENGINE_VERSION RDS_HANDOFF_DIR RDS_ENGINE_HOST RDS_ENGINE_PORT \
-            RDS_PG_ISREADY RDS_POLL_WAIT RDS_DATA_MOUNT
+            RDS_ENGINE RDS_ENGINE_VERSION RDS_HANDOFF_DIR RDS_ENGINE_HOST RDS_ENGINE_PORT \
+            RDS_POLL_WAIT RDS_DATA_MOUNT
     fi
 }
 

scripts/images/rds-postgres/rds-agent_initd_test.sh

@@ -9,7 +9,10 @@ source_time_handoff="${CASE}/source-time"
 configured_handoff="${CASE}/configured"
 mkdir -p "${configured_handoff}"
 touch "${configured_handoff}/bootstrap.env"
-printf 'RDS_HANDOFF_DIR=%s\n' "${configured_handoff}" >"${CASE}/agent.env"
+{
+    printf 'RDS_HANDOFF_DIR=%s\n' "${configured_handoff}"
+    printf 'RDS_ENGINE=postgres\n'
+} >"${CASE}/agent.env"
 
 # Simulate OpenRC sourcing the service before start_pre loads agent.env.
 export RDS_HANDOFF_DIR="${source_time_handoff}"
@@ -34,5 +37,12 @@ if [ "${EEND_STATUS}" != "0" ]; then
     echo "FAIL: post-start wait did not find the configured handoff" >&2
     exit 1
 fi
+# The agent checks this against the engine its own image bakes. Left out of the
+# export list it would read as no assertion at all, so a VM launched as the
+# wrong engine would bootstrap instead of refusing.
+if [ "${RDS_ENGINE:-}" != "postgres" ]; then
+    echo "FAIL: start_pre did not export RDS_ENGINE from agent.env" >&2
+    exit 1
+fi
 
 echo "rds-agent.initd: all tests passed"

scripts/images/rds-postgres/rds-datadir.initd

@@ -14,11 +14,16 @@ depend() {
 }
 
 start_pre() {
+    # This image's layout, stated here because the mount script is shared by
+    # every engine preset. A delivered RDS_DATA_MOUNT still wins, so the control
+    # plane can move the mount point without rebuilding the image.
+    RDS_DATA_MOUNT="/var/lib/postgresql"
+    RDS_ENGINE_USER="postgres"
     if [ -f "${AGENT_ENV}" ]; then
         # shellcheck disable=SC1090
         . "${AGENT_ENV}"
-        export RDS_HANDOFF_DIR RDS_DATA_MOUNT
     fi
+    export RDS_HANDOFF_DIR RDS_DATA_MOUNT RDS_ENGINE_USER
 }
 
 start() {

scripts/images/rds-postgres/rds-engine-stamp_test.sh

@@ -0,0 +1,37 @@
+#!/bin/sh
+set -eu
+
+# The engine this preset bakes is stated in four places that have to agree: the
+# stamp setup.sh writes, the manifest tag resolveEngineAMI selects on, the name
+# rds-agent's layout table is keyed by, and the stamp rds-init puts on the data
+# volume. A drift between them launches the wrong image, refuses the right one,
+# or stamps a volume with an engine that did not write it.
+
+SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
+REPO_ROOT=$(CDPATH='' cd -- "${SCRIPT_DIR}/../../.." && pwd)
+ENGINE="postgres"
+
+stamped=$(sed -n "s|^printf '\\(.*\\)\\\\n' > /etc/spinifex-rds/engine\$|\\1|p" "${SCRIPT_DIR}/setup.sh")
+if [ "${stamped}" != "${ENGINE}" ]; then
+    echo "FAIL: setup.sh stamps engine '${stamped}', want '${ENGINE}'" >&2
+    exit 1
+fi
+
+if ! grep -q "engine=${ENGINE} " "${SCRIPT_DIR}/manifest.conf"; then
+    echo "FAIL: manifest.conf SYSTEM_TAG does not carry engine=${ENGINE}" >&2
+    grep '^SYSTEM_TAG' "${SCRIPT_DIR}/manifest.conf" >&2
+    exit 1
+fi
+
+if ! grep -q "enginePostgres = \"${ENGINE}\"" "${REPO_ROOT}/cmd/rds-agent/engine.go"; then
+    echo "FAIL: rds-agent does not implement an engine named ${ENGINE}" >&2
+    exit 1
+fi
+
+if ! grep -q "^ENGINE=\"${ENGINE}\"\$" "${SCRIPT_DIR}/rds-init"; then
+    echo "FAIL: rds-init does not stamp the data volume '${ENGINE}'" >&2
+    grep '^ENGINE=' "${SCRIPT_DIR}/rds-init" >&2
+    exit 1
+fi
+
+echo "rds-engine-stamp: all tests passed"

scripts/images/rds-postgres/rds-init

@@ -26,6 +26,10 @@
 # Nothing there survives a reboot: the next boot re-fetches and the cert is
 # re-minted rather than persisted.
 #
+# The data volume also carries an engine stamp, written before the datadir is
+# inspected and compared on every boot, so another engine's volume is refused
+# rather than initialised over.
+#
 # rds-init keys off the datadir, never off RDS_MODE. An initialised datadir
 # means skip initdb and only refresh config, so a replaced VM comes back on its
 # existing data volume without re-bootstrapping. RDS_BOOTSTRAP_PENDING is not an
@@ -37,6 +41,11 @@ PG_VERSION="${RDS_PG_VERSION:-18}"
 PG_BIN="${RDS_PG_BIN:-/usr/libexec/postgresql${PG_VERSION}}"
 PG_USER="${RDS_PG_USER:-postgres}"
 
+# The engine this image bakes, stamped onto the data volume and compared against
+# it on every boot. Held here rather than read from /etc/spinifex-rds/engine:
+# this script is the PostgreSQL half of the seam, so it is the same constant.
+ENGINE="postgres"
+
 # Mount point of the data volume, and the datadir one level inside it. The
 # datadir is deliberately not the mount point itself: initdb refuses a non-empty
 # directory and a freshly formatted ext4 volume already holds lost+found.
@@ -54,6 +63,11 @@ BOOTSTRAP_SENTINEL="${PGDATA}/rds-bootstrap-incomplete"
 RECEIPT_DIR="${RDS_RECEIPT_DIR:-${DATA_MOUNT}/.spinifex-rds/bootstrap}"
 RECEIPT_FILE="${RECEIPT_DIR}/receipt.env"
 
+# Which engine's data this volume holds. Outside PGDATA for the same reason the
+# receipt is, and beside it rather than inside the bootstrap directory: it
+# outlives any one bootstrap and is read before the datadir is ever inspected.
+ENGINE_STAMP="${RDS_ENGINE_STAMP:-${DATA_MOUNT}/.spinifex-rds/engine}"
+
 HANDOFF_DIR="${RDS_HANDOFF_DIR:-/run/spinifex-rds}"
 ENV_FILE="${HANDOFF_DIR}/bootstrap.env"
 SOCKET_DIR="${RDS_SOCKET_DIR:-/run/postgresql}"
@@ -71,6 +85,10 @@ MOUNTS_FILE="${RDS_MOUNTS_FILE:-/proc/mounts}"
 # AWS, so a customer's `GRANT rds_superuser TO ...` works here too.
 MASTER_GROUP_ROLE="rds_superuser"
 
+# NAMEDATALEN-1: the server truncates a longer identifier, so the database
+# created would not be the one the customer asked for.
+MAX_DB_NAME_LEN=63
+
 log() { echo "[rds-init] $*"; }
 fail() { echo "[rds-init] ERROR: $*" >&2; exit 1; }
 
@@ -134,6 +152,20 @@ load_handoff() {
             ;;
     esac
 
+    # The same rule the control plane applies, for the same reason the reserved
+    # name is re-checked: an unusable name reaches initdb, fails at CREATE
+    # DATABASE and spends the one-shot password on the way. psql quotes the
+    # identifier, so here this is defence in depth rather than the barrier.
+    case "${RDS_DB_NAME}" in
+        '') ;;
+        [!A-Za-z]* | *[!A-Za-z0-9_]*)
+            fail "RDS_DB_NAME ${RDS_DB_NAME} must begin with a letter and hold only letters, digits and underscores; the control plane should have rejected it"
+            ;;
+    esac
+    if [ "${#RDS_DB_NAME}" -gt "${MAX_DB_NAME_LEN}" ]; then
+        fail "RDS_DB_NAME is longer than the ${MAX_DB_NAME_LEN} characters PostgreSQL allows; the control plane should have rejected it"
+    fi
+
     case "${RDS_PORT}" in
         '' | *[!0-9]*) fail "bootstrap config has a non-numeric RDS_PORT: ${RDS_PORT}" ;;
     esac
@@ -162,6 +194,52 @@ require_data_volume() {
     log "warning: ${DATA_MOUNT} is not a mount point; data will not survive a VM replace (RDS_ALLOW_LOCAL_DATADIR=1)"
 }
 
+# Written before the datadir is inspected, so a volume this image goes on to
+# initialise carries the stamp from the first boot onwards. Fatal on failure:
+# an unstamped volume is one the next boot cannot tell from another engine's.
+write_engine_stamp() {
+    _stamp_dir=$(dirname "${ENGINE_STAMP}")
+    install -d -m 0700 -o "${PG_USER}" -g "${PG_USER}" "${_stamp_dir}" ||
+        fail "the datadir engine stamp directory ${_stamp_dir} could not be created; refusing to use a data volume whose engine cannot be recorded"
+
+    _tmp="${ENGINE_STAMP}.new"
+    printf '%s\n' "${ENGINE}" > "${_tmp}" ||
+        fail "the datadir engine stamp ${ENGINE_STAMP} could not be written; refusing to use a data volume whose engine cannot be recorded"
+    chown "${PG_USER}:${PG_USER}" "${_tmp}"
+    chmod 0600 "${_tmp}"
+
+    # Renamed rather than written in place, so a crash leaves either the old
+    # stamp or none — and none is backfilled, where a torn one would be fatal.
+    sync "${_tmp}" 2>/dev/null || sync
+    mv -f "${_tmp}" "${ENGINE_STAMP}" ||
+        fail "the datadir engine stamp ${ENGINE_STAMP} could not be installed; refusing to use a data volume whose engine cannot be recorded"
+    sync "${_stamp_dir}" 2>/dev/null || sync
+}
+
+# The volume mounts cleanly whichever engine wrote it, and another engine's
+# datadir reads as uninitialised here: rds-init would initdb beside the
+# customer's data and serve an empty database that passes every health probe.
+require_engine_stamp() {
+    if [ -e "${ENGINE_STAMP}" ]; then
+        _stamped=""
+        _stamp_read=0
+        _stamped=$(head -n 1 -- "${ENGINE_STAMP}") && _stamp_read=1
+        [ "${_stamp_read}" = "1" ] ||
+            fail "the datadir engine stamp ${ENGINE_STAMP} exists but could not be read; refusing to attach to a data volume whose engine cannot be established"
+        if [ "${_stamped}" != "${ENGINE}" ]; then
+            fail "the data volume holds a '${_stamped}' datadir but this image runs ${ENGINE}; refusing to touch it. Attach it to a '${_stamped}' instance, or recover the volume out of band."
+        fi
+        log "data volume is stamped ${ENGINE}"
+        return 0
+    fi
+
+    # No stamp is either a volume predating it or one about to be initialised.
+    # Both are ours here — PostgreSQL had unstamped volumes in the field, so a
+    # non-empty datadir is backfilled rather than refused.
+    log "data volume carries no engine stamp — recording ${ENGINE}"
+    write_engine_stamp
+}
+
 # Runs initdb into an empty datadir and lays down the cluster config written
 # once: the include_dir hook and pg_hba.
 initialise_cluster() {
@@ -565,6 +643,7 @@ consume_secrets() {
 
 load_handoff
 require_data_volume
+require_engine_stamp
 
 if [ -s "${PGDATA}/PG_VERSION" ]; then
     # The engine would start cleanly on such a datadir and pass every health

scripts/images/rds-postgres/rds-init_test.sh

@@ -160,11 +160,15 @@ EOF
 
 # sync stub: a no-op, except that SYNC_KILL_PARENT signals rds-init from the
 # directory sync that follows the receipt rename. That is the one window where
-# an installed receipt and an armed sweep coexist.
+# an installed receipt and an armed sweep coexist. Matched on the receipt
+# directory by name: the engine stamp syncs its directory too, long before the
+# traps are armed, and killing there would leave the window untested.
 cat > "${STUBBIN}/sync" <<'EOF'
 #!/bin/sh
-if [ "${SYNC_KILL_PARENT:-0}" = "1" ] && [ -d "${1:-}" ]; then
-    kill -TERM "$(cat "${KILL_PID_FILE}")"
+if [ "${SYNC_KILL_PARENT:-0}" = "1" ]; then
+    case "${1:-}" in
+        */bootstrap) kill -TERM "$(cat "${KILL_PID_FILE}")" ;;
+    esac
 fi
 exit 0
 EOF
@@ -182,6 +186,7 @@ PGDATA="${DATA_MOUNT}/18/data"
 SENTINEL="${PGDATA}/rds-bootstrap-incomplete"
 RECEIPT_DIR="${DATA_MOUNT}/.spinifex-rds/bootstrap"
 RECEIPT="${RECEIPT_DIR}/receipt.env"
+STAMP="${DATA_MOUNT}/.spinifex-rds/engine"
 HANDOFF="${WORK}/run/spinifex-rds"
 MOUNTS="${WORK}/mounts"
 KILL_PID_FILE="${WORK}/rds-init.pid"
@@ -207,7 +212,10 @@ write_handoff() {
         # password carrying shell metacharacters — or a newline — survive the
         # handoff intact and reach the checks that exist for it.
         [ -n "$2" ] && echo "RDS_MASTER_PASSWORD='$2'"
-        [ -n "$3" ] && echo "RDS_DB_NAME=$3"
+        # Quoted as rds-agent's shellQuote writes it, so a name carrying a space
+        # or a shell metacharacter survives the handoff and reaches the check
+        # that exists for it rather than being mangled on the way.
+        [ -n "$3" ] && echo "RDS_DB_NAME='$3'"
         [ -n "${PENDING:-}" ] && echo "RDS_BOOTSTRAP_PENDING=${PENDING}"
         [ -n "${PAYLOAD_ID:-}" ] && echo "RDS_PAYLOAD_ID=${PAYLOAD_ID}"
         # Last, and unconditional: an optional line failing its test would make
@@ -239,7 +247,7 @@ reset_state() {
     : > "${PGCTL_CALLS}"
     : > "${PSQL_CALLS}"
     unset INITDB_FAIL PG_HBA_AS_DIR PSQL_FAIL LS_FAIL RDS_ALLOW_LOCAL_DATADIR MASTER_USER || true
-    unset PENDING PAYLOAD_ID DB_ID RECEIPT_DIR_OVERRIDE PSQL_STATUS_CORRUPT || true
+    unset PENDING PAYLOAD_ID DB_ID RECEIPT_DIR_OVERRIDE PSQL_STATUS_CORRUPT STAMP_OVERRIDE || true
 }
 
 # run_env: invoke a command with every path knob pointed into the temp dir.
@@ -252,6 +260,7 @@ run_env() {
         RDS_LOG_DIR="${WORK}/log" \
         RDS_MOUNTS_FILE="${MOUNTS}" \
         RDS_RECEIPT_DIR="${RECEIPT_DIR_OVERRIDE:-${RECEIPT_DIR}}" \
+        RDS_ENGINE_STAMP="${STAMP_OVERRIDE:-${STAMP}}" \
         RDS_ALLOW_LOCAL_DATADIR="${RDS_ALLOW_LOCAL_DATADIR:-0}" \
         INITDB_FAIL="${INITDB_FAIL:-0}" \
         PG_HBA_AS_DIR="${PG_HBA_AS_DIR:-0}" \
@@ -811,6 +820,122 @@ for reserved in postgres rds_superuser rdsadmin pg_toast_owner PostGres; do
     unset MASTER_USER
 done
 
+# --- Case 10: an initial database name the control plane would have refused ---
+# The name is interpolated into a CREATE DATABASE, and a failure there is a
+# failure after initdb: the datadir is cleared and the one-shot password is
+# gone. Refusing before initdb is what keeps the create retryable.
+for badname in 'my-db' 'my db' 'my/db' 'my.db' '1db' \
+    'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'; do
+    reset_state
+    write_handoff initialize 's3cr3t' "${badname}"
+    run_fails "bad-dbname-${badname}"
+    [ -s "${INITDB_CALLS}" ] \
+        && fail "bad-dbname-${badname}: initdb ran before the name was refused" \
+        || pass "bad-dbname-${badname}: refused before initdb"
+done
+
+# The rule is a rejection of malformed names, not of every name: the boundary
+# length and an underscore have to keep working.
+reset_state
+write_handoff initialize 's3cr3t' \
+    'a_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
+write_parameters
+if run_ok "dbname-at-the-limit"; then
+    grep -q 'CREATE DATABASE' "${PSQL_CALLS}" \
+        && pass "dbname-at-the-limit: accepted" || fail "dbname-at-the-limit: no CREATE DATABASE"
+fi
+
+# --- Case 11: the data volume records which engine wrote it ---
+# Another engine's datadir mounts cleanly and reads as uninitialised here, so
+# without the stamp rds-init would initdb beside the customer's data and serve
+# an empty database that passes every health probe while the real data sits
+# unreferenced — until the first automated snapshot captures the wrong one.
+reset_state
+write_handoff initialize 's3cr3t' appdb
+if run_ok "stamp-absent-empty"; then
+    [ "$(cat "${STAMP}" 2>/dev/null)" = "postgres" ] \
+        && pass "stamp-absent-empty: fresh volume stamped postgres" \
+        || fail "stamp-absent-empty: no stamp written at initialisation"
+    case "${STAMP}" in
+        "${PGDATA}"/*) fail "stamp-absent-empty: written inside the datadir the traps clear" ;;
+        *) pass "stamp-absent-empty: written outside PGDATA" ;;
+    esac
+
+    # --- Case 11a: a matching stamp attaches ---
+    : > "${INITDB_CALLS}"
+    write_handoff attach '' appdb
+    if run_ok "stamp-match"; then
+        [ -s "${INITDB_CALLS}" ] \
+            && fail "stamp-match: re-ran initdb" || pass "stamp-match: attached"
+    fi
+
+    # --- Case 11b: a disagreeing stamp is fatal and touches nothing ---
+    echo 'customer table data' > "${PGDATA}/base-relation"
+    printf 'mariadb\n' > "${STAMP}"
+    : > "${INITDB_CALLS}"
+    run_fails "stamp-mismatch"
+    [ -s "${INITDB_CALLS}" ] \
+        && fail "stamp-mismatch: initdb ran on another engine's volume" \
+        || pass "stamp-mismatch: no initdb"
+    [ -f "${PGDATA}/base-relation" ] \
+        && pass "stamp-mismatch: datadir preserved" || fail "stamp-mismatch: datadir cleared"
+    [ "$(cat "${STAMP}")" = "mariadb" ] \
+        && pass "stamp-mismatch: the other engine's stamp is left as it stands" \
+        || fail "stamp-mismatch: stamp overwritten"
+    grep -q "holds a 'mariadb' datadir" "${WORK}/out" \
+        && pass "stamp-mismatch: refusal names both engines" \
+        || fail "stamp-mismatch: no refusal naming the stamped engine"
+
+    # --- Case 11c: a stamp that cannot be read is not an absent one ---
+    # Reading it as absent would backfill our own engine over a volume whose
+    # engine was never established, turning the check into a rubber stamp.
+    rm -f "${STAMP}"
+    mkdir "${STAMP}"
+    run_fails "stamp-unreadable"
+    grep -q 'could not be read' "${WORK}/out" \
+        && pass "stamp-unreadable: refusal names the unreadable stamp" \
+        || fail "stamp-unreadable: no refusal message"
+    rmdir "${STAMP}"
+
+    # --- Case 11c2: a zero-length stamp does not read as a match ---
+    : > "${STAMP}"
+    run_fails "stamp-empty"
+    [ -f "${PGDATA}/base-relation" ] \
+        && pass "stamp-empty: datadir preserved" || fail "stamp-empty: datadir cleared"
+
+    # --- Case 11d: an unstamped datadir with data in it is backfilled ---
+    # PostgreSQL volumes predate the stamp, so the check becomes total over time
+    # rather than refusing every instance created before it existed.
+    rm -f "${STAMP}"
+    : > "${INITDB_CALLS}"
+    if run_ok "stamp-absent-nonempty"; then
+        [ "$(cat "${STAMP}" 2>/dev/null)" = "postgres" ] \
+            && pass "stamp-absent-nonempty: existing datadir gains a stamp" \
+            || fail "stamp-absent-nonempty: stamp not backfilled"
+        [ -s "${INITDB_CALLS}" ] \
+            && fail "stamp-absent-nonempty: re-ran initdb" || pass "stamp-absent-nonempty: attached"
+        [ -f "${PGDATA}/base-relation" ] \
+            && pass "stamp-absent-nonempty: datadir preserved" \
+            || fail "stamp-absent-nonempty: datadir cleared"
+    fi
+fi
+
+# --- Case 11e: a stamp that cannot be written is fatal, before initdb ---
+# The refusal is before the one-shot password is spent, so the create is still
+# retryable; continuing would leave a volume the next boot cannot identify.
+reset_state
+write_handoff initialize 's3cr3t' appdb
+: > "${WORK}/not-a-dir-stamp"
+STAMP_OVERRIDE="${WORK}/not-a-dir-stamp/engine"
+export STAMP_OVERRIDE
+run_fails "stamp-unwritable"
+[ -s "${INITDB_CALLS}" ] \
+    && fail "stamp-unwritable: initdb ran before the stamp was recorded" \
+    || pass "stamp-unwritable: refused before initdb"
+grep -q 'engine stamp' "${WORK}/out" \
+    && pass "stamp-unwritable: refusal names the stamp" || fail "stamp-unwritable: no refusal message"
+unset STAMP_OVERRIDE
+
 if [ "${FAILS}" -eq 0 ]; then
     echo "PASS: all rds-init cases"
     exit 0

scripts/images/rds-postgres/setup.sh

@@ -22,6 +22,13 @@ rc-update add mulga-mgmt-net boot
 # so the delivery lands in a root-only directory.
 install -d -m 0700 /etc/spinifex-rds
 
+# The engine this image bakes. rds-agent builds its engine implementation from
+# this file, so running the wrong implementation is structurally impossible
+# rather than merely validated against; a VM launched as another engine is
+# refused before anything touches the datadir.
+printf 'postgres\n' > /etc/spinifex-rds/engine
+chmod 0444 /etc/spinifex-rds/engine
+
 # Empty in the image: rds-datadir mounts over it at boot. Postgres-owned so the
 # engine can traverse it if the volume's own root is stricter.
 install -d -m 0750 -o postgres -g postgres /var/lib/postgresql

scripts/node-reset.sh

@@ -181,14 +181,6 @@ for dir in "${WIPE_DIRS[@]}"; do
         ! \( "${KEEP_TOP[@]}" \) -delete 2>/dev/null || true
 done
 
-# The wipe above keeps directories, so the pre-cutover cluster/host-N/node-M
-# skeleton would survive beside the node-<id> dirs written now. predastore owns
-# this tree, not setup.sh, so removing it outright loses no installed ownership.
-if ! $KEEP_DATA && [ -d "$DATA_DIR/predastore/cluster" ]; then
-    log "removing the predastore cluster tree"
-    run sudo rm -rf "$DATA_DIR/predastore/cluster"
-fi
-
 # Directories may survive — a mountpoint cannot be removed, and neither can the
 # path leading down to one. Files may not: a surviving file is state that would
 # carry into the new cluster, which is the exact failure this script exists to

spinifex/admin/admin.go

@@ -951,54 +951,39 @@ func SetupAWSCredentials(accessKey, secretKey, region, certPath, bindIP string)
 	return nil
 }
 
-// PredastoreClusterNode is one [[host.node]] entry of the Predastore topology:
-// a role pinned to a host, answering on a port unique within it.
+// PredastoreClusterNode is one [[node]] entry of the Predastore topology: a
+// role pinned to a host.
 type PredastoreClusterNode struct {
 	ID     int
 	HostID int
 	Role   string
-	Port   int
 }
 
-// Predastore node roles: a gate serves the S3 API, blob nodes hold
-// erasure-coded object shards, and meta nodes form the Raft quorum over global
-// state.
+// Predastore node roles: shard storage holds erasure-coded object shards,
+// state replicas form the Raft quorum over global state.
 const (
-	predastoreRoleGate = "gate"
-	predastoreRoleBlob = "blob"
-	predastoreRoleMeta = "meta"
-)
-
-// Predastore node ports. Only a gate binds a socket clients reach; the others
-// are still required and must not collide within a host, so each role gets its
-// own base and colocated siblings count up from it.
-const (
-	predastoreGatePort     = 8443
-	predastoreBlobBasePort = 6660
-	predastoreMetaBasePort = 7660
+	predastoreRoleShardStorage = "shard-storage"
+	predastoreRoleStateReplica = "state-replica"
 )
 
 // PredastoreTopology derives the cluster nodes for a set of machines: each
-// machine hosts a gate, one blob node and one meta node. Node IDs are unique
-// across the whole file, so gates take 1..n, blobs n+1..2n and metas 2n+1..3n.
+// machine hosts one shard-storage node and one state replica. Node IDs are
+// unique across roles, so storage takes 1..n and the replicas n+1..2n.
 func PredastoreTopology(nodes []PredastoreNodeConfig) []PredastoreClusterNode {
-	out := make([]PredastoreClusterNode, 0, len(nodes)*3)
+	out := make([]PredastoreClusterNode, 0, len(nodes)*2)
 	for _, n := range nodes {
-		out = append(out, PredastoreClusterNode{ID: n.ID, HostID: n.ID, Role: predastoreRoleGate, Port: predastoreGatePort})
+		out = append(out, PredastoreClusterNode{ID: n.ID, HostID: n.ID, Role: predastoreRoleShardStorage})
 	}
 	for _, n := range nodes {
-		out = append(out, PredastoreClusterNode{ID: n.ID + len(nodes), HostID: n.ID, Role: predastoreRoleBlob, Port: predastoreBlobBasePort})
-	}
-	for _, n := range nodes {
-		out = append(out, PredastoreClusterNode{ID: n.ID + 2*len(nodes), HostID: n.ID, Role: predastoreRoleMeta, Port: predastoreMetaBasePort})
+		out = append(out, PredastoreClusterNode{ID: n.ID + len(nodes), HostID: n.ID, Role: predastoreRoleStateReplica})
 	}
 	return out
 }
 
 // GenerateMultiNodePredastoreConfig produces a complete predastore.toml for a
 // multi-node Predastore cluster. Each machine becomes one [[host]] — a
-// predastore process owning a data directory and a TLS identity — carrying the
-// gate, blob and meta nodes pinned to it under [[host.node]].
+// predastore process owning a socket on port 6660 and a data directory —
+// carrying the shard-storage and state-replica nodes pinned to it.
 //
 // dataDir is the Spinifex data root; the hosts' data directories are absolute
 // beneath it, since the service runs under systemd with no dependable working
@@ -1064,14 +1049,14 @@ func FindNodeIDByIP(nodes []PredastoreNodeConfig, ip string) int {
 }
 
 // ParsePredastoreHostIDFromConfig parses a predastore.toml string and returns
-// the ID of the [[host]] whose addr matches the given IP, or 0 if not found.
-// A host addr names one machine and never carries a port; the ports belong to
-// the nodes pinned to it.
+// the ID of the [[host]] whose public address matches the given IP, or 0 if
+// not found. The host is what an operator places on a machine; the nodes
+// pinned to it follow from the topology.
 func ParsePredastoreHostIDFromConfig(tomlContent string, ip string) int {
 	var cfg struct {
 		Hosts []struct {
-			ID   int    `toml:"id"`
-			Addr string `toml:"addr"`
+			ID         int    `toml:"id"`
+			PublicAddr string `toml:"public_addr"`
 		} `toml:"host"`
 	}
 	if err := toml.Unmarshal([]byte(tomlContent), &cfg); err != nil {
@@ -1079,7 +1064,12 @@ func ParsePredastoreHostIDFromConfig(tomlContent string, ip string) int {
 		return 0
 	}
 	for _, h := range cfg.Hosts {
-		if h.Addr == ip {
+		host, _, err := net.SplitHostPort(h.PublicAddr)
+		if err != nil {
+			// A bare address without a port is still a usable match.
+			host = h.PublicAddr
+		}
+		if host == ip {
 			return h.ID
 		}
 	}

spinifex/admin/admin_test.go

@@ -802,7 +802,7 @@ func TestCreateServiceDirectories_Idempotent(t *testing.T) {
 func TestGenerateMultiNodePredastoreConfig_Success(t *testing.T) {
 	tmpl := `{{range .Nodes}}[[host]]
 id = {{.ID}}
-addr = "{{.Host}}"
+public_addr = "{{.Host}}:6660"
 data_dir = "{{$.PredastoreDataDir}}"
 {{end}}`
 	nodes := []PredastoreNodeConfig{
@@ -813,19 +813,18 @@ data_dir = "{{$.PredastoreDataDir}}"
 
 	result, err := GenerateMultiNodePredastoreConfig(tmpl, nodes, "AK", "SK", "us-east-1", "nats-token", "/config", "/var/lib/spinifex", "10.0.0.1", 0, NorthstarCredentials{})
 	require.NoError(t, err)
-	assert.Contains(t, result, `addr = "10.0.0.1"`)
-	assert.Contains(t, result, `addr = "10.0.0.3"`)
+	assert.Contains(t, result, `public_addr = "10.0.0.1:6660"`)
+	assert.Contains(t, result, `public_addr = "10.0.0.3:6660"`)
 	assert.Contains(t, result, `data_dir = "/var/lib/spinifex/predastore/cluster"`)
 }
 
-// Each machine hosts a gate, a blob node and a meta node, with node IDs unique
-// across all three roles so the topology validates.
+// Each machine hosts one shard-storage node and one state replica, with node
+// IDs unique across both roles so the topology validates.
 func TestGenerateMultiNodePredastoreConfig_Topology(t *testing.T) {
-	tmpl := `{{range .ClusterNodes}}[[host.node]]
+	tmpl := `{{range .ClusterNodes}}[[node]]
 id = {{.ID}}
 host_id = {{.HostID}}
 role = "{{.Role}}"
-port = {{.Port}}
 {{end}}`
 	nodes := []PredastoreNodeConfig{
 		{ID: 1, Host: "10.0.0.1"},
@@ -836,11 +835,10 @@ port = {{.Port}}
 	result, err := GenerateMultiNodePredastoreConfig(tmpl, nodes, "AK", "SK", "us-east-1", "nats-token", "/config", "/var/lib/spinifex", "10.0.0.1", 0, NorthstarCredentials{})
 	require.NoError(t, err)
 
-	assert.Contains(t, result, "id = 1\nhost_id = 1\nrole = \"gate\"\nport = 8443")
-	assert.Contains(t, result, "id = 4\nhost_id = 1\nrole = \"blob\"\nport = 6660")
-	assert.Contains(t, result, "id = 6\nhost_id = 3\nrole = \"blob\"\nport = 6660")
-	assert.Contains(t, result, "id = 7\nhost_id = 1\nrole = \"meta\"\nport = 7660")
-	assert.Contains(t, result, "id = 9\nhost_id = 3\nrole = \"meta\"\nport = 7660")
+	assert.Contains(t, result, "id = 1\nhost_id = 1\nrole = \"shard-storage\"")
+	assert.Contains(t, result, "id = 3\nhost_id = 3\nrole = \"shard-storage\"")
+	assert.Contains(t, result, "id = 4\nhost_id = 1\nrole = \"state-replica\"")
+	assert.Contains(t, result, "id = 6\nhost_id = 3\nrole = \"state-replica\"")
 }
 
 func TestPredastoreTopology_UniqueIDsAcrossRoles(t *testing.T) {
@@ -849,16 +847,15 @@ func TestPredastoreTopology_UniqueIDsAcrossRoles(t *testing.T) {
 		{ID: 2, Host: "10.0.0.2"},
 	})
 
-	require.Len(t, topology, 6)
+	require.Len(t, topology, 4)
 	seen := map[int]bool{}
 	for _, n := range topology {
 		assert.False(t, seen[n.ID], "duplicate node id %d", n.ID)
 		seen[n.ID] = true
 		assert.Contains(t, []int{1, 2}, n.HostID)
 	}
-	assert.Equal(t, "gate", topology[0].Role)
-	assert.Equal(t, "blob", topology[2].Role)
-	assert.Equal(t, "meta", topology[4].Role)
+	assert.Equal(t, "shard-storage", topology[0].Role)
+	assert.Equal(t, "state-replica", topology[2].Role)
 }
 
 // The northstar template fields were declared but never assigned, so every
@@ -930,23 +927,18 @@ func TestParsePredastoreHostIDFromConfig(t *testing.T) {
 	tomlContent := `
 [[host]]
 id = 1
-bind_addr = "0.0.0.0"
-addr = "10.0.0.1"
-
-[[host.node]]
-id = 1
-role = "gate"
-port = 8443
+bind_addr = "0.0.0.0:6660"
+public_addr = "10.0.0.1:6660"
 
 [[host]]
 id = 2
-bind_addr = "0.0.0.0"
-addr = "10.0.0.2"
+bind_addr = "0.0.0.0:6660"
+public_addr = "10.0.0.2:6660"
 
 [[host]]
 id = 3
-bind_addr = "0.0.0.0"
-addr = "10.0.0.3"
+bind_addr = "0.0.0.0:6660"
+public_addr = "10.0.0.3:6660"
 `
 	assert.Equal(t, 2, ParsePredastoreHostIDFromConfig(tomlContent, "10.0.0.2"))
 	assert.Equal(t, 0, ParsePredastoreHostIDFromConfig(tomlContent, "10.0.0.99"))
@@ -954,15 +946,15 @@ addr = "10.0.0.3"
 	assert.Equal(t, 0, ParsePredastoreHostIDFromConfig("", "10.0.0.1"))
 }
 
-// A host addr never carries a port, so one written with a node's port attached
-// names no machine and must not resolve to a host.
-func TestParsePredastoreHostIDFromConfig_AddressWithPortDoesNotMatch(t *testing.T) {
+// A public_addr without a port must still match, so a hand-edited config that
+// omits it resolves to a host rather than silently to zero.
+func TestParsePredastoreHostIDFromConfig_AddressWithoutPort(t *testing.T) {
 	tomlContent := `
 [[host]]
 id = 7
-addr = "10.0.0.7:6660"
+public_addr = "10.0.0.7"
 `
-	assert.Equal(t, 0, ParsePredastoreHostIDFromConfig(tomlContent, "10.0.0.7"))
+	assert.Equal(t, 7, ParsePredastoreHostIDFromConfig(tomlContent, "10.0.0.7"))
 }
 
 // --- Integration: Full config generation flow ---

spinifex/daemon/daemon_handlers.go

@@ -347,10 +347,13 @@ func (d *Daemon) handleNodeStatus(msg *nats.Msg) {
 	// OVN roles are probed only on DB quorum members; compute-only nodes skip
 	// the shell-out entirely.
 	var wg sync.WaitGroup
-	wg.Go(func() { resp.NATSRole = d.queryNATSRole() })
+	wg.Add(2)
+	go func() { defer wg.Done(); resp.NATSRole = d.queryNATSRole() }()
+	go func() { defer wg.Done(); resp.PredastoreRole = d.queryPredastoreRole() }()
 	if d.isOVNDBQuorumMember() {
-		wg.Go(func() { resp.OVNNBRole = host.OVNDBRole(host.OVNNBTarget, host.OVNNBSchema) })
-		wg.Go(func() { resp.OVNSBRole = host.OVNDBRole(host.OVNSBTarget, host.OVNSBSchema) })
+		wg.Add(2)
+		go func() { defer wg.Done(); resp.OVNNBRole = host.OVNDBRole(host.OVNNBTarget, host.OVNNBSchema) }()
+		go func() { defer wg.Done(); resp.OVNSBRole = host.OVNDBRole(host.OVNSBTarget, host.OVNSBSchema) }()
 	}
 	wg.Wait()
 
@@ -361,7 +364,8 @@ const (
 	roleLeader   = "leader"
 	roleFollower = "follower"
 
-	natsMonitorPort = 8222
+	natsMonitorPort  = 8222
+	predastoreDBPort = 6660
 )
 
 // queryNATSRole queries the local NATS monitoring endpoint to determine this
@@ -374,6 +378,16 @@ func (d *Daemon) queryNATSRole() string {
 	return fetchNATSRole(url, roleHTTPClient)
 }
 
+// queryPredastoreRole queries the local Predastore DB status endpoint to
+// determine this node's Raft role. Returns "leader", "follower", or "".
+func (d *Daemon) queryPredastoreRole() string {
+	if !d.config.HasService("predastore") {
+		return ""
+	}
+	url := "https://" + net.JoinHostPort(d.daemonIP(), strconv.Itoa(predastoreDBPort)) + "/status"
+	return fetchPredastoreRole(url, roleTLSHTTPClient)
+}
+
 // isOVNDBQuorumMember reports whether this node hosts the clustered OVN NB/SB
 // databases. Compute-only nodes short-circuit OVN role probes to "" without
 // shelling out to ovn-appctl.
@@ -387,6 +401,10 @@ func (d *Daemon) isOVNDBQuorumMember() bool {
 
 var roleHTTPClient = &http.Client{Timeout: 500 * time.Millisecond}
 
+var roleTLSHTTPClient = &http.Client{
+	Timeout: 500 * time.Millisecond,
+}
+
 // fetchNATSRole queries a NATS /varz endpoint and returns "leader", "follower", or "".
 func fetchNATSRole(url string, client *http.Client) string {
 	resp, err := client.Get(url) //nolint:noctx // internal monitoring call
@@ -507,6 +525,29 @@ func resolveVMGPU(att gpu.GPUAttachment, byMdev, byPCI map[string]gpu.PoolEntry)
 	return nil
 }
 
+// fetchPredastoreRole queries a Predastore /status endpoint and returns "leader", "follower", or "".
+func fetchPredastoreRole(url string, client *http.Client) string {
+	resp, err := client.Get(url) //nolint:noctx // internal monitoring call
+	if err != nil {
+		slog.Debug("Failed to query Predastore status", "err", err)
+		return ""
+	}
+	defer resp.Body.Close()
+
+	var status struct {
+		IsLeader bool `json:"is_leader"`
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&status); err != nil {
+		slog.Debug("Failed to decode Predastore status", "err", err)
+		return ""
+	}
+
+	if status.IsLeader {
+		return roleLeader
+	}
+	return roleFollower
+}
+
 // handleNodeVMs responds with the list of VMs running on this node.
 // Used by the CLI: spx get vms.
 func (d *Daemon) handleNodeVMs(msg *nats.Msg) {

spinifex/daemon/daemon_handlers_storage.go

@@ -2,8 +2,10 @@ package daemon
 
 import (
 	"log/slog"
+	"net"
 	"os"
 	"path/filepath"
+	"strconv"
 
 	"github.com/mulgadc/spinifex/spinifex/types"
 	"github.com/nats-io/nats.go"
@@ -15,7 +17,8 @@ import (
 type predastoreTOML struct {
 	RS      predastoreRS       `toml:"rs"`
 	Hosts   []predastoreHost   `toml:"host"`
-	Buckets []predastoreBucket `toml:"bucket"`
+	Nodes   []predastoreNode   `toml:"node"`
+	Buckets []predastoreBucket `toml:"buckets"`
 }
 
 type predastoreRS struct {
@@ -23,32 +26,30 @@ type predastoreRS struct {
 	Parity int `toml:"parity"`
 }
 
-// predastoreHost is one predastore process, as written under [[host]]. Addr
-// names the machine and carries no port; the nodes pinned to it supply their
-// own.
+// predastoreHost is one predastore process; the nodes pinned to it are all
+// reachable at its public address.
 type predastoreHost struct {
-	ID    int              `toml:"id"`
-	Addr  string           `toml:"addr"`
-	Nodes []predastoreNode `toml:"node"`
+	ID         int    `toml:"id"`
+	PublicAddr string `toml:"public_addr"`
 }
 
-// predastoreNode is a role pinned to the host it is written under. Its address
-// is that host's addr joined with this port.
+// predastoreNode is a role pinned to a host. Its address is the host's, so
+// reporting a node means resolving its host_id.
 type predastoreNode struct {
-	ID   int    `toml:"id"`
-	Role string `toml:"role"`
-	Port int    `toml:"port"`
+	ID     int    `toml:"id"`
+	HostID int    `toml:"host_id"`
+	Role   string `toml:"role"`
 }
 
-// Predastore node roles, as written in the [[host.node]] blocks.
+// Predastore node roles, as written in the [[node]] blocks.
 const (
-	predastoreRoleGate = "gate"
-	predastoreRoleBlob = "blob"
-	predastoreRoleMeta = "meta"
+	predastoreRoleShardStorage = "shard-storage"
+	predastoreRoleStateReplica = "state-replica"
 )
 
 type predastoreBucket struct {
 	Name   string `toml:"name"`
+	Type   string `toml:"type"`
 	Region string `toml:"region"`
 }
 
@@ -79,33 +80,35 @@ func (d *Daemon) handleStorageConfig(msg *nats.Msg) {
 		},
 	}
 
-	// A node's address is its host's addr plus its own port: colocated nodes
-	// share a machine but never a port.
+	// A node's address is its host's: every node pinned to a host is served
+	// by that one process, keyed apart within it.
+	hostAddrs := make(map[int]predastoreHost, len(cfg.Hosts))
 	for _, h := range cfg.Hosts {
-		for _, n := range h.Nodes {
-			switch n.Role {
-			case predastoreRoleMeta:
-				resp.MetaNodes = append(resp.MetaNodes, types.StorageMetaNode{ID: n.ID, Host: h.Addr, Port: n.Port})
-			case predastoreRoleBlob:
-				resp.BlobNodes = append(resp.BlobNodes, types.StorageBlobNode{ID: n.ID, Host: h.Addr, Port: n.Port})
-			case predastoreRoleGate:
-				// A gate serves the S3 API and holds no cluster state, so it
-				// belongs to neither table this report carries.
-			default:
-				slog.Warn("handleStorageConfig: unknown predastore node role", "node", n.ID, "role", n.Role)
-			}
+		hostAddrs[h.ID] = h
+	}
+
+	for _, n := range cfg.Nodes {
+		host, port := splitPredastoreAddr(hostAddrs[n.HostID].PublicAddr)
+		switch n.Role {
+		case predastoreRoleStateReplica:
+			resp.DBNodes = append(resp.DBNodes, types.StorageDBNode{ID: n.ID, Host: host, Port: port})
+		case predastoreRoleShardStorage:
+			resp.ShardNodes = append(resp.ShardNodes, types.StorageShardNode{ID: n.ID, Host: host, Port: port})
+		default:
+			slog.Warn("handleStorageConfig: unknown predastore node role", "node", n.ID, "role", n.Role)
 		}
 	}
-	if resp.MetaNodes == nil {
-		resp.MetaNodes = []types.StorageMetaNode{}
+	if resp.DBNodes == nil {
+		resp.DBNodes = []types.StorageDBNode{}
 	}
-	if resp.BlobNodes == nil {
-		resp.BlobNodes = []types.StorageBlobNode{}
+	if resp.ShardNodes == nil {
+		resp.ShardNodes = []types.StorageShardNode{}
 	}
 
 	for _, b := range cfg.Buckets {
 		resp.Buckets = append(resp.Buckets, types.StorageBucket{
 			Name:   b.Name,
+			Type:   b.Type,
 			Region: b.Region,
 		})
 	}
@@ -115,3 +118,18 @@ func (d *Daemon) handleStorageConfig(msg *nats.Msg) {
 
 	respondWithJSON(msg, resp)
 }
+
+// splitPredastoreAddr splits a host's "ip:port" into its parts. An address
+// missing or malformed in the config yields the raw value and a zero port
+// rather than dropping the node from the report entirely.
+func splitPredastoreAddr(addr string) (string, int) {
+	host, portStr, err := net.SplitHostPort(addr)
+	if err != nil {
+		return addr, 0
+	}
+	port, err := strconv.Atoi(portStr)
+	if err != nil {
+		return host, 0
+	}
+	return host, port
+}

spinifex/daemon/daemon_handlers_storage_test.go

@@ -1,165 +1,82 @@
 package daemon
 
 import (
-	"encoding/json"
-	"os"
-	"path/filepath"
 	"testing"
-	"time"
 
-	"github.com/mulgadc/spinifex/spinifex/types"
-	"github.com/nats-io/nats.go"
 	toml "github.com/pelletier/go-toml/v2"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-// clusterTOML is the two-host topology both tests below read: host 1 runs a
-// gate, a blob node and a meta replica; host 2 runs a second blob node.
-const clusterTOML = `
-version = 1
-region = "ap-southeast-2"
-
+// The storage status report is built by parsing predastore.toml directly, so
+// it degrades silently when the schema moves under it: a config that no longer
+// matches yields an empty topology rather than an error. This pins the parse
+// to the [[host]]/[[node]] dialect the templates actually emit.
+func TestPredastoreTOML_ParsesClusterTopology(t *testing.T) {
+	content := `
 [rs]
 data = 2
 parity = 1
 
 [[host]]
 id = 1
-bind_addr = "0.0.0.0"
-addr = "10.0.0.1"
+bind_addr = "0.0.0.0:6660"
+public_addr = "10.0.0.1:6660"
 data_dir = "/var/lib/spinifex/predastore/cluster"
 
-[[host.node]]
-id = 1
-role = "gate"
-port = 8443
-
-[[host.node]]
-id = 2
-role = "blob"
-port = 6660
-
-[[host.node]]
-id = 3
-role = "meta"
-port = 7660
-
 [[host]]
 id = 2
-bind_addr = "0.0.0.0"
-addr = "10.0.0.2"
+bind_addr = "0.0.0.0:6660"
+public_addr = "10.0.0.2:6660"
 data_dir = "/var/lib/spinifex/predastore/cluster"
 
-[[host.node]]
-id = 4
-role = "blob"
-port = 6660
+[[node]]
+id = 1
+host_id = 1
+role = "shard-storage"
 
-[[bucket]]
+[[node]]
+id = 2
+host_id = 2
+role = "shard-storage"
+
+[[node]]
+id = 3
+host_id = 1
+role = "state-replica"
+
+[[buckets]]
 name = "predastore"
+type = "distributed"
 region = "ap-southeast-2"
 `
 
-// storageConfigResponse installs cfg as the node's predastore.toml, drives the
-// handler over NATS and returns what it answered.
-func storageConfigResponse(t *testing.T, cfg string, reply string) []byte {
-	t.Helper()
-
-	dir := t.TempDir()
-	if cfg != "" {
-		require.NoError(t, os.MkdirAll(filepath.Join(dir, "predastore"), 0o750))
-		require.NoError(t, os.WriteFile(filepath.Join(dir, "predastore", "predastore.toml"), []byte(cfg), 0o600))
-	}
-
-	nc, err := nats.Connect(sharedNATSURL)
-	require.NoError(t, err)
-	t.Cleanup(nc.Close)
-
-	replies := make(chan []byte, 1)
-	sub, err := nc.Subscribe(reply, func(msg *nats.Msg) { replies <- msg.Data })
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = sub.Unsubscribe() })
-
-	d := &Daemon{configPath: filepath.Join(dir, "spinifex.toml")}
-	d.handleStorageConfig(&nats.Msg{Subject: "storage.config", Reply: reply, Sub: sub})
-
-	select {
-	case payload := <-replies:
-		return payload
-	case <-time.After(5 * time.Second):
-		t.Fatal("handleStorageConfig sent no response")
-		return nil
-	}
-}
-
-// TestHandleStorageConfig_SplitsNodesByRole pins what the gateway builds its
-// storage report from: every node is reported at its host address and its own
-// port, meta and blob in separate tables, and the gate in neither — it serves
-// the S3 API and holds no cluster state to report.
-func TestHandleStorageConfig_SplitsNodesByRole(t *testing.T) {
-	payload := storageConfigResponse(t, clusterTOML, "test.storage.config.ok")
-
-	var resp types.StorageConfigResponse
-	require.NoError(t, json.Unmarshal(payload, &resp))
-
-	assert.Equal(t, 2, resp.Encoding.DataShards)
-	assert.Equal(t, 1, resp.Encoding.ParityShards)
-
-	require.Len(t, resp.MetaNodes, 1)
-	assert.Equal(t, types.StorageMetaNode{ID: 3, Host: "10.0.0.1", Port: 7660}, resp.MetaNodes[0])
-
-	require.Len(t, resp.BlobNodes, 2)
-	assert.Equal(t, types.StorageBlobNode{ID: 2, Host: "10.0.0.1", Port: 6660}, resp.BlobNodes[0])
-	assert.Equal(t, types.StorageBlobNode{ID: 4, Host: "10.0.0.2", Port: 6660}, resp.BlobNodes[1])
-
-	require.Len(t, resp.Buckets, 1)
-	assert.Equal(t, "predastore", resp.Buckets[0].Name)
-	assert.Equal(t, "ap-southeast-2", resp.Buckets[0].Region)
-}
-
-// TestHandleStorageConfig_ReportsAMissingConfig covers the node that has no
-// predastore config: the report is an error rather than an empty topology,
-// which would read as a cluster with no storage at all.
-func TestHandleStorageConfig_ReportsAMissingConfig(t *testing.T) {
-	payload := storageConfigResponse(t, "", "test.storage.config.absent")
-
-	assert.Contains(t, string(payload), "InternalError")
-}
-
-// TestHandleStorageConfig_ReportsAMalformedConfig covers a config that parses
-// as neither the current schema nor anything else: the same error, rather than
-// a half-read topology.
-func TestHandleStorageConfig_ReportsAMalformedConfig(t *testing.T) {
-	payload := storageConfigResponse(t, "not = [toml", "test.storage.config.malformed")
-
-	assert.Contains(t, string(payload), "InternalError")
-}
-
-// The storage status report is built by parsing predastore.toml directly, so
-// it degrades silently when the schema moves under it: a config that no longer
-// matches yields an empty topology rather than an error. This pins the parse
-// to the [[host]]/[[host.node]] dialect the templates actually emit.
-func TestPredastoreTOML_ParsesClusterTopology(t *testing.T) {
 	var cfg predastoreTOML
-	require.NoError(t, toml.Unmarshal([]byte(clusterTOML), &cfg))
+	require.NoError(t, toml.Unmarshal([]byte(content), &cfg))
 
 	assert.Equal(t, 2, cfg.RS.Data)
 	assert.Equal(t, 1, cfg.RS.Parity)
 	require.Len(t, cfg.Hosts, 2)
-	require.Len(t, cfg.Buckets, 1)
+	require.Len(t, cfg.Nodes, 3)
 
-	assert.Equal(t, "10.0.0.2", cfg.Hosts[1].Addr)
-	require.Len(t, cfg.Hosts[0].Nodes, 3)
-	assert.Equal(t, predastoreRoleGate, cfg.Hosts[0].Nodes[0].Role)
-	assert.Equal(t, 8443, cfg.Hosts[0].Nodes[0].Port)
-	assert.Equal(t, predastoreRoleBlob, cfg.Hosts[0].Nodes[1].Role)
-	assert.Equal(t, predastoreRoleMeta, cfg.Hosts[0].Nodes[2].Role)
-	assert.Equal(t, 7660, cfg.Hosts[0].Nodes[2].Port)
-
-	require.Len(t, cfg.Hosts[1].Nodes, 1)
-	assert.Equal(t, 4, cfg.Hosts[1].Nodes[0].ID)
-	assert.Equal(t, predastoreRoleBlob, cfg.Hosts[1].Nodes[0].Role)
-	assert.Equal(t, 6660, cfg.Hosts[1].Nodes[0].Port)
-	assert.Equal(t, "predastore", cfg.Buckets[0].Name)
+	assert.Equal(t, "10.0.0.2:6660", cfg.Hosts[1].PublicAddr)
+	assert.Equal(t, predastoreRoleShardStorage, cfg.Nodes[0].Role)
+	assert.Equal(t, 1, cfg.Nodes[2].HostID)
+	assert.Equal(t, predastoreRoleStateReplica, cfg.Nodes[2].Role)
+}
+
+func TestSplitPredastoreAddr(t *testing.T) {
+	host, port := splitPredastoreAddr("10.0.0.1:6660")
+	assert.Equal(t, "10.0.0.1", host)
+	assert.Equal(t, 6660, port)
+
+	// A hand-edited address without a port must still name its node rather
+	// than dropping it from the report.
+	host, port = splitPredastoreAddr("10.0.0.1")
+	assert.Equal(t, "10.0.0.1", host)
+	assert.Equal(t, 0, port)
+
+	host, port = splitPredastoreAddr("")
+	assert.Empty(t, host)
+	assert.Equal(t, 0, port)
 }

spinifex/daemon/roles_test.go

@@ -76,3 +76,38 @@ func TestFetchNATSRole_BadJSON(t *testing.T) {
 	role := fetchNATSRole(srv.URL, srv.Client())
 	assert.Empty(t, role)
 }
+
+func TestFetchPredastoreRole_Leader(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte(`{"node_id":"1","state":"Leader","is_leader":true}`))
+	}))
+	defer srv.Close()
+
+	role := fetchPredastoreRole(srv.URL, srv.Client())
+	assert.Equal(t, "leader", role)
+}
+
+func TestFetchPredastoreRole_Follower(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte(`{"node_id":"2","state":"Follower","is_leader":false}`))
+	}))
+	defer srv.Close()
+
+	role := fetchPredastoreRole(srv.URL, srv.Client())
+	assert.Equal(t, "follower", role)
+}
+
+func TestFetchPredastoreRole_Unreachable(t *testing.T) {
+	role := fetchPredastoreRole("https://127.0.0.1:1/status", &http.Client{})
+	assert.Empty(t, role)
+}
+
+func TestFetchPredastoreRole_BadJSON(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte(`not json`))
+	}))
+	defer srv.Close()
+
+	role := fetchPredastoreRole(srv.URL, srv.Client())
+	assert.Empty(t, role)
+}

spinifex/gateway/ecr_principal.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/iam"
-	"github.com/mulgadc/predastore/pkg/auth"
+	"github.com/mulgadc/predastore/auth"
 	"github.com/mulgadc/spinifex/spinifex/awserrors"
 	gateway_ecrauth "github.com/mulgadc/spinifex/spinifex/gateway/ecrauth"
 	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"

spinifex/gateway/gateway.go

@@ -16,10 +16,10 @@ import (
 	"github.com/go-chi/chi/v5"
 	"github.com/go-chi/chi/v5/middleware"
 	"github.com/google/uuid"
-	"github.com/mulgadc/predastore/pkg/auth"
+	"github.com/mulgadc/predastore/auth"
 	"github.com/mulgadc/predastore/pkg/iampolicy"
-	"github.com/mulgadc/predastore/pkg/ratelimit"
 	"github.com/mulgadc/predastore/pkg/sigv4"
+	"github.com/mulgadc/predastore/ratelimit"
 	"github.com/mulgadc/spinifex/spinifex/awserrors"
 	gateway_bedrock "github.com/mulgadc/spinifex/spinifex/gateway/bedrock"
 	gateway_ecr "github.com/mulgadc/spinifex/spinifex/gateway/ecr"

spinifex/gateway/gateway_test.go

@@ -17,7 +17,7 @@ import (
 	awscreds "github.com/aws/aws-sdk-go/aws/credentials"
 	awssession "github.com/aws/aws-sdk-go/aws/session"
 	awsec2 "github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/mulgadc/predastore/pkg/ratelimit"
+	"github.com/mulgadc/predastore/ratelimit"
 	"github.com/mulgadc/spinifex/spinifex/awserrors"
 	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
 	"github.com/mulgadc/spinifex/spinifex/testutil"

spinifex/gateway/rds.go

@@ -5,7 +5,7 @@ import (
 	"log/slog"
 	"net/http"
 
-	"github.com/mulgadc/predastore/pkg/auth"
+	"github.com/mulgadc/predastore/auth"
 	"github.com/mulgadc/spinifex/spinifex/awserrors"
 	gateway_rds "github.com/mulgadc/spinifex/spinifex/gateway/rds"
 )

spinifex/gateway/spx/storage.go

@@ -15,8 +15,8 @@ import (
 	"github.com/nats-io/nats.go"
 )
 
-// MetaNodeStatus is a single meta node's live status merged with its config.
-type MetaNodeStatus struct {
+// DBNodeStatus is a single DB node's live status merged with its config.
+type DBNodeStatus struct {
 	ID         int    `json:"id"`
 	Host       string `json:"host"`
 	Port       int    `json:"port"`
@@ -32,10 +32,10 @@ type MetaNodeStatus struct {
 
 // StorageStatusOutput is the response for GetStorageStatus.
 type StorageStatusOutput struct {
-	Encoding  StorageEncodingOutput   `json:"encoding"`
-	MetaNodes []MetaNodeStatus        `json:"meta_nodes"`
-	BlobNodes []types.StorageBlobNode `json:"blob_nodes"`
-	Buckets   []types.StorageBucket   `json:"buckets"`
+	Encoding   StorageEncodingOutput    `json:"encoding"`
+	DBNodes    []DBNodeStatus           `json:"db_nodes"`
+	ShardNodes []types.StorageShardNode `json:"shard_nodes"`
+	Buckets    []types.StorageBucket    `json:"buckets"`
 }
 
 // StorageEncodingOutput adds the type label to the encoding config.
@@ -49,8 +49,8 @@ var storageHTTPClient = &http.Client{
 	Timeout: 1 * time.Second,
 }
 
-// GetStorageStatus fetches predastore topology via NATS, then queries each
-// meta node's /status and /health endpoints in parallel.
+// GetStorageStatus fetches predastore topology via NATS, then queries each DB
+// node's /status and /health endpoints in parallel.
 func GetStorageStatus(nc *nats.Conn) (*StorageStatusOutput, error) {
 	msg, err := nc.Request("spinifex.storage.config", []byte("{}"), 3*time.Second)
 	if err != nil {
@@ -62,22 +62,22 @@ func GetStorageStatus(nc *nats.Conn) (*StorageStatusOutput, error) {
 		return nil, fmt.Errorf("parse storage config: %w", err)
 	}
 
-	metaStatuses := make([]MetaNodeStatus, len(cfg.MetaNodes))
+	dbStatuses := make([]DBNodeStatus, len(cfg.DBNodes))
 	var wg sync.WaitGroup
 	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
 	defer cancel()
 
-	for i, meta := range cfg.MetaNodes {
-		metaStatuses[i] = MetaNodeStatus{
-			ID:   meta.ID,
-			Host: meta.Host,
-			Port: meta.Port,
+	for i, db := range cfg.DBNodes {
+		dbStatuses[i] = DBNodeStatus{
+			ID:   db.ID,
+			Host: db.Host,
+			Port: db.Port,
 		}
 		wg.Add(1)
 		go func(idx int, host string, port int) {
 			defer wg.Done()
-			queryMetaNodeStatus(ctx, &metaStatuses[idx], host, port)
-		}(i, meta.Host, meta.Port)
+			queryDBNodeStatus(ctx, &dbStatuses[idx], host, port)
+		}(i, db.Host, db.Port)
 	}
 	wg.Wait()
 
@@ -87,9 +87,9 @@ func GetStorageStatus(nc *nats.Conn) (*StorageStatusOutput, error) {
 			DataShards:   cfg.Encoding.DataShards,
 			ParityShards: cfg.Encoding.ParityShards,
 		},
-		MetaNodes: metaStatuses,
-		BlobNodes: cfg.BlobNodes,
-		Buckets:   cfg.Buckets,
+		DBNodes:    dbStatuses,
+		ShardNodes: cfg.ShardNodes,
+		Buckets:    cfg.Buckets,
 	}, nil
 }
 
@@ -105,7 +105,7 @@ type predastoreStatusResponse struct {
 	IsLeader   bool   `json:"is_leader"`
 }
 
-func queryMetaNodeStatus(ctx context.Context, out *MetaNodeStatus, host string, port int) {
+func queryDBNodeStatus(ctx context.Context, out *DBNodeStatus, host string, port int) {
 	// Resolve 0.0.0.0 to a routable address for HTTPS.
 	queryHost := host
 	if queryHost == "0.0.0.0" {
@@ -119,7 +119,7 @@ func queryMetaNodeStatus(ctx context.Context, out *MetaNodeStatus, host string,
 	}
 	resp, err := storageHTTPClient.Do(req)
 	if err != nil {
-		slog.Debug("queryMetaNodeStatus: health check failed", "host", host, "port", port, "err", err)
+		slog.Debug("queryDBNodeStatus: health check failed", "host", host, "port", port, "err", err)
 		return
 	}
 	resp.Body.Close()
@@ -132,14 +132,14 @@ func queryMetaNodeStatus(ctx context.Context, out *MetaNodeStatus, host string,
 	}
 	resp, err = storageHTTPClient.Do(req)
 	if err != nil {
-		slog.Debug("queryMetaNodeStatus: status check failed", "host", host, "port", port, "err", err)
+		slog.Debug("queryDBNodeStatus: status check failed", "host", host, "port", port, "err", err)
 		return
 	}
 	defer resp.Body.Close()
 
 	var status predastoreStatusResponse
 	if err := json.NewDecoder(resp.Body).Decode(&status); err != nil {
-		slog.Debug("queryMetaNodeStatus: failed to decode status", "host", host, "port", port, "err", err)
+		slog.Debug("queryDBNodeStatus: failed to decode status", "host", host, "port", port, "err", err)
 		return
 	}
 

spinifex/gateway/spx/storage_test.go

@@ -22,15 +22,15 @@ func TestGetStorageStatus_Success(t *testing.T) {
 	sub, err := nc.Subscribe("spinifex.storage.config", func(msg *nats.Msg) {
 		resp := types.StorageConfigResponse{
 			Encoding: types.StorageEncoding{DataShards: 2, ParityShards: 1},
-			MetaNodes: []types.StorageMetaNode{
+			DBNodes: []types.StorageDBNode{
 				{ID: 1, Host: "127.0.0.1", Port: 0}, // port set below
 			},
-			BlobNodes: []types.StorageBlobNode{
+			ShardNodes: []types.StorageShardNode{
 				{ID: 1, Host: "0.0.0.0", Port: 9991},
 				{ID: 2, Host: "0.0.0.0", Port: 9992},
 			},
 			Buckets: []types.StorageBucket{
-				{Name: "predastore", Region: "ap-southeast-2"},
+				{Name: "predastore", Type: "distributed", Region: "ap-southeast-2"},
 			},
 		}
 		data, _ := json.Marshal(resp)
@@ -46,13 +46,13 @@ func TestGetStorageStatus_Success(t *testing.T) {
 	assert.Equal(t, "Reed-Solomon", out.Encoding.Type)
 	assert.Equal(t, 2, out.Encoding.DataShards)
 	assert.Equal(t, 1, out.Encoding.ParityShards)
-	assert.Len(t, out.BlobNodes, 2)
+	assert.Len(t, out.ShardNodes, 2)
 	assert.Len(t, out.Buckets, 1)
 	assert.Equal(t, "predastore", out.Buckets[0].Name)
-	// Meta node health check will fail (no real predastore running) — expected
-	require.Len(t, out.MetaNodes, 1)
-	assert.Equal(t, 1, out.MetaNodes[0].ID)
-	assert.False(t, out.MetaNodes[0].Healthy) // no predastore server to respond
+	// DB node health check will fail (no real predastore running) — that's expected
+	require.Len(t, out.DBNodes, 1)
+	assert.Equal(t, 1, out.DBNodes[0].ID)
+	assert.False(t, out.DBNodes[0].Healthy) // no predastore server to respond
 }
 
 func TestGetStorageStatus_NoNATSResponse(t *testing.T) {
@@ -63,8 +63,8 @@ func TestGetStorageStatus_NoNATSResponse(t *testing.T) {
 	assert.Contains(t, err.Error(), "storage config request")
 }
 
-func TestQueryMetaNodeStatus_HealthyNode(t *testing.T) {
-	// Start a mock predastore meta node server
+func TestQueryDBNodeStatus_HealthyNode(t *testing.T) {
+	// Start a mock predastore DB server
 	mux := http.NewServeMux()
 	mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Content-Type", "application/json")
@@ -104,8 +104,8 @@ func TestQueryMetaNodeStatus_HealthyNode(t *testing.T) {
 	port, err := strconv.Atoi(u.Port())
 	require.NoError(t, err)
 
-	out := &MetaNodeStatus{ID: 1, Host: host, Port: port}
-	queryMetaNodeStatus(t.Context(), out, host, port)
+	out := &DBNodeStatus{ID: 1, Host: host, Port: port}
+	queryDBNodeStatus(t.Context(), out, host, port)
 
 	assert.True(t, out.Healthy)
 	assert.Equal(t, "Leader", out.State)
@@ -115,9 +115,9 @@ func TestQueryMetaNodeStatus_HealthyNode(t *testing.T) {
 	assert.Equal(t, "998", out.AppliedIdx)
 }
 
-func TestQueryMetaNodeStatus_UnreachableNode(t *testing.T) {
-	out := &MetaNodeStatus{ID: 1, Host: "127.0.0.1", Port: 1} // port 1 won't respond
-	queryMetaNodeStatus(t.Context(), out, "127.0.0.1", 1)
+func TestQueryDBNodeStatus_UnreachableNode(t *testing.T) {
+	out := &DBNodeStatus{ID: 1, Host: "127.0.0.1", Port: 1} // port 1 won't respond
+	queryDBNodeStatus(t.Context(), out, "127.0.0.1", 1)
 
 	assert.False(t, out.Healthy)
 	assert.Empty(t, out.State)

spinifex/handlers/rds/create.go

@@ -44,7 +44,7 @@ func (s *Service) CreateDBInstance(ctx context.Context, input *rds.CreateDBInsta
 	// does not exist leaves no record behind. The set is literals only: the
 	// class's memory has already been folded into every size-derived default, so
 	// the agent never sees a formula.
-	parameters, err := s.resolveGroupParameters(ctx, kv, accountID, req.DBParameterGroupName, req.InstanceClass)
+	parameters, err := s.resolveGroupParameters(ctx, kv, accountID, req.Engine, req.DBParameterGroupName, req.InstanceClass)
 	if err != nil {
 		return nil, err
 	}
@@ -99,6 +99,7 @@ func (s *Service) CreateDBInstance(ctx context.Context, input *rds.CreateDBInsta
 			GatewayCACert:        s.deps.GatewayCACert,
 			Region:               s.region,
 			DBInstanceIdentifier: req.Identifier,
+			Engine:               req.Engine.Name,
 			EngineVersion:        req.EngineVersion,
 			EnginePort:           req.Port,
 		}),