diff --git a/scripts/images/rds-postgres/rds-engine-stamp_test.sh b/scripts/images/rds-postgres/rds-engine-stamp_test.sh index 0fd68d240..229c74c69 100755 --- a/scripts/images/rds-postgres/rds-engine-stamp_test.sh +++ b/scripts/images/rds-postgres/rds-engine-stamp_test.sh @@ -1,10 +1,11 @@ #!/bin/sh set -eu -# The engine this preset bakes is stated in three places that have to agree: the -# stamp setup.sh writes, the manifest tag resolveEngineAMI selects on, and the -# name rds-agent's layout table is keyed by. A drift between them launches the -# wrong image or refuses the right one. +# The engine this preset bakes is stated in four places that have to agree: the +# stamp setup.sh writes, the manifest tag resolveEngineAMI selects on, the name +# rds-agent's layout table is keyed by, and the stamp rds-init puts on the data +# volume. A drift between them launches the wrong image, refuses the right one, +# or stamps a volume with an engine that did not write it. SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) REPO_ROOT=$(CDPATH='' cd -- "${SCRIPT_DIR}/../../.." && pwd) @@ -27,4 +28,10 @@ if ! grep -q "enginePostgres = \"${ENGINE}\"" "${REPO_ROOT}/cmd/rds-agent/engine exit 1 fi +if ! grep -q "^ENGINE=\"${ENGINE}\"\$" "${SCRIPT_DIR}/rds-init"; then + echo "FAIL: rds-init does not stamp the data volume '${ENGINE}'" >&2 + grep '^ENGINE=' "${SCRIPT_DIR}/rds-init" >&2 + exit 1 +fi + echo "rds-engine-stamp: all tests passed" diff --git a/scripts/images/rds-postgres/rds-init b/scripts/images/rds-postgres/rds-init index 84a135fee..db5b5b97c 100644 --- a/scripts/images/rds-postgres/rds-init +++ b/scripts/images/rds-postgres/rds-init @@ -26,6 +26,10 @@ # Nothing there survives a reboot: the next boot re-fetches and the cert is # re-minted rather than persisted. # +# The data volume also carries an engine stamp, written before the datadir is +# inspected and compared on every boot, so another engine's volume is refused +# rather than initialised over. +# # rds-init keys off the datadir, never off RDS_MODE. An initialised datadir # means skip initdb and only refresh config, so a replaced VM comes back on its # existing data volume without re-bootstrapping. RDS_BOOTSTRAP_PENDING is not an @@ -37,6 +41,11 @@ PG_VERSION="${RDS_PG_VERSION:-18}" PG_BIN="${RDS_PG_BIN:-/usr/libexec/postgresql${PG_VERSION}}" PG_USER="${RDS_PG_USER:-postgres}" +# The engine this image bakes, stamped onto the data volume and compared against +# it on every boot. Held here rather than read from /etc/spinifex-rds/engine: +# this script is the PostgreSQL half of the seam, so it is the same constant. +ENGINE="postgres" + # Mount point of the data volume, and the datadir one level inside it. The # datadir is deliberately not the mount point itself: initdb refuses a non-empty # directory and a freshly formatted ext4 volume already holds lost+found. @@ -54,6 +63,11 @@ BOOTSTRAP_SENTINEL="${PGDATA}/rds-bootstrap-incomplete" RECEIPT_DIR="${RDS_RECEIPT_DIR:-${DATA_MOUNT}/.spinifex-rds/bootstrap}" RECEIPT_FILE="${RECEIPT_DIR}/receipt.env" +# Which engine's data this volume holds. Outside PGDATA for the same reason the +# receipt is, and beside it rather than inside the bootstrap directory: it +# outlives any one bootstrap and is read before the datadir is ever inspected. +ENGINE_STAMP="${RDS_ENGINE_STAMP:-${DATA_MOUNT}/.spinifex-rds/engine}" + HANDOFF_DIR="${RDS_HANDOFF_DIR:-/run/spinifex-rds}" ENV_FILE="${HANDOFF_DIR}/bootstrap.env" SOCKET_DIR="${RDS_SOCKET_DIR:-/run/postgresql}" @@ -180,6 +194,52 @@ require_data_volume() { log "warning: ${DATA_MOUNT} is not a mount point; data will not survive a VM replace (RDS_ALLOW_LOCAL_DATADIR=1)" } +# Written before the datadir is inspected, so a volume this image goes on to +# initialise carries the stamp from the first boot onwards. Fatal on failure: +# an unstamped volume is one the next boot cannot tell from another engine's. +write_engine_stamp() { + _stamp_dir=$(dirname "${ENGINE_STAMP}") + install -d -m 0700 -o "${PG_USER}" -g "${PG_USER}" "${_stamp_dir}" || + fail "the datadir engine stamp directory ${_stamp_dir} could not be created; refusing to use a data volume whose engine cannot be recorded" + + _tmp="${ENGINE_STAMP}.new" + printf '%s\n' "${ENGINE}" > "${_tmp}" || + fail "the datadir engine stamp ${ENGINE_STAMP} could not be written; refusing to use a data volume whose engine cannot be recorded" + chown "${PG_USER}:${PG_USER}" "${_tmp}" + chmod 0600 "${_tmp}" + + # Renamed rather than written in place, so a crash leaves either the old + # stamp or none — and none is backfilled, where a torn one would be fatal. + sync "${_tmp}" 2>/dev/null || sync + mv -f "${_tmp}" "${ENGINE_STAMP}" || + fail "the datadir engine stamp ${ENGINE_STAMP} could not be installed; refusing to use a data volume whose engine cannot be recorded" + sync "${_stamp_dir}" 2>/dev/null || sync +} + +# The volume mounts cleanly whichever engine wrote it, and another engine's +# datadir reads as uninitialised here: rds-init would initdb beside the +# customer's data and serve an empty database that passes every health probe. +require_engine_stamp() { + if [ -e "${ENGINE_STAMP}" ]; then + _stamped="" + _stamp_read=0 + _stamped=$(head -n 1 -- "${ENGINE_STAMP}") && _stamp_read=1 + [ "${_stamp_read}" = "1" ] || + fail "the datadir engine stamp ${ENGINE_STAMP} exists but could not be read; refusing to attach to a data volume whose engine cannot be established" + if [ "${_stamped}" != "${ENGINE}" ]; then + fail "the data volume holds a '${_stamped}' datadir but this image runs ${ENGINE}; refusing to touch it. Attach it to a '${_stamped}' instance, or recover the volume out of band." + fi + log "data volume is stamped ${ENGINE}" + return 0 + fi + + # No stamp is either a volume predating it or one about to be initialised. + # Both are ours here — PostgreSQL had unstamped volumes in the field, so a + # non-empty datadir is backfilled rather than refused. + log "data volume carries no engine stamp — recording ${ENGINE}" + write_engine_stamp +} + # Runs initdb into an empty datadir and lays down the cluster config written # once: the include_dir hook and pg_hba. initialise_cluster() { @@ -583,6 +643,7 @@ consume_secrets() { load_handoff require_data_volume +require_engine_stamp if [ -s "${PGDATA}/PG_VERSION" ]; then # The engine would start cleanly on such a datadir and pass every health diff --git a/scripts/images/rds-postgres/rds-init_test.sh b/scripts/images/rds-postgres/rds-init_test.sh index a93bda521..fc7fe73e4 100644 --- a/scripts/images/rds-postgres/rds-init_test.sh +++ b/scripts/images/rds-postgres/rds-init_test.sh @@ -160,11 +160,15 @@ EOF # sync stub: a no-op, except that SYNC_KILL_PARENT signals rds-init from the # directory sync that follows the receipt rename. That is the one window where -# an installed receipt and an armed sweep coexist. +# an installed receipt and an armed sweep coexist. Matched on the receipt +# directory by name: the engine stamp syncs its directory too, long before the +# traps are armed, and killing there would leave the window untested. cat > "${STUBBIN}/sync" <<'EOF' #!/bin/sh -if [ "${SYNC_KILL_PARENT:-0}" = "1" ] && [ -d "${1:-}" ]; then - kill -TERM "$(cat "${KILL_PID_FILE}")" +if [ "${SYNC_KILL_PARENT:-0}" = "1" ]; then + case "${1:-}" in + */bootstrap) kill -TERM "$(cat "${KILL_PID_FILE}")" ;; + esac fi exit 0 EOF @@ -182,6 +186,7 @@ PGDATA="${DATA_MOUNT}/18/data" SENTINEL="${PGDATA}/rds-bootstrap-incomplete" RECEIPT_DIR="${DATA_MOUNT}/.spinifex-rds/bootstrap" RECEIPT="${RECEIPT_DIR}/receipt.env" +STAMP="${DATA_MOUNT}/.spinifex-rds/engine" HANDOFF="${WORK}/run/spinifex-rds" MOUNTS="${WORK}/mounts" KILL_PID_FILE="${WORK}/rds-init.pid" @@ -242,7 +247,7 @@ reset_state() { : > "${PGCTL_CALLS}" : > "${PSQL_CALLS}" unset INITDB_FAIL PG_HBA_AS_DIR PSQL_FAIL LS_FAIL RDS_ALLOW_LOCAL_DATADIR MASTER_USER || true - unset PENDING PAYLOAD_ID DB_ID RECEIPT_DIR_OVERRIDE PSQL_STATUS_CORRUPT || true + unset PENDING PAYLOAD_ID DB_ID RECEIPT_DIR_OVERRIDE PSQL_STATUS_CORRUPT STAMP_OVERRIDE || true } # run_env: invoke a command with every path knob pointed into the temp dir. @@ -255,6 +260,7 @@ run_env() { RDS_LOG_DIR="${WORK}/log" \ RDS_MOUNTS_FILE="${MOUNTS}" \ RDS_RECEIPT_DIR="${RECEIPT_DIR_OVERRIDE:-${RECEIPT_DIR}}" \ + RDS_ENGINE_STAMP="${STAMP_OVERRIDE:-${STAMP}}" \ RDS_ALLOW_LOCAL_DATADIR="${RDS_ALLOW_LOCAL_DATADIR:-0}" \ INITDB_FAIL="${INITDB_FAIL:-0}" \ PG_HBA_AS_DIR="${PG_HBA_AS_DIR:-0}" \ @@ -839,6 +845,97 @@ if run_ok "dbname-at-the-limit"; then && pass "dbname-at-the-limit: accepted" || fail "dbname-at-the-limit: no CREATE DATABASE" fi +# --- Case 11: the data volume records which engine wrote it --- +# Another engine's datadir mounts cleanly and reads as uninitialised here, so +# without the stamp rds-init would initdb beside the customer's data and serve +# an empty database that passes every health probe while the real data sits +# unreferenced — until the first automated snapshot captures the wrong one. +reset_state +write_handoff initialize 's3cr3t' appdb +if run_ok "stamp-absent-empty"; then + [ "$(cat "${STAMP}" 2>/dev/null)" = "postgres" ] \ + && pass "stamp-absent-empty: fresh volume stamped postgres" \ + || fail "stamp-absent-empty: no stamp written at initialisation" + case "${STAMP}" in + "${PGDATA}"/*) fail "stamp-absent-empty: written inside the datadir the traps clear" ;; + *) pass "stamp-absent-empty: written outside PGDATA" ;; + esac + + # --- Case 11a: a matching stamp attaches --- + : > "${INITDB_CALLS}" + write_handoff attach '' appdb + if run_ok "stamp-match"; then + [ -s "${INITDB_CALLS}" ] \ + && fail "stamp-match: re-ran initdb" || pass "stamp-match: attached" + fi + + # --- Case 11b: a disagreeing stamp is fatal and touches nothing --- + echo 'customer table data' > "${PGDATA}/base-relation" + printf 'mariadb\n' > "${STAMP}" + : > "${INITDB_CALLS}" + run_fails "stamp-mismatch" + [ -s "${INITDB_CALLS}" ] \ + && fail "stamp-mismatch: initdb ran on another engine's volume" \ + || pass "stamp-mismatch: no initdb" + [ -f "${PGDATA}/base-relation" ] \ + && pass "stamp-mismatch: datadir preserved" || fail "stamp-mismatch: datadir cleared" + [ "$(cat "${STAMP}")" = "mariadb" ] \ + && pass "stamp-mismatch: the other engine's stamp is left as it stands" \ + || fail "stamp-mismatch: stamp overwritten" + grep -q "holds a 'mariadb' datadir" "${WORK}/out" \ + && pass "stamp-mismatch: refusal names both engines" \ + || fail "stamp-mismatch: no refusal naming the stamped engine" + + # --- Case 11c: a stamp that cannot be read is not an absent one --- + # Reading it as absent would backfill our own engine over a volume whose + # engine was never established, turning the check into a rubber stamp. + rm -f "${STAMP}" + mkdir "${STAMP}" + run_fails "stamp-unreadable" + grep -q 'could not be read' "${WORK}/out" \ + && pass "stamp-unreadable: refusal names the unreadable stamp" \ + || fail "stamp-unreadable: no refusal message" + rmdir "${STAMP}" + + # --- Case 11c2: a zero-length stamp does not read as a match --- + : > "${STAMP}" + run_fails "stamp-empty" + [ -f "${PGDATA}/base-relation" ] \ + && pass "stamp-empty: datadir preserved" || fail "stamp-empty: datadir cleared" + + # --- Case 11d: an unstamped datadir with data in it is backfilled --- + # PostgreSQL volumes predate the stamp, so the check becomes total over time + # rather than refusing every instance created before it existed. + rm -f "${STAMP}" + : > "${INITDB_CALLS}" + if run_ok "stamp-absent-nonempty"; then + [ "$(cat "${STAMP}" 2>/dev/null)" = "postgres" ] \ + && pass "stamp-absent-nonempty: existing datadir gains a stamp" \ + || fail "stamp-absent-nonempty: stamp not backfilled" + [ -s "${INITDB_CALLS}" ] \ + && fail "stamp-absent-nonempty: re-ran initdb" || pass "stamp-absent-nonempty: attached" + [ -f "${PGDATA}/base-relation" ] \ + && pass "stamp-absent-nonempty: datadir preserved" \ + || fail "stamp-absent-nonempty: datadir cleared" + fi +fi + +# --- Case 11e: a stamp that cannot be written is fatal, before initdb --- +# The refusal is before the one-shot password is spent, so the create is still +# retryable; continuing would leave a volume the next boot cannot identify. +reset_state +write_handoff initialize 's3cr3t' appdb +: > "${WORK}/not-a-dir-stamp" +STAMP_OVERRIDE="${WORK}/not-a-dir-stamp/engine" +export STAMP_OVERRIDE +run_fails "stamp-unwritable" +[ -s "${INITDB_CALLS}" ] \ + && fail "stamp-unwritable: initdb ran before the stamp was recorded" \ + || pass "stamp-unwritable: refused before initdb" +grep -q 'engine stamp' "${WORK}/out" \ + && pass "stamp-unwritable: refusal names the stamp" || fail "stamp-unwritable: no refusal message" +unset STAMP_OVERRIDE + if [ "${FAILS}" -eq 0 ]; then echo "PASS: all rds-init cases" exit 0