mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-19 10:52:04 +00:00
feat: stamp the RDS data volume with the engine that wrote it
Another engine's datadir mounts cleanly and reads as uninitialised to
rds-init, which would initialise beside the customer's data and serve an
empty database that passes every health probe while the real data sits
unreferenced.
rds-init now writes the engine to ${DATA_MOUNT}/.spinifex-rds/engine
before the datadir is inspected, backfills it on attach for volumes
predating it, and refuses a stamp that disagrees.
This commit is contained in:
Julian Sommer
@@ -1,10 +1,11 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
# The engine this preset bakes is stated in three places that have to agree: the
|
||||
# stamp setup.sh writes, the manifest tag resolveEngineAMI selects on, and the
|
||||
# name rds-agent's layout table is keyed by. A drift between them launches the
|
||||
# wrong image or refuses the right one.
|
||||
# The engine this preset bakes is stated in four places that have to agree: the
|
||||
# stamp setup.sh writes, the manifest tag resolveEngineAMI selects on, the name
|
||||
# rds-agent's layout table is keyed by, and the stamp rds-init puts on the data
|
||||
# volume. A drift between them launches the wrong image, refuses the right one,
|
||||
# or stamps a volume with an engine that did not write it.
|
||||
|
||||
SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
|
||||
REPO_ROOT=$(CDPATH='' cd -- "${SCRIPT_DIR}/../../.." && pwd)
|
||||
@@ -27,4 +28,10 @@ if ! grep -q "enginePostgres = \"${ENGINE}\"" "${REPO_ROOT}/cmd/rds-agent/engine
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! grep -q "^ENGINE=\"${ENGINE}\"\$" "${SCRIPT_DIR}/rds-init"; then
|
||||
echo "FAIL: rds-init does not stamp the data volume '${ENGINE}'" >&2
|
||||
grep '^ENGINE=' "${SCRIPT_DIR}/rds-init" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "rds-engine-stamp: all tests passed"
|
||||
|
||||
@@ -26,6 +26,10 @@
|
||||
# Nothing there survives a reboot: the next boot re-fetches and the cert is
|
||||
# re-minted rather than persisted.
|
||||
#
|
||||
# The data volume also carries an engine stamp, written before the datadir is
|
||||
# inspected and compared on every boot, so another engine's volume is refused
|
||||
# rather than initialised over.
|
||||
#
|
||||
# rds-init keys off the datadir, never off RDS_MODE. An initialised datadir
|
||||
# means skip initdb and only refresh config, so a replaced VM comes back on its
|
||||
# existing data volume without re-bootstrapping. RDS_BOOTSTRAP_PENDING is not an
|
||||
@@ -37,6 +41,11 @@ PG_VERSION="${RDS_PG_VERSION:-18}"
|
||||
PG_BIN="${RDS_PG_BIN:-/usr/libexec/postgresql${PG_VERSION}}"
|
||||
PG_USER="${RDS_PG_USER:-postgres}"
|
||||
|
||||
# The engine this image bakes, stamped onto the data volume and compared against
|
||||
# it on every boot. Held here rather than read from /etc/spinifex-rds/engine:
|
||||
# this script is the PostgreSQL half of the seam, so it is the same constant.
|
||||
ENGINE="postgres"
|
||||
|
||||
# Mount point of the data volume, and the datadir one level inside it. The
|
||||
# datadir is deliberately not the mount point itself: initdb refuses a non-empty
|
||||
# directory and a freshly formatted ext4 volume already holds lost+found.
|
||||
@@ -54,6 +63,11 @@ BOOTSTRAP_SENTINEL="${PGDATA}/rds-bootstrap-incomplete"
|
||||
RECEIPT_DIR="${RDS_RECEIPT_DIR:-${DATA_MOUNT}/.spinifex-rds/bootstrap}"
|
||||
RECEIPT_FILE="${RECEIPT_DIR}/receipt.env"
|
||||
|
||||
# Which engine's data this volume holds. Outside PGDATA for the same reason the
|
||||
# receipt is, and beside it rather than inside the bootstrap directory: it
|
||||
# outlives any one bootstrap and is read before the datadir is ever inspected.
|
||||
ENGINE_STAMP="${RDS_ENGINE_STAMP:-${DATA_MOUNT}/.spinifex-rds/engine}"
|
||||
|
||||
HANDOFF_DIR="${RDS_HANDOFF_DIR:-/run/spinifex-rds}"
|
||||
ENV_FILE="${HANDOFF_DIR}/bootstrap.env"
|
||||
SOCKET_DIR="${RDS_SOCKET_DIR:-/run/postgresql}"
|
||||
@@ -180,6 +194,52 @@ require_data_volume() {
|
||||
log "warning: ${DATA_MOUNT} is not a mount point; data will not survive a VM replace (RDS_ALLOW_LOCAL_DATADIR=1)"
|
||||
}
|
||||
|
||||
# Written before the datadir is inspected, so a volume this image goes on to
|
||||
# initialise carries the stamp from the first boot onwards. Fatal on failure:
|
||||
# an unstamped volume is one the next boot cannot tell from another engine's.
|
||||
write_engine_stamp() {
|
||||
_stamp_dir=$(dirname "${ENGINE_STAMP}")
|
||||
install -d -m 0700 -o "${PG_USER}" -g "${PG_USER}" "${_stamp_dir}" ||
|
||||
fail "the datadir engine stamp directory ${_stamp_dir} could not be created; refusing to use a data volume whose engine cannot be recorded"
|
||||
|
||||
_tmp="${ENGINE_STAMP}.new"
|
||||
printf '%s\n' "${ENGINE}" > "${_tmp}" ||
|
||||
fail "the datadir engine stamp ${ENGINE_STAMP} could not be written; refusing to use a data volume whose engine cannot be recorded"
|
||||
chown "${PG_USER}:${PG_USER}" "${_tmp}"
|
||||
chmod 0600 "${_tmp}"
|
||||
|
||||
# Renamed rather than written in place, so a crash leaves either the old
|
||||
# stamp or none — and none is backfilled, where a torn one would be fatal.
|
||||
sync "${_tmp}" 2>/dev/null || sync
|
||||
mv -f "${_tmp}" "${ENGINE_STAMP}" ||
|
||||
fail "the datadir engine stamp ${ENGINE_STAMP} could not be installed; refusing to use a data volume whose engine cannot be recorded"
|
||||
sync "${_stamp_dir}" 2>/dev/null || sync
|
||||
}
|
||||
|
||||
# The volume mounts cleanly whichever engine wrote it, and another engine's
|
||||
# datadir reads as uninitialised here: rds-init would initdb beside the
|
||||
# customer's data and serve an empty database that passes every health probe.
|
||||
require_engine_stamp() {
|
||||
if [ -e "${ENGINE_STAMP}" ]; then
|
||||
_stamped=""
|
||||
_stamp_read=0
|
||||
_stamped=$(head -n 1 -- "${ENGINE_STAMP}") && _stamp_read=1
|
||||
[ "${_stamp_read}" = "1" ] ||
|
||||
fail "the datadir engine stamp ${ENGINE_STAMP} exists but could not be read; refusing to attach to a data volume whose engine cannot be established"
|
||||
if [ "${_stamped}" != "${ENGINE}" ]; then
|
||||
fail "the data volume holds a '${_stamped}' datadir but this image runs ${ENGINE}; refusing to touch it. Attach it to a '${_stamped}' instance, or recover the volume out of band."
|
||||
fi
|
||||
log "data volume is stamped ${ENGINE}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# No stamp is either a volume predating it or one about to be initialised.
|
||||
# Both are ours here — PostgreSQL had unstamped volumes in the field, so a
|
||||
# non-empty datadir is backfilled rather than refused.
|
||||
log "data volume carries no engine stamp — recording ${ENGINE}"
|
||||
write_engine_stamp
|
||||
}
|
||||
|
||||
# Runs initdb into an empty datadir and lays down the cluster config written
|
||||
# once: the include_dir hook and pg_hba.
|
||||
initialise_cluster() {
|
||||
@@ -583,6 +643,7 @@ consume_secrets() {
|
||||
|
||||
load_handoff
|
||||
require_data_volume
|
||||
require_engine_stamp
|
||||
|
||||
if [ -s "${PGDATA}/PG_VERSION" ]; then
|
||||
# The engine would start cleanly on such a datadir and pass every health
|
||||
|
||||
@@ -160,11 +160,15 @@ EOF
|
||||
|
||||
# sync stub: a no-op, except that SYNC_KILL_PARENT signals rds-init from the
|
||||
# directory sync that follows the receipt rename. That is the one window where
|
||||
# an installed receipt and an armed sweep coexist.
|
||||
# an installed receipt and an armed sweep coexist. Matched on the receipt
|
||||
# directory by name: the engine stamp syncs its directory too, long before the
|
||||
# traps are armed, and killing there would leave the window untested.
|
||||
cat > "${STUBBIN}/sync" <<'EOF'
|
||||
#!/bin/sh
|
||||
if [ "${SYNC_KILL_PARENT:-0}" = "1" ] && [ -d "${1:-}" ]; then
|
||||
kill -TERM "$(cat "${KILL_PID_FILE}")"
|
||||
if [ "${SYNC_KILL_PARENT:-0}" = "1" ]; then
|
||||
case "${1:-}" in
|
||||
*/bootstrap) kill -TERM "$(cat "${KILL_PID_FILE}")" ;;
|
||||
esac
|
||||
fi
|
||||
exit 0
|
||||
EOF
|
||||
@@ -182,6 +186,7 @@ PGDATA="${DATA_MOUNT}/18/data"
|
||||
SENTINEL="${PGDATA}/rds-bootstrap-incomplete"
|
||||
RECEIPT_DIR="${DATA_MOUNT}/.spinifex-rds/bootstrap"
|
||||
RECEIPT="${RECEIPT_DIR}/receipt.env"
|
||||
STAMP="${DATA_MOUNT}/.spinifex-rds/engine"
|
||||
HANDOFF="${WORK}/run/spinifex-rds"
|
||||
MOUNTS="${WORK}/mounts"
|
||||
KILL_PID_FILE="${WORK}/rds-init.pid"
|
||||
@@ -242,7 +247,7 @@ reset_state() {
|
||||
: > "${PGCTL_CALLS}"
|
||||
: > "${PSQL_CALLS}"
|
||||
unset INITDB_FAIL PG_HBA_AS_DIR PSQL_FAIL LS_FAIL RDS_ALLOW_LOCAL_DATADIR MASTER_USER || true
|
||||
unset PENDING PAYLOAD_ID DB_ID RECEIPT_DIR_OVERRIDE PSQL_STATUS_CORRUPT || true
|
||||
unset PENDING PAYLOAD_ID DB_ID RECEIPT_DIR_OVERRIDE PSQL_STATUS_CORRUPT STAMP_OVERRIDE || true
|
||||
}
|
||||
|
||||
# run_env: invoke a command with every path knob pointed into the temp dir.
|
||||
@@ -255,6 +260,7 @@ run_env() {
|
||||
RDS_LOG_DIR="${WORK}/log" \
|
||||
RDS_MOUNTS_FILE="${MOUNTS}" \
|
||||
RDS_RECEIPT_DIR="${RECEIPT_DIR_OVERRIDE:-${RECEIPT_DIR}}" \
|
||||
RDS_ENGINE_STAMP="${STAMP_OVERRIDE:-${STAMP}}" \
|
||||
RDS_ALLOW_LOCAL_DATADIR="${RDS_ALLOW_LOCAL_DATADIR:-0}" \
|
||||
INITDB_FAIL="${INITDB_FAIL:-0}" \
|
||||
PG_HBA_AS_DIR="${PG_HBA_AS_DIR:-0}" \
|
||||
@@ -839,6 +845,97 @@ if run_ok "dbname-at-the-limit"; then
|
||||
&& pass "dbname-at-the-limit: accepted" || fail "dbname-at-the-limit: no CREATE DATABASE"
|
||||
fi
|
||||
|
||||
# --- Case 11: the data volume records which engine wrote it ---
|
||||
# Another engine's datadir mounts cleanly and reads as uninitialised here, so
|
||||
# without the stamp rds-init would initdb beside the customer's data and serve
|
||||
# an empty database that passes every health probe while the real data sits
|
||||
# unreferenced — until the first automated snapshot captures the wrong one.
|
||||
reset_state
|
||||
write_handoff initialize 's3cr3t' appdb
|
||||
if run_ok "stamp-absent-empty"; then
|
||||
[ "$(cat "${STAMP}" 2>/dev/null)" = "postgres" ] \
|
||||
&& pass "stamp-absent-empty: fresh volume stamped postgres" \
|
||||
|| fail "stamp-absent-empty: no stamp written at initialisation"
|
||||
case "${STAMP}" in
|
||||
"${PGDATA}"/*) fail "stamp-absent-empty: written inside the datadir the traps clear" ;;
|
||||
*) pass "stamp-absent-empty: written outside PGDATA" ;;
|
||||
esac
|
||||
|
||||
# --- Case 11a: a matching stamp attaches ---
|
||||
: > "${INITDB_CALLS}"
|
||||
write_handoff attach '' appdb
|
||||
if run_ok "stamp-match"; then
|
||||
[ -s "${INITDB_CALLS}" ] \
|
||||
&& fail "stamp-match: re-ran initdb" || pass "stamp-match: attached"
|
||||
fi
|
||||
|
||||
# --- Case 11b: a disagreeing stamp is fatal and touches nothing ---
|
||||
echo 'customer table data' > "${PGDATA}/base-relation"
|
||||
printf 'mariadb\n' > "${STAMP}"
|
||||
: > "${INITDB_CALLS}"
|
||||
run_fails "stamp-mismatch"
|
||||
[ -s "${INITDB_CALLS}" ] \
|
||||
&& fail "stamp-mismatch: initdb ran on another engine's volume" \
|
||||
|| pass "stamp-mismatch: no initdb"
|
||||
[ -f "${PGDATA}/base-relation" ] \
|
||||
&& pass "stamp-mismatch: datadir preserved" || fail "stamp-mismatch: datadir cleared"
|
||||
[ "$(cat "${STAMP}")" = "mariadb" ] \
|
||||
&& pass "stamp-mismatch: the other engine's stamp is left as it stands" \
|
||||
|| fail "stamp-mismatch: stamp overwritten"
|
||||
grep -q "holds a 'mariadb' datadir" "${WORK}/out" \
|
||||
&& pass "stamp-mismatch: refusal names both engines" \
|
||||
|| fail "stamp-mismatch: no refusal naming the stamped engine"
|
||||
|
||||
# --- Case 11c: a stamp that cannot be read is not an absent one ---
|
||||
# Reading it as absent would backfill our own engine over a volume whose
|
||||
# engine was never established, turning the check into a rubber stamp.
|
||||
rm -f "${STAMP}"
|
||||
mkdir "${STAMP}"
|
||||
run_fails "stamp-unreadable"
|
||||
grep -q 'could not be read' "${WORK}/out" \
|
||||
&& pass "stamp-unreadable: refusal names the unreadable stamp" \
|
||||
|| fail "stamp-unreadable: no refusal message"
|
||||
rmdir "${STAMP}"
|
||||
|
||||
# --- Case 11c2: a zero-length stamp does not read as a match ---
|
||||
: > "${STAMP}"
|
||||
run_fails "stamp-empty"
|
||||
[ -f "${PGDATA}/base-relation" ] \
|
||||
&& pass "stamp-empty: datadir preserved" || fail "stamp-empty: datadir cleared"
|
||||
|
||||
# --- Case 11d: an unstamped datadir with data in it is backfilled ---
|
||||
# PostgreSQL volumes predate the stamp, so the check becomes total over time
|
||||
# rather than refusing every instance created before it existed.
|
||||
rm -f "${STAMP}"
|
||||
: > "${INITDB_CALLS}"
|
||||
if run_ok "stamp-absent-nonempty"; then
|
||||
[ "$(cat "${STAMP}" 2>/dev/null)" = "postgres" ] \
|
||||
&& pass "stamp-absent-nonempty: existing datadir gains a stamp" \
|
||||
|| fail "stamp-absent-nonempty: stamp not backfilled"
|
||||
[ -s "${INITDB_CALLS}" ] \
|
||||
&& fail "stamp-absent-nonempty: re-ran initdb" || pass "stamp-absent-nonempty: attached"
|
||||
[ -f "${PGDATA}/base-relation" ] \
|
||||
&& pass "stamp-absent-nonempty: datadir preserved" \
|
||||
|| fail "stamp-absent-nonempty: datadir cleared"
|
||||
fi
|
||||
fi
|
||||
|
||||
# --- Case 11e: a stamp that cannot be written is fatal, before initdb ---
|
||||
# The refusal is before the one-shot password is spent, so the create is still
|
||||
# retryable; continuing would leave a volume the next boot cannot identify.
|
||||
reset_state
|
||||
write_handoff initialize 's3cr3t' appdb
|
||||
: > "${WORK}/not-a-dir-stamp"
|
||||
STAMP_OVERRIDE="${WORK}/not-a-dir-stamp/engine"
|
||||
export STAMP_OVERRIDE
|
||||
run_fails "stamp-unwritable"
|
||||
[ -s "${INITDB_CALLS}" ] \
|
||||
&& fail "stamp-unwritable: initdb ran before the stamp was recorded" \
|
||||
|| pass "stamp-unwritable: refused before initdb"
|
||||
grep -q 'engine stamp' "${WORK}/out" \
|
||||
&& pass "stamp-unwritable: refusal names the stamp" || fail "stamp-unwritable: no refusal message"
|
||||
unset STAMP_OVERRIDE
|
||||
|
||||
if [ "${FAILS}" -eq 0 ]; then
|
||||
echo "PASS: all rds-init cases"
|
||||
exit 0
|
||||
|
||||
Reference in New Issue
Block a user