#!/bin/sh # rds-init — PostgreSQL bootstrap for the spinifex-rds-postgres AMI. # # A boot oneshot ordered before the postgresql service. This is the PostgreSQL # half of the engine seam: initdb, ALTER ROLE, the datadir and the 5432 default # live here and nowhere else in the platform. # # Handoff contract, written by rds-agent into a root-owned tmpfs directory: # # /run/spinifex-rds/bootstrap.env 0600 KEY=value shell fragment: # RDS_MODE=initialize|attach # RDS_DB_INSTANCE_IDENTIFIER= # RDS_MASTER_USERNAME= # RDS_MASTER_PASSWORD= # RDS_DB_NAME= # RDS_PORT= # RDS_DATA_VOLUME_ID / RDS_DATA_VOLUME_SERIAL= # RDS_VM_GENERATION / RDS_FORMAT_AUTHORIZED= # RDS_BOOTSTRAP_PENDING / RDS_PAYLOAD_ID= # /run/spinifex-rds/parameters.conf 0600 resolved parameter-group values in # postgresql.conf syntax # /run/spinifex-rds/server.crt 0600 serving cert, cluster-CA-signed # /run/spinifex-rds/server.key 0600 its private key # # Nothing there survives a reboot: the next boot re-fetches and the cert is # re-minted rather than persisted. # # rds-init keys off the datadir, never off RDS_MODE. An initialised datadir # means skip initdb and only refresh config, so a replaced VM comes back on its # existing data volume without re-bootstrapping. RDS_BOOTSTRAP_PENDING is not an # exception: it is a control-plane-resolved precondition compared against # on-disk evidence, the same shape as RDS_FORMAT_AUTHORIZED. set -eu PG_VERSION="${RDS_PG_VERSION:-18}" PG_BIN="${RDS_PG_BIN:-/usr/libexec/postgresql${PG_VERSION}}" PG_USER="${RDS_PG_USER:-postgres}" # Mount point of the data volume, and the datadir one level inside it. The # datadir is deliberately not the mount point itself: initdb refuses a non-empty # directory and a freshly formatted ext4 volume already holds lost+found. DATA_MOUNT="${RDS_DATA_MOUNT:-/var/lib/postgresql}" PGDATA="${RDS_PGDATA:-${DATA_MOUNT}/${PG_VERSION}/data}" # Written between initdb and a working master role. It lives on the data volume # because no trap survives a crash or a hard kill, and it is what stops the next # boot attaching to a datadir whose master role was never created. BOOTSTRAP_SENTINEL="${PGDATA}/rds-bootstrap-incomplete" # Proof that the master role was durably applied, written once initdb and the # role are both done. Deliberately outside PGDATA, which clear_unfinished_datadir # rm -rf's: no initdb-adjacent path can touch it and PGDATA stays a stock datadir. RECEIPT_DIR="${RDS_RECEIPT_DIR:-${DATA_MOUNT}/.spinifex-rds/bootstrap}" RECEIPT_FILE="${RECEIPT_DIR}/receipt.env" HANDOFF_DIR="${RDS_HANDOFF_DIR:-/run/spinifex-rds}" ENV_FILE="${HANDOFF_DIR}/bootstrap.env" SOCKET_DIR="${RDS_SOCKET_DIR:-/run/postgresql}" TLS_DIR="${RDS_TLS_DIR:-${SOCKET_DIR}/tls}" LOG_DIR="${RDS_LOG_DIR:-/var/log/postgresql}" # The bootstrap server listens on a unix socket in this directory and nowhere # else. It is 0700 and postgres-owned, so only postgres and root can reach the # short trust-authenticated window before the master password exists. BOOTSTRAP_DIR="${RDS_BOOTSTRAP_DIR:-/run/rds-init}" MOUNTS_FILE="${RDS_MOUNTS_FILE:-/proc/mounts}" # The group role the master's administrative privileges hang off. Named as on # AWS, so a customer's `GRANT rds_superuser TO ...` works here too. MASTER_GROUP_ROLE="rds_superuser" log() { echo "[rds-init] $*"; } fail() { echo "[rds-init] ERROR: $*" >&2; exit 1; } # Between these two, the datadir carries PG_VERSION but no usable master role. _created_datadir=0 _bootstrapped=0 # Runs from a trap rather than a statement, so neither `set -e` on an unguarded # step between initdb and a working master role nor a signal can skip it — either # would leave a healthy-looking engine with no usable master role, unrepairable. clear_unfinished_datadir() { [ "${_created_datadir}" = "1" ] || return 0 [ "${_bootstrapped}" = "1" ] && return 0 # The receipt is installed before the traps retire, so a sweep in between # would leave one vouching for a datadir that is about to be deleted. The # agent would acknowledge it and the control plane would destroy the only # copy of the password this instance can ever be initialised with. rm -f -- "${RECEIPT_FILE}" if rm -rf -- "${PGDATA}" && [ ! -e "${PGDATA}" ]; then echo "[rds-init] ERROR: bootstrap did not complete; datadir ${PGDATA} cleared" >&2 return 0 fi echo "[rds-init] ERROR: bootstrap did not complete and datadir ${PGDATA} could not be cleared; recover it out of band before retrying" >&2 } # The server refuses to run as root. as_pg() { su "${PG_USER}" -c "$1"; } load_handoff() { [ -f "${ENV_FILE}" ] || fail "no bootstrap handoff at ${ENV_FILE} — rds-agent has not delivered the bootstrap config" # shellcheck disable=SC1090 . "${ENV_FILE}" RDS_MODE="${RDS_MODE:-}" RDS_DB_INSTANCE_IDENTIFIER="${RDS_DB_INSTANCE_IDENTIFIER:-}" RDS_MASTER_USERNAME="${RDS_MASTER_USERNAME:-}" RDS_MASTER_PASSWORD="${RDS_MASTER_PASSWORD:-}" RDS_DB_NAME="${RDS_DB_NAME:-}" RDS_PORT="${RDS_PORT:-5432}" RDS_VM_GENERATION="${RDS_VM_GENERATION:-}" # Absent from an agent that predates the receipt protocol, which reads as # "nothing is pending" and leaves every path below unchanged. RDS_BOOTSTRAP_PENDING="${RDS_BOOTSTRAP_PENDING:-0}" RDS_PAYLOAD_ID="${RDS_PAYLOAD_ID:-}" [ -n "${RDS_MASTER_USERNAME}" ] || fail "bootstrap config has no RDS_MASTER_USERNAME" # The master role is created NOSUPERUSER, so a master named after the # bootstrap superuser would leave the cluster without one and strip rds-agent # of the privileged SQL it reaches over peer auth — unrepairable, since the # datadir bootstraps once. # # The same set the control plane reserves, matched the same way: anything it # refuses must be refused here too, or the name reaches initdb, fails at role # creation and spends the one-shot password on the way. _master_lower=$(printf '%s' "${RDS_MASTER_USERNAME}" | tr '[:upper:]' '[:lower:]') case "${_master_lower}" in "${PG_USER}" | "${MASTER_GROUP_ROLE}" | rdsadmin | pg_*) fail "RDS_MASTER_USERNAME ${RDS_MASTER_USERNAME} is reserved by the platform; the control plane should have rejected it" ;; esac case "${RDS_PORT}" in '' | *[!0-9]*) fail "bootstrap config has a non-numeric RDS_PORT: ${RDS_PORT}" ;; esac log "bootstrap config loaded (mode=${RDS_MODE:-unset}, port=${RDS_PORT})" } # Every replace boots a fresh clone of this AMI, so a datadir off the data volume # disappears at the next class change or recovery — silently, as the engine looks # fine. require_data_volume() { if awk -v mnt="${DATA_MOUNT}" '$2 == mnt { found = 1 } END { exit !found }' "${MOUNTS_FILE}"; then return 0 fi [ "${RDS_ALLOW_LOCAL_DATADIR:-0}" = "1" ] || fail "${DATA_MOUNT} is not a mount point — the data volume is not attached; refusing to initialise on the boot volume" log "warning: ${DATA_MOUNT} is not a mount point; data will not survive a VM replace (RDS_ALLOW_LOCAL_DATADIR=1)" } # Runs initdb into an empty datadir and lays down the cluster config written # once: the include_dir hook and pg_hba. initialise_cluster() { # An empty datadir in `attach` mode means the data volume did not come back. # Creating a cluster there would report success and hand the customer an # empty database in place of theirs. if [ "${RDS_MODE}" = "attach" ]; then fail "datadir ${PGDATA} is empty but the bootstrap config is Mode=attach — the existing data volume is missing; refusing to create an empty cluster" fi [ -n "${RDS_MASTER_PASSWORD}" ] || fail "Mode=${RDS_MODE:-unset} carries no master password; cannot initialise the cluster" install -d -m 0700 -o "${PG_USER}" -g "${PG_USER}" "$(dirname "${PGDATA}")" "${PGDATA}" # Whether the datadir was empty beforehand gates the failure cleanup below. # A torn write can leave a missing PG_VERSION over intact data, and deleting # then would destroy it in response to the one signal that it is still there. _was_empty=0 if _contents=$(ls -A -- "${PGDATA}" 2>/dev/null); then [ -z "${_contents}" ] && _was_empty=1 else fail "could not inspect datadir ${PGDATA}; refusing to decide whether failure cleanup is safe" fi log "initialising PostgreSQL ${PG_VERSION} cluster at ${PGDATA}" # ICU collation, not libc: musl has no locale support, so a libc-provider # cluster would silently behave as C. Local peer auth keeps root/postgres # administration working once the bootstrap server is gone. if ! as_pg "'${PG_BIN}/initdb' --pgdata='${PGDATA}' -E UTF-8 \ --locale-provider=icu --icu-locale=en-001-x-icu --data-checksums \ --auth-local=peer --auth-host=scram-sha-256"; then # A half-written datadir carries PG_VERSION and would look initialised # forever after, but only what this invocation created may be cleared. if [ "${_was_empty}" = "1" ]; then if rm -rf -- "${PGDATA}" && [ ! -e "${PGDATA}" ]; then fail "initdb failed; datadir cleared" fi fail "initdb failed and datadir ${PGDATA} could not be cleared; recover it out of band before retrying" fi fail "initdb failed and ${PGDATA} was not empty beforehand — refusing to clear it, since the contents may be a datadir whose PG_VERSION was lost; recover the data volume out of band" fi # Only a datadir initdb built over an empty directory may be swept later. _created_datadir="${_was_empty}" trap clear_unfinished_datadir EXIT # An EXIT trap does not run on a signal in a POSIX shell, so the same sweep # is armed for the signals a shutdown, an ACPI powerdown or a host-side # force-stop delivers. EXIT is disarmed first so the sweep runs once. trap 'trap - EXIT; clear_unfinished_datadir; exit 1' INT TERM HUP # No trap survives a crash or a SIGKILL, so the incomplete state is also # recorded on the volume itself. Cleared once the master role exists. printf '%s\n' \ 'rds-init created this datadir and did not finish bootstrapping the master role.' \ 'While this file exists rds-init refuses to start the engine: the datadir carries no' \ 'usable master role, so it must not be attached to. Recover out of band.' \ > "${BOOTSTRAP_SENTINEL}" chown "${PG_USER}:${PG_USER}" "${BOOTSTRAP_SENTINEL}" chmod 0600 "${BOOTSTRAP_SENTINEL}" cat >> "${PGDATA}/pg_hba.conf" <<'EOF' # Managed by rds-init. Clients reach the instance over its customer ENI, gated # by the DB security group as on AWS RDS, and authenticate with scram-sha-256. # TLS is offered, not enforced — clients opt in with sslmode=require and up. host all all 0.0.0.0/0 scram-sha-256 host all all ::/0 scram-sha-256 EOF } # Parameters live beside the data rather than in /etc, since a class change boots # a fresh root volume and would quietly revert them to engine defaults. install_parameters() { if [ -f "${HANDOFF_DIR}/parameters.conf" ]; then install -m 0600 -o "${PG_USER}" -g "${PG_USER}" \ "${HANDOFF_DIR}/parameters.conf" "${PGDATA}/conf.d/10-rds-parameters.conf" return 0 fi # A handoff without parameters keeps the previous include: dropping it would # silently reset the instance to engine defaults. if [ -f "${PGDATA}/conf.d/10-rds-parameters.conf" ]; then log "warning: no parameters in the handoff — keeping the parameters already installed" fi } # The key stays on tmpfs: it is re-minted every fetch, so persisting it would # spread a private key into every snapshot. install_tls() { if [ -f "${HANDOFF_DIR}/server.crt" ] && [ -f "${HANDOFF_DIR}/server.key" ]; then install -d -m 0700 -o "${PG_USER}" -g "${PG_USER}" "${TLS_DIR}" install -m 0600 -o "${PG_USER}" -g "${PG_USER}" "${HANDOFF_DIR}/server.crt" "${TLS_DIR}/server.crt" install -m 0600 -o "${PG_USER}" -g "${PG_USER}" "${HANDOFF_DIR}/server.key" "${TLS_DIR}/server.key" rm -f "${HANDOFF_DIR}/server.crt" "${HANDOFF_DIR}/server.key" SSL_MODE="on" return 0 fi # A re-run after the cert was consumed must not turn TLS off underneath # clients connecting with sslmode=require. if [ -f "${TLS_DIR}/server.crt" ] && [ -f "${TLS_DIR}/server.key" ]; then SSL_MODE="on" return 0 fi log "warning: no serving cert in the handoff — the engine will start without TLS" SSL_MODE="off" } # The 90- prefix sorts last within include_dir and the last setting wins, so a # resolved parameter can never move the endpoint's port or turn off TLS. write_rds_conf() { _ssl="$1" _tmp="${PGDATA}/conf.d/.90-rds-init.conf.new" cat > "${_tmp}" <> "${_tmp}" <> "${PGDATA}/postgresql.conf" fi install_parameters install_tls write_rds_conf "${SSL_MODE}" } # Brings the cluster up on a private socket with trust auth and no TLS, so the # master password can be set before the engine is reachable. Never listens on TCP. start_bootstrap_server() { install -d -m 0700 -o "${PG_USER}" -g "${PG_USER}" "${BOOTSTRAP_DIR}" printf 'local all all trust\n' > "${BOOTSTRAP_DIR}/pg_hba.conf" chown "${PG_USER}:${PG_USER}" "${BOOTSTRAP_DIR}/pg_hba.conf" chmod 0600 "${BOOTSTRAP_DIR}/pg_hba.conf" as_pg "'${PG_BIN}/pg_ctl' --pgdata='${PGDATA}' --wait --timeout=60 \ --log='${LOG_DIR}/rds-init.log' \ -o \"-c listen_addresses='' -c unix_socket_directories='${BOOTSTRAP_DIR}' \ -c unix_socket_permissions=0700 -c hba_file='${BOOTSTRAP_DIR}/pg_hba.conf' -c ssl=off\" \ start" } # The trust-authenticated pg_hba is removed by the caller, not here: a caller # reading this function's status must be reading pg_ctl's. stop_bootstrap_server() { as_pg "'${PG_BIN}/pg_ctl' --pgdata='${PGDATA}' --wait --timeout=60 --mode=fast stop" } # Identifiers and the password ride the environment and are re-quoted by psql, # never reaching a shell word or argv. psql_bootstrap() { RDS_MASTER_USERNAME="${RDS_MASTER_USERNAME}" \ RDS_MASTER_PASSWORD="${RDS_MASTER_PASSWORD}" \ RDS_DB_NAME="${RDS_DB_NAME}" \ RDS_GROUP_ROLE="${MASTER_GROUP_ROLE}" \ "${PG_BIN}/psql" --no-psqlrc --quiet --no-align --tuples-only \ -v ON_ERROR_STOP=1 \ -h "${BOOTSTRAP_DIR}" -p "${RDS_PORT}" -U "${PG_USER}" -d postgres -f - } # Runs on a freshly initialised cluster only. bootstrap_master() { log "starting bootstrap server on ${BOOTSTRAP_DIR}" start_bootstrap_server _rc=0 apply_master_role || _rc=$? if [ "${_rc}" -eq 0 ] && [ -n "${RDS_DB_NAME}" ]; then create_database || _rc=$? fi # `set -e` is suspended for a function called from an AND-OR list, so the # stop status is captured rather than propagated. A failed stop must also not # abort before the bootstrap check below, which names the real cause. _stop_rc=0 stop_bootstrap_server || _stop_rc=$? rm -f "${BOOTSTRAP_DIR}/pg_hba.conf" # The datadir is swept by the traps, which also cover the steps here that # abort under `set -e` before this check is ever reached. if [ "${_rc}" -ne 0 ]; then fail "bootstrapping the master role/database failed (rc=${_rc})" fi # Proof written while the traps are still armed, so a receipt that cannot be # made durable clears the datadir instead of leaving one it cannot vouch for. write_completion_receipt # The master role exists from here, so the datadir is complete and nothing # may clear it — including on the stop failure reported below. mark_bootstrap_complete # A postmaster still holding the datadir blocks the postgresql service for # the life of the boot, so this is a failure and not a warning. if [ "${_stop_rc}" -ne 0 ]; then fail "the bootstrap server did not stop (rc=${_stop_rc}) and still holds ${PGDATA}; the postgresql service cannot start until it is gone" fi } # The value of one receipt key. Never sourced: the receipt is on the data volume, # so a restored volume can carry one written by another instance, and executing # it would run whatever that instance's control plane put there. receipt_value() { [ -f "${RECEIPT_FILE}" ] || return 0 sed -n "s/^$1=//p" "${RECEIPT_FILE}" 2>/dev/null | head -n 1 } # Written between the master role and the retirement of the traps, so a receipt # that cannot be made durable still clears the datadir rather than leaving a # bootstrapped one nobody can prove. # # Fatal on failure, the mirror image of the sentinel removal below: a healthy # engine whose receipt never landed would come up available while the agent # blocked forever on a receipt that will never appear, leaving a working database # that can never accept a password change or a parameter reload. write_completion_receipt() { install -d -m 0700 -o "${PG_USER}" -g "${PG_USER}" "${RECEIPT_DIR}" || fail "the master role exists but the bootstrap receipt directory ${RECEIPT_DIR} could not be created" _tmp="${RECEIPT_FILE}.new" { echo "# Written by rds-init once the master role was applied. Holds no secrets." echo "RDS_RECEIPT_PAYLOAD_ID=${RDS_PAYLOAD_ID}" echo "RDS_RECEIPT_DB_INSTANCE_IDENTIFIER=${RDS_DB_INSTANCE_IDENTIFIER}" echo "RDS_RECEIPT_VM_GENERATION=${RDS_VM_GENERATION}" echo "RDS_RECEIPT_COMPLETED_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)" } > "${_tmp}" || fail "the master role exists but the bootstrap receipt ${RECEIPT_FILE} could not be written" chown "${PG_USER}:${PG_USER}" "${_tmp}" chmod 0600 "${_tmp}" # A receipt still in the page cache is lost by exactly the crash it exists to # survive, so it is on disk before it is renamed into place. sync "${_tmp}" 2>/dev/null || sync mv -f "${_tmp}" "${RECEIPT_FILE}" || fail "the master role exists but the bootstrap receipt ${RECEIPT_FILE} could not be installed" sync "${RECEIPT_DIR}" 2>/dev/null || sync } # PG_VERSION with no sentinel and no receipt is ambiguous from the datadir alone: # either a legacy instance that bootstrapped before receipts existed, which must # attach, or one that applied the master role and crashed before the receipt was # durable, which must not. Only the control plane's view of whether the payload # is still staged separates the two, and that is what RDS_BOOTSTRAP_PENDING says. require_bootstrap_receipt() { [ "${RDS_BOOTSTRAP_PENDING}" = "1" ] || return 0 if [ -n "${RDS_PAYLOAD_ID}" ] && [ "$(receipt_value RDS_RECEIPT_PAYLOAD_ID)" = "${RDS_PAYLOAD_ID}" ] && [ "$(receipt_value RDS_RECEIPT_DB_INSTANCE_IDENTIFIER)" = "${RDS_DB_INSTANCE_IDENTIFIER}" ]; then return 0 fi fail "the control plane has no record of a completed bootstrap for this datadir; refusing to attach to a datadir whose master role cannot be proven. Recover the volume out of band, or delete and recreate the instance." } # Retires the guards armed at initdb, in an order where no signal in between can # clear a datadir that now carries a usable master role. mark_bootstrap_complete() { _bootstrapped=1 rm -f "${BOOTSTRAP_SENTINEL}" || fail "the master role exists but ${BOOTSTRAP_SENTINEL} could not be removed; the next boot will refuse to start until it is deleted" trap - EXIT INT TERM HUP } # The master role is administrative but deliberately not a PostgreSQL SUPERUSER, # as on AWS: a superuser reaches outside the database entirely — COPY FROM # PROGRAM, pg_read_file/pg_write_file, untrusted languages — so master # credentials would be a shell in the DB VM rather than a database privilege. # The cluster's own superuser stays the bootstrap role, which only rds-init and # rds-agent reach, over the unix socket under peer auth. apply_master_role() { log "creating the master role ${RDS_MASTER_USERNAME}" # PG16 hardened CREATEROLE: it can neither grant itself SUPERUSER nor take # membership in a predefined role it does not administer, so CREATEROLE here # is not a route back to the privileges above. psql_bootstrap <<'SQL' \getenv master RDS_MASTER_USERNAME \getenv password RDS_MASTER_PASSWORD \getenv grp RDS_GROUP_ROLE SELECT NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = :'grp') AS create_group \gset \if :create_group CREATE ROLE :"grp" NOLOGIN; \endif GRANT pg_monitor, pg_signal_backend, pg_checkpoint TO :"grp"; SELECT NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = :'master') AS create_role \gset \if :create_role CREATE ROLE :"master" WITH LOGIN NOSUPERUSER CREATEDB CREATEROLE; \endif ALTER ROLE :"master" WITH LOGIN NOSUPERUSER CREATEDB CREATEROLE PASSWORD :'password'; GRANT :"grp" TO :"master" WITH ADMIN OPTION; SQL } # As on AWS, an instance created without a DBName gets no initial database. create_database() { log "creating database ${RDS_DB_NAME}" psql_bootstrap <<'SQL' \getenv master RDS_MASTER_USERNAME \getenv dbname RDS_DB_NAME SELECT NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = :'dbname') AS create_db \gset \if :create_db CREATE DATABASE :"dbname" OWNER :"master"; \endif SQL } # Drops the master password from the handoff once applied, so it is not left # readable for the life of the boot. The rest is tmpfs and goes at reboot. consume_secrets() { [ -f "${ENV_FILE}" ] || return 0 _tmp="${ENV_FILE}.new" install -m 0600 /dev/null "${_tmp}" grep -v '^RDS_MASTER_PASSWORD=' "${ENV_FILE}" >> "${_tmp}" || true mv -f "${_tmp}" "${ENV_FILE}" } load_handoff require_data_volume if [ -s "${PGDATA}/PG_VERSION" ]; then # The engine would start cleanly on such a datadir and pass every health # probe with no master role to log in as. The sentinel says the bootstrap did # not finish, not that the volume is empty, so nothing here clears it. if [ -e "${BOOTSTRAP_SENTINEL}" ]; then fail "datadir ${PGDATA} carries ${BOOTSTRAP_SENTINEL}: an earlier bootstrap was interrupted before the master role existed; recover the data volume out of band rather than starting the engine" fi require_bootstrap_receipt log "datadir ${PGDATA} is already initialised — attaching" install_runtime_config else initialise_cluster install_runtime_config bootstrap_master fi consume_secrets log "bootstrap complete; handing off to the postgresql service"