spinifex/scripts/images/rds-postgres/rds-init

#!/bin/sh
# rds-init — PostgreSQL bootstrap for the spinifex-rds-postgres AMI.
#
# A boot oneshot ordered before the postgresql service. This is the PostgreSQL
# half of the engine seam: initdb, ALTER ROLE, the datadir and the 5432 default
# live here and nowhere else in the platform.
#
# Handoff contract, written by rds-agent into a root-owned tmpfs directory:
#
#   /run/spinifex-rds/bootstrap.env    0600  KEY=value shell fragment:
#       RDS_MODE=initialize|attach
#       RDS_DB_INSTANCE_IDENTIFIER=<the DB instance this VM backs>
#       RDS_MASTER_USERNAME=<master role name>
#       RDS_MASTER_PASSWORD=<present only when RDS_MODE=initialize>
#       RDS_DB_NAME=<initial database; optional, as on AWS>
#       RDS_PORT=<listen port; defaults to 5432>
#       RDS_DATA_VOLUME_ID / RDS_DATA_VOLUME_SERIAL=<expected volume identity>
#       RDS_VM_GENERATION / RDS_FORMAT_AUTHORIZED=<create-only format grant>
#       RDS_BOOTSTRAP_PENDING / RDS_PAYLOAD_ID=<the control plane is still
#                                            waiting for this payload to be applied>
#   /run/spinifex-rds/parameters.conf  0600  resolved parameter-group values in
#                                            postgresql.conf syntax
#   /run/spinifex-rds/server.crt       0600  serving cert, cluster-CA-signed
#   /run/spinifex-rds/server.key       0600  its private key
#
# Nothing there survives a reboot: the next boot re-fetches and the cert is
# re-minted rather than persisted.
#
# The data volume also carries an engine stamp, written before the datadir is
# inspected and compared on every boot, so another engine's volume is refused
# rather than initialised over.
#
# rds-init keys off the datadir, never off RDS_MODE. An initialised datadir
# means skip initdb and only refresh config, so a replaced VM comes back on its
# existing data volume without re-bootstrapping. RDS_BOOTSTRAP_PENDING is not an
# exception: it is a control-plane-resolved precondition compared against
# on-disk evidence, the same shape as RDS_FORMAT_AUTHORIZED.
set -eu

PG_VERSION="${RDS_PG_VERSION:-18}"
PG_BIN="${RDS_PG_BIN:-/usr/libexec/postgresql${PG_VERSION}}"
PG_USER="${RDS_PG_USER:-postgres}"

# The engine this image bakes, stamped onto the data volume and compared against
# it on every boot. Held here rather than read from /etc/spinifex-rds/engine:
# this script is the PostgreSQL half of the seam, so it is the same constant.
ENGINE="postgres"

# Mount point of the data volume, and the datadir one level inside it. The
# datadir is deliberately not the mount point itself: initdb refuses a non-empty
# directory and a freshly formatted ext4 volume already holds lost+found.
DATA_MOUNT="${RDS_DATA_MOUNT:-/var/lib/postgresql}"
PGDATA="${RDS_PGDATA:-${DATA_MOUNT}/${PG_VERSION}/data}"

# Written between initdb and a working master role. It lives on the data volume
# because no trap survives a crash or a hard kill, and it is what stops the next
# boot attaching to a datadir whose master role was never created.
BOOTSTRAP_SENTINEL="${PGDATA}/rds-bootstrap-incomplete"

# Proof that the master role was durably applied, written once initdb and the
# role are both done. Deliberately outside PGDATA, which clear_unfinished_datadir
# rm -rf's: no initdb-adjacent path can touch it and PGDATA stays a stock datadir.
RECEIPT_DIR="${RDS_RECEIPT_DIR:-${DATA_MOUNT}/.spinifex-rds/bootstrap}"
RECEIPT_FILE="${RECEIPT_DIR}/receipt.env"

# Which engine's data this volume holds. Outside PGDATA for the same reason the
# receipt is, and beside it rather than inside the bootstrap directory: it
# outlives any one bootstrap and is read before the datadir is ever inspected.
ENGINE_STAMP="${RDS_ENGINE_STAMP:-${DATA_MOUNT}/.spinifex-rds/engine}"

HANDOFF_DIR="${RDS_HANDOFF_DIR:-/run/spinifex-rds}"
ENV_FILE="${HANDOFF_DIR}/bootstrap.env"
SOCKET_DIR="${RDS_SOCKET_DIR:-/run/postgresql}"
TLS_DIR="${RDS_TLS_DIR:-${SOCKET_DIR}/tls}"
LOG_DIR="${RDS_LOG_DIR:-/var/log/postgresql}"

# The bootstrap server listens on a unix socket in this directory and nowhere
# else. It is 0700 and postgres-owned, so only postgres and root can reach the
# short trust-authenticated window before the master password exists.
BOOTSTRAP_DIR="${RDS_BOOTSTRAP_DIR:-/run/rds-init}"

MOUNTS_FILE="${RDS_MOUNTS_FILE:-/proc/mounts}"

# The group role the master's administrative privileges hang off. Named as on
# AWS, so a customer's `GRANT rds_superuser TO ...` works here too.
MASTER_GROUP_ROLE="rds_superuser"

# The Ochre vector store appliance is the one platform (non-tenant) consumer of
# this image, provisioned with this fixed master name. Tenant instances take a
# different master, so the pgvector install below is scoped to the appliance.
OCHRE_APPLIANCE_MASTER="ochre_vector_admin"

# NAMEDATALEN-1: the server truncates a longer identifier, so the database
# created would not be the one the customer asked for.
MAX_DB_NAME_LEN=63

log() { echo "[rds-init] $*"; }
fail() { echo "[rds-init] ERROR: $*" >&2; exit 1; }

# Between these two, the datadir carries PG_VERSION but no usable master role.
_created_datadir=0
_bootstrapped=0

# Runs from a trap rather than a statement, so neither `set -e` on an unguarded
# step between initdb and a working master role nor a signal can skip it — either
# would leave a healthy-looking engine with no usable master role, unrepairable.
clear_unfinished_datadir() {
    [ "${_created_datadir}" = "1" ] || return 0
    [ "${_bootstrapped}" = "1" ] && return 0
    # The receipt is installed before the traps retire, so a sweep in between
    # would leave one vouching for a datadir that is about to be deleted. The
    # agent would acknowledge it and the control plane would destroy the only
    # copy of the password this instance can ever be initialised with.
    rm -f -- "${RECEIPT_FILE}"
    if rm -rf -- "${PGDATA}" && [ ! -e "${PGDATA}" ]; then
        echo "[rds-init] ERROR: bootstrap did not complete; datadir ${PGDATA} cleared" >&2
        return 0
    fi
    echo "[rds-init] ERROR: bootstrap did not complete and datadir ${PGDATA} could not be cleared; recover it out of band before retrying" >&2
}

# The server refuses to run as root.
as_pg() { su "${PG_USER}" -c "$1"; }

load_handoff() {
    [ -f "${ENV_FILE}" ] || fail "no bootstrap handoff at ${ENV_FILE} — rds-agent has not delivered the bootstrap config"

    # shellcheck disable=SC1090
    . "${ENV_FILE}"

    RDS_MODE="${RDS_MODE:-}"
    RDS_DB_INSTANCE_IDENTIFIER="${RDS_DB_INSTANCE_IDENTIFIER:-}"
    RDS_MASTER_USERNAME="${RDS_MASTER_USERNAME:-}"
    RDS_MASTER_PASSWORD="${RDS_MASTER_PASSWORD:-}"
    RDS_DB_NAME="${RDS_DB_NAME:-}"
    RDS_PORT="${RDS_PORT:-5432}"
    RDS_VM_GENERATION="${RDS_VM_GENERATION:-}"
    # Absent from an agent that predates the receipt protocol, which reads as
    # "nothing is pending" and leaves every path below unchanged.
    RDS_BOOTSTRAP_PENDING="${RDS_BOOTSTRAP_PENDING:-0}"
    RDS_PAYLOAD_ID="${RDS_PAYLOAD_ID:-}"

    [ -n "${RDS_MASTER_USERNAME}" ] || fail "bootstrap config has no RDS_MASTER_USERNAME"

    # The master role is created NOSUPERUSER, so a master named after the
    # bootstrap superuser would leave the cluster without one and strip rds-agent
    # of the privileged SQL it reaches over peer auth — unrepairable, since the
    # datadir bootstraps once.
    #
    # The same set the control plane reserves, matched the same way: anything it
    # refuses must be refused here too, or the name reaches initdb, fails at role
    # creation and spends the one-shot password on the way.
    _master_lower=$(printf '%s' "${RDS_MASTER_USERNAME}" | tr '[:upper:]' '[:lower:]')
    case "${_master_lower}" in
        "${PG_USER}" | "${MASTER_GROUP_ROLE}" | rdsadmin | pg_*)
            fail "RDS_MASTER_USERNAME ${RDS_MASTER_USERNAME} is reserved by the platform; the control plane should have rejected it"
            ;;
    esac

    # Scopes the platform pgvector install to the appliance; a tenant master
    # takes a different name and skips it.
    RDS_IS_OCHRE_APPLIANCE=0
    [ "${_master_lower}" = "${OCHRE_APPLIANCE_MASTER}" ] && RDS_IS_OCHRE_APPLIANCE=1

    # The same rule the control plane applies, for the same reason the reserved
    # name is re-checked: an unusable name reaches initdb, fails at CREATE
    # DATABASE and spends the one-shot password on the way. psql quotes the
    # identifier, so here this is defence in depth rather than the barrier.
    case "${RDS_DB_NAME}" in
        '') ;;
        [!A-Za-z]* | *[!A-Za-z0-9_]*)
            fail "RDS_DB_NAME ${RDS_DB_NAME} must begin with a letter and hold only letters, digits and underscores; the control plane should have rejected it"
            ;;
    esac
    if [ "${#RDS_DB_NAME}" -gt "${MAX_DB_NAME_LEN}" ]; then
        fail "RDS_DB_NAME is longer than the ${MAX_DB_NAME_LEN} characters PostgreSQL allows; the control plane should have rejected it"
    fi

    case "${RDS_PORT}" in
        '' | *[!0-9]*) fail "bootstrap config has a non-numeric RDS_PORT: ${RDS_PORT}" ;;
    esac

    # redact_stream is line-oriented, so a password spanning a line boundary
    # would match neither half and reach /dev/console in the clear. The control
    # plane rejects one; refusing here too keeps that an asserted property of
    # the handoff rather than a silent dependency on another service.
    case "${RDS_MASTER_PASSWORD}" in
        *'
'*) fail "the master password in the bootstrap config spans more than one line; the control plane should have rejected it" ;;
    esac

    log "bootstrap config loaded (mode=${RDS_MODE:-unset}, port=${RDS_PORT})"
}

# Every replace boots a fresh clone of this AMI, so a datadir off the data volume
# disappears at the next class change or recovery — silently, as the engine looks
# fine.
require_data_volume() {
    if awk -v mnt="${DATA_MOUNT}" '$2 == mnt { found = 1 } END { exit !found }' "${MOUNTS_FILE}"; then
        return 0
    fi
    [ "${RDS_ALLOW_LOCAL_DATADIR:-0}" = "1" ] ||
        fail "${DATA_MOUNT} is not a mount point — the data volume is not attached; refusing to initialise on the boot volume"
    log "warning: ${DATA_MOUNT} is not a mount point; data will not survive a VM replace (RDS_ALLOW_LOCAL_DATADIR=1)"
}

# Written before the datadir is inspected, so a volume this image goes on to
# initialise carries the stamp from the first boot onwards. Fatal on failure:
# an unstamped volume is one the next boot cannot tell from another engine's.
write_engine_stamp() {
    _stamp_dir=$(dirname "${ENGINE_STAMP}")
    install -d -m 0700 -o "${PG_USER}" -g "${PG_USER}" "${_stamp_dir}" ||
        fail "the datadir engine stamp directory ${_stamp_dir} could not be created; refusing to use a data volume whose engine cannot be recorded"

    _tmp="${ENGINE_STAMP}.new"
    printf '%s\n' "${ENGINE}" > "${_tmp}" ||
        fail "the datadir engine stamp ${ENGINE_STAMP} could not be written; refusing to use a data volume whose engine cannot be recorded"
    chown "${PG_USER}:${PG_USER}" "${_tmp}"
    chmod 0600 "${_tmp}"

    # Renamed rather than written in place, so a crash leaves either the old
    # stamp or none — and none is backfilled, where a torn one would be fatal.
    sync "${_tmp}" 2>/dev/null || sync
    mv -f "${_tmp}" "${ENGINE_STAMP}" ||
        fail "the datadir engine stamp ${ENGINE_STAMP} could not be installed; refusing to use a data volume whose engine cannot be recorded"
    sync "${_stamp_dir}" 2>/dev/null || sync
}

# The volume mounts cleanly whichever engine wrote it, and another engine's
# datadir reads as uninitialised here: rds-init would initdb beside the
# customer's data and serve an empty database that passes every health probe.
require_engine_stamp() {
    if [ -e "${ENGINE_STAMP}" ]; then
        _stamped=""
        _stamp_read=0
        _stamped=$(head -n 1 -- "${ENGINE_STAMP}") && _stamp_read=1
        [ "${_stamp_read}" = "1" ] ||
            fail "the datadir engine stamp ${ENGINE_STAMP} exists but could not be read; refusing to attach to a data volume whose engine cannot be established"
        if [ "${_stamped}" != "${ENGINE}" ]; then
            fail "the data volume holds a '${_stamped}' datadir but this image runs ${ENGINE}; refusing to touch it. Attach it to a '${_stamped}' instance, or recover the volume out of band."
        fi
        log "data volume is stamped ${ENGINE}"
        return 0
    fi

    # No stamp is either a volume predating it or one about to be initialised.
    # Both are ours here — PostgreSQL had unstamped volumes in the field, so a
    # non-empty datadir is backfilled rather than refused.
    log "data volume carries no engine stamp — recording ${ENGINE}"
    write_engine_stamp
}

# Runs initdb into an empty datadir and lays down the cluster config written
# once: the include_dir hook and pg_hba.
initialise_cluster() {
    # An empty datadir in `attach` mode means the data volume did not come back.
    # Creating a cluster there would report success and hand the customer an
    # empty database in place of theirs.
    if [ "${RDS_MODE}" = "attach" ]; then
        fail "datadir ${PGDATA} is empty but the bootstrap config is Mode=attach — the existing data volume is missing; refusing to create an empty cluster"
    fi
    [ -n "${RDS_MASTER_PASSWORD}" ] || fail "Mode=${RDS_MODE:-unset} carries no master password; cannot initialise the cluster"

    install -d -m 0700 -o "${PG_USER}" -g "${PG_USER}" "$(dirname "${PGDATA}")" "${PGDATA}"

    # Whether the datadir was empty beforehand gates the failure cleanup below.
    # A torn write can leave a missing PG_VERSION over intact data, and deleting
    # then would destroy it in response to the one signal that it is still there.
    _was_empty=0
    if _contents=$(ls -A -- "${PGDATA}" 2>/dev/null); then
        [ -z "${_contents}" ] && _was_empty=1
    else
        fail "could not inspect datadir ${PGDATA}; refusing to decide whether failure cleanup is safe"
    fi

    log "initialising PostgreSQL ${PG_VERSION} cluster at ${PGDATA}"
    # ICU collation, not libc: musl has no locale support, so a libc-provider
    # cluster would silently behave as C. Local peer auth keeps root/postgres
    # administration working once the bootstrap server is gone.
    if ! as_pg "'${PG_BIN}/initdb' --pgdata='${PGDATA}' -E UTF-8 \
        --locale-provider=icu --icu-locale=en-001-x-icu --data-checksums \
        --auth-local=peer --auth-host=scram-sha-256"; then
        # A half-written datadir carries PG_VERSION and would look initialised
        # forever after, but only what this invocation created may be cleared.
        if [ "${_was_empty}" = "1" ]; then
            if rm -rf -- "${PGDATA}" && [ ! -e "${PGDATA}" ]; then
                fail "initdb failed; datadir cleared"
            fi
            fail "initdb failed and datadir ${PGDATA} could not be cleared; recover it out of band before retrying"
        fi
        fail "initdb failed and ${PGDATA} was not empty beforehand — refusing to clear it, since the contents may be a datadir whose PG_VERSION was lost; recover the data volume out of band"
    fi

    # Only a datadir initdb built over an empty directory may be swept later.
    _created_datadir="${_was_empty}"
    trap clear_unfinished_datadir EXIT
    # An EXIT trap does not run on a signal in a POSIX shell, so the same sweep
    # is armed for the signals a shutdown, an ACPI powerdown or a host-side
    # force-stop delivers. EXIT is disarmed first so the sweep runs once.
    trap 'trap - EXIT; clear_unfinished_datadir; exit 1' INT TERM HUP

    # No trap survives a crash or a SIGKILL, so the incomplete state is also
    # recorded on the volume itself. Cleared once the master role exists.
    printf '%s\n' \
        'rds-init created this datadir and did not finish bootstrapping the master role.' \
        'While this file exists rds-init refuses to start the engine: the datadir carries no' \
        'usable master role, so it must not be attached to. Recover out of band.' \
        > "${BOOTSTRAP_SENTINEL}"
    chown "${PG_USER}:${PG_USER}" "${BOOTSTRAP_SENTINEL}"
    chmod 0600 "${BOOTSTRAP_SENTINEL}"

    cat >> "${PGDATA}/pg_hba.conf" <<'EOF'

# Managed by rds-init. Clients reach the instance over its customer ENI, gated
# by the DB security group as on AWS RDS, and authenticate with scram-sha-256.
# TLS is offered, not enforced — clients opt in with sslmode=require and up.
host all all 0.0.0.0/0 scram-sha-256
host all all ::/0     scram-sha-256
EOF
}

# Parameters live beside the data rather than in /etc, since a class change boots
# a fresh root volume and would quietly revert them to engine defaults.
install_parameters() {
    if [ -f "${HANDOFF_DIR}/parameters.conf" ]; then
        install -m 0600 -o "${PG_USER}" -g "${PG_USER}" \
            "${HANDOFF_DIR}/parameters.conf" "${PGDATA}/conf.d/10-rds-parameters.conf"
        return 0
    fi
    # A handoff without parameters keeps the previous include: dropping it would
    # silently reset the instance to engine defaults.
    if [ -f "${PGDATA}/conf.d/10-rds-parameters.conf" ]; then
        log "warning: no parameters in the handoff — keeping the parameters already installed"
    fi
}

# The key stays on tmpfs: it is re-minted every fetch, so persisting it would
# spread a private key into every snapshot.
install_tls() {
    if [ -f "${HANDOFF_DIR}/server.crt" ] && [ -f "${HANDOFF_DIR}/server.key" ]; then
        install -d -m 0700 -o "${PG_USER}" -g "${PG_USER}" "${TLS_DIR}"
        install -m 0600 -o "${PG_USER}" -g "${PG_USER}" "${HANDOFF_DIR}/server.crt" "${TLS_DIR}/server.crt"
        install -m 0600 -o "${PG_USER}" -g "${PG_USER}" "${HANDOFF_DIR}/server.key" "${TLS_DIR}/server.key"
        rm -f "${HANDOFF_DIR}/server.crt" "${HANDOFF_DIR}/server.key"
        SSL_MODE="on"
        return 0
    fi
    # A re-run after the cert was consumed must not turn TLS off underneath
    # clients connecting with sslmode=require.
    if [ -f "${TLS_DIR}/server.crt" ] && [ -f "${TLS_DIR}/server.key" ]; then
        SSL_MODE="on"
        return 0
    fi
    log "warning: no serving cert in the handoff — the engine will start without TLS"
    SSL_MODE="off"
}

# The 90- prefix sorts last within include_dir and the last setting wins, so a
# resolved parameter can never move the endpoint's port or turn off TLS.
write_rds_conf() {
    _ssl="$1"
    _tmp="${PGDATA}/conf.d/.90-rds-init.conf.new"
    cat > "${_tmp}" <<EOF
# Managed by rds-init — regenerated on every boot from the bootstrap config.
# Edits here are lost; instance parameters belong in the parameter group.
listen_addresses = '*'
port = ${RDS_PORT}
unix_socket_directories = '${SOCKET_DIR}'
ssl = ${_ssl}
EOF
    if [ "${_ssl}" = "on" ]; then
        cat >> "${_tmp}" <<EOF
ssl_cert_file = '${TLS_DIR}/server.crt'
ssl_key_file = '${TLS_DIR}/server.key'
EOF
    fi
    chown "${PG_USER}:${PG_USER}" "${_tmp}"
    chmod 0600 "${_tmp}"
    mv -f "${_tmp}" "${PGDATA}/conf.d/90-rds-init.conf"
}

# Everything refreshed on every boot: the include hook, the resolved parameters,
# the serving cert and the platform's own connectivity settings.
install_runtime_config() {
    install -d -m 0700 -o "${PG_USER}" -g "${PG_USER}" "${PGDATA}/conf.d"
    install -d -m 1775 -o "${PG_USER}" -g "${PG_USER}" "${SOCKET_DIR}"
    install -d -m 0755 -o "${PG_USER}" -g "${PG_USER}" "${LOG_DIR}"

    # postgresql.conf is on the data volume, so the hook is asserted every boot
    # rather than only at initdb — a restored volume may predate it.
    if ! grep -q "^include_dir = 'conf.d'" "${PGDATA}/postgresql.conf"; then
        printf "\n# Managed by rds-init: parameter-group values (10-) then platform settings (90-).\ninclude_dir = 'conf.d'\n" \
            >> "${PGDATA}/postgresql.conf"
    fi

    install_parameters
    install_tls
    write_rds_conf "${SSL_MODE}"
}

# Brings the cluster up on a private socket with trust auth and no TLS, so the
# master password can be set before the engine is reachable. Never listens on TCP.
start_bootstrap_server() {
    install -d -m 0700 -o "${PG_USER}" -g "${PG_USER}" "${BOOTSTRAP_DIR}"
    printf 'local all all trust\n' > "${BOOTSTRAP_DIR}/pg_hba.conf"
    chown "${PG_USER}:${PG_USER}" "${BOOTSTRAP_DIR}/pg_hba.conf"
    chmod 0600 "${BOOTSTRAP_DIR}/pg_hba.conf"

    as_pg "'${PG_BIN}/pg_ctl' --pgdata='${PGDATA}' --wait --timeout=60 \
        --log='${LOG_DIR}/rds-init.log' \
        -o \"-c listen_addresses='' -c unix_socket_directories='${BOOTSTRAP_DIR}' \
             -c unix_socket_permissions=0700 -c hba_file='${BOOTSTRAP_DIR}/pg_hba.conf' -c ssl=off\" \
        start"
}

# The trust-authenticated pg_hba is removed by the caller, not here: a caller
# reading this function's status must be reading pg_ctl's.
stop_bootstrap_server() {
    as_pg "'${PG_BIN}/pg_ctl' --pgdata='${PGDATA}' --wait --timeout=60 --mode=fast stop"
}

# Prepended to every bootstrap script. psql interpolates the password into the
# ALTER ROLE before the server sees it, and these three are what would then write
# it to the log: log_min_error_statement does so on any failure at its own
# default. All are SUSET and set for this session only, so nothing the customer
# puts in the parameter group overrides them.
BOOTSTRAP_LOG_GUARD="SET log_statement = 'none';
SET log_min_duration_statement = -1;
SET log_min_error_statement = 'panic';"

# Replaces every occurrence of the secret with a placeholder. Parameter
# expansion rather than sed, whose script would carry the secret in an argv any
# process on the guest can read.
redact_stream() {
    _secret="$1"
    while IFS= read -r _line || [ -n "${_line}" ]; do
        _out=""
        while [ -n "${_secret}" ]; do
            case "${_line}" in
                *"${_secret}"*)
                    _out="${_out}${_line%%"${_secret}"*}[REDACTED]"
                    _line="${_line#*"${_secret}"}"
                    ;;
                *) break ;;
            esac
        done
        printf '%s\n' "${_out}${_line}"
    done
}

# Identifiers and the password ride the environment and are re-quoted by psql,
# never reaching a shell word or argv. Both streams are redacted before they
# reach the caller: psql echoes the statement it failed on, and the initd sends
# everything here to /dev/console, which the host captures off ttyS0.
psql_bootstrap() {
    # A pipeline's status is its last command's and POSIX sh has no pipefail, so
    # psql's own status leaves the subshell through the filesystem. The guard
    # shifts psql's <stdin> line numbers by the number of lines it adds.
    _status_file="${BOOTSTRAP_DIR}/psql.status"
    rm -f "${_status_file}"
    {
        _psql_rc=0
        { printf '%s\n' "${BOOTSTRAP_LOG_GUARD}"; cat; } |
            RDS_MASTER_USERNAME="${RDS_MASTER_USERNAME}" \
            RDS_MASTER_PASSWORD="${RDS_MASTER_PASSWORD}" \
            RDS_DB_NAME="${RDS_DB_NAME}" \
            RDS_GROUP_ROLE="${MASTER_GROUP_ROLE}" \
                "${PG_BIN}/psql" --no-psqlrc --quiet --no-align --tuples-only \
                    -v ON_ERROR_STOP=1 \
                    -h "${BOOTSTRAP_DIR}" -p "${RDS_PORT}" -U "${PG_USER}" -d postgres -f - ||
            _psql_rc=$?
        printf '%s\n' "${_psql_rc}" > "${_status_file}"
    } 2>&1 | redact_stream "${RDS_MASTER_PASSWORD}"

    # Every way of not recovering psql's status is a failed bootstrap rather
    # than a silent success, and each says so in its own words: blaming psql for
    # a status file that could not be written would send an operator looking at
    # a PostgreSQL that did exactly what it was asked.
    #
    # A torn write leaves the file created but empty, so the value is checked
    # before it is returned: `return ""` is fatal in the guest's ash, and would
    # kill rds-init here — past the sweep, before the diagnostics.
    _psql_status=1
    if [ -f "${_status_file}" ]; then
        _psql_status=$(cat "${_status_file}" 2>/dev/null) || _psql_status=""
        case "${_psql_status}" in
            '' | *[!0-9]*)
                echo "[rds-init] ERROR: psql ran but its exit status could not be read back from ${_status_file} (got '${_psql_status}'); treating the bootstrap as failed" >&2
                _psql_status=1
                ;;
        esac
        rm -f "${_status_file}"
    else
        echo "[rds-init] ERROR: psql ran but its exit status was never recorded at ${_status_file}; treating the bootstrap as failed" >&2
    fi
    return "${_psql_status}"
}

# Runs on a freshly initialised cluster only.
bootstrap_master() {
    log "starting bootstrap server on ${BOOTSTRAP_DIR}"
    start_bootstrap_server

    _rc=0
    apply_master_role || _rc=$?
    if [ "${_rc}" -eq 0 ] && [ -n "${RDS_DB_NAME}" ]; then
        create_database || _rc=$?
    fi
    if [ "${_rc}" -eq 0 ]; then
        create_platform_extensions || _rc=$?
    fi

    # `set -e` is suspended for a function called from an AND-OR list, so the
    # stop status is captured rather than propagated. A failed stop must also not
    # abort before the bootstrap check below, which names the real cause.
    _stop_rc=0
    stop_bootstrap_server || _stop_rc=$?
    rm -f "${BOOTSTRAP_DIR}/pg_hba.conf"

    # The datadir is swept by the traps, which also cover the steps here that
    # abort under `set -e` before this check is ever reached.
    if [ "${_rc}" -ne 0 ]; then
        fail "bootstrapping the master role/database failed (rc=${_rc})"
    fi

    # Proof written while the traps are still armed, so a receipt that cannot be
    # made durable clears the datadir instead of leaving one it cannot vouch for.
    write_completion_receipt

    # The master role exists from here, so the datadir is complete and nothing
    # may clear it — including on the stop failure reported below.
    mark_bootstrap_complete

    # A postmaster still holding the datadir blocks the postgresql service for
    # the life of the boot, so this is a failure and not a warning.
    if [ "${_stop_rc}" -ne 0 ]; then
        fail "the bootstrap server did not stop (rc=${_stop_rc}) and still holds ${PGDATA}; the postgresql service cannot start until it is gone"
    fi
}

# The value of one receipt key. Never sourced: the receipt is on the data volume,
# so a restored volume can carry one written by another instance, and executing
# it would run whatever that instance's control plane put there.
receipt_value() {
    [ -f "${RECEIPT_FILE}" ] || return 0
    sed -n "s/^$1=//p" "${RECEIPT_FILE}" 2>/dev/null | head -n 1
}

# Written between the master role and the retirement of the traps, so a receipt
# that cannot be made durable still clears the datadir rather than leaving a
# bootstrapped one nobody can prove.
#
# Fatal on failure, the mirror image of the sentinel removal below: a healthy
# engine whose receipt never landed would come up available while the agent
# blocked forever on a receipt that will never appear, leaving a working database
# that can never accept a password change or a parameter reload.
write_completion_receipt() {
    install -d -m 0700 -o "${PG_USER}" -g "${PG_USER}" "${RECEIPT_DIR}" ||
        fail "the master role exists but the bootstrap receipt directory ${RECEIPT_DIR} could not be created"

    _tmp="${RECEIPT_FILE}.new"
    {
        echo "# Written by rds-init once the master role was applied. Holds no secrets."
        echo "RDS_RECEIPT_PAYLOAD_ID=${RDS_PAYLOAD_ID}"
        echo "RDS_RECEIPT_DB_INSTANCE_IDENTIFIER=${RDS_DB_INSTANCE_IDENTIFIER}"
        echo "RDS_RECEIPT_VM_GENERATION=${RDS_VM_GENERATION}"
        echo "RDS_RECEIPT_COMPLETED_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
    } > "${_tmp}" ||
        fail "the master role exists but the bootstrap receipt ${RECEIPT_FILE} could not be written"
    chown "${PG_USER}:${PG_USER}" "${_tmp}"
    chmod 0600 "${_tmp}"

    # A receipt still in the page cache is lost by exactly the crash it exists to
    # survive, so it is on disk before it is renamed into place.
    sync "${_tmp}" 2>/dev/null || sync
    mv -f "${_tmp}" "${RECEIPT_FILE}" ||
        fail "the master role exists but the bootstrap receipt ${RECEIPT_FILE} could not be installed"
    sync "${RECEIPT_DIR}" 2>/dev/null || sync
}

# PG_VERSION with no sentinel and no receipt is ambiguous from the datadir alone:
# either a legacy instance that bootstrapped before receipts existed, which must
# attach, or one that applied the master role and crashed before the receipt was
# durable, which must not. Only the control plane's view of whether the payload
# is still staged separates the two, and that is what RDS_BOOTSTRAP_PENDING says.
require_bootstrap_receipt() {
    [ "${RDS_BOOTSTRAP_PENDING}" = "1" ] || return 0
    if [ -n "${RDS_PAYLOAD_ID}" ] &&
        [ "$(receipt_value RDS_RECEIPT_PAYLOAD_ID)" = "${RDS_PAYLOAD_ID}" ] &&
        [ "$(receipt_value RDS_RECEIPT_DB_INSTANCE_IDENTIFIER)" = "${RDS_DB_INSTANCE_IDENTIFIER}" ]; then
        return 0
    fi
    fail "the control plane has no record of a completed bootstrap for this datadir; refusing to attach to a datadir whose master role cannot be proven. Recover the volume out of band, or delete and recreate the instance."
}

# Retires the guards armed at initdb, in an order where no signal in between can
# clear a datadir that now carries a usable master role.
mark_bootstrap_complete() {
    _bootstrapped=1
    rm -f "${BOOTSTRAP_SENTINEL}" ||
        fail "the master role exists but ${BOOTSTRAP_SENTINEL} could not be removed; the next boot will refuse to start until it is deleted"
    trap - EXIT INT TERM HUP
}

# The master role is administrative but deliberately not a PostgreSQL SUPERUSER,
# as on AWS: a superuser reaches outside the database entirely — COPY FROM
# PROGRAM, pg_read_file/pg_write_file, untrusted languages — so master
# credentials would be a shell in the DB VM rather than a database privilege.
# The cluster's own superuser stays the bootstrap role, which only rds-init and
# rds-agent reach, over the unix socket under peer auth.
apply_master_role() {
    log "creating the master role ${RDS_MASTER_USERNAME}"
    # PG16 hardened CREATEROLE: it can neither grant itself SUPERUSER nor take
    # membership in a predefined role it does not administer, so CREATEROLE here
    # is not a route back to the privileges above.
    psql_bootstrap <<'SQL'
\getenv master RDS_MASTER_USERNAME
\getenv password RDS_MASTER_PASSWORD
\getenv grp RDS_GROUP_ROLE
SELECT NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = :'grp') AS create_group \gset
\if :create_group
CREATE ROLE :"grp" NOLOGIN;
\endif
GRANT pg_monitor, pg_signal_backend, pg_checkpoint TO :"grp";
SELECT NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = :'master') AS create_role \gset
\if :create_role
CREATE ROLE :"master" WITH LOGIN NOSUPERUSER CREATEDB CREATEROLE;
\endif
ALTER ROLE :"master" WITH LOGIN NOSUPERUSER CREATEDB CREATEROLE PASSWORD :'password';
GRANT :"grp" TO :"master" WITH ADMIN OPTION;
SQL
}

# As on AWS, an instance created without a DBName gets no initial database.
create_database() {
    log "creating database ${RDS_DB_NAME}"
    psql_bootstrap <<'SQL'
\getenv master RDS_MASTER_USERNAME
\getenv dbname RDS_DB_NAME
SELECT NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = :'dbname') AS create_db \gset
\if :create_db
CREATE DATABASE :"dbname" OWNER :"master";
\endif
SQL
}

# pgvector is baked in but left uninstalled so tenant databases stay free of it.
# The appliance master is NOSUPERUSER, so the bootstrap superuser installs the
# extension into a read-only extensions schema and grants the master the database
# and role privileges its daemon needs to create a per-account schema and role.
# Scoped to the appliance master alone; tenant masters skip this entirely.
create_platform_extensions() {
    [ "${RDS_IS_OCHRE_APPLIANCE:-0}" = "1" ] || return 0
    log "configuring pgvector and platform privileges for the Ochre vector appliance"
    psql_bootstrap <<SQL
CREATE SCHEMA IF NOT EXISTS extensions;
CREATE EXTENSION IF NOT EXISTS vector SCHEMA extensions;
GRANT USAGE ON SCHEMA extensions TO PUBLIC;
GRANT CREATE ON DATABASE postgres TO "${OCHRE_APPLIANCE_MASTER}";
ALTER ROLE "${OCHRE_APPLIANCE_MASTER}" SET createrole_self_grant = 'set, inherit';
SQL
}

# Drops the master password from the handoff once applied, so it is not left
# readable for the life of the boot. The rest is tmpfs and goes at reboot.
consume_secrets() {
    [ -f "${ENV_FILE}" ] || return 0
    _tmp="${ENV_FILE}.new"
    install -m 0600 /dev/null "${_tmp}"
    grep -v '^RDS_MASTER_PASSWORD=' "${ENV_FILE}" >> "${_tmp}" || true
    mv -f "${_tmp}" "${ENV_FILE}"
}

load_handoff
require_data_volume
require_engine_stamp

if [ -s "${PGDATA}/PG_VERSION" ]; then
    # The engine would start cleanly on such a datadir and pass every health
    # probe with no master role to log in as. The sentinel says the bootstrap did
    # not finish, not that the volume is empty, so nothing here clears it.
    if [ -e "${BOOTSTRAP_SENTINEL}" ]; then
        fail "datadir ${PGDATA} carries ${BOOTSTRAP_SENTINEL}: an earlier bootstrap was interrupted before the master role existed; recover the data volume out of band rather than starting the engine"
    fi
    require_bootstrap_receipt
    log "datadir ${PGDATA} is already initialised — attaching"
    install_runtime_config
else
    initialise_cluster
    install_runtime_config
    bootstrap_master
fi

consume_secrets
log "bootstrap complete; handing off to the postgresql service"