From 32ad16a24db4ed2feed33d3efaf35009fb5077bb Mon Sep 17 00:00:00 2001 From: Julian Sommer <115516918+juliansommer@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:13:13 +1000 Subject: [PATCH] step 8 --- scripts/images/rds-mariadb/manifest.conf | 13 +- scripts/images/rds-mariadb/mariadb.confd | 15 + .../rds-mariadb/rds-engine-stamp_test.sh | 9 +- scripts/images/rds-mariadb/rds-init | 846 ++++++++++++++ scripts/images/rds-mariadb/rds-init.initd | 35 + scripts/images/rds-mariadb/rds-init_test.sh | 1036 +++++++++++++++++ scripts/images/rds-mariadb/rds-openrc_test.sh | 24 + scripts/images/rds-mariadb/setup.sh | 16 +- 8 files changed, 1984 insertions(+), 10 deletions(-) create mode 100644 scripts/images/rds-mariadb/mariadb.confd create mode 100644 scripts/images/rds-mariadb/rds-init create mode 100644 scripts/images/rds-mariadb/rds-init.initd create mode 100755 scripts/images/rds-mariadb/rds-init_test.sh diff --git a/scripts/images/rds-mariadb/manifest.conf b/scripts/images/rds-mariadb/manifest.conf index c7ca26a22..f4488ffc0 100644 --- a/scripts/images/rds-mariadb/manifest.conf +++ b/scripts/images/rds-mariadb/manifest.conf @@ -29,11 +29,14 @@ INSTALL_BINARIES="bin/rds-agent:/usr/local/bin/rds-agent" INSTALL_FILES="\ scripts/images/common/rds-datadir:/usr/local/sbin/rds-datadir \ scripts/images/rds-mariadb/rds-datadir.initd:/etc/init.d/rds-datadir \ +scripts/images/rds-mariadb/rds-init:/usr/local/sbin/rds-init \ +scripts/images/rds-mariadb/rds-init.initd:/etc/init.d/rds-init \ scripts/images/rds-mariadb/rds-agent.initd:/etc/init.d/rds-agent \ scripts/images/common/mulga-mgmt-net.initd:/etc/init.d/mulga-mgmt-net \ scripts/images/common/mulga-mgmt-net-routes.initd:/etc/init.d/mulga-mgmt-net-routes \ scripts/images/common/mulga-mgmt-net.sh:/usr/local/sbin/mulga-mgmt-net \ -scripts/images/rds-mariadb/zz-rds-include.cnf:/etc/my.cnf.d/zz-rds-include.cnf" +scripts/images/rds-mariadb/zz-rds-include.cnf:/etc/my.cnf.d/zz-rds-include.cnf \ +scripts/images/rds-mariadb/mariadb.confd:/etc/conf.d/mariadb" # Each service's own depend() asserts the ordering; this lists it to match. # mulga-mgmt-net is NOT here: setup.sh adds it to the boot runlevel, since a @@ -41,10 +44,10 @@ scripts/images/rds-mariadb/zz-rds-include.cnf:/etc/my.cnf.d/zz-rds-include.cnf" # route-enforcement second pass is the opposite — it has to follow cloud-init, so # the default runlevel is exactly where it belongs. # -# mariadb is deliberately absent while this image has no bootstrap: the packaged -# service only warns about an empty datadir instead of refusing, so an enabled -# engine would launch mysqld_safe against an unbootstrapped data volume. -ENABLE_SERVICES="rds-datadir rds-agent mulga-mgmt-net-routes" +# The packaged mariadb service only warns about an empty datadir instead of +# refusing, so it is not ordering that keeps it off an unbootstrapped volume: +# /etc/conf.d/mariadb makes it need rds-init, which fails closed. +ENABLE_SERVICES="rds-datadir rds-agent rds-init mariadb mulga-mgmt-net-routes" SETUP_SCRIPT="scripts/images/rds-mariadb/setup.sh" diff --git a/scripts/images/rds-mariadb/mariadb.confd b/scripts/images/rds-mariadb/mariadb.confd new file mode 100644 index 000000000..b18a0fac0 --- /dev/null +++ b/scripts/images/rds-mariadb/mariadb.confd @@ -0,0 +1,15 @@ +# /etc/conf.d/mariadb — Spinifex RDS overrides for the stock Alpine mariadb +# service. The package ships no conf.d file of its own, so this adds one rather +# than replacing one. + +# The packaged service's start_pre only prints "Datadir is empty or invalid" and +# starts anyway, so an rds-init that failed — an unreachable gateway, a data +# volume that did not attach, a datadir stamped for another engine — would be +# answered by launching mysqld_safe against whatever is mounted. OpenRC reads +# rc_need from here, so the engine depends on the bootstrap without the packaged +# init script being patched. +rc_need="rds-init" + +# Everything else the service resolves from the configuration itself: datadir, +# socket and port come from the platform drop-in rds-init writes onto the data +# volume, which the packaged !includedir reads last. diff --git a/scripts/images/rds-mariadb/rds-engine-stamp_test.sh b/scripts/images/rds-mariadb/rds-engine-stamp_test.sh index 965c6c0b2..d367e2bbe 100755 --- a/scripts/images/rds-mariadb/rds-engine-stamp_test.sh +++ b/scripts/images/rds-mariadb/rds-engine-stamp_test.sh @@ -7,8 +7,8 @@ set -eu # the control plane launches instances with. A drift between them launches the # wrong image, refuses the right one, or reports a version the VM is not running. # -# This preset carries no bootstrap yet, so the datadir stamp and the agent's -# layout table are checked by the postgres preset's copy of this test only. +# The agent's layout table is checked by the postgres preset's copy of this test +# only, until this engine has a guest implementation to key it by. SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) REPO_ROOT=$(CDPATH='' cd -- "${SCRIPT_DIR}/../../.." && pwd) @@ -32,6 +32,11 @@ if ! grep -q "Name: *\"${ENGINE}\"" "${ENGINE_GO}"; then fail "the control plane has no engine named ${ENGINE}" fi +if ! grep -q "^ENGINE=\"${ENGINE}\"\$" "${SCRIPT_DIR}/rds-init"; then + fail "rds-init does not stamp the data volume '${ENGINE}'" + grep '^ENGINE=' "${SCRIPT_DIR}/rds-init" >&2 +fi + # The tag is what an instance's recorded EngineVersion resolves an AMI by, so it # has to be the same series the control plane pins and the packages install. version=$(sed -n 's/^[[:space:]]*MajorVersion: *"\([^"]*\)".*/\1/p' "${ENGINE_GO}") diff --git a/scripts/images/rds-mariadb/rds-init b/scripts/images/rds-mariadb/rds-init new file mode 100644 index 000000000..57d3298be --- /dev/null +++ b/scripts/images/rds-mariadb/rds-init @@ -0,0 +1,846 @@ +#!/bin/sh +# rds-init — MariaDB bootstrap for the spinifex-rds-mariadb AMI. +# +# A boot oneshot ordered before the mariadb service. This is the MariaDB half of +# the engine seam: mariadb-install-db, the master grant, the datadir and the 3306 +# default live here and nowhere else in the platform. +# +# Handoff contract, written by rds-agent into a root-owned tmpfs directory: +# +# /run/spinifex-rds/bootstrap.env 0600 KEY=value shell fragment: +# RDS_MODE=initialize|attach +# RDS_DB_INSTANCE_IDENTIFIER= +# RDS_MASTER_USERNAME= +# RDS_MASTER_PASSWORD= +# RDS_DB_NAME= +# RDS_PORT= +# RDS_DATA_VOLUME_ID / RDS_DATA_VOLUME_SERIAL= +# RDS_VM_GENERATION / RDS_FORMAT_AUTHORIZED= +# RDS_BOOTSTRAP_PENDING / RDS_PAYLOAD_ID= +# /run/spinifex-rds/parameters.conf 0600 resolved parameter-group values, one +# name = 'value' per line +# /run/spinifex-rds/server.crt 0600 serving cert, cluster-CA-signed +# /run/spinifex-rds/server.key 0600 its private key +# +# Nothing there survives a reboot: the next boot re-fetches and the cert is +# re-minted rather than persisted. +# +# The data volume also carries an engine stamp, written before the datadir is +# inspected and compared on every boot, so another engine's volume is refused +# rather than initialised over. +# +# rds-init keys off the datadir, never off RDS_MODE. A datadir holding the mysql +# system database means skip mariadb-install-db and only refresh config, so a +# replaced VM comes back on its existing data volume without re-bootstrapping. +# RDS_BOOTSTRAP_PENDING is not an exception: it is a control-plane-resolved +# precondition compared against on-disk evidence, the same shape as +# RDS_FORMAT_AUTHORIZED. +set -eu + +# The engine this image bakes, stamped onto the data volume and compared against +# it on every boot. Held here rather than read from /etc/spinifex-rds/engine: +# this script is the MariaDB half of the seam, so it is the same constant. +ENGINE="mariadb" + +MARIADB_BIN="${RDS_MARIADB_BIN:-/usr/bin}" +ENGINE_USER="${RDS_ENGINE_USER:-mysql}" + +# Mount point of the data volume, and the datadir one level inside it. The +# datadir is deliberately not the mount point itself: a failed bootstrap clears +# it, and the include directory, the engine stamp and the completion receipt all +# have to outlive that. It also keeps lost+found and the include directory out of +# a directory whose every subdirectory MariaDB lists as a database. +DATA_MOUNT="${RDS_DATA_MOUNT:-/var/lib/mysql}" +DATADIR="${RDS_DATADIR:-${DATA_MOUNT}/data}" + +# Read through the image's baked /etc/my.cnf.d/zz-rds-include.cnf. Generated +# configuration lives here rather than under /etc because a class change is a VM +# replacement — fresh boot volume, same data volume — and parameters under /etc +# would be discarded by it while the API still reported the customer's values. +CONF_DIR="${RDS_CONF_DIR:-${DATA_MOUNT}/conf.d}" +PARAM_FILE="${CONF_DIR}/10-rds-parameters.cnf" +# What the server actually started on. The agent computes pending-restart by +# comparing the installed file against this one, so writing it here — before the +# engine starts — is what makes the two equal by construction on every boot. +# Deliberately not a .cnf name: !includedir reads *.cnf and this is comparison +# material, not a second set of settings. +SERVING_FILE="${CONF_DIR}/10-rds-parameters.serving" +PLATFORM_FILE="${CONF_DIR}/90-rds-init.cnf" + +# Written between mariadb-install-db and a working master user. It lives on the +# data volume because no trap survives a crash or a hard kill, and it is what +# stops the next boot attaching to a datadir whose master user was never created. +BOOTSTRAP_SENTINEL="${DATADIR}/rds-bootstrap-incomplete" + +# Proof that the master user was durably applied, written once the datadir and +# the user are both done. Deliberately outside DATADIR, which +# clear_unfinished_datadir rm -rf's. +RECEIPT_DIR="${RDS_RECEIPT_DIR:-${DATA_MOUNT}/.spinifex-rds/bootstrap}" +RECEIPT_FILE="${RECEIPT_DIR}/receipt.env" + +# Which engine's data this volume holds. Outside DATADIR for the same reason the +# receipt is, and beside it rather than inside the bootstrap directory: it +# outlives any one bootstrap and is read before the datadir is ever inspected. +ENGINE_STAMP="${RDS_ENGINE_STAMP:-${DATA_MOUNT}/.spinifex-rds/engine}" + +HANDOFF_DIR="${RDS_HANDOFF_DIR:-/run/spinifex-rds}" +ENV_FILE="${HANDOFF_DIR}/bootstrap.env" + +SOCKET_DIR="${RDS_SOCKET_DIR:-/run/mysqld}" +SOCKET="${SOCKET_DIR}/mysqld.sock" +TLS_DIR="${RDS_TLS_DIR:-${SOCKET_DIR}/tls}" +LOG_DIR="${RDS_LOG_DIR:-/var/log/mysql}" + +# The bootstrap server listens on a unix socket in this directory and nowhere +# else. It is 0700 and mysql-owned, so only mysql and root can reach the short +# window before the master user exists. +BOOTSTRAP_DIR="${RDS_BOOTSTRAP_DIR:-/run/rds-init}" +BOOTSTRAP_SOCKET="${BOOTSTRAP_DIR}/mysqld.sock" +BOOTSTRAP_PID="${BOOTSTRAP_DIR}/mysqld.pid" + +# secure_file_priv names the one directory FILE-privileged statements may read +# and write. It must be a real directory: the empty string means no restriction +# at all. Root-owned and empty, on the boot volume, so it stays empty. +SECURE_FILE_DIR="${RDS_SECURE_FILE_DIR:-/var/lib/mysql-files}" + +MOUNTS_FILE="${RDS_MOUNTS_FILE:-/proc/mounts}" + +# How long the private bootstrap server gets to open its socket, as probes one +# BOOTSTRAP_POLL apart. InnoDB creates its redo log and system tablespace on this +# first start, so it is not instant. +BOOTSTRAP_PROBES="${RDS_BOOTSTRAP_PROBES:-60}" +BOOTSTRAP_POLL="${RDS_BOOTSTRAP_POLL:-1}" + +# The server truncates a longer name, so the database created would not be the +# one the customer asked for, and MariaDB maps the name onto a directory name. +MAX_DB_NAME_LEN=64 +MAX_USERNAME_LEN=80 + +log() { echo "[rds-init] $*"; } +fail() { echo "[rds-init] ERROR: $*" >&2; exit 1; } + +# Between these two, the datadir carries a mysql system database but no usable +# master user. _server_pid is the private bootstrap server, if one is running. +_created_datadir=0 +_bootstrapped=0 +_server_pid="" + +# Runs from a trap rather than a statement, so neither `set -e` on an unguarded +# step between mariadb-install-db and a working master user nor a signal can skip +# it — either would leave a healthy-looking engine with no usable master user, +# unrepairable. +clear_unfinished_datadir() { + [ "${_created_datadir}" = "1" ] || return 0 + [ "${_bootstrapped}" = "1" ] && return 0 + # A server left holding a datadir that is about to be deleted keeps its + # socket and pidfile and blocks the mariadb service for the life of the boot. + kill_bootstrap_server + # The receipt is installed before the traps retire, so a sweep in between + # would leave one vouching for a datadir that is about to be deleted. The + # agent would acknowledge it and the control plane would destroy the only + # copy of the password this instance can ever be initialised with. + rm -f -- "${RECEIPT_FILE}" + if rm -rf -- "${DATADIR}" && [ ! -e "${DATADIR}" ]; then + echo "[rds-init] ERROR: bootstrap did not complete; datadir ${DATADIR} cleared" >&2 + return 0 + fi + echo "[rds-init] ERROR: bootstrap did not complete and datadir ${DATADIR} could not be cleared; recover it out of band before retrying" >&2 +} + +kill_bootstrap_server() { + [ -n "${_server_pid}" ] || return 0 + kill -TERM "${_server_pid}" 2>/dev/null || true + _server_pid="" +} + +load_handoff() { + [ -f "${ENV_FILE}" ] || fail "no bootstrap handoff at ${ENV_FILE} — rds-agent has not delivered the bootstrap config" + + # shellcheck disable=SC1090 + . "${ENV_FILE}" + + RDS_MODE="${RDS_MODE:-}" + RDS_DB_INSTANCE_IDENTIFIER="${RDS_DB_INSTANCE_IDENTIFIER:-}" + RDS_MASTER_USERNAME="${RDS_MASTER_USERNAME:-}" + RDS_MASTER_PASSWORD="${RDS_MASTER_PASSWORD:-}" + RDS_DB_NAME="${RDS_DB_NAME:-}" + RDS_PORT="${RDS_PORT:-3306}" + RDS_VM_GENERATION="${RDS_VM_GENERATION:-}" + # Absent from an agent that predates the receipt protocol, which reads as + # "nothing is pending" and leaves every path below unchanged. + RDS_BOOTSTRAP_PENDING="${RDS_BOOTSTRAP_PENDING:-0}" + RDS_PAYLOAD_ID="${RDS_PAYLOAD_ID:-}" + + [ -n "${RDS_MASTER_USERNAME}" ] || fail "bootstrap config has no RDS_MASTER_USERNAME" + + # The name is interpolated into CREATE USER by the shell — the mariadb client + # has no identifier quoting of its own — so the rule the control plane + # applies is enforced here rather than trusted. + case "${RDS_MASTER_USERNAME}" in + [!A-Za-z]* | *[!A-Za-z0-9_]*) + fail "RDS_MASTER_USERNAME ${RDS_MASTER_USERNAME} must begin with a letter and hold only letters, digits and underscores; the control plane should have rejected it" + ;; + esac + if [ "${#RDS_MASTER_USERNAME}" -gt "${MAX_USERNAME_LEN}" ]; then + fail "RDS_MASTER_USERNAME is longer than the ${MAX_USERNAME_LEN} characters MariaDB allows; the control plane should have rejected it" + fi + + # root is the superuser mariadb-install-db creates and rds-init keeps for + # itself, mysql is the unix_socket account beside it, mariadb.sys owns the sys + # schema, rdsadmin is reserved as on AWS, and PUBLIC is a role name the server + # refuses to create a user under. Matched the same way the control plane + # matches them: anything it refuses must be refused here too, or the name + # reaches mariadb-install-db, fails at CREATE USER and spends the one-shot + # password on the way. + _master_lower=$(printf '%s' "${RDS_MASTER_USERNAME}" | tr '[:upper:]' '[:lower:]') + case "${_master_lower}" in + root | mysql | mariadb.sys | rdsadmin | public | mysql.* | mariadb.*) + fail "RDS_MASTER_USERNAME ${RDS_MASTER_USERNAME} is reserved by the platform; the control plane should have rejected it" + ;; + esac + + # The same rule the control plane applies. Here it is the barrier rather than + # defence in depth: the mariadb client has no identifier interpolation, so the + # name is built into CREATE DATABASE by the shell, and MariaDB maps a database + # name onto a directory name — which makes /, \, . and a trailing space + # structurally dangerous rather than merely awkward. + case "${RDS_DB_NAME}" in + '') ;; + [!A-Za-z]* | *[!A-Za-z0-9_]*) + fail "RDS_DB_NAME ${RDS_DB_NAME} must begin with a letter and hold only letters, digits and underscores; the control plane should have rejected it" + ;; + esac + if [ "${#RDS_DB_NAME}" -gt "${MAX_DB_NAME_LEN}" ]; then + fail "RDS_DB_NAME is longer than the ${MAX_DB_NAME_LEN} characters MariaDB allows; the control plane should have rejected it" + fi + + case "${RDS_PORT}" in + '' | *[!0-9]*) fail "bootstrap config has a non-numeric RDS_PORT: ${RDS_PORT}" ;; + esac + + # redact_stream is line-oriented, so a password spanning a line boundary + # would match neither half and reach /dev/console in the clear. The control + # plane rejects one; refusing here too keeps that an asserted property of + # the handoff rather than a silent dependency on another service. + case "${RDS_MASTER_PASSWORD}" in + *' +'*) fail "the master password in the bootstrap config spans more than one line; the control plane should have rejected it" ;; + esac + + # The form the password takes inside the SQL, computed once beside it. A + # statement echoed on failure carries that form and not the raw secret, so + # redacting only the raw one would put a trivially unescaped password on the + # console for exactly the passwords that needed escaping. + RDS_MASTER_PASSWORD_SQL="" + if [ -n "${RDS_MASTER_PASSWORD}" ]; then + RDS_MASTER_PASSWORD_SQL=$(sql_literal "${RDS_MASTER_PASSWORD}") + fi + + log "bootstrap config loaded (mode=${RDS_MODE:-unset}, port=${RDS_PORT})" +} + +# Every replace boots a fresh clone of this AMI, so a datadir off the data volume +# disappears at the next class change or recovery — silently, as the engine looks +# fine. +require_data_volume() { + if awk -v mnt="${DATA_MOUNT}" '$2 == mnt { found = 1 } END { exit !found }' "${MOUNTS_FILE}"; then + return 0 + fi + [ "${RDS_ALLOW_LOCAL_DATADIR:-0}" = "1" ] || + fail "${DATA_MOUNT} is not a mount point — the data volume is not attached; refusing to initialise on the boot volume" + log "warning: ${DATA_MOUNT} is not a mount point; data will not survive a VM replace (RDS_ALLOW_LOCAL_DATADIR=1)" +} + +# True when the datadir holds anything at all. A failed enumeration is neither +# "empty" nor "has content": it is a refusal, because both callers below decide +# whether something may be created or deleted from the answer. +datadir_has_content() { + [ -d "${DATADIR}" ] || return 1 + _contents=$(ls -A -- "${DATADIR}" 2>/dev/null) || + fail "could not inspect datadir ${DATADIR}; refusing to decide whether it is ours" + [ -n "${_contents}" ] +} + +# Written before the datadir is inspected, so a volume this image goes on to +# initialise carries the stamp from the first boot onwards. Fatal on failure: +# an unstamped volume is one the next boot cannot tell from another engine's. +write_engine_stamp() { + _stamp_dir=$(dirname "${ENGINE_STAMP}") + install -d -m 0700 -o "${ENGINE_USER}" -g "${ENGINE_USER}" "${_stamp_dir}" || + fail "the datadir engine stamp directory ${_stamp_dir} could not be created; refusing to use a data volume whose engine cannot be recorded" + + _tmp="${ENGINE_STAMP}.new" + printf '%s\n' "${ENGINE}" > "${_tmp}" || + fail "the datadir engine stamp ${ENGINE_STAMP} could not be written; refusing to use a data volume whose engine cannot be recorded" + chown "${ENGINE_USER}:${ENGINE_USER}" "${_tmp}" + chmod 0600 "${_tmp}" + + # Renamed rather than written in place, so a crash leaves either the old + # stamp or none — and none is decided on the datadir, where a torn one would + # be fatal. + sync "${_tmp}" 2>/dev/null || sync + mv -f "${_tmp}" "${ENGINE_STAMP}" || + fail "the datadir engine stamp ${ENGINE_STAMP} could not be installed; refusing to use a data volume whose engine cannot be recorded" + sync "${_stamp_dir}" 2>/dev/null || sync +} + +# The volume mounts cleanly whichever engine wrote it, and another engine's +# datadir reads as uninitialised here: rds-init would run mariadb-install-db +# beside the customer's data and serve an empty database that passes every health +# probe. +require_engine_stamp() { + if [ -e "${ENGINE_STAMP}" ]; then + _stamped="" + _stamp_read=0 + _stamped=$(head -n 1 -- "${ENGINE_STAMP}") && _stamp_read=1 + [ "${_stamp_read}" = "1" ] || + fail "the datadir engine stamp ${ENGINE_STAMP} exists but could not be read; refusing to attach to a data volume whose engine cannot be established" + if [ "${_stamped}" != "${ENGINE}" ]; then + fail "the data volume holds a '${_stamped}' datadir but this image runs ${ENGINE}; refusing to touch it. Attach it to a '${_stamped}' instance, or recover the volume out of band." + fi + log "data volume is stamped ${ENGINE}" + return 0 + fi + + # No MariaDB volume in the field predates the stamp, so unlike PostgreSQL + # there is nothing to backfill onto: an unstamped datadir with content in it + # is by definition not ours, and stamping it would turn the check into a + # rubber stamp on exactly the volume it exists to refuse. + if datadir_has_content; then + fail "${DATADIR} holds data but the volume carries no ${ENGINE} engine stamp; refusing to touch it. Recover the volume out of band." + fi + + log "data volume carries no engine stamp — recording ${ENGINE}" + write_engine_stamp +} + +# Creates the system tables in an empty datadir. Root authenticates through the +# unix_socket plugin — the direct analogue of PostgreSQL's peer auth, and the +# reason no bootstrap password is needed. +initialise_datadir() { + # An empty datadir in `attach` mode means the data volume did not come back. + # Initialising there would report success and hand the customer an empty + # database in place of theirs. + if [ "${RDS_MODE}" = "attach" ]; then + fail "datadir ${DATADIR} holds no MariaDB system database but the bootstrap config is Mode=attach — the existing data volume is missing; refusing to create an empty database" + fi + [ -n "${RDS_MASTER_PASSWORD}" ] || fail "Mode=${RDS_MODE:-unset} carries no master password; cannot initialise the datadir" + + install -d -m 0700 -o "${ENGINE_USER}" -g "${ENGINE_USER}" "${DATADIR}" + + # Whether the datadir was empty beforehand gates the failure cleanup below. + # A torn write can leave the mysql system database missing over intact user + # data, and deleting then would destroy it in response to the one signal that + # it is still there. + _was_empty=0 + if _contents=$(ls -A -- "${DATADIR}" 2>/dev/null); then + [ -z "${_contents}" ] && _was_empty=1 + else + fail "could not inspect datadir ${DATADIR}; refusing to decide whether failure cleanup is safe" + fi + + log "initialising the MariaDB datadir at ${DATADIR}" + # --no-defaults: the installed configuration includes the customer's own + # parameter file, and a static value this server will not start under would + # spend the one-shot master password on a datadir that never gets one. The + # real server starts on those parameters, where a rollback exists for them. + if ! "${MARIADB_BIN}/mariadb-install-db" --no-defaults \ + --user="${ENGINE_USER}" --datadir="${DATADIR}" \ + --auth-root-authentication-method=socket --skip-test-db; then + # A half-written datadir carries a mysql directory and would look + # initialised forever after, but only what this invocation created may be + # cleared. + if [ "${_was_empty}" = "1" ]; then + if rm -rf -- "${DATADIR}" && [ ! -e "${DATADIR}" ]; then + fail "mariadb-install-db failed; datadir cleared" + fi + fail "mariadb-install-db failed and datadir ${DATADIR} could not be cleared; recover it out of band before retrying" + fi + fail "mariadb-install-db failed and ${DATADIR} was not empty beforehand — refusing to clear it, since the contents may be a datadir whose system tables were lost; recover the data volume out of band" + fi + + # Only a datadir built over an empty directory may be swept later. + _created_datadir="${_was_empty}" + trap clear_unfinished_datadir EXIT + # An EXIT trap does not run on a signal in a POSIX shell, so the same sweep + # is armed for the signals a shutdown, an ACPI powerdown or a host-side + # force-stop delivers. EXIT is disarmed first so the sweep runs once. + trap 'trap - EXIT; clear_unfinished_datadir; exit 1' INT TERM HUP + + # No trap survives a crash or a SIGKILL, so the incomplete state is also + # recorded on the volume itself. Cleared once the master user exists. + printf '%s\n' \ + 'rds-init created this datadir and did not finish bootstrapping the master user.' \ + 'While this file exists rds-init refuses to start the engine: the datadir carries no' \ + 'usable master user, so it must not be attached to. Recover out of band.' \ + > "${BOOTSTRAP_SENTINEL}" + chown "${ENGINE_USER}:${ENGINE_USER}" "${BOOTSTRAP_SENTINEL}" + chmod 0600 "${BOOTSTRAP_SENTINEL}" +} + +# The resolved parameter group, prefixed with the group header. MariaDB parses +# each file in an include directory on its own and treats a setting before any +# group as a fatal error, so the header belongs to whoever writes the file rather +# than to the engine-neutral renderer that produced its body. +install_parameters() { + if [ -f "${HANDOFF_DIR}/parameters.conf" ]; then + _tmp="${CONF_DIR}/.10-rds-parameters.cnf.new" + { printf '[mysqld]\n'; cat "${HANDOFF_DIR}/parameters.conf"; } > "${_tmp}" || + fail "the resolved parameter group could not be written to ${PARAM_FILE}" + chown "${ENGINE_USER}:${ENGINE_USER}" "${_tmp}" + chmod 0600 "${_tmp}" + mv -f "${_tmp}" "${PARAM_FILE}" + record_serving_parameters + return 0 + fi + # A handoff without parameters keeps the previous include: dropping it would + # silently reset the instance to engine defaults. + if [ -f "${PARAM_FILE}" ]; then + log "warning: no parameters in the handoff — keeping the parameters already installed" + record_serving_parameters + fi +} + +# The engine is about to start on whatever is installed, so the two are equal by +# construction here. The agent refreshes this copy only after a restart it +# performs itself: a live SET GLOBAL apply deliberately leaves it alone, since +# that value was adopted without a restart, which is what "not pending" means. +record_serving_parameters() { + install -m 0600 -o "${ENGINE_USER}" -g "${ENGINE_USER}" "${PARAM_FILE}" "${SERVING_FILE}" || + fail "the serving copy of the parameters could not be written to ${SERVING_FILE}" +} + +# The key stays on tmpfs: it is re-minted every fetch, so persisting it would +# spread a private key into every snapshot. +install_tls() { + if [ -f "${HANDOFF_DIR}/server.crt" ] && [ -f "${HANDOFF_DIR}/server.key" ]; then + install -d -m 0700 -o "${ENGINE_USER}" -g "${ENGINE_USER}" "${TLS_DIR}" + install -m 0600 -o "${ENGINE_USER}" -g "${ENGINE_USER}" "${HANDOFF_DIR}/server.crt" "${TLS_DIR}/server.crt" + install -m 0600 -o "${ENGINE_USER}" -g "${ENGINE_USER}" "${HANDOFF_DIR}/server.key" "${TLS_DIR}/server.key" + rm -f "${HANDOFF_DIR}/server.crt" "${HANDOFF_DIR}/server.key" + SSL_MODE="on" + return 0 + fi + # A re-run after the cert was consumed must not turn TLS off underneath + # clients connecting with --ssl-verify-server-cert. + if [ -f "${TLS_DIR}/server.crt" ] && [ -f "${TLS_DIR}/server.key" ]; then + SSL_MODE="on" + return 0 + fi + log "warning: no serving cert in the handoff — the engine will start without TLS" + SSL_MODE="off" +} + +# The 90- prefix sorts last within the include directory and MariaDB takes the +# last occurrence of a setting, so a resolved parameter can never move the +# endpoint's port, redirect the datadir or turn off TLS. +# +# innodb_buffer_pool_chunk_size and innodb_buffer_pool_instances are deliberately +# absent although they are platform-owned: 11.8 removed the second outright, so +# naming it is a startup failure rather than an override, and the first is +# deprecated with no effect. The granularity they used to pin is fixed by the +# server itself in this series. +write_platform_config() { + _ssl="$1" + _tmp="${CONF_DIR}/.90-rds-init.cnf.new" + cat > "${_tmp}" <> "${_tmp}" < /dev/null 2>&1; do + # The server writes why it gave up to its error log, which is on the boot + # volume and readable after the fact; the console only gets the refusal. + if ! kill -0 "${_server_pid}" 2>/dev/null; then + _server_pid="" + fail "the bootstrap server exited before it opened ${BOOTSTRAP_SOCKET}; see ${LOG_DIR}/rds-init.log" + fi + if [ "${_probes}" -ge "${BOOTSTRAP_PROBES}" ]; then + fail "the bootstrap server did not open ${BOOTSTRAP_SOCKET} after ${BOOTSTRAP_PROBES} probes ${BOOTSTRAP_POLL}s apart; see ${LOG_DIR}/rds-init.log" + fi + sleep "${BOOTSTRAP_POLL}" + _probes=$((_probes + 1)) + done +} + +bootstrap_admin() { + "${MARIADB_BIN}/mariadb-admin" --no-defaults \ + --socket="${BOOTSTRAP_SOCKET}" --user=root "$@" +} + +# Returns the shutdown's status, then the server's own: a caller reading this +# must be able to tell "it refused to stop" from "it stopped badly", and both +# from a server still holding the datadir. +stop_bootstrap_server() { + [ -n "${_server_pid}" ] || return 0 + bootstrap_admin shutdown || return 1 + + # `wait` rather than a kill -0 poll: the server is this shell's own child, so + # until it is reaped it stays a zombie that kill -0 reports as alive. + _wait_rc=0 + wait "${_server_pid}" || _wait_rc=$? + _server_pid="" + return "${_wait_rc}" +} + +# Prepended to every bootstrap script. The two logs that would copy the password +# out of the statement are turned off — global rather than session variables, and +# safe to set globally because nothing but rds-init can reach this server. The +# mode is pinned because the password is escaped below for a mode that has +# NO_BACKSLASH_ESCAPES off. +BOOTSTRAP_SQL_GUARD="SET GLOBAL general_log = 0; +SET GLOBAL slow_query_log = 0; +SET SESSION sql_mode = 'NO_ENGINE_SUBSTITUTION';" + +# Replaces every occurrence of the secret with a placeholder. Parameter +# expansion rather than sed, whose script would carry the secret in an argv any +# process on the guest can read. +redact_stream() { + _secret="$1" + while IFS= read -r _line || [ -n "${_line}" ]; do + _out="" + while [ -n "${_secret}" ]; do + case "${_line}" in + *"${_secret}"*) + _out="${_out}${_line%%"${_secret}"*}[REDACTED]" + _line="${_line#*"${_secret}"}" + ;; + *) break ;; + esac + done + printf '%s\n' "${_out}${_line}" + done +} + +# A value safe to interpolate into a single-quoted SQL literal. The quote is +# doubled, which holds in every sql_mode, and the backslash is doubled, which is +# why the guard above pins one where backslash escapes are on. The secret reaches +# sed on stdin: an argv would be readable by every process in the guest. +sql_literal() { + printf '%s' "$1" | sed -e 's/\\/\\\\/g' -e "s/'/''/g" +} + +# The script rides stdin and never an argv, and both streams are redacted before +# they reach the caller: the client echoes the statement it failed on, and the +# initd sends everything here to /dev/console, which the host captures off ttyS0. +mariadb_bootstrap() { + # A pipeline's status is its last command's and POSIX sh has no pipefail, so + # the client's own status leaves the subshell through the filesystem. The + # guard shifts the client's reported line numbers by the lines it adds. + _status_file="${BOOTSTRAP_DIR}/client.status" + rm -f "${_status_file}" + { + _client_rc=0 + # --batch stops at the first error and exits non-zero, so a failed + # statement cannot leave the rest of the script running. + { printf '%s\n' "${BOOTSTRAP_SQL_GUARD}"; cat; } | + "${MARIADB_BIN}/mariadb" --no-defaults --batch --skip-column-names \ + --socket="${BOOTSTRAP_SOCKET}" --user=root || + _client_rc=$? + printf '%s\n' "${_client_rc}" > "${_status_file}" + } 2>&1 | redact_stream "${RDS_MASTER_PASSWORD}" | redact_stream "${RDS_MASTER_PASSWORD_SQL}" + + # Every way of not recovering the client's status is a failed bootstrap + # rather than a silent success, and each says so in its own words: blaming + # the client for a status file that could not be written would send an + # operator looking at a MariaDB that did exactly what it was asked. + # + # A torn write leaves the file created but empty, so the value is checked + # before it is returned: `return ""` is fatal in the guest's ash, and would + # kill rds-init here — past the sweep, before the diagnostics. + _client_status=1 + if [ -f "${_status_file}" ]; then + _client_status=$(cat "${_status_file}" 2>/dev/null) || _client_status="" + case "${_client_status}" in + '' | *[!0-9]*) + echo "[rds-init] ERROR: the mariadb client ran but its exit status could not be read back from ${_status_file} (got '${_client_status}'); treating the bootstrap as failed" >&2 + _client_status=1 + ;; + esac + rm -f "${_status_file}" + else + echo "[rds-init] ERROR: the mariadb client ran but its exit status was never recorded at ${_status_file}; treating the bootstrap as failed" >&2 + fi + return "${_client_status}" +} + +# 11.8 creates no anonymous account and --skip-test-db leaves no test database, +# so on a current image both of these are no-ops. They run anyway: the properties +# are asserted by the image tests, and an engine or packaging change that +# reintroduced either would otherwise reach a customer's instance. +remove_insecure_defaults() { + mariadb_bootstrap <<'SQL' +DELETE FROM mysql.global_priv WHERE User = ''; +DROP DATABASE IF EXISTS test; +FLUSH PRIVILEGES; +SQL +} + +# The master user is administrative but deliberately holds none of the +# privileges that reach outside the database: FILE is SELECT INTO OUTFILE and +# LOAD_FILE against any path as the mysql OS user, INSERT on mysql.* is native +# code execution through CREATE FUNCTION ... SONAME, SUPER and its 11.8 split-out +# successors reach SET GLOBAL and plugin loading, and SHUTDOWN stops the server +# underneath the control plane's lifecycle. Master credentials must be a database +# privilege, not a shell in the DB VM. +apply_master_user() { + log "creating the master user ${RDS_MASTER_USERNAME}" + # Enumerated rather than GRANT ALL followed by revocations: a MariaDB release + # that adds a privilege would otherwise hand it to the master silently. + # '%' because the customer reaches the instance over its customer ENI. + mariadb_bootstrap </dev/null | head -n 1 +} + +# Written between the master user and the retirement of the traps, so a receipt +# that cannot be made durable still clears the datadir rather than leaving a +# bootstrapped one nobody can prove. +# +# Fatal on failure, the mirror image of the sentinel removal below: a healthy +# engine whose receipt never landed would come up available while the agent +# blocked forever on a receipt that will never appear, leaving a working database +# that can never accept a password change or a parameter reload. +write_completion_receipt() { + install -d -m 0700 -o "${ENGINE_USER}" -g "${ENGINE_USER}" "${RECEIPT_DIR}" || + fail "the master user exists but the bootstrap receipt directory ${RECEIPT_DIR} could not be created" + + _tmp="${RECEIPT_FILE}.new" + { + echo "# Written by rds-init once the master user was applied. Holds no secrets." + echo "RDS_RECEIPT_PAYLOAD_ID=${RDS_PAYLOAD_ID}" + echo "RDS_RECEIPT_DB_INSTANCE_IDENTIFIER=${RDS_DB_INSTANCE_IDENTIFIER}" + echo "RDS_RECEIPT_VM_GENERATION=${RDS_VM_GENERATION}" + echo "RDS_RECEIPT_COMPLETED_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + } > "${_tmp}" || + fail "the master user exists but the bootstrap receipt ${RECEIPT_FILE} could not be written" + chown "${ENGINE_USER}:${ENGINE_USER}" "${_tmp}" + chmod 0600 "${_tmp}" + + # A receipt still in the page cache is lost by exactly the crash it exists to + # survive, so it is on disk before it is renamed into place. + sync "${_tmp}" 2>/dev/null || sync + mv -f "${_tmp}" "${RECEIPT_FILE}" || + fail "the master user exists but the bootstrap receipt ${RECEIPT_FILE} could not be installed" + sync "${RECEIPT_DIR}" 2>/dev/null || sync +} + +# A system database with no sentinel and no receipt is ambiguous from the datadir +# alone: either an instance that bootstrapped before receipts existed, which must +# attach, or one that applied the master user and crashed before the receipt was +# durable, which must not. Only the control plane's view of whether the payload +# is still staged separates the two, and that is what RDS_BOOTSTRAP_PENDING says. +require_bootstrap_receipt() { + [ "${RDS_BOOTSTRAP_PENDING}" = "1" ] || return 0 + if [ -n "${RDS_PAYLOAD_ID}" ] && + [ "$(receipt_value RDS_RECEIPT_PAYLOAD_ID)" = "${RDS_PAYLOAD_ID}" ] && + [ "$(receipt_value RDS_RECEIPT_DB_INSTANCE_IDENTIFIER)" = "${RDS_DB_INSTANCE_IDENTIFIER}" ]; then + return 0 + fi + fail "the control plane has no record of a completed bootstrap for this datadir; refusing to attach to a datadir whose master user cannot be proven. Recover the volume out of band, or delete and recreate the instance." +} + +# Retires the guards armed at initialisation, in an order where no signal in +# between can clear a datadir that now carries a usable master user. +mark_bootstrap_complete() { + _bootstrapped=1 + rm -f "${BOOTSTRAP_SENTINEL}" || + fail "the master user exists but ${BOOTSTRAP_SENTINEL} could not be removed; the next boot will refuse to start until it is deleted" + trap - EXIT INT TERM HUP +} + +# Drops the master password from the handoff once applied, so it is not left +# readable for the life of the boot. The rest is tmpfs and goes at reboot. +consume_secrets() { + [ -f "${ENV_FILE}" ] || return 0 + _tmp="${ENV_FILE}.new" + install -m 0600 /dev/null "${_tmp}" + grep -v '^RDS_MASTER_PASSWORD=' "${ENV_FILE}" >> "${_tmp}" || true + mv -f "${_tmp}" "${ENV_FILE}" +} + +load_handoff +require_data_volume +require_engine_stamp + +if [ -d "${DATADIR}/mysql" ]; then + # The engine would start cleanly on such a datadir and pass every health + # probe with no master user to log in as. The sentinel says the bootstrap did + # not finish, not that the volume is empty, so nothing here clears it. + if [ -e "${BOOTSTRAP_SENTINEL}" ]; then + fail "datadir ${DATADIR} carries ${BOOTSTRAP_SENTINEL}: an earlier bootstrap was interrupted before the master user existed; recover the data volume out of band rather than starting the engine" + fi + require_bootstrap_receipt + log "datadir ${DATADIR} is already initialised — attaching" + install_runtime_config +else + initialise_datadir + install_runtime_config + bootstrap_master +fi + +consume_secrets +log "bootstrap complete; handing off to the mariadb service" diff --git a/scripts/images/rds-mariadb/rds-init.initd b/scripts/images/rds-mariadb/rds-init.initd new file mode 100644 index 000000000..b422573c4 --- /dev/null +++ b/scripts/images/rds-mariadb/rds-init.initd @@ -0,0 +1,35 @@ +#!/sbin/openrc-run +# rds-init — boot oneshot that bootstraps or attaches the MariaDB datadir before +# the engine starts. +# +# `before mariadb` orders the two, and /etc/conf.d/mariadb makes the engine need +# this oneshot: Alpine's packaged mariadb service only prints a warning when the +# datadir is empty and starts anyway, so ordering alone would let mysqld_safe +# launch against an unbootstrapped data volume. `after rds-agent`, not `need`, so +# a failed fetch fails this oneshot visibly rather than skipping it. + +description="Spinifex RDS MariaDB bootstrap" + +AGENT_ENV="/etc/spinifex-rds/agent.env" + +depend() { + need net + after rds-agent rds-datadir + before mariadb +} + +start_pre() { + if [ -f "${AGENT_ENV}" ]; then + # shellcheck disable=SC1090 + . "${AGENT_ENV}" + export RDS_HANDOFF_DIR RDS_DATA_MOUNT + fi +} + +start() { + ebegin "Bootstrapping the RDS MariaDB instance" + # The VM has no operator login, so the host-side serial log is the only view + # of a bootstrap that failed before the engine came up. + /usr/local/sbin/rds-init > /dev/console 2>&1 + eend $? +} diff --git a/scripts/images/rds-mariadb/rds-init_test.sh b/scripts/images/rds-mariadb/rds-init_test.sh new file mode 100755 index 000000000..89d13ae1f --- /dev/null +++ b/scripts/images/rds-mariadb/rds-init_test.sh @@ -0,0 +1,1036 @@ +#!/bin/sh +# Self-contained POSIX test for the MariaDB rds-init: the initialize path +# (mariadb-install-db, the master grant, the initial database), the idempotent +# attach path, the engine stamp, the data-volume guard, TLS install and the +# parameter include. No MariaDB, no root: mariadb-install-db, mariadbd, +# mariadb-admin and the mariadb client are stubbed on PATH alongside +# install/chown (the script sets file ownership), and every path it touches is +# redirected into a temp dir via its env knobs. +# +# Run: sh scripts/images/rds-mariadb/rds-init_test.sh +set -eu + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +SCRIPT="${SCRIPT_DIR}/rds-init" +WORK=$(mktemp -d) +trap 'rm -rf "${WORK}"' EXIT + +REAL_LS=$(command -v ls) +REAL_CAT=$(command -v cat) +export REAL_LS REAL_CAT + +STUBBIN="${WORK}/bin" +mkdir -p "${STUBBIN}" + +# install stub: honours -d and -m, drops -o/-g (the harness is not root). +cat > "${STUBBIN}/install" <<'EOF' +#!/bin/sh +mode=""; dirmode=0; args="" +while [ $# -gt 0 ]; do + case "$1" in + -d) dirmode=1; shift ;; + -m) mode="$2"; shift 2 ;; + -o|-g) shift 2 ;; + *) args="${args} $1"; shift ;; + esac +done +# shellcheck disable=SC2086 +set -- ${args} +if [ "${dirmode}" = 1 ]; then + mkdir -p "$@" || exit 1 + [ -n "${mode}" ] && chmod "${mode}" "$@" + exit 0 +fi +dest="" +for a in "$@"; do dest="${a}"; done +src="" +for a in "$@"; do [ "${a}" = "${dest}" ] || src="${a}"; done +cp "${src}" "${dest}" || exit 1 +[ -n "${mode}" ] && chmod "${mode}" "${dest}" +exit 0 +EOF + +# chown stub: file ownership is a no-op outside the guest. +cat > "${STUBBIN}/chown" <<'EOF' +#!/bin/sh +exit 0 +EOF + +# ls stub: delegates normally, but lets the datadir safety tests prove that a +# failed enumeration is not mistaken for an empty directory. +cat > "${STUBBIN}/ls" <<'EOF' +#!/bin/sh +[ "${LS_FAIL:-0}" = "1" ] && exit 74 +exec "${REAL_LS}" "$@" +EOF + +# mariadb-install-db stub: records the call and materialises what a real one +# leaves behind. The mysql system database is what rds-init keys "already +# initialised" off. +cat > "${STUBBIN}/mariadb-install-db" <<'EOF' +#!/bin/sh +echo "mariadb-install-db $*" >> "${INSTALLDB_CALLS}" +[ "${INSTALLDB_FAIL:-0}" = "1" ] && exit 1 +datadir="" +for a in "$@"; do + case "$a" in --datadir=*) datadir="${a#--datadir=}" ;; esac +done +mkdir -p "${datadir}/mysql" +echo "global_priv" > "${datadir}/mysql/global_priv.MAD" +echo "ibdata" > "${datadir}/ibdata1" +exit 0 +EOF + +# mariadbd stub: the private bootstrap server. Opens its socket and pidfile and +# serves until mariadb-admin shutdown removes the socket. MARIADBD_START_FAIL is +# a server that will not come up, e.g. a datadir it cannot open; +# MARIADBD_KILL_PARENT signals rds-init from the one window between the datadir +# and the master user. +cat > "${STUBBIN}/mariadbd" <<'EOF' +#!/bin/sh +echo "mariadbd $*" >> "${MARIADBD_CALLS}" +[ "${MARIADBD_START_FAIL:-0}" = "1" ] && exit 1 +sock=""; pid="" +for a in "$@"; do + case "$a" in + --socket=*) sock="${a#--socket=}" ;; + --pid-file=*) pid="${a#--pid-file=}" ;; + esac +done +if [ "${MARIADBD_KILL_PARENT:-0}" = "1" ]; then + kill -TERM "$(cat "${KILL_PID_FILE}")" + exit 0 +fi +: > "${sock}" +echo "$$" > "${pid}" +# The bound keeps a refused shutdown from leaving a process behind the run. +_left=200 +while [ -e "${sock}" ] && [ "${_left}" -gt 0 ]; do + sleep 0.05 + _left=$((_left - 1)) +done +rm -f "${pid}" +exit 0 +EOF + +# mariadb-admin stub: ping answers only while the socket is open, and shutdown +# closes it. ADMIN_SHUTDOWN_FAIL is a server that will not let go of the datadir. +cat > "${STUBBIN}/mariadb-admin" <<'EOF' +#!/bin/sh +sock=""; action="" +for a in "$@"; do + case "$a" in + --socket=*) sock="${a#--socket=}" ;; + ping|shutdown) action="$a" ;; + esac +done +echo "mariadb-admin ${action} $*" >> "${ADMIN_CALLS}" +[ -e "${sock}" ] || exit 1 +case "${action}" in + ping) exit 0 ;; + shutdown) + [ "${ADMIN_SHUTDOWN_FAIL:-0}" = "1" ] && exit 1 + rm -f "${sock}" + exit 0 + ;; +esac +exit 1 +EOF + +# mariadb client stub: records the SQL it was fed on stdin plus its argv, so the +# test can prove the password rode neither an argument nor the environment. +cat > "${STUBBIN}/mariadb" <<'EOF' +#!/bin/sh +_in="${WORKDIR}/client.stdin.$$" +cat > "${_in}" +{ + echo "--- mariadb $* [env-password=${RDS_MASTER_PASSWORD:-}]" + cat "${_in}" +} >> "${CLIENT_CALLS}" +# --batch aborts at the first error and echoes the statement it failed on, +# password and all, which is what the redaction filter exists to catch. Only the +# session carrying the password fails, so the case reaches the statement it is +# about. +if [ "${CLIENT_FAIL:-0}" = "1" ] && grep -q 'IDENTIFIED BY' "${_in}"; then + _stmt=$(grep -m 1 'IDENTIFIED BY' "${_in}") + rm -f "${_in}" + # printf rather than echo: a real client echoes the statement verbatim, and + # this shell's echo would eat the very backslash the escaping put there. + printf 'ERROR 1064 (42000) at line 6: You have an error in your SQL syntax near "%s"\n' \ + "${_stmt}" >&2 + exit 1 +fi +rm -f "${_in}" +# CLIENT_STATUS_CORRUPT=absent: the client succeeds, but rds-init cannot create +# the status file it recovers the exit code through — the shape a read-only or +# full /run gives it. +if [ "${CLIENT_STATUS_CORRUPT:-}" = "absent" ]; then + for a in "$@"; do + case "$a" in + --socket=*) mkdir -p "$(dirname "${a#--socket=}")/client.status" ;; + esac + done +fi +exit 0 +EOF + +# cat stub: delegates normally, but lets the read-back of the client's exit +# status return what a torn write or an I/O error would. rds-init's other uses of +# cat read a file or stdin and are untouched. +cat > "${STUBBIN}/cat" <<'EOF' +#!/bin/sh +case "${1:-}" in + *client.status) + case "${CLIENT_STATUS_CORRUPT:-}" in + empty) exit 0 ;; + garbage) echo "not-a-number"; exit 0 ;; + esac + ;; +esac +exec "${REAL_CAT}" "$@" +EOF + +# sync stub: a no-op, except that SYNC_KILL_PARENT signals rds-init from the +# directory sync that follows the receipt rename. That is the one window where +# an installed receipt and an armed sweep coexist. Matched on the receipt +# directory by name: the engine stamp syncs its directory too, long before the +# traps are armed, and killing there would leave the window untested. +cat > "${STUBBIN}/sync" <<'EOF' +#!/bin/sh +if [ "${SYNC_KILL_PARENT:-0}" = "1" ]; then + case "${1:-}" in + */bootstrap) kill -TERM "$(cat "${KILL_PID_FILE}")" ;; + esac +fi +exit 0 +EOF + +chmod +x "${STUBBIN}"/* +PATH="${STUBBIN}:${PATH}" +export PATH + +FAILS=0 +fail() { echo "FAIL: $*"; FAILS=$((FAILS + 1)); } +pass() { echo "ok: $*"; } + +DATA_MOUNT="${WORK}/data" +DATADIR="${DATA_MOUNT}/data" +CONF_DIR="${DATA_MOUNT}/conf.d" +PARAM_FILE="${CONF_DIR}/10-rds-parameters.cnf" +SERVING_FILE="${CONF_DIR}/10-rds-parameters.serving" +PLATFORM_FILE="${CONF_DIR}/90-rds-init.cnf" +SENTINEL="${DATADIR}/rds-bootstrap-incomplete" +RECEIPT_DIR="${DATA_MOUNT}/.spinifex-rds/bootstrap" +RECEIPT="${RECEIPT_DIR}/receipt.env" +STAMP="${DATA_MOUNT}/.spinifex-rds/engine" +HANDOFF="${WORK}/run/spinifex-rds" +SECURE_FILE_DIR="${WORK}/mysql-files" +MOUNTS="${WORK}/mounts" +KILL_PID_FILE="${WORK}/rds-init.pid" +WORKDIR="${WORK}" +export KILL_PID_FILE WORKDIR + +INSTALLDB_CALLS="${WORK}/install-db.calls" +MARIADBD_CALLS="${WORK}/mariadbd.calls" +ADMIN_CALLS="${WORK}/admin.calls" +CLIENT_CALLS="${WORK}/client.calls" +export INSTALLDB_CALLS MARIADBD_CALLS ADMIN_CALLS CLIENT_CALLS + +# write_handoff : lay down the bootstrap.env rds-agent +# would have written. An empty password omits the field, as `attach` does. +# MASTER_USER overrides the master user name, for the reserved-name cases. +# PENDING/PAYLOAD_ID carry the control plane's "this payload is still staged" +# assertion, which an agent predating the receipt protocol omits entirely. +write_handoff() { + mkdir -p "${HANDOFF}" + { + echo "RDS_MODE=$1" + echo "RDS_DB_INSTANCE_IDENTIFIER=${DB_ID:-db1}" + echo "RDS_MASTER_USERNAME=${MASTER_USER:-mulgamaster}" + # Single-quoted as rds-agent's shellQuote writes it, which is what lets a + # password carrying shell metacharacters — or a newline — survive the + # handoff intact and reach the checks that exist for it. An embedded + # quote is closed, escaped and reopened, exactly as the agent does. + [ -n "$2" ] && printf "RDS_MASTER_PASSWORD='%s'\n" "$(printf '%s' "$2" | sed "s/'/'\\\\''/g")" + [ -n "$3" ] && echo "RDS_DB_NAME='$3'" + [ -n "${PENDING:-}" ] && echo "RDS_BOOTSTRAP_PENDING=${PENDING}" + [ -n "${PAYLOAD_ID:-}" ] && echo "RDS_PAYLOAD_ID=${PAYLOAD_ID}" + # Last, and unconditional: an optional line failing its test would make + # the whole group's status non-zero and `set -e` would end the run here. + echo "RDS_PORT=6543" + echo "RDS_VM_GENERATION=1" + } > "${HANDOFF}/bootstrap.env" +} + +write_tls() { + mkdir -p "${HANDOFF}" + echo "CERT" > "${HANDOFF}/server.crt" + echo "KEY" > "${HANDOFF}/server.key" +} + +# The resolved parameter group rds-agent hands over, in the engine-neutral +# `name = 'value'` rendering and with no group header — rds-init is what adds the +# one MariaDB needs. general_log is in it to prove the bootstrap server does not +# read it: the password is interpolated into a statement, and a customer's +# logging parameter must not be able to write that statement to a log. +write_parameters() { + mkdir -p "${HANDOFF}" + { + echo "# Resolved parameter group, written by rds-agent." + echo "max_connections = '100'" + echo "general_log = 'on'" + echo "character_set_server = 'utf8mb4'" + echo "collation_server = 'utf8mb4_general_ci'" + } > "${HANDOFF}/parameters.conf" +} + +# reset_state clears everything the previous case left behind: a fresh data +# volume, an attached mount, and empty stub-call logs. +reset_state() { + rm -rf "${WORK}/data" "${WORK}/run" "${WORK}/log" "${SECURE_FILE_DIR}" + mkdir -p "${DATA_MOUNT}" + printf '/dev/vdb %s ext4 rw,relatime 0 0\n' "${DATA_MOUNT}" > "${MOUNTS}" + : > "${INSTALLDB_CALLS}" + : > "${MARIADBD_CALLS}" + : > "${ADMIN_CALLS}" + : > "${CLIENT_CALLS}" + unset INSTALLDB_FAIL CLIENT_FAIL LS_FAIL RDS_ALLOW_LOCAL_DATADIR MASTER_USER || true + unset PENDING PAYLOAD_ID DB_ID RECEIPT_DIR_OVERRIDE CLIENT_STATUS_CORRUPT STAMP_OVERRIDE || true + unset MARIADBD_START_FAIL MARIADBD_KILL_PARENT ADMIN_SHUTDOWN_FAIL SYNC_KILL_PARENT || true +} + +# run_env: invoke a command with every path knob pointed into the temp dir. The +# probe interval is driven down so the socket wait costs milliseconds rather than +# a real second per case. +run_env() { + env RDS_MARIADB_BIN="${STUBBIN}" \ + RDS_DATA_MOUNT="${DATA_MOUNT}" \ + RDS_HANDOFF_DIR="${HANDOFF}" \ + RDS_SOCKET_DIR="${WORK}/run/mysqld" \ + RDS_BOOTSTRAP_DIR="${WORK}/run/rds-init" \ + RDS_LOG_DIR="${WORK}/log" \ + RDS_SECURE_FILE_DIR="${SECURE_FILE_DIR}" \ + RDS_MOUNTS_FILE="${MOUNTS}" \ + RDS_RECEIPT_DIR="${RECEIPT_DIR_OVERRIDE:-${RECEIPT_DIR}}" \ + RDS_ENGINE_STAMP="${STAMP_OVERRIDE:-${STAMP}}" \ + RDS_ALLOW_LOCAL_DATADIR="${RDS_ALLOW_LOCAL_DATADIR:-0}" \ + RDS_BOOTSTRAP_POLL=0.05 \ + RDS_BOOTSTRAP_PROBES=200 \ + "$@" "${WORK}/out" 2>&1 || { fail "$1: non-zero exit: $(cat "${WORK}/out")"; return 1; }; } +run_fails() { run > "${WORK}/out" 2>&1 && fail "$1: expected a non-zero exit" || pass "$1: refused"; } + +# run_signalled: run rds-init in the background with its PID published for the +# stub that signals it. The wrapper writes the PID before exec'ing the script, +# so the stub cannot read the file before it exists. +run_signalled() { + run_env sh -c 'echo $$ > "$1"; shift; exec sh "$@"' _ "${KILL_PID_FILE}" "${SCRIPT}" \ + > "${WORK}/out" 2>&1 & + _bg=$! + wait "${_bg}" +} + +run_signalled_fails() { + run_signalled && fail "$1: expected a non-zero exit" || pass "$1: refused" +} + +# --- Case 1: initialize on a fresh data volume --- +reset_state +write_handoff initialize 's3cr3t' appdb +write_tls +write_parameters +if run_ok "initialize"; then + grep -q 'mariadb-install-db' "${INSTALLDB_CALLS}" \ + && pass "initialize: the datadir was created" || fail "initialize: no mariadb-install-db" + grep -q -- '--skip-test-db' "${INSTALLDB_CALLS}" \ + && pass "initialize: no test database installed" || fail "initialize: no --skip-test-db" + grep -q -- '--auth-root-authentication-method=socket' "${INSTALLDB_CALLS}" \ + && pass "initialize: root authenticates through unix_socket" \ + || fail "initialize: root was not given socket authentication" + grep -q -- "--datadir=${DATADIR}" "${INSTALLDB_CALLS}" \ + && pass "initialize: the datadir is one level inside the mount" \ + || fail "initialize: wrong datadir" + + # The mount point itself is not the datadir: a failed bootstrap clears the + # datadir, and the include directory, the stamp and the receipt outlive it. + [ -d "${CONF_DIR}" ] && [ -d "${DATADIR}" ] \ + && pass "initialize: the include directory sits beside the datadir" \ + || fail "initialize: the include directory is not beside the datadir" + + grep -q -- '--skip-networking' "${MARIADBD_CALLS}" \ + && pass "initialize: bootstrap server is socket-only" \ + || fail "initialize: bootstrap server listened on TCP" + grep -q -- "--socket=${WORK}/run/rds-init/" "${MARIADBD_CALLS}" \ + && pass "initialize: bootstrap socket is in the private directory" \ + || fail "initialize: bootstrap socket outside the private directory" + grep -q 'shutdown' "${ADMIN_CALLS}" \ + && pass "initialize: bootstrap server stopped" || fail "initialize: bootstrap server left running" + + grep -q "CREATE USER IF NOT EXISTS 'mulgamaster'@'%'" "${CLIENT_CALLS}" \ + && pass "initialize: master user created for the customer ENI" \ + || fail "initialize: no CREATE USER" + grep -q "IDENTIFIED BY 's3cr3t'" "${CLIENT_CALLS}" \ + && pass "initialize: master password applied" || fail "initialize: password not applied" + grep -q 'env-password=s3cr3t' "${CLIENT_CALLS}" \ + && fail "initialize: the password was also handed over in the environment" \ + || pass "initialize: the password rode stdin only" + grep -q 'CREATE DATABASE IF NOT EXISTS `appdb`' "${CLIENT_CALLS}" \ + && pass "initialize: initial database created" || fail "initialize: no CREATE DATABASE" + # A database keeps the character set it was created under, and the private + # server runs on --no-defaults rather than on the resolved group. + grep -q 'CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci' "${CLIENT_CALLS}" \ + && pass "initialize: the database takes the group's character set" \ + || fail "initialize: the database was created under the compiled default" + + # Withheld deliberately: FILE is arbitrary host file access as the mysql user, + # SUPER reaches SET GLOBAL and plugin loading, and SHUTDOWN stops the server + # underneath the control plane's lifecycle. + for privilege in SUPER FILE SHUTDOWN 'ALL PRIVILEGES'; do + grep -q "${privilege}" "${CLIENT_CALLS}" \ + && fail "initialize: the master grant includes ${privilege}" \ + || pass "initialize: master holds no ${privilege}" + done + grep -q 'GRANT SELECT, INSERT, UPDATE, DELETE' "${CLIENT_CALLS}" \ + && pass "initialize: the grant is enumerated" || fail "initialize: no enumerated grant" + grep -q "WITH GRANT OPTION" "${CLIENT_CALLS}" \ + && pass "initialize: master can delegate its own privileges" \ + || fail "initialize: master cannot grant" + grep -q "DELETE FROM mysql.global_priv WHERE User = ''" "${CLIENT_CALLS}" \ + && pass "initialize: anonymous accounts removed" || fail "initialize: anonymous accounts left" + grep -q 'DROP DATABASE IF EXISTS test' "${CLIENT_CALLS}" \ + && pass "initialize: the test database is dropped" || fail "initialize: test database left" + + # Every session carries the guard, so a statement added later cannot end up + # outside it. `grep -c` exits non-zero on no match, which `set -e` would turn + # into an aborted run in place of a reported failure. + guard_count=$(grep -cF "SET GLOBAL general_log = 0;" "${CLIENT_CALLS}" || true) + session_count=$(grep -c '^--- mariadb ' "${CLIENT_CALLS}" || true) + [ "${guard_count}" = "${session_count}" ] \ + && pass "log-guard: all ${session_count} bootstrap sessions guarded" \ + || fail "log-guard: ${guard_count} of ${session_count} bootstrap sessions guarded" + grep -qF "SET SESSION sql_mode = 'NO_ENGINE_SUBSTITUTION';" "${CLIENT_CALLS}" \ + && pass "log-guard: the escaping mode is pinned" \ + || fail "log-guard: the session mode the escaping assumes is not pinned" + # The customer's group carries general_log = on. Nothing in the bootstrap + # reads it, so their parameter cannot log the statement carrying the + # password, and a static value they set cannot stop this server starting and + # spend the one-shot password with it. + for stub in "${MARIADBD_CALLS}" "${INSTALLDB_CALLS}" "${CLIENT_CALLS}"; do + grep -q -- '--no-defaults' "${stub}" \ + && pass "log-guard: $(basename "${stub}" .calls) ignores the installed configuration" \ + || fail "log-guard: $(basename "${stub}" .calls) read the customer's parameters" + done + + head -n 1 "${PARAM_FILE}" | grep -qx '\[mysqld\]' \ + && pass "initialize: the parameter file carries the group header" \ + || fail "initialize: a setting before any group is a fatal parse error" + grep -q "max_connections = '100'" "${PARAM_FILE}" \ + && pass "initialize: resolved parameters installed" || fail "initialize: no parameter include" + cmp -s "${PARAM_FILE}" "${SERVING_FILE}" \ + && pass "initialize: the serving copy is what the engine starts on" \ + || fail "initialize: the serving copy differs from the installed file" + case "${SERVING_FILE}" in + *.cnf) fail "initialize: the serving copy is read as a second settings file" ;; + *) pass "initialize: the serving copy is not read by the include" ;; + esac + + grep -q "^port = 6543" "${PLATFORM_FILE}" \ + && pass "initialize: delivered port applied" || fail "initialize: port not applied" + grep -q "^datadir = ${DATADIR}" "${PLATFORM_FILE}" \ + && pass "initialize: datadir pinned by the platform" || fail "initialize: no datadir" + grep -q "^bind_address = 0.0.0.0" "${PLATFORM_FILE}" \ + && pass "initialize: the engine listens for the customer ENI" \ + || fail "initialize: no bind_address" + grep -q "^default_storage_engine = InnoDB" "${PLATFORM_FILE}" \ + && pass "initialize: tables land in the engine snapshots recover" \ + || fail "initialize: default_storage_engine not pinned" + grep -q "^skip-log-bin" "${PLATFORM_FILE}" \ + && pass "initialize: binary logging off" || fail "initialize: binary logging left on" + grep -q "^secure_file_priv = ${SECURE_FILE_DIR}" "${PLATFORM_FILE}" \ + && pass "initialize: secure_file_priv names a directory, not the empty string" \ + || fail "initialize: secure_file_priv unset or empty" + [ -d "${SECURE_FILE_DIR}" ] \ + && pass "initialize: the secure_file_priv directory exists" \ + || fail "initialize: secure_file_priv points at nothing" + grep -q "^ssl_cert = " "${PLATFORM_FILE}" && grep -q "^ssl_key = " "${PLATFORM_FILE}" \ + && pass "initialize: TLS offered" || fail "initialize: TLS not configured" + grep -q "require_secure_transport" "${PLATFORM_FILE}" \ + && fail "initialize: TLS enforced before the enforcement phase" \ + || pass "initialize: TLS offered rather than enforced" + # Removed outright in 11.8, so naming it is a startup failure rather than an + # override — every instance would boot-loop on it. + grep -q "innodb_buffer_pool_instances" "${PLATFORM_FILE}" \ + && fail "initialize: the platform file names a setting 11.8 removed" \ + || pass "initialize: no removed InnoDB settings" + + [ -e "${SENTINEL}" ] \ + && fail "initialize: incomplete-bootstrap sentinel left behind" \ + || pass "initialize: sentinel cleared once the master user exists" +fi + +# --- Case 2: the master password does not outlive its use --- +grep -q '^RDS_MASTER_PASSWORD=' "${HANDOFF}/bootstrap.env" \ + && fail "consume: password still in the handoff" || pass "consume: password dropped from the handoff" +grep -q '^RDS_MASTER_USERNAME=mulgamaster' "${HANDOFF}/bootstrap.env" \ + && pass "consume: the rest of the config is kept" || fail "consume: non-secret fields lost" +[ -f "${HANDOFF}/server.key" ] \ + && fail "consume: serving key left in the handoff" || pass "consume: serving key moved out of the handoff" +[ -f "${WORK}/run/mysqld/tls/server.key" ] \ + && pass "tls: key installed on tmpfs, not the data volume" || fail "tls: key not installed" +# Anything on the data volume rides into every snapshot of it. +if grep -rq 's3cr3t' "${DATA_MOUNT}" 2>/dev/null; then + fail "consume: the master password is on the data volume" + grep -rl 's3cr3t' "${DATA_MOUNT}" 2>/dev/null +else + pass "consume: no password anywhere on the data volume" +fi +grep -rq 'KEY' "${DATA_MOUNT}" 2>/dev/null \ + && fail "tls: the serving key reached the data volume" || pass "tls: data volume free of the key" +# A client option file holding the password would hand it to anything that runs +# the mariadb client as root, and would survive the boot. +grep -q '\.my\.cnf' "${SCRIPT}" \ + && fail "consume: rds-init writes a client option file" || pass "consume: no /root/.my.cnf written" + +# --- Case 3: second boot (attach) is idempotent --- +: > "${INSTALLDB_CALLS}"; : > "${CLIENT_CALLS}"; : > "${MARIADBD_CALLS}" +write_handoff attach '' appdb +write_parameters +if run_ok "attach"; then + [ -s "${INSTALLDB_CALLS}" ] \ + && fail "attach: re-initialised an existing datadir" || pass "attach: initialisation skipped" + [ -s "${CLIENT_CALLS}" ] \ + && fail "attach: re-applied the master user" || pass "attach: no user/database work" + [ -s "${MARIADBD_CALLS}" ] \ + && fail "attach: started a bootstrap server it had no work for" \ + || pass "attach: no bootstrap server" + grep -q "^ssl_cert = " "${PLATFORM_FILE}" \ + && pass "attach: TLS survives a handoff without a new cert" || fail "attach: TLS turned off" + cmp -s "${PARAM_FILE}" "${SERVING_FILE}" \ + && pass "attach: the serving copy tracks the boot" \ + || fail "attach: the serving copy is stale" +fi + +# --- Case 3a: a handoff with no parameters keeps the installed ones --- +: > "${INSTALLDB_CALLS}" +rm -f "${HANDOFF}/parameters.conf" +write_handoff attach '' appdb +if run_ok "no-parameters"; then + grep -q "max_connections = '100'" "${PARAM_FILE}" \ + && pass "no-parameters: the installed group is kept" \ + || fail "no-parameters: the instance was reset to engine defaults" + cmp -s "${PARAM_FILE}" "${SERVING_FILE}" \ + && pass "no-parameters: the serving copy still matches" \ + || fail "no-parameters: the serving copy drifted" +fi + +# --- Case 3b: a failing client must not echo the password to the console --- +# rds-init.initd sends this script's output to /dev/console, which the host +# captures off ttyS0, and the client under --batch echoes the statement it failed +# on. The redaction must not swallow the failure: the datadir is cleared. +reset_state +write_handoff initialize 's3cr3t' appdb +write_parameters +CLIENT_FAIL=1 +export CLIENT_FAIL +run_fails "console-redaction" +grep -q 's3cr3t' "${WORK}/out" \ + && fail "console-redaction: the console capture carries the master password" \ + || pass "console-redaction: no password on the console" +grep -q '\[REDACTED\]' "${WORK}/out" \ + && pass "console-redaction: the redaction is marked" \ + || fail "console-redaction: the client's stderr never reached the console" +grep -q 'ERROR 1064' "${WORK}/out" \ + && pass "console-redaction: the diagnostic itself survives" \ + || fail "console-redaction: the redaction swallowed the diagnostic" +[ -e "${DATADIR}/mysql" ] \ + && fail "console-redaction: datadir kept after a failed master bootstrap" \ + || pass "console-redaction: the failure still propagates" +unset CLIENT_FAIL + +# --- Case 3c: the client's status must cross the redaction pipeline or fail closed --- +# The status file is how the client's exit code leaves the subshell, POSIX sh +# having no pipefail. A torn write leaves it created but empty, and `return ""` +# is fatal in the guest's ash: rds-init would die past the datadir sweep and +# before any diagnostic. +for corruption in empty garbage; do + reset_state + write_handoff initialize 's3cr3t' appdb + CLIENT_STATUS_CORRUPT="${corruption}" + export CLIENT_STATUS_CORRUPT + run_fails "client-status-${corruption}" + grep -q 'exit status could not be read back' "${WORK}/out" \ + && pass "client-status-${corruption}: names the unreadable status rather than blaming the client" \ + || fail "client-status-${corruption}: no diagnostic naming the status file" + grep -q 'Illegal number\|numeric argument required' "${WORK}/out" \ + && fail "client-status-${corruption}: an unvalidated status reached return" \ + || pass "client-status-${corruption}: nothing unvalidated reached return" + [ -e "${DATADIR}/mysql" ] \ + && fail "client-status-${corruption}: datadir kept after an unrecoverable status" \ + || pass "client-status-${corruption}: datadir cleared" + grep -q 'shutdown' "${ADMIN_CALLS}" \ + && pass "client-status-${corruption}: bootstrap server stopped" \ + || fail "client-status-${corruption}: bootstrap server left holding the datadir" + unset CLIENT_STATUS_CORRUPT +done + +# --- Case 3d: a status file that could never be written is not the client's fault --- +reset_state +write_handoff initialize 's3cr3t' appdb +CLIENT_STATUS_CORRUPT=absent +export CLIENT_STATUS_CORRUPT +run_fails "client-status-absent" +grep -q 'exit status was never recorded' "${WORK}/out" \ + && pass "client-status-absent: names the missing status file" \ + || fail "client-status-absent: no diagnostic distinguishing it from a client failure" +[ -e "${DATADIR}/mysql" ] \ + && fail "client-status-absent: datadir kept" || pass "client-status-absent: datadir cleared" +unset CLIENT_STATUS_CORRUPT + +# --- Case 3e: a multi-line master password is refused before it is spent --- +reset_state +write_handoff initialize 'top +secret' appdb +run_fails "multiline-password" +[ -s "${INSTALLDB_CALLS}" ] \ + && fail "multiline-password: the datadir was created before the password was refused" \ + || pass "multiline-password: refused before the datadir was created" +grep -q 'spans more than one line' "${WORK}/out" \ + && pass "multiline-password: refusal names the reason" \ + || fail "multiline-password: no refusal message" + +# --- Case 3f: a password holding a quote and a backslash is escaped, not mangled --- +# ValidateMasterUserPassword permits both, and the mariadb client has no +# equivalent of psql's :'password' quoting, so the literal is built here. +reset_state +write_handoff initialize "pa'ss\\word" appdb +write_parameters +if run_ok "password-escaping"; then + grep -qF "IDENTIFIED BY 'pa''ss\\\\word'" "${CLIENT_CALLS}" \ + && pass "password-escaping: the quote is doubled and the backslash escaped" \ + || fail "password-escaping: the literal is not correctly escaped" + grep -qF "IDENTIFIED BY 'pa'ss" "${CLIENT_CALLS}" \ + && fail "password-escaping: an unescaped quote closed the literal early" \ + || pass "password-escaping: the literal is not closed early" +fi + +# The escaped form is what a failing statement echoes, and it is trivially +# unescaped by anyone reading the console, so it has to be redacted too. +reset_state +CLIENT_FAIL=1 +export CLIENT_FAIL +write_handoff initialize "pa'ss\\word" appdb +run_fails "password-escaping-redaction" +grep -qF "pa''ss\\\\word" "${WORK}/out" \ + && fail "password-escaping-redaction: the escaped password reached the console" \ + || pass "password-escaping-redaction: the escaped form is redacted too" +grep -qF "pa'ss\\word" "${WORK}/out" \ + && fail "password-escaping-redaction: the raw password reached the console" \ + || pass "password-escaping-redaction: the raw form is redacted" +grep -q '\[REDACTED\]' "${WORK}/out" \ + && pass "password-escaping-redaction: the statement was echoed and caught" \ + || fail "password-escaping-redaction: nothing was redacted, so this proves nothing" +unset CLIENT_FAIL + +# --- Case 4: an empty datadir in attach mode means the data volume is missing --- +reset_state +write_handoff attach '' appdb +run_fails "attach-empty" +[ -s "${INSTALLDB_CALLS}" ] \ + && fail "attach-empty: created an empty database over missing data" || pass "attach-empty: nothing created" + +# --- Case 5: the datadir must be on the attached data volume --- +reset_state +: > "${MOUNTS}" +write_handoff initialize 's3cr3t' appdb +run_fails "no-volume" +[ -s "${INSTALLDB_CALLS}" ] \ + && fail "no-volume: initialised on the boot volume" || pass "no-volume: nothing created" + +RDS_ALLOW_LOCAL_DATADIR=1 +export RDS_ALLOW_LOCAL_DATADIR +if run_ok "local-datadir"; then + grep -q 'mariadb-install-db' "${INSTALLDB_CALLS}" \ + && pass "local-datadir: explicit override initialises anyway" || fail "local-datadir: nothing created" +fi +unset RDS_ALLOW_LOCAL_DATADIR + +# --- Case 6: a failed initialisation leaves nothing that looks initialised --- +reset_state +write_handoff initialize 's3cr3t' appdb +INSTALLDB_FAIL=1 +export INSTALLDB_FAIL +run_fails "install-db-fail" +[ -e "${DATADIR}/mysql" ] \ + && fail "install-db-fail: half-written datadir kept" || pass "install-db-fail: datadir cleared" +unset INSTALLDB_FAIL + +# --- Case 6a: a failed initialisation over a NON-empty datadir must not delete it --- +# A datadir whose system tables were lost over otherwise intact data takes the +# initialise path, and "directory not empty" is one way the install then fails. +reset_state +write_handoff initialize 's3cr3t' appdb +mkdir -p "${DATADIR}" +echo 'customer table data' > "${DATADIR}/appdb.ibd" +printf 'mariadb\n' > "${WORK}/stamp-seed" +mkdir -p "$(dirname "${STAMP}")" +cp "${WORK}/stamp-seed" "${STAMP}" +INSTALLDB_FAIL=1 +export INSTALLDB_FAIL +run_fails "install-db-fail-nonempty" +[ -f "${DATADIR}/appdb.ibd" ] \ + && pass "install-db-fail-nonempty: pre-existing datadir preserved" \ + || fail "install-db-fail-nonempty: pre-existing datadir deleted" +grep -q 'refusing to clear' "${WORK}/out" \ + && pass "install-db-fail-nonempty: refusal explains why" \ + || fail "install-db-fail-nonempty: no refusal message" +unset INSTALLDB_FAIL + +# --- Case 6b: a failed emptiness probe must abort before anything is created --- +reset_state +write_handoff initialize 's3cr3t' appdb +LS_FAIL=1 +export LS_FAIL +run_fails "datadir-probe-fail" +unset LS_FAIL +[ -s "${INSTALLDB_CALLS}" ] \ + && fail "datadir-probe-fail: initialised after the probe failed" \ + || pass "datadir-probe-fail: nothing created" +grep -q 'could not inspect datadir' "${WORK}/out" \ + && pass "datadir-probe-fail: refusal explains why" || fail "datadir-probe-fail: no refusal message" + +# --- Case 6c: the bootstrap server never starts --- +# The datadir already carries the system tables and the password is spent, so +# leaving it would attach on the next boot and serve a database whose master user +# was never created. +reset_state +write_handoff initialize 's3cr3t' appdb +MARIADBD_START_FAIL=1 +export MARIADBD_START_FAIL +run_fails "bootstrap-nostart" +[ -e "${DATADIR}/mysql" ] \ + && fail "bootstrap-nostart: datadir kept after the server failed to start" \ + || pass "bootstrap-nostart: datadir cleared" +grep -q 'exited before it opened' "${WORK}/out" \ + && pass "bootstrap-nostart: refusal names the server" || fail "bootstrap-nostart: no refusal message" +unset MARIADBD_START_FAIL + +# --- Case 6d: a bootstrap server that will not stop is a failure --- +# It holds the datadir, so the mariadb service cannot start for the life of the +# boot. The master user exists by then, so the datadir itself must survive. +reset_state +write_handoff initialize 's3cr3t' appdb +ADMIN_SHUTDOWN_FAIL=1 +export ADMIN_SHUTDOWN_FAIL +run_fails "stop-fail" +grep -q 'did not stop' "${WORK}/out" \ + && pass "stop-fail: reported rather than swallowed" || fail "stop-fail: no failure message" +[ -e "${DATADIR}/mysql" ] \ + && pass "stop-fail: bootstrapped datadir kept" || fail "stop-fail: cleared a datadir with a master user" +[ -e "${SENTINEL}" ] \ + && fail "stop-fail: sentinel kept although the master user exists" \ + || pass "stop-fail: sentinel cleared" +[ -e "${RECEIPT}" ] \ + && pass "stop-fail: the completed bootstrap is still proven" \ + || fail "stop-fail: no receipt for a bootstrap that completed" +unset ADMIN_SHUTDOWN_FAIL + +# --- Case 6e: a SIGTERM between the datadir and the master user clears it --- +# An EXIT trap does not run on a signal in a POSIX shell, so a shutdown, an ACPI +# powerdown from a stop arriving while the instance is still creating, or a +# host-side force-stop would otherwise leave a datadir with no master user. +reset_state +write_handoff initialize 's3cr3t' appdb +MARIADBD_KILL_PARENT=1 +export MARIADBD_KILL_PARENT +run_signalled_fails "sigterm-midbootstrap" +[ -e "${DATADIR}/mysql" ] \ + && fail "sigterm-midbootstrap: datadir kept after a signal mid-bootstrap" \ + || pass "sigterm-midbootstrap: datadir cleared" +[ -s "${CLIENT_CALLS}" ] \ + && fail "sigterm-midbootstrap: the master user work ran after the signal" \ + || pass "sigterm-midbootstrap: stopped before the master user" +unset MARIADBD_KILL_PARENT + +# --- Case 6f: a datadir carrying the sentinel is refused, not started --- +# No trap survives a crash or a SIGKILL, so the sentinel is the only record that +# the bootstrap did not finish. It says the master user is missing, not that the +# volume is empty, so the refusal must not clear anything. +reset_state +write_handoff initialize 's3cr3t' appdb +if run_ok "sentinel-setup"; then + echo 'customer table data' > "${DATADIR}/appdb.ibd" + : > "${SENTINEL}" + write_handoff attach '' appdb + run_fails "sentinel-refused" + [ -f "${DATADIR}/appdb.ibd" ] \ + && pass "sentinel-refused: datadir preserved" || fail "sentinel-refused: datadir cleared" + grep -q 'recover the data volume out of band' "${WORK}/out" \ + && pass "sentinel-refused: refusal points at out-of-band recovery" \ + || fail "sentinel-refused: no refusal message" +fi + +# --- Case 7: a completed bootstrap leaves a receipt proving it --- +reset_state +PENDING=1 PAYLOAD_ID=bp-alpha DB_ID=db-alpha +export PENDING PAYLOAD_ID DB_ID +write_handoff initialize 's3cr3t' appdb +if run_ok "receipt"; then + [ -f "${RECEIPT}" ] \ + && pass "receipt: written on the initialize path" || fail "receipt: no receipt written" + grep -q '^RDS_RECEIPT_PAYLOAD_ID=bp-alpha$' "${RECEIPT}" \ + && pass "receipt: names the payload it applied" || fail "receipt: wrong or missing payload id" + grep -q '^RDS_RECEIPT_DB_INSTANCE_IDENTIFIER=db-alpha$' "${RECEIPT}" \ + && pass "receipt: names its DB instance" || fail "receipt: wrong or missing DB instance" + grep -q 'PASSWORD\|s3cr3t' "${RECEIPT}" \ + && fail "receipt: carries a secret" || pass "receipt: carries no secrets" + case "${RECEIPT}" in + "${DATADIR}"/*) fail "receipt: written inside the datadir the traps clear" ;; + *) pass "receipt: written outside the datadir" ;; + esac +fi + +# --- Case 7a: a receipt that cannot be written is fatal --- +reset_state +PENDING=1 PAYLOAD_ID=bp-beta +export PENDING PAYLOAD_ID +write_handoff initialize 's3cr3t' appdb +: > "${WORK}/not-a-dir" +RECEIPT_DIR_OVERRIDE="${WORK}/not-a-dir/bootstrap" +export RECEIPT_DIR_OVERRIDE +run_fails "receipt-fail" +[ -e "${DATADIR}/mysql" ] \ + && fail "receipt-fail: datadir kept although its bootstrap cannot be proven" \ + || pass "receipt-fail: datadir cleared" +grep -q 'bootstrap receipt' "${WORK}/out" \ + && pass "receipt-fail: reported rather than swallowed" || fail "receipt-fail: no failure message" +unset RECEIPT_DIR_OVERRIDE + +# --- Case 7b: a sweep between the receipt and the retired traps takes both --- +reset_state +PENDING=1 PAYLOAD_ID=bp-gamma +export PENDING PAYLOAD_ID +write_handoff initialize 's3cr3t' appdb +SYNC_KILL_PARENT=1 +export SYNC_KILL_PARENT +run_signalled_fails "receipt-swept" +[ -e "${DATADIR}/mysql" ] \ + && fail "receipt-swept: datadir kept after a signal mid-bootstrap" \ + || pass "receipt-swept: datadir cleared" +[ -e "${RECEIPT}" ] \ + && fail "receipt-swept: receipt survived the datadir it vouches for" \ + || pass "receipt-swept: receipt cleared with the datadir" +unset SYNC_KILL_PARENT + +# --- Case 7c: pending payload, initialised datadir, no receipt -> fail closed --- +reset_state +PENDING=1 PAYLOAD_ID=bp-delta DB_ID=db-delta +export PENDING PAYLOAD_ID DB_ID +write_handoff initialize 's3cr3t' appdb +if run_ok "pending-setup"; then + echo 'customer table data' > "${DATADIR}/appdb.ibd" + rm -f "${RECEIPT}" + run_fails "pending-no-receipt" + [ -f "${DATADIR}/appdb.ibd" ] \ + && pass "pending-no-receipt: datadir preserved" || fail "pending-no-receipt: datadir cleared" + grep -q 'master user cannot be proven' "${WORK}/out" \ + && pass "pending-no-receipt: refusal names the real reason" \ + || fail "pending-no-receipt: no refusal message" + + # --- Case 7d: the same boot with a matching receipt attaches --- + : > "${INSTALLDB_CALLS}"; : > "${CLIENT_CALLS}" + mkdir -p "${RECEIPT_DIR}" + printf 'RDS_RECEIPT_PAYLOAD_ID=bp-delta\nRDS_RECEIPT_DB_INSTANCE_IDENTIFIER=db-delta\n' > "${RECEIPT}" + if run_ok "pending-with-receipt"; then + [ -s "${INSTALLDB_CALLS}" ] \ + && fail "pending-with-receipt: re-initialised" || pass "pending-with-receipt: attached" + fi + + # --- Case 7e: a receipt naming another instance is treated as absent --- + # The receipt is on the data volume, so it rides along in every snapshot of + # it and a restored volume carries the source instance's receipt. + printf 'RDS_RECEIPT_PAYLOAD_ID=bp-delta\nRDS_RECEIPT_DB_INSTANCE_IDENTIFIER=db-somebody-else\n' > "${RECEIPT}" + run_fails "pending-foreign-receipt" + [ -f "${DATADIR}/appdb.ibd" ] \ + && pass "pending-foreign-receipt: datadir preserved" || fail "pending-foreign-receipt: datadir cleared" +fi +unset PENDING PAYLOAD_ID DB_ID + +# --- Case 7f: no pending payload leaves the attach path unchanged --- +reset_state +write_handoff initialize 's3cr3t' appdb +if run_ok "legacy-setup"; then + rm -rf "${RECEIPT_DIR}" + : > "${INSTALLDB_CALLS}" + write_handoff attach '' appdb + if run_ok "legacy-attach"; then + [ -s "${INSTALLDB_CALLS}" ] \ + && fail "legacy-attach: re-initialised" || pass "legacy-attach: attached without a receipt" + fi +fi + +# --- Case 8: no serving cert -> TLS off rather than a failed start --- +reset_state +write_handoff initialize 's3cr3t' '' +if run_ok "no-cert"; then + grep -q '^ssl_cert' "${PLATFORM_FILE}" \ + && fail "no-cert: points at a cert that was never delivered" || pass "no-cert: no cert paths" + grep -q 'CREATE DATABASE' "${CLIENT_CALLS}" \ + && fail "no-dbname: created a database without a DBName" || pass "no-dbname: no initial database" +fi + +# --- Case 9: no handoff at all --- +reset_state +rm -rf "${HANDOFF}" +run_fails "no-handoff" + +# --- Case 10: a master username the platform reserves --- +# The master user is created without SUPER, so one named after an account +# mariadb-install-db already made would either fail outright or rewrite the +# account rds-init and rds-agent reach the server through — on a datadir that +# bootstraps once. The set and the case-insensitive matching are the control +# plane's: a name it refuses must not reach the install here. +for reserved in root mysql rdsadmin public Root mariadb.sys; do + reset_state + MASTER_USER="${reserved}" + write_handoff initialize 's3cr3t' appdb + run_fails "reserved-master-${reserved}" + [ -s "${INSTALLDB_CALLS}" ] \ + && fail "reserved-master-${reserved}: initialised before the name was refused" \ + || pass "reserved-master-${reserved}: refused before the datadir was created" + unset MASTER_USER +done + +# --- Case 10a: a master username that is not safe to interpolate --- +# It is built into CREATE USER by the shell, the client having no identifier +# quoting of its own. +for badname in 'my-user' 'my user' "o'brien" '1user' 'user;DROP' \ + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'; do + reset_state + MASTER_USER="${badname}" + write_handoff initialize 's3cr3t' appdb + run_fails "bad-master-${badname}" + [ -s "${INSTALLDB_CALLS}" ] \ + && fail "bad-master-${badname}: initialised before the name was refused" \ + || pass "bad-master-${badname}: refused before the datadir was created" + unset MASTER_USER +done + +# --- Case 11: an initial database name the control plane would have refused --- +# MariaDB maps a database name onto a directory name and the client cannot quote +# an identifier for us, so this rule is the barrier rather than defence in depth. +for badname in 'my-db' 'my db' 'my/db' 'my.db' '1db' 'db`x' \ + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'; do + reset_state + write_handoff initialize 's3cr3t' "${badname}" + run_fails "bad-dbname-${badname}" + [ -s "${INSTALLDB_CALLS}" ] \ + && fail "bad-dbname-${badname}: initialised before the name was refused" \ + || pass "bad-dbname-${badname}: refused before the datadir was created" +done + +# The rule is a rejection of malformed names, not of every name: the boundary +# length and an underscore have to keep working. +reset_state +write_handoff initialize 's3cr3t' \ + 'a_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' +write_parameters +if run_ok "dbname-at-the-limit"; then + grep -q 'CREATE DATABASE' "${CLIENT_CALLS}" \ + && pass "dbname-at-the-limit: accepted" || fail "dbname-at-the-limit: no CREATE DATABASE" +fi + +# --- Case 12: the data volume records which engine wrote it --- +reset_state +write_handoff initialize 's3cr3t' appdb +if run_ok "stamp-absent-empty"; then + [ "$(cat "${STAMP}" 2>/dev/null)" = "mariadb" ] \ + && pass "stamp-absent-empty: fresh volume stamped mariadb" \ + || fail "stamp-absent-empty: no stamp written at initialisation" + case "${STAMP}" in + "${DATADIR}"/*) fail "stamp-absent-empty: written inside the datadir the traps clear" ;; + *) pass "stamp-absent-empty: written outside the datadir" ;; + esac + + # --- Case 12a: a matching stamp attaches --- + : > "${INSTALLDB_CALLS}" + write_handoff attach '' appdb + if run_ok "stamp-match"; then + [ -s "${INSTALLDB_CALLS}" ] \ + && fail "stamp-match: re-initialised" || pass "stamp-match: attached" + fi + + # --- Case 12b: another engine's stamp is fatal and touches nothing --- + # A PostgreSQL data volume mounts cleanly here and its datadir reads as + # uninitialised, so without this rds-init would bootstrap beside the + # customer's data and serve an empty database that passes every probe. + echo 'customer table data' > "${DATADIR}/appdb.ibd" + printf 'postgres\n' > "${STAMP}" + : > "${INSTALLDB_CALLS}" + run_fails "stamp-mismatch" + [ -s "${INSTALLDB_CALLS}" ] \ + && fail "stamp-mismatch: initialised over another engine's volume" \ + || pass "stamp-mismatch: nothing created" + [ -f "${DATADIR}/appdb.ibd" ] \ + && pass "stamp-mismatch: datadir preserved" || fail "stamp-mismatch: datadir cleared" + [ "$(cat "${STAMP}")" = "postgres" ] \ + && pass "stamp-mismatch: the other engine's stamp is left as it stands" \ + || fail "stamp-mismatch: stamp overwritten" + grep -q "holds a 'postgres' datadir" "${WORK}/out" \ + && pass "stamp-mismatch: refusal names both engines" \ + || fail "stamp-mismatch: no refusal naming the stamped engine" + + # --- Case 12c: a stamp that cannot be read is not an absent one --- + rm -f "${STAMP}" + mkdir "${STAMP}" + run_fails "stamp-unreadable" + grep -q 'could not be read' "${WORK}/out" \ + && pass "stamp-unreadable: refusal names the unreadable stamp" \ + || fail "stamp-unreadable: no refusal message" + rmdir "${STAMP}" + + # --- Case 12d: an unstamped datadir with data in it is refused --- + # PostgreSQL backfills here, because it had unstamped volumes in the field. + # MariaDB never did, so a datadir with content and no stamp is by definition + # not ours, and stamping it would rubber-stamp the volume the check exists + # to refuse. + rm -f "${STAMP}" + : > "${INSTALLDB_CALLS}" + run_fails "stamp-absent-nonempty" + [ -s "${INSTALLDB_CALLS}" ] \ + && fail "stamp-absent-nonempty: initialised over an unidentified datadir" \ + || pass "stamp-absent-nonempty: nothing created" + [ -f "${DATADIR}/appdb.ibd" ] \ + && pass "stamp-absent-nonempty: datadir preserved" \ + || fail "stamp-absent-nonempty: datadir cleared" + [ -e "${STAMP}" ] \ + && fail "stamp-absent-nonempty: an unidentified volume was stamped ours" \ + || pass "stamp-absent-nonempty: no stamp written" + grep -q 'no mariadb engine stamp' "${WORK}/out" \ + && pass "stamp-absent-nonempty: refusal names the missing stamp" \ + || fail "stamp-absent-nonempty: no refusal message" +fi + +# --- Case 12e: a stamp that cannot be written is fatal, before initialisation --- +reset_state +write_handoff initialize 's3cr3t' appdb +: > "${WORK}/not-a-dir-stamp" +STAMP_OVERRIDE="${WORK}/not-a-dir-stamp/engine" +export STAMP_OVERRIDE +run_fails "stamp-unwritable" +[ -s "${INSTALLDB_CALLS}" ] \ + && fail "stamp-unwritable: initialised before the stamp was recorded" \ + || pass "stamp-unwritable: refused before the datadir was created" +grep -q 'engine stamp' "${WORK}/out" \ + && pass "stamp-unwritable: refusal names the stamp" || fail "stamp-unwritable: no refusal message" +unset STAMP_OVERRIDE + +if [ "${FAILS}" -eq 0 ]; then + echo "PASS: all rds-init cases" + exit 0 +fi +echo "FAILED: ${FAILS} case(s)" +exit 1 diff --git a/scripts/images/rds-mariadb/rds-openrc_test.sh b/scripts/images/rds-mariadb/rds-openrc_test.sh index 104f9f88d..03e93c76a 100755 --- a/scripts/images/rds-mariadb/rds-openrc_test.sh +++ b/scripts/images/rds-mariadb/rds-openrc_test.sh @@ -29,16 +29,40 @@ assert_line() { capture_dependencies rds-agent.initd > "${WORK}/agent" capture_dependencies rds-datadir.initd > "${WORK}/datadir" +capture_dependencies rds-init.initd > "${WORK}/init" assert_line "${WORK}/agent" "need net" assert_line "${WORK}/agent" "after localmount" assert_line "${WORK}/agent" "before rds-datadir rds-init mariadb" assert_line "${WORK}/datadir" "after localmount rds-agent" assert_line "${WORK}/datadir" "before rds-init mariadb" +assert_line "${WORK}/init" "need net" +assert_line "${WORK}/init" "after rds-agent rds-datadir" +assert_line "${WORK}/init" "before mariadb" if grep -Fq "before rds-agent" "${WORK}/datadir"; then echo "FAIL: rds-datadir still starts before the bootstrap handoff" >&2 exit 1 fi +# Ordering alone is not enough for this engine: Alpine's mariadb service only +# warns about an empty datadir and starts anyway, so a failed rds-init would be +# answered by launching mysqld_safe against an unbootstrapped volume. OpenRC +# reads the dependency out of the service's conf.d file. +grep -Fq 'rc_need="rds-init"' "${SCRIPT_DIR}/mariadb.confd" || { + echo "FAIL: /etc/conf.d/mariadb does not make the engine need rds-init" >&2 + exit 1 +} +grep -Fq "mariadb.confd:/etc/conf.d/mariadb" "${SCRIPT_DIR}/manifest.conf" || { + echo "FAIL: manifest.conf does not install mariadb.confd" >&2 + exit 1 +} +for svc in rds-init mariadb; do + grep -qE "^ENABLE_SERVICES=.*[\" ]${svc}[\" ]" "${SCRIPT_DIR}/manifest.conf" || { + echo "FAIL: manifest.conf does not enable ${svc}" >&2 + grep '^ENABLE_SERVICES' "${SCRIPT_DIR}/manifest.conf" >&2 + exit 1 + } +done + echo "rds-openrc: dependency order tests passed" diff --git a/scripts/images/rds-mariadb/setup.sh b/scripts/images/rds-mariadb/setup.sh index 6a599e1bf..0152a72a7 100644 --- a/scripts/images/rds-mariadb/setup.sh +++ b/scripts/images/rds-mariadb/setup.sh @@ -5,9 +5,10 @@ set -eu # the libguestfs appliance under build-system-image.sh, after packages and # INSTALL_FILES are placed. -# INSTALL_FILES land 0644; OpenRC requires 0755 on init scripts, and rds-datadir -# is executed directly by its service. -chmod 0755 /etc/init.d/rds-datadir /etc/init.d/rds-agent /usr/local/sbin/rds-datadir +# INSTALL_FILES land 0644; OpenRC requires 0755 on init scripts, and rds-init +# and rds-datadir are executed directly by their services. +chmod 0755 /etc/init.d/rds-datadir /etc/init.d/rds-init /etc/init.d/rds-agent \ + /usr/local/sbin/rds-datadir /usr/local/sbin/rds-init chmod 0755 /etc/init.d/mulga-mgmt-net /etc/init.d/mulga-mgmt-net-routes \ /usr/local/sbin/mulga-mgmt-net @@ -61,6 +62,15 @@ if grep -rEq '^[[:space:]]*skip[-_]networking' /etc/my.cnf /etc/my.cnf.d; then exit 1 fi +# The packaged service starts on a warning when the datadir is empty, so the +# dependency is what keeps it off an unbootstrapped volume. Asserted here, since +# an image whose engine can come up beside a failed bootstrap is the one failure +# this preset must not have. +grep -q '^rc_need="rds-init"' /etc/conf.d/mariadb || { + echo "[rds-mariadb-setup] /etc/conf.d/mariadb does not make the engine need rds-init" + exit 1 +} + # Bind /dev/console to the serial port so userspace boot output reaches ttyS0, # which the orchestrator captures host-side. Linux makes the last console= the # controlling one, and stock Alpine lists tty0 last — reorder so ttyS0 wins.