fix: harden MariaDB guest bootstrap

2026-10-20 03:11:46 +00:00 · 2026-08-12 09:50:38 +10:00
parent 8c5e5467e3
commit 16aa888d10
12 changed files with 247 additions and 94 deletions
@@ -16,8 +16,9 @@ const (
 	defaultEngineHost = "127.0.0.1"
 	// Where setup.sh stamps the engine the image bakes. The agent builds its
 	// engine implementation from this file rather than from anything delivered.
-	defaultEngineFile = "/etc/spinifex-rds/engine"
-	defaultRCService  = "rc-service"
+	defaultEngineFile         = "/etc/spinifex-rds/engine"
+	defaultRCService          = "rc-service"
+	defaultEngineProbeTimeout = 5 * time.Second
 	// The long-poll window the agent asks the gateway to hold a request open
 	// for. The gateway caps it at 20s.
 	defaultPollWait = 20 * time.Second
@@ -60,7 +61,8 @@ type config struct {
 	// Where the engine records the pid its probe checks for liveness, for an
 	// engine whose own client cannot tell a server still recovering from one that
 	// is not running at all.
-	EnginePidFile string
+	EnginePidFile      string
+	EngineProbeTimeout time.Duration

 	// Where the data volume is mounted, and the two kernel surfaces a storage
 	// grow resolves its device from. Overridable so a test can point them at
@@ -93,6 +95,7 @@ func loadConfig(envFile string) config {
 		EnginePidFile:        get("RDS_ENGINE_PIDFILE"),
 		DataMount:            get("RDS_DATA_MOUNT"),
 		PollWait:             defaultPollWait,
+		EngineProbeTimeout:   defaultEngineProbeTimeout,
 	}
 	if cfg.GatewayCA == "" {
 		cfg.GatewayCA = defaultGatewayCA
@@ -131,6 +134,11 @@ func loadConfig(envFile string) config {
 			cfg.PollWait = d
 		}
 	}
+	if v := get("RDS_ENGINE_PROBE_TIMEOUT"); v != "" {
+		if d, err := time.ParseDuration(v); err == nil && d > 0 {
+			cfg.EngineProbeTimeout = d
+		}
+	}
 	return cfg
 }

@@ -29,6 +29,9 @@ func TestLoadConfig_Defaults(t *testing.T) {
 	if cfg.EngineHost != defaultEngineHost || cfg.PollWait != defaultPollWait {
 		t.Errorf("engine host/pollWait = %q/%v, want the built-in defaults", cfg.EngineHost, cfg.PollWait)
 	}
+	if cfg.EngineProbeTimeout != defaultEngineProbeTimeout {
+		t.Errorf("engine probe timeout = %v, want %v", cfg.EngineProbeTimeout, defaultEngineProbeTimeout)
+	}
 	// Identity is optional: the gateway resolves it from the caller's
 	// credentials, so an agent with no configured identifier is normal.
 	if cfg.DBInstanceIdentifier != "" {
@@ -74,7 +74,8 @@ const (
 	mariadbAdminBinary  = "mariadb-admin"
 	// The account mariadb-install-db creates for the unix_socket plugin, which
 	// rds-init keeps for the platform and never hands to the customer.
-	mariadbSuperuser = "root"
+	mariadbSuperuser                  = "root"
+	mariadbProbeConnectTimeoutSeconds = 3
 	// Named by the platform drop-in rds-init writes, so both halves reach the
 	// same socket without either asserting it to the other.
 	mariadbSocketFile = "mysqld.sock"
@@ -175,7 +176,20 @@ func newMariaDBProbe(cfg config, run probeRunner) *engineProbe {
 func mariadbProbeState(cfg config, run probeRunner, alive processLivenessFn) probeStateFn {
 	pidFile, socket := cfg.EnginePidFile, mariadbSocketPath(cfg)
 	admin, client := filepath.Join(cfg.EngineBinDir, mariadbAdminBinary), filepath.Join(cfg.EngineBinDir, mariadbClientBinary)
-	connect := []string{"--no-defaults", "--protocol=socket", "--socket=" + socket, "--user=" + mariadbSuperuser}
+	connect := []string{
+		"--no-defaults", "--protocol=socket", "--socket=" + socket,
+		"--user=" + mariadbSuperuser,
+		fmt.Sprintf("--connect-timeout=%d", mariadbProbeConnectTimeoutSeconds),
+	}
+	probeTimeout := cfg.EngineProbeTimeout
+	if probeTimeout <= 0 {
+		probeTimeout = defaultEngineProbeTimeout
+	}
+	runBounded := func(ctx context.Context, name string, args ...string) (int, error) {
+		probeCtx, cancel := context.WithTimeout(ctx, probeTimeout)
+		defer cancel()
+		return run(probeCtx, name, args...)
+	}

 	return func(ctx context.Context, _ int64) (engineState, string) {
 		pid, err := readPidFile(pidFile)
@@ -192,7 +206,7 @@ func mariadbProbeState(cfg config, run probeRunner, alive processLivenessFn) pro
 		// that is not serving yet rather than one that is gone — including a probe
 		// binary that will not run at all. Reporting absent against a live server
 		// would have the rollback guard restart one that is making progress.
-		switch code, err := run(ctx, admin, append(slices.Clone(connect), "ping")...); {
+		switch code, err := runBounded(ctx, admin, append(slices.Clone(connect), "ping")...); {
 		case err != nil:
 			return engineRecovering, fmt.Sprintf("engine probe could not run: %v", err)
 		case code != 0:
@@ -204,7 +218,7 @@ func mariadbProbeState(cfg config, run probeRunner, alive processLivenessFn) pro
 		// statement is a literal in the argv rather than on stdin because it
 		// carries nothing secret and a probe reads no result back.
 		query := append(slices.Clone(connect), "--batch", "--skip-column-names", "--execute=SELECT 1")
-		switch code, err := run(ctx, client, query...); {
+		switch code, err := runBounded(ctx, client, query...); {
 		case err != nil:
 			return engineRecovering, fmt.Sprintf("engine probe could not run: %v", err)
 		case code != 0:
@@ -551,6 +551,33 @@ func TestMariaDBProbe_SeparatesRecoveringFromAbsent(t *testing.T) {
 // The failure the guard exists for still behaves correctly: a static value
 // mariadbd will not accept makes it exit during startup, so the pidfile names a
 // process that is gone and the deadline runs.
+func TestMariaDBProbe_BoundsAStalledClient(t *testing.T) {
+	cfg := mariadbTestProbeConfig(t)
+	cfg.EngineProbeTimeout = 20 * time.Millisecond
+	writePid(t, cfg.EnginePidFile, 4242)
+
+	started := time.Now()
+	state, message := mariadbProbeState(cfg,
+		func(ctx context.Context, _ string, args ...string) (int, error) {
+			if !slices.Contains(args, "--connect-timeout=3") {
+				t.Errorf("probe args = %v, want a client connection timeout", args)
+			}
+			<-ctx.Done()
+			return -1, ctx.Err()
+		},
+		func(int) bool { return true })(t.Context(), int64(cfg.EnginePort))
+
+	if state != engineRecovering {
+		t.Errorf("state = %v, want recovering after a live engine's probe timed out", state)
+	}
+	if !strings.Contains(message, context.DeadlineExceeded.Error()) {
+		t.Errorf("message = %q, want the probe deadline", message)
+	}
+	if elapsed := time.Since(started); elapsed > time.Second {
+		t.Errorf("probe took %v, want the configured deadline to bound it", elapsed)
+	}
+}
+
 func TestMariaDBProbe_MapsToTheHealthTheControlPlaneReads(t *testing.T) {
 	cfg := mariadbTestProbeConfig(t)
 	writePid(t, cfg.EnginePidFile, 4242)
@@ -18,6 +18,9 @@ func execProbeRunner(ctx context.Context, name string, args ...string) (int, err
 	if err == nil {
 		return 0, nil
 	}
+	if ctxErr := ctx.Err(); ctxErr != nil {
+		return -1, ctxErr
+	}
 	var exitErr *exec.ExitError
 	if errors.As(err, &exitErr) {
 		return exitErr.ExitCode(), nil
@@ -5,6 +5,7 @@ import (
 	"errors"
 	"strings"
 	"testing"
+	"time"

 	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
 )
@@ -81,6 +82,19 @@ func TestEngineProbe_ProbesTheAssignedPort(t *testing.T) {
 	}
 }

+func TestExecProbeRunner_PreservesTheProbeDeadline(t *testing.T) {
+	ctx, cancel := context.WithTimeout(t.Context(), 20*time.Millisecond)
+	defer cancel()
+
+	code, err := execProbeRunner(ctx, "sh", "-c", "while :; do :; done")
+	if code != -1 {
+		t.Errorf("exit code = %d, want -1 for a timed-out probe", code)
+	}
+	if !errors.Is(err, context.DeadlineExceeded) {
+		t.Errorf("error = %v, want the context deadline", err)
+	}
+}
+
 func testProbeConfig() config {
 	return config{EngineHost: defaultEngineHost, EnginePort: engineLayouts[enginePostgres].port}
 }
@@ -871,7 +871,7 @@ Everything below applies to both engines unless it names one. `Engine` is fixed
 - **`DBName`:** optional; an omitted name creates no initial database. When supplied it must begin with a letter and hold only letters, digits and underscores, within the per-engine limit above. The rule is narrower than either engine's own, because the guest interpolates the name into a `CREATE DATABASE`.
 - **TLS:** offered, not enforced, on both engines. The engine serves a per-instance certificate signed by the cluster CA (the `ca.pem` baked into AMIs) carrying both the ENI IP and the DNS name in its SAN set, so PostgreSQL's `sslmode=verify-full` verifies by name or by address, and MariaDB's `--ssl-ca` plus `--ssl-verify-server-cert` verifies against the endpoint name. A deployment with no cluster CA starts the engine without TLS.
 - **PostgreSQL master user:** administrative but **not a PostgreSQL superuser**, as on AWS. It gets `CREATEDB` and `CREATEROLE`, owns the initial database, and holds `rds_superuser` (`pg_monitor`, `pg_signal_backend`, `pg_checkpoint`) with `ADMIN OPTION`, so it can grant that set onward. It cannot `COPY ... FROM/TO PROGRAM`, touch server-side files, run `ALTER SYSTEM`, or install **untrusted** extensions — those need the cluster superuser, which no customer credential is. Trusted extensions install normally in a database it owns.
- **MariaDB master user:** created as `'<name>'@'%'` with an explicitly enumerated grant list `ON *.*` `WITH GRANT OPTION` — `SELECT`, `INSERT`, `UPDATE`, `DELETE`, `CREATE`, `DROP`, `RELOAD`, `PROCESS`, `REFERENCES`, `INDEX`, `ALTER`, `SHOW DATABASES`, `CREATE TEMPORARY TABLES`, `LOCK TABLES`, `EXECUTE`, `REPLICATION SLAVE`, `REPLICATION CLIENT`, `CREATE VIEW`, `SHOW VIEW`, `CREATE ROUTINE`, `ALTER ROUTINE`, `CREATE USER`, `EVENT`, `TRIGGER`. Deliberately withheld are `SUPER` and its 11.8 split-out successors, `FILE`, `SHUTDOWN` and everything scoped to `mysql.*`: `FILE` would give `SELECT ... INTO OUTFILE` and `LOAD_FILE()` as the OS user, and `INSERT` on `mysql.*` would allow `CREATE FUNCTION ... SONAME` to load native code into the server.
+- **MariaDB master user:** created as `'<name>'@'%'` with global `RELOAD`, `PROCESS`, `SHOW DATABASES`, `REPLICATION SLAVE` and `REPLICATION CLIENT`, while application DML, DDL, routines, events, triggers and delegation are granted only on customer databases. MariaDB has no deny or partial-revoke mechanism, so granting those object privileges `ON *.*` would also grant them on `mysql.global_priv` and let the master acquire `SUPER`, `FILE` or `SHUTDOWN`. The initial `DBName` is granted automatically. Create another customer database with `CALL _spinifex_rds.create_database('name')`; plain `CREATE DATABASE`, arbitrary account administration and access to `mysql.*` are intentionally unavailable on the stock MariaDB build.
 - **MariaDB storage engine:** `default_storage_engine` is platform-owned and pinned to **InnoDB**, and is absent from the parameter catalog — a deliberate divergence from AWS, which exposes it as modifiable. Only InnoDB keeps a redo log, so only InnoDB is covered by the snapshot guarantee. An explicit `ENGINE=Aria` or `ENGINE=MyISAM` on `CREATE TABLE` still works — the limitation is documented rather than prevented — but such a table has no redo log, so a snapshot taken without a successful quiesce can leave it inconsistent with no way back. Use InnoDB, as AWS also recommends.
 - **Backups:** daily COW snapshots of the data volume inside `PreferredBackupWindow`. Retention defaults to 7 days and caps at 7; `0` disables automated backups. No point-in-time recovery, and MariaDB's binary log is off.
 - **Availability:** single-AZ. Engine crashes restart in-guest and VM crashes on a live host restart via `ec2-health-restart`, but an instance whose host is lost is reported `failed` with a reason in `StatusInfos` and needs operator recovery.
@@ -111,11 +111,15 @@ MOUNTS_FILE="${RDS_MOUNTS_FILE:-/proc/mounts}"
 # first start, so it is not instant.
 BOOTSTRAP_PROBES="${RDS_BOOTSTRAP_PROBES:-60}"
 BOOTSTRAP_POLL="${RDS_BOOTSTRAP_POLL:-1}"
+# Failure cleanup gives the private server this long to honor TERM before KILL.
+BOOTSTRAP_STOP_GRACE="${RDS_BOOTSTRAP_STOP_GRACE:-10}"

 # The server truncates a longer name, so the database created would not be the
 # one the customer asked for, and MariaDB maps the name onto a directory name.
 MAX_DB_NAME_LEN=64
 MAX_USERNAME_LEN=80
+MANAGEMENT_DB="_spinifex_rds"
+CREATE_DATABASE_PROCEDURE="create_database"

 log() { echo "[rds-init] $*"; }
 fail() { echo "[rds-init] ERROR: $*" >&2; exit 1; }
@@ -150,8 +154,23 @@ clear_unfinished_datadir() {

 kill_bootstrap_server() {
    [ -n "${_server_pid}" ] || return 0
-    kill -TERM "${_server_pid}" 2>/dev/null || true
+    _pid="${_server_pid}"
+
+    # wait has no timeout in POSIX sh. A watchdog bounds it and escalates, while
+    # this shell remains the one that waits and reaps its own child.
+    kill -TERM "${_pid}" 2>/dev/null || true
+    (
+        sleep "${BOOTSTRAP_STOP_GRACE}"
+        kill -KILL "${_pid}" 2>/dev/null || true
+    ) &
+    _watchdog_pid=$!
+
+    # A signal exit is expected here: cleanup cares that the process is reaped
+    # and cannot touch the datadir, not whether its incomplete startup was clean.
+    wait "${_pid}" 2>/dev/null || true
    _server_pid=""
+    kill -TERM "${_watchdog_pid}" 2>/dev/null || true
+    wait "${_watchdog_pid}" 2>/dev/null || true
 }

 load_handoff() {
@@ -654,38 +673,49 @@ FLUSH PRIVILEGES;
 SQL
 }

-# The master user is administrative but is not granted the privileges that reach
-# outside the database: FILE is SELECT INTO OUTFILE and LOAD_FILE against any
-# path as the mysql OS user, SUPER and its 11.8 split-out successors reach SET
-# GLOBAL and plugin loading, and SHUTDOWN stops the server underneath the control
-# plane's lifecycle. Master credentials must be a database privilege, not a shell
-# in the DB VM.
-#
-# What the grant list cannot do is withhold the mysql schema. MariaDB privileges
-# are additive and it has no DENY, so the DML below reaches mysql.global_priv,
-# and REVOKE ... ON mysql.* is refused outright (ER_NONEXISTING_GRANT) because no
-# database-level row exists to revoke from. A master that writes that table and
-# runs FLUSH PRIVILEGES therefore holds ALL PRIVILEGES, including the three above.
-#
-# What still holds after that is the boundary this preset actually depends on:
-# secure_file_priv and plugin_dir are read-only variables, so 90-rds-init.cnf
-# pins them past any SET GLOBAL and FILE reaches only an empty root-owned
-# directory. The residual exposure is SHUTDOWN and unrecorded SET GLOBAL, not a
-# shell in the DB VM.
+# MariaDB has no DENY or partial revocation: an object privilege ON *.* also
+# reaches mysql.global_priv. Application privileges are therefore database-scoped,
+# while this protected definer procedure creates and grants future databases.
 apply_master_user() {
    log "creating the master user ${RDS_MASTER_USERNAME}"
-    # Enumerated rather than GRANT ALL followed by revocations: a MariaDB release
-    # that adds a privilege would otherwise hand it to the master silently.
-    # '%' because the customer reaches the instance over its customer ENI.
    mariadb_bootstrap <<SQL
 CREATE USER IF NOT EXISTS '${RDS_MASTER_USERNAME}'@'%' IDENTIFIED BY '${RDS_MASTER_PASSWORD_SQL}';
 ALTER USER '${RDS_MASTER_USERNAME}'@'%' IDENTIFIED BY '${RDS_MASTER_PASSWORD_SQL}';
-GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS,
-      REFERENCES, INDEX, ALTER, SHOW DATABASES, CREATE TEMPORARY TABLES,
-      LOCK TABLES, EXECUTE, REPLICATION SLAVE, REPLICATION CLIENT,
-      CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE,
-      CREATE USER, EVENT, TRIGGER
-  ON *.* TO '${RDS_MASTER_USERNAME}'@'%' WITH GRANT OPTION;
+GRANT RELOAD, PROCESS, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT
+  ON *.* TO '${RDS_MASTER_USERNAME}'@'%';
+CREATE DATABASE IF NOT EXISTS \`${MANAGEMENT_DB}\`;
+DROP PROCEDURE IF EXISTS \`${MANAGEMENT_DB}\`.\`${CREATE_DATABASE_PROCEDURE}\`;
+DELIMITER //
+CREATE DEFINER='root'@'localhost' PROCEDURE \`${MANAGEMENT_DB}\`.\`${CREATE_DATABASE_PROCEDURE}\`(
+    IN database_name VARCHAR(64) CHARACTER SET ascii
+)
+SQL SECURITY DEFINER
+BEGIN
+    IF database_name IS NULL
+       OR BINARY database_name NOT REGEXP '^[A-Za-z][A-Za-z0-9_]{0,63}$'
+       OR LOWER(database_name) IN ('mysql', 'information_schema', 'performance_schema', 'sys', '${MANAGEMENT_DB}') THEN
+        SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'database name must be a non-system identifier of at most 64 characters';
+    END IF;
+
+    SET @rds_database = REPLACE(database_name, '\`', '\`\`');
+    SET @rds_statement = CONCAT('CREATE DATABASE IF NOT EXISTS \`', @rds_database, '\`');
+    PREPARE rds_create_database FROM @rds_statement;
+    EXECUTE rds_create_database;
+    DEALLOCATE PREPARE rds_create_database;
+
+    SET @rds_statement = CONCAT(
+        'GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, ',
+        'CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, ',
+        'CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON \`', @rds_database,
+        '\`.* TO ''${RDS_MASTER_USERNAME}''@''%'' WITH GRANT OPTION'
+    );
+    PREPARE rds_grant_database FROM @rds_statement;
+    EXECUTE rds_grant_database;
+    DEALLOCATE PREPARE rds_grant_database;
+END//
+DELIMITER ;
+GRANT EXECUTE ON PROCEDURE \`${MANAGEMENT_DB}\`.\`${CREATE_DATABASE_PROCEDURE}\`
+  TO '${RDS_MASTER_USERNAME}'@'%';
 SQL
 }

@@ -722,14 +752,15 @@ database_charset_clause() {
    printf 'CHARACTER SET %s' "${_charset}"
 }

-# As on AWS, an instance created without a DBName gets no initial database. The
-# master reaches it through the global grant above, so nothing further is granted
-# on it here.
+# As on AWS, an instance created without a DBName gets no initial database. A
+# supplied database is created under its resolved charset before the protected
+# routine applies the same scope future databases receive.
 create_database() {
    log "creating database ${RDS_DB_NAME}"
    _charset_clause=$(database_charset_clause)
    mariadb_bootstrap <<SQL
 CREATE DATABASE IF NOT EXISTS \`${RDS_DB_NAME}\` ${_charset_clause};
+CALL \`${MANAGEMENT_DB}\`.\`${CREATE_DATABASE_PROCEDURE}\`('${RDS_DB_NAME}');
 SQL
 }

@@ -97,12 +97,14 @@ for a in "$@"; do
        --pid-file=*) pid="${a#--pid-file=}" ;;
    esac
 done
+[ "${MARIADBD_IGNORE_TERM:-0}" = "1" ] && trap '' TERM
 if [ "${MARIADBD_KILL_PARENT:-0}" = "1" ]; then
    kill -TERM "$(cat "${KILL_PID_FILE}")"
    exit 0
 fi
 : > "${sock}"
 echo "$$" > "${pid}"
+[ -n "${MARIADBD_TEST_PID_FILE:-}" ] && echo "$$" > "${MARIADBD_TEST_PID_FILE}"
 # The bound keeps a refused shutdown from leaving a process behind the run.
 _left=200
 while [ -e "${sock}" ] && [ "${_left}" -gt 0 ]; do
@@ -143,6 +145,10 @@ cat > "${STUBBIN}/mariadb" <<'EOF'
 #!/bin/sh
 _in="${WORKDIR}/client.stdin.$$"
 cat > "${_in}"
+if [ "${CLIENT_KILL_PARENT:-0}" = "1" ]; then
+    kill -TERM "$(cat "${KILL_PID_FILE}")"
+    exit 1
+fi
 {
    echo "--- mariadb $* [env-password=${RDS_MASTER_PASSWORD:-}]"
    cat "${_in}"
@@ -227,8 +233,9 @@ HANDOFF="${WORK}/run/spinifex-rds"
 SECURE_FILE_DIR="${WORK}/mysql-files"
 MOUNTS="${WORK}/mounts"
 KILL_PID_FILE="${WORK}/rds-init.pid"
+MARIADBD_TEST_PID_FILE="${WORK}/mariadbd.pid"
 WORKDIR="${WORK}"
-export KILL_PID_FILE WORKDIR
+export KILL_PID_FILE MARIADBD_TEST_PID_FILE WORKDIR

 INSTALLDB_CALLS="${WORK}/install-db.calls"
 MARIADBD_CALLS="${WORK}/mariadbd.calls"
@@ -296,7 +303,9 @@ reset_state() {
    : > "${CLIENT_CALLS}"
    unset INSTALLDB_FAIL CLIENT_FAIL LS_FAIL RDS_ALLOW_LOCAL_DATADIR MASTER_USER || true
    unset PENDING PAYLOAD_ID DB_ID RECEIPT_DIR_OVERRIDE CLIENT_STATUS_CORRUPT STAMP_OVERRIDE || true
-    unset MARIADBD_START_FAIL MARIADBD_KILL_PARENT ADMIN_SHUTDOWN_FAIL SYNC_KILL_PARENT || true
+    unset MARIADBD_START_FAIL MARIADBD_KILL_PARENT MARIADBD_IGNORE_TERM CLIENT_KILL_PARENT || true
+    unset ADMIN_SHUTDOWN_FAIL SYNC_KILL_PARENT || true
+    rm -f "${MARIADBD_TEST_PID_FILE}"
 }

 # run_env: invoke a command with every path knob pointed into the temp dir. The
@@ -316,6 +325,7 @@ run_env() {
        RDS_ALLOW_LOCAL_DATADIR="${RDS_ALLOW_LOCAL_DATADIR:-0}" \
        RDS_BOOTSTRAP_POLL=0.05 \
        RDS_BOOTSTRAP_PROBES=200 \
+        RDS_BOOTSTRAP_STOP_GRACE=0.1 \
        "$@" </dev/null
 }

@@ -386,39 +396,38 @@ if run_ok "initialize"; then
        && pass "initialize: the database takes the group's character set" \
        || fail "initialize: the database was created under the compiled default"

-    # Withheld deliberately: FILE is arbitrary host file access as the mysql user,
-    # SUPER and its 11.8 split-out successors reach SET GLOBAL and plugin loading,
-    # and SHUTDOWN stops the server underneath the control plane's lifecycle.
-    for privilege in SUPER FILE SHUTDOWN 'ALL PRIVILEGES' 'SET USER' \
-        'BINLOG ADMIN' 'BINLOG REPLAY' 'CONNECTION ADMIN' 'FEDERATED ADMIN' \
-        'READ_ONLY ADMIN' 'REPLICATION MASTER ADMIN' 'REPLICATION SLAVE ADMIN' \
-        'SLAVE MONITOR'; do
-        grep -q "${privilege}" "${CLIENT_CALLS}" \
-            && fail "initialize: the master grant includes ${privilege}" \
-            || pass "initialize: master holds no ${privilege}"
+    # MariaDB cannot partially revoke mysql.* from a global grant, so only the
+    # monitoring privileges are global and they carry no grant option.
+    _global_grant=$(sed -n '/^GRANT RELOAD,/,/^  ON .*;$/p' "${CLIENT_CALLS}" | tr '\n' ' ' | tr -s ' ')
+    _expected_global="GRANT RELOAD, PROCESS, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'mulgamaster'@'%'; "
+    [ "${_global_grant}" = "${_expected_global}" ] \
+        && pass "initialize: global privileges cannot modify system schemas" \
+        || fail "initialize: unsafe global grant: ${_global_grant}"
+
+    grep -q 'CREATE DEFINER='"'"'root'"'"'@'"'"'localhost'"'"' PROCEDURE `_spinifex_rds`.`create_database`' "${CLIENT_CALLS}" \
+        && pass "initialize: future databases use the root-owned routine" \
+        || fail "initialize: no protected create-database routine"
+    grep -q "GRANT EXECUTE ON PROCEDURE \`_spinifex_rds\`.\`create_database\`" "${CLIENT_CALLS}" \
+        && pass "initialize: master can call only the protected routine" \
+        || fail "initialize: routine execute grant missing"
+    grep -q "CALL \`_spinifex_rds\`.\`create_database\`('appdb')" "${CLIENT_CALLS}" \
+        && pass "initialize: initial database receives the scoped grant" \
+        || fail "initialize: initial database was not granted through the routine"
+    grep -q "LOWER(database_name) IN ('mysql', 'information_schema', 'performance_schema', 'sys', '_spinifex_rds')" "${CLIENT_CALLS}" \
+        && pass "initialize: routine refuses every system schema" \
+        || fail "initialize: routine does not protect system schemas"
+    grep -q "CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON" "${CLIENT_CALLS}" && \
+        grep -q "TO ''mulgamaster''@''%'' WITH GRANT OPTION" "${CLIENT_CALLS}" \
+        && pass "initialize: application privileges are database-scoped" \
+        || fail "initialize: database-scoped privilege grant missing"
+
+    # The prohibited names may appear in comments or the procedure's validation,
+    # so inspect the actual global grant rather than grepping the whole script.
+    for privilege in SELECT INSERT UPDATE DELETE CREATE DROP EXECUTE 'CREATE USER' 'WITH GRANT OPTION'; do
+        printf '%s\n' "${_global_grant}" | grep -q "${privilege}" \
+            && fail "initialize: global grant includes ${privilege}" \
+            || pass "initialize: ${privilege} is not global"
    done
-
-    # Asserted whole rather than by prefix. A privilege appended to the list is a
-    # privilege the master silently gains, and the enumeration exists precisely so
-    # that widening it has to be deliberate.
-    _grant=$(sed -n '/^GRANT /,/WITH GRANT OPTION;/p' "${CLIENT_CALLS}" | tr '\n' ' ' | tr -s ' ')
-    _expected_grant="GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, \
-REFERENCES, INDEX, ALTER, SHOW DATABASES, CREATE TEMPORARY TABLES, \
-LOCK TABLES, EXECUTE, REPLICATION SLAVE, REPLICATION CLIENT, \
-CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, \
-CREATE USER, EVENT, TRIGGER ON *.* TO 'mulgamaster'@'%' WITH GRANT OPTION; "
-    [ "${_grant}" = "${_expected_grant}" ] \
-        && pass "initialize: the grant is exactly the enumerated list" \
-        || fail "initialize: the master grant has changed: ${_grant}"
-
-    # MariaDB privileges are additive and it has no DENY, so scope is the only
-    # control left: a second ON clause would widen the master beyond this list.
-    [ "$(grep -c '^  ON ' "${CLIENT_CALLS}")" = "1" ] \
-        && pass "initialize: the grant is scoped ON *.* and nothing else" \
-        || fail "initialize: the master carries more than one grant scope"
-    grep -q "WITH GRANT OPTION" "${CLIENT_CALLS}" \
-        && pass "initialize: master can delegate its own privileges" \
-        || fail "initialize: master cannot grant"
    grep -q "DELETE FROM mysql.global_priv WHERE User = ''" "${CLIENT_CALLS}" \
        && pass "initialize: anonymous accounts removed" || fail "initialize: anonymous accounts left"
    grep -q 'DROP DATABASE IF EXISTS test' "${CLIENT_CALLS}" \
@@ -775,7 +784,27 @@ run_signalled_fails "sigterm-midbootstrap"
    || pass "sigterm-midbootstrap: stopped before the master user"
 unset MARIADBD_KILL_PARENT

-# --- Case 6f: a datadir carrying the sentinel is refused, not started ---
+# --- Case 6f: cleanup reaps a bootstrap server that ignores TERM ---
+reset_state
+write_handoff initialize 's3cr3t' appdb
+MARIADBD_IGNORE_TERM=1
+CLIENT_KILL_PARENT=1
+export MARIADBD_IGNORE_TERM CLIENT_KILL_PARENT
+run_signalled_fails "sigterm-stubborn-bootstrap"
+if [ -s "${MARIADBD_TEST_PID_FILE}" ]; then
+    _stubborn_pid=$(cat "${MARIADBD_TEST_PID_FILE}")
+    kill -0 "${_stubborn_pid}" 2>/dev/null \
+        && fail "sigterm-stubborn-bootstrap: mariadbd survived cleanup" \
+        || pass "sigterm-stubborn-bootstrap: mariadbd was killed and reaped"
+else
+    fail "sigterm-stubborn-bootstrap: the private server never published its pid"
+fi
+[ -e "${DATADIR}/mysql" ] \
+    && fail "sigterm-stubborn-bootstrap: datadir kept after the server was reaped" \
+    || pass "sigterm-stubborn-bootstrap: datadir cleared after process death"
+unset MARIADBD_IGNORE_TERM CLIENT_KILL_PARENT
+
+# --- Case 6g: a datadir carrying the sentinel is refused, not started ---
 # No trap survives a crash or a SIGKILL, so the sentinel is the only record that
 # the bootstrap did not finish. It says the master user is missing, not that the
 # volume is empty, so the refusal must not clear anything.
@@ -898,7 +927,7 @@ write_handoff initialize 's3cr3t' ''
 if run_ok "no-cert"; then
    grep -q '^ssl_cert' "${PLATFORM_FILE}" \
        && fail "no-cert: points at a cert that was never delivered" || pass "no-cert: no cert paths"
-    grep -q 'CREATE DATABASE' "${CLIENT_CALLS}" \
+    grep -q '^CALL `_spinifex_rds`.`create_database`' "${CLIENT_CALLS}" \
        && fail "no-dbname: created a database without a DBName" || pass "no-dbname: no initial database"
 fi

@@ -4,6 +4,7 @@ import (
 	"slices"
 	"strconv"
 	"strings"
+	"time"

 	"github.com/mulgadc/spinifex/spinifex/awserrors"
 )
@@ -456,26 +457,13 @@ func validateMariaDBTimeZone(value string) error {
 }

 func parseTimeZoneOffset(value string) (int, error) {
-	sign := 1
-	switch {
-	case strings.HasPrefix(value, "-"):
-		sign = -1
-	case !strings.HasPrefix(value, "+"):
-		return 0, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "offset must be signed")
+	parsed, err := time.Parse("-07:00", value)
+	if err != nil {
+		return 0, awserrors.Errorf(awserrors.ErrorInvalidParameterValue,
+			"offset must use signed HH:MM syntax: %v", err)
 	}
-	hours, minutes, ok := strings.Cut(value[1:], ":")
-	if !ok {
-		return 0, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "offset must be hours and minutes")
-	}
-	h, err := strconv.Atoi(hours)
-	if err != nil || h < 0 {
-		return 0, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "offset hours are not a number")
-	}
-	m, err := strconv.Atoi(minutes)
-	if err != nil || len(minutes) != 2 || m < 0 || m > 59 {
-		return 0, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "offset minutes are not a number")
-	}
-	return sign * (h*60 + m), nil
+	_, offsetSeconds := parsed.Zone()
+	return offsetSeconds / int(time.Minute/time.Second), nil
 }

 func validateMariaDBParameterCombinations(params []Parameter) error {
@@ -256,6 +256,8 @@ func TestMariaDBCatalog_RejectsBadValues(t *testing.T) {
 		{"OffsetOutOfRange", "time_zone", "+15:00", "does not accept"},
 		{"UnsignedOffset", "time_zone", "10:00", "does not accept"},
 		{"OffsetWithoutMinutes", "time_zone", "+10", "does not accept"},
+		{"OffsetWithShortHour", "time_zone", "+1:00", "does not accept"},
+		{"OffsetWithLongHour", "time_zone", "+0001:00", "does not accept"},
 		{"Empty", "sort_buffer_size", "", "empty value"},
 		{"Formula", "innodb_buffer_pool_size", "{DBInstanceClassMemory*3/4}", "is a formula"},
 	}
@@ -154,6 +154,40 @@ func TestMariaDBInstance(t *testing.T) {
 			"a table created with no ENGINE clause must be InnoDB")
 	})

+	// Stock MariaDB cannot exclude mysql.* from a global object grant. The
+	// protected routine is the only path to privileges on another database.
+	t.Run("MasterPrivilegesCannotReachSystemAccounts", func(t *testing.T) {
+		for name, sql := range map[string]string{
+			"ReadGrantTable":  "SELECT Priv FROM mysql.global_priv LIMIT 1;",
+			"WriteGrantTable": "UPDATE mysql.global_priv SET Priv = Priv WHERE User = 'root';",
+			"AlterRoot":       "ALTER USER 'root'@'localhost' IDENTIFIED BY 'not-allowed';",
+			"DropRoutine":     "DROP PROCEDURE `_spinifex_rds`.`create_database`;",
+			"PlainCreate":     "CREATE DATABASE forbidden_plain_database;",
+		} {
+			t.Run(name, func(t *testing.T) {
+				out, err := harness.TryMariaDB(client, conn, sql)
+				require.Error(t, err, "master statement unexpectedly succeeded: %s", out)
+				assert.Contains(t, strings.ToLower(out), "denied")
+			})
+		}
+
+		harness.MariaDB(t, client, conn, "FLUSH PRIVILEGES;")
+		futureDB := "e2e_future_database"
+		harness.MariaDB(t, client, conn,
+			fmt.Sprintf("CALL `_spinifex_rds`.`create_database`('%s');", futureDB))
+		future := conn
+		future.DBName = futureDB
+		harness.MariaDB(t, client, future,
+			"CREATE TABLE future_table (id int primary key); INSERT INTO future_table VALUES (1);")
+		assert.Equal(t, "1", strings.TrimSpace(harness.MariaDB(t, client, future,
+			"SELECT id FROM future_table;")))
+
+		out, err := harness.TryMariaDB(client, conn,
+			"CALL `_spinifex_rds`.`create_database`('mysql');")
+		require.Error(t, err, "system database creation unexpectedly succeeded: %s", out)
+		assert.Contains(t, out, "non-system identifier")
+	})
+
 	// TLS is offered rather than enforced, so what is asserted is that a client
 	// asking for it gets it.
 	t.Run("AClientCanConnectOverTLS", func(t *testing.T) {