feat(eks): control-plane disaster recovery — etcd snapshot + restore-snapshot (#503)

* fix(eks): write etcd-snapshot.env at CP launch so snapshots actually run

* test(eks): cover resolveSystemPredastoreURL and buildEKSServiceDeps

Diff-coverage gate for 852f61d9 was unmet (48.8% < 60%) because the
mgmt-bridge branches and EKSServiceDeps assembly had no direct test.

* feat(eks): restore-snapshot DR path (fresh CP + cluster-reset restore + NLB re-point)

* fix(eks): make restore-snapshot fail-safe (validate snapshot, unwind on partial failure, honest status)

* fix(eks): sign etcd-snapshot uploads with deployment region (was 403ing)

* fix(eks): apply recovery directive on DR-seed first boot + single-node replacement host

* fix(daemon): log NATS handler failures at Error (were only on the trace span)

scripts/images/eks-node/eks-node-role.sh

@@ -48,14 +48,26 @@ if [ -z "${ROLE}" ] && [ -f "${AGENT_ENVFILE}" ]; then
     ROLE="agent"
 fi
 
+case "${ROLE}" in
+    server | server-join | agent) ;;
+    *)
+        die "SPINIFEX_K3S_ROLE missing or invalid: '${ROLE}'"
+        ;;
+esac
+
+# Written before the rc-service calls below: mulga-eks-k3s-recovery, started
+# inline for server/server-join, reads ROLE_FILE at the top of its own logic
+# and must find it already in place, not empty.
+mkdir -p "$(dirname "${ROLE_FILE}")"
+printf '%s\n' "${ROLE}" > "${ROLE_FILE}"
+
 case "${ROLE}" in
     server)
         log "configuring server role"
         rc-update add eks-token-webhook default
-        # Pre-k3s etcd recovery: added to the runlevel so it runs before k3s on
-        # every subsequent boot, but not started now — a fresh cluster has no
-        # directive and the role file it guards on is written at the end of this
-        # script. It first acts on the reboot the reconciler triggers.
+        # Pre-k3s etcd recovery: enabled for later boots and started inline
+        # now too, so a restore-snapshot DR seed's directive applies before
+        # k3s starts on this first boot. A plain create has no directive, so it's a no-op.
         rc-update add mulga-eks-k3s-recovery default
         rc-update add k3s default
         # konnectivity-server fronts apiserver egress; every server replica runs
@@ -69,6 +81,7 @@ case "${ROLE}" in
         # (mulga-siv-231.7); server-join nodes do not run it.
         rc-update add mulga-eks-addon-sync default
         rc-service eks-token-webhook start
+        rc-service mulga-eks-k3s-recovery start
         rc-service k3s start
         rc-service konnectivity-server start
         rc-service k3s-first-boot start
@@ -79,12 +92,14 @@ case "${ROLE}" in
         log "configuring server-join role"
         rc-update add eks-token-webhook default
         # Pre-k3s etcd recovery (see server role): server-join members are the
-        # wipe-rejoin followers of a cluster-reset, so they need it too.
+        # wipe-rejoin followers of a cluster-reset, so they need it too, and for
+        # the same first-boot reason must be started here, not deferred.
         rc-update add mulga-eks-k3s-recovery default
         rc-update add k3s default
         rc-update add konnectivity-server default
         rc-update add mulga-eks-state-report default
         rc-service eks-token-webhook start
+        rc-service mulga-eks-k3s-recovery start
         rc-service k3s start
         rc-service konnectivity-server start
         rc-service mulga-eks-state-report start
@@ -99,7 +114,5 @@ case "${ROLE}" in
         ;;
 esac
 
-mkdir -p "$(dirname "${ROLE_FILE}")"
-printf '%s\n' "${ROLE}" > "${ROLE_FILE}"
 rc-update del eks-node-role default 2>/dev/null || true
 log "role '${ROLE}' configured"

scripts/images/eks-node/eks-node-role_test.sh

@@ -65,8 +65,11 @@ check "server: enables k3s-first-boot" "yes" "$(grep -q 'rc-update add k3s-first
 check "server: starts k3s" "yes" "$(grep -q 'rc-service k3s start' "${CALLS}" && echo yes || echo no)"
 check "server: enables konnectivity-server" "yes" "$(grep -q 'rc-update add konnectivity-server default' "${CALLS}" && echo yes || echo no)"
 check "server: enables k3s-recovery" "yes" "$(grep -q 'rc-update add mulga-eks-k3s-recovery default' "${CALLS}" && echo yes || echo no)"
-# Recovery is added to the runlevel (pre-k3s on later boots) but not started now.
-check "server: does not start k3s-recovery" "no" "$(grep -q 'rc-service mulga-eks-k3s-recovery' "${CALLS}" && echo yes || echo no)"
+# Recovery is added to the runlevel (so it also runs pre-k3s on later boots)
+# AND started inline now — a restore-snapshot DR seed's directive must apply
+# on this very first boot, before k3s starts.
+check "server: starts k3s-recovery" "yes" "$(grep -q 'rc-service mulga-eks-k3s-recovery start' "${CALLS}" && echo yes || echo no)"
+check "server: starts k3s-recovery before k3s" "yes" "$(awk '/rc-service mulga-eks-k3s-recovery start/{r=NR} /rc-service k3s start/{k=NR} END{print (r>0 && k>0 && r<k) ? "yes" : "no"}' "${CALLS}")"
 check "server: no k3s-agent" "no" "$(grep -q 'k3s-agent' "${CALLS}" && echo yes || echo no)"
 check "server: self-disables" "yes" "$(grep -q 'rc-update del eks-node-role default' "${CALLS}" && echo yes || echo no)"
 
@@ -81,6 +84,7 @@ check "server-join: enables k3s" "yes" "$(grep -q 'rc-update add k3s default' "$
 check "server-join: enables state-report" "yes" "$(grep -q 'rc-update add mulga-eks-state-report default' "${CALLS}" && echo yes || echo no)"
 check "server-join: enables konnectivity-server" "yes" "$(grep -q 'rc-update add konnectivity-server default' "${CALLS}" && echo yes || echo no)"
 check "server-join: enables k3s-recovery" "yes" "$(grep -q 'rc-update add mulga-eks-k3s-recovery default' "${CALLS}" && echo yes || echo no)"
+check "server-join: starts k3s-recovery" "yes" "$(grep -q 'rc-service mulga-eks-k3s-recovery start' "${CALLS}" && echo yes || echo no)"
 # Join servers must NOT re-publish bootstrap; the first server already did.
 check "server-join: no k3s-first-boot" "no" "$(grep -q 'k3s-first-boot' "${CALLS}" && echo yes || echo no)"
 check "server-join: no k3s-agent" "no" "$(grep -q 'k3s-agent' "${CALLS}" && echo yes || echo no)"

scripts/images/eks-node/k3s.initd

@@ -28,6 +28,16 @@ depend() {
 }
 
 start_pre() {
+    # mulga-eks-k3s-recovery drops this marker when a required-snapshot restore
+    # could not fetch its etcd snapshot. Refuse to start: a fresh DR seed with no
+    # snapshot would cluster-init an EMPTY datastore (silent total data loss). The
+    # marker is in /run tmpfs, cleared on reboot, so recovery retries the fetch
+    # next boot and k3s starts once the snapshot restores.
+    if [ -f /run/spinifex-eks/k3s-restore-block ]; then
+        eerror "restore-snapshot could not fetch the required etcd snapshot; refusing to start k3s on an empty datastore (see mulga-eks-k3s-recovery log)"
+        return 1
+    fi
+
     # config.yaml is written by k3s-first-boot from the skeleton + cloud-init
     # data. Refuse to start if neither exists.
     if [ ! -f /etc/rancher/k3s/config.yaml ]; then

scripts/images/eks-node/mulga-eks-etcd-snapshot.sh

@@ -66,14 +66,24 @@ fi
 #
 # Note: --aws-sigv4 requires curl >= 7.75 (Alpine 3.21 ships 8.x).
 URL="${SPINIFEX_PREDASTORE_ENDPOINT%/}/${BUCKET}/${KEY}"
-curl -fsSL \
+# Capture curl's own exit: piping straight into logger masks it behind logger's
+# (near-always 0), so a failed upload would log "uploaded" and delete the only
+# copy — snapshots would silently never land. Keep the local snapshot and exit
+# non-zero on failure so cron surfaces it and the next run still has data on disk.
+if UPLOAD_OUT=$(curl -fsSL \
     --aws-sigv4 "aws:amz:${AWS_REGION:-au-mel-1}:s3" \
     --user "${SPINIFEX_PREDASTORE_AKID}:${SPINIFEX_PREDASTORE_SECRET}" \
     -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \
     --upload-file "${SNAPSHOT_FILE}" \
-    "${URL}" 2>&1 | logger -t mulga-eks-etcd-snapshot
-
-# Remove the local snapshot to bound disk usage; the canonical copy lives in
-# predastore. Restore path (`spx admin eks restore-snapshot`) pulls from there.
-rm -f "${SNAPSHOT_FILE}"
-logger -t mulga-eks-etcd-snapshot "uploaded ${KEY}"
+    "${URL}" 2>&1); then
+    [ -n "${UPLOAD_OUT}" ] && printf '%s\n' "${UPLOAD_OUT}" | logger -t mulga-eks-etcd-snapshot
+    # Remove the local snapshot only after a confirmed upload to bound disk usage;
+    # the canonical copy lives in predastore (`spx admin eks restore-snapshot`).
+    rm -f "${SNAPSHOT_FILE}"
+    logger -t mulga-eks-etcd-snapshot "uploaded ${KEY}"
+else
+    UPLOAD_RC=$?
+    [ -n "${UPLOAD_OUT}" ] && printf '%s\n' "${UPLOAD_OUT}" | logger -t mulga-eks-etcd-snapshot
+    logger -t mulga-eks-etcd-snapshot "ERROR: upload of ${KEY} failed (curl rc=${UPLOAD_RC}); keeping local snapshot ${SNAPSHOT_FILE}"
+    exit 1
+fi

scripts/images/eks-node/mulga-eks-etcd-snapshot_test.sh

@@ -29,10 +29,13 @@ done
 :
 EOF
 
-# curl stub: record the full argument line (carries --upload-file + URL).
+# curl stub: record the full argument line (carries --upload-file + URL). Exits
+# non-zero when CURL_FAIL=1 so the harness can exercise the failed-upload path.
 cat > "${STUBBIN}/curl" <<'EOF'
 #!/bin/sh
 echo "curl $*" >> "${CURL_CALLS}"
+[ "${CURL_FAIL:-0}" = "1" ] && { echo "curl: (7) connection refused" >&2; exit 7; }
+exit 0
 EOF
 
 # logger stub: swallow.
@@ -102,6 +105,24 @@ run server "${WORK}/15min/mulga-eks-etcd-snapshot"
 [ -s "${WORK}/curl.calls" ] && fail "no-env: must not upload" || pass "no-env: no upload without creds"
 ENVFILE="${ENVFILE_SAVE}"
 
+# --- Case 5: upload FAILS -> non-zero exit + local snapshot KEPT (not masked) ---
+# Set up like run() but expect a non-zero exit and assert the snapshot survives.
+K3S_CALLS="${WORK}/k3s.calls"; CURL_CALLS="${WORK}/curl.calls"
+SNAPSHOT_DIR="${WORK}/snapshots"; ROLE_FILE="${WORK}/role"; K3S_BIN=k3s
+: > "${K3S_CALLS}"; : > "${CURL_CALLS}"; rm -rf "${SNAPSHOT_DIR}"
+printf '%s' server > "${ROLE_FILE}"
+export K3S_CALLS CURL_CALLS SNAPSHOT_DIR ROLE_FILE K3S_BIN ENVFILE
+export TIER=frequent CURL_FAIL=1
+if sh "${WORK}/15min/mulga-eks-etcd-snapshot" </dev/null; then
+    fail "upload-fail: must exit non-zero when curl fails"
+else
+    pass "upload-fail: non-zero exit surfaces the failure"
+fi
+ls "${SNAPSHOT_DIR}"/etcd-frequent-* >/dev/null 2>&1 \
+    && pass "upload-fail: local snapshot kept for the next run" \
+    || fail "upload-fail: local snapshot deleted despite failed upload"
+unset TIER CURL_FAIL
+
 if [ "${FAILS}" -eq 0 ]; then
     echo "PASS: all mulga-eks-etcd-snapshot cases"
     exit 0

scripts/images/eks-node/mulga-eks-k3s-recovery.sh

@@ -7,10 +7,10 @@ set -eu
 # but embedded etcd never reformed quorum after a simultaneous restart).
 #
 # The reconciler cannot push into the guest, so recovery is pull-based: this
-# fetches a per-member directive epoch<TAB>action<TAB>snapshot through the AWS
-# gateway (eks-gateway-fetch -resource recovery), epoch-guarded so a directive
-# applies exactly once. Ordered `before k3s` so the etcd datastore is mutated
-# while the apiserver is stopped. Actions:
+# fetches a per-member directive epoch<TAB>action<TAB>snapshot<TAB>required
+# through the AWS gateway (eks-gateway-fetch -resource recovery), epoch-guarded
+# so a directive applies exactly once. Ordered `before k3s` so the etcd datastore
+# is mutated while the apiserver is stopped. Actions:
 #   none          steady state — nothing to do
 #   cluster-reset seed member: `k3s server --cluster-reset` reforms a
 #                 single-member etcd from this node's intact data (optionally
@@ -18,6 +18,13 @@ set -eu
 #   wipe-rejoin   follower: delete the local etcd datastore so k3s re-joins the
 #                 reset seed's quorum fresh on the next k3s start
 #
+# required=1 marks a snapshot that MUST restore (fresh DR seed via
+# `spx admin eks restore-snapshot`): a fresh seed has no intact local etcd, so a
+# cluster-reset without the snapshot would boot an EMPTY cluster. On a fetch
+# failure this drops BLOCK_MARKER and aborts boot; k3s.initd refuses to start on
+# the marker, so k3s never resets into an empty datastore. The fetch is retried
+# on the next boot (BLOCK_MARKER lives in /run tmpfs, cleared on reboot).
+#
 # Env knobs default to production paths; overridable so the logic is unit-testable
 # without root, IMDS, a gateway, or a real k3s.
 
@@ -25,6 +32,7 @@ ROLE_FILE=${ROLE_FILE:-/etc/spinifex-eks/role}
 ENVFILE=${ENVFILE:-/etc/spinifex-eks/first-boot.env}
 SNAPSHOT_ENVFILE=${SNAPSHOT_ENVFILE:-/etc/spinifex-eks/etcd-snapshot.env}
 EPOCH_FILE=${EPOCH_FILE:-/var/lib/spinifex-eks/recovery.epoch}
+BLOCK_MARKER=${BLOCK_MARKER:-/run/spinifex-eks/k3s-restore-block}
 K3S_BIN=${K3S_BIN:-/usr/local/bin/k3s}
 K3S_CONFIG=${K3S_CONFIG:-/etc/rancher/k3s/config.yaml}
 ETCD_DIR=${ETCD_DIR:-/var/lib/rancher/k3s/server/db/etcd}
@@ -72,12 +80,26 @@ fetch_snapshot() {
     [ -n "${SPINIFEX_PREDASTORE_SECRET:-}" ] || return 1
     _url="${SPINIFEX_PREDASTORE_ENDPOINT%/}/eks-backups-system/${EKS_ACCOUNT_ID}/${EKS_CLUSTER_NAME}/${_name}"
     mkdir -p "${SNAPSHOT_DIR}"
-    curl -fsSL \
+    # Capture curl's own exit — piping straight into logger would mask it behind
+    # logger's (near-always 0), so a failed download would look like success and
+    # the caller would wrongly cluster-reset from local etcd.
+    if _out=$(curl -fsSL \
         --aws-sigv4 "aws:amz:${AWS_REGION:-au-mel-1}:s3" \
         --user "${SPINIFEX_PREDASTORE_AKID}:${SPINIFEX_PREDASTORE_SECRET}" \
         -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \
         -o "${SNAPSHOT_DIR}/${_name}" \
-        "${_url}" 2>&1 | logger -t "${LOGTAG}"
+        "${_url}" 2>&1); then
+        [ -n "${_out}" ] && printf '%s\n' "${_out}" | logger -t "${LOGTAG}"
+        return 0
+    else
+        # Capture rc as the first else statement: $? read after `fi` would be the
+        # if's own 0, re-masking the failure we are here to surface.
+        _rc=$?
+        [ -n "${_out}" ] && printf '%s\n' "${_out}" | logger -t "${LOGTAG}"
+        rm -f "${SNAPSHOT_DIR}/${_name}"
+        log "snapshot ${_name} download failed (curl rc=${_rc})"
+        return 1
+    fi
 }
 
 # Instance ID from IMDS keys the per-member directive. IMDSv2 token if enforced,
@@ -109,8 +131,11 @@ fi
 epoch=$(printf '%s' "${line}" | cut -f1)
 action=$(printf '%s' "${line}" | cut -f2)
 snapshot=$(printf '%s' "${line}" | cut -f3)
+snapshot_required=$(printf '%s' "${line}" | cut -f4)
 [ -n "${action}" ] || action=none
 case "${epoch}" in '' | *[!0-9]*) epoch=0 ;; esac
+# Absent 4th field (older gateway) reads as not-required, preserving HA-reform semantics.
+case "${snapshot_required}" in 1) snapshot_required=1 ;; *) snapshot_required=0 ;; esac
 
 if [ "${action}" = "none" ]; then
     log "directive action=none (epoch ${epoch}); nothing to do"
@@ -136,6 +161,13 @@ case "${action}" in
             if fetch_snapshot "${snapshot}"; then
                 restore_arg="--cluster-reset-restore-path=${SNAPSHOT_DIR}/${snapshot}"
                 log "cluster-reset restoring snapshot ${snapshot}"
+            elif [ "${snapshot_required}" = "1" ]; then
+                # Fresh DR seed: no intact local etcd, so a reset without the snapshot
+                # would boot an EMPTY cluster. Block k3s and abort; retry next boot.
+                log "ERROR: required snapshot ${snapshot} could not be fetched; aborting boot to avoid resetting into an EMPTY datastore (retries next boot)"
+                mkdir -p "$(dirname "${BLOCK_MARKER}")"
+                : > "${BLOCK_MARKER}"
+                exit 1
             else
                 log "WARN: snapshot ${snapshot} fetch failed; cluster-reset from local etcd only"
             fi

scripts/images/eks-node/mulga-eks-k3s-recovery_test.sh

@@ -24,7 +24,9 @@ printf '%s\n' "${DIRECTIVE_LINE}"
 EOF
 
 # curl stub: answers the IMDS token PUT + instance-id GET, and materialises a
-# snapshot file for any `-o <path>` download. Records every call.
+# snapshot file for any `-o <path>` download. Records every call. A snapshot
+# download (an `-o` arg) fails with a non-zero exit when CURL_SNAP_FAIL=1, so the
+# harness can exercise the fetch-failure path.
 cat > "${STUBBIN}/curl" <<'EOF'
 #!/bin/sh
 echo "curl $*" >> "${CURL_CALLS}"
@@ -36,7 +38,10 @@ for a in "$@"; do
 done
 prev=""
 for a in "$@"; do
-    [ "$prev" = "-o" ] && echo snapdata > "$a"
+    if [ "$prev" = "-o" ]; then
+        [ "${CURL_SNAP_FAIL:-0}" = "1" ] && { echo "curl: (22) 404" >&2; exit 22; }
+        echo snapdata > "$a"
+    fi
     prev="$a"
 done
 exit 0
@@ -80,7 +85,7 @@ pass() { echo "ok: $*"; }
 
 # run <role> <directive-line>: reset capture files + state, then run the script
 # with every path knob pointed into the temp dir.
-run() {
+_setup_run() {
     _role=$1
     DIRECTIVE_LINE=$2
     K3S_CALLS="${WORK}/k3s.calls"
@@ -89,13 +94,29 @@ run() {
     EPOCH_FILE="${WORK}/recovery.epoch"
     ETCD_DIR="${WORK}/etcd"
     SNAPSHOT_DIR="${WORK}/snapshots"
+    BLOCK_MARKER="${WORK}/run/k3s-restore-block"
     : > "${K3S_CALLS}"
     : > "${CURL_CALLS}"
+    rm -f "${BLOCK_MARKER}"
     printf '%s' "${_role}" > "${ROLE_FILE}"
-    export DIRECTIVE_LINE K3S_CALLS CURL_CALLS ROLE_FILE EPOCH_FILE ETCD_DIR SNAPSHOT_DIR
+    export DIRECTIVE_LINE K3S_CALLS CURL_CALLS ROLE_FILE EPOCH_FILE ETCD_DIR SNAPSHOT_DIR BLOCK_MARKER
     export ENVFILE SNAPSHOT_ENVFILE
     export K3S_BIN=k3s FETCH_BIN=eks-gateway-fetch
-    sh "${SCRIPT}" </dev/null || fail "role=${_role}: non-zero exit"
+}
+
+# run <role> <directive-line>: expects a zero exit (the steady-state path).
+run() {
+    _setup_run "$1" "$2"
+    sh "${SCRIPT}" </dev/null || fail "role=${1}: non-zero exit"
+}
+
+# run_abort <role> <directive-line>: expects a NON-zero exit (the required-snapshot
+# fetch-failure abort). CURL_SNAP_FAIL must be set by the caller.
+run_abort() {
+    _setup_run "$1" "$2"
+    if sh "${SCRIPT}" </dev/null; then
+        fail "role=${1}: expected non-zero exit (required-snapshot abort)"
+    fi
 }
 
 TAB=$(printf '\t')
@@ -149,6 +170,36 @@ grep -q 'aws-sigv4' "${WORK}/curl.calls" \
 [ -f "${WORK}/snapshots/etcd-daily-20260706T000000Z.snap" ] \
     && pass "snapshot: fetched into snapshot dir" || fail "snapshot: file not materialised"
 
+# --- Case 7: REQUIRED snapshot, fetch OK -> restore-path + no block marker ---
+rm -f "${WORK}/recovery.epoch"
+CURL_SNAP_FAIL=0; export CURL_SNAP_FAIL
+run server "5${TAB}cluster-reset${TAB}etcd-frequent-20260709T010000Z.snap${TAB}1"
+grep -q 'cluster-reset-restore-path=.*etcd-frequent-20260709T010000Z.snap' "${WORK}/k3s.calls" \
+    && pass "required-ok: restore-path passed" || fail "required-ok: no restore-path: $(cat "${WORK}/k3s.calls")"
+[ -f "${WORK}/run/k3s-restore-block" ] && fail "required-ok: must not block k3s on success" || pass "required-ok: no block marker"
+
+# --- Case 8: REQUIRED snapshot, fetch FAILS -> abort: block marker, no k3s, no epoch ---
+rm -f "${WORK}/recovery.epoch"
+CURL_SNAP_FAIL=1; export CURL_SNAP_FAIL
+run_abort server "6${TAB}cluster-reset${TAB}etcd-frequent-20260709T010000Z.snap${TAB}1"
+[ -f "${WORK}/run/k3s-restore-block" ] \
+    && pass "required-fail: block marker dropped to stop k3s" || fail "required-fail: no block marker"
+[ -s "${WORK}/k3s.calls" ] && fail "required-fail: must NOT cluster-reset into empty datastore" || pass "required-fail: k3s not reset"
+[ -f "${WORK}/recovery.epoch" ] && fail "required-fail: epoch must not record (retry next boot)" || pass "required-fail: epoch not recorded"
+
+# --- Case 9: NON-required snapshot, fetch FAILS -> local reset, no block, epoch recorded ---
+rm -f "${WORK}/recovery.epoch"
+CURL_SNAP_FAIL=1; export CURL_SNAP_FAIL
+run server "7${TAB}cluster-reset${TAB}etcd-frequent-20260709T010000Z.snap${TAB}0"
+grep -q 'server --config .* --cluster-reset' "${WORK}/k3s.calls" \
+    && pass "optional-fail: falls back to local cluster-reset" || fail "optional-fail: no reset: $(cat "${WORK}/k3s.calls")"
+grep -q 'cluster-reset-restore-path' "${WORK}/k3s.calls" \
+    && fail "optional-fail: must not pass a restore-path for a failed fetch" || pass "optional-fail: local-data reset only"
+[ -f "${WORK}/run/k3s-restore-block" ] && fail "optional-fail: must not block k3s (HA-reform semantics)" || pass "optional-fail: no block marker"
+[ "$(cat "${WORK}/recovery.epoch" 2>/dev/null)" = 7 ] \
+    && pass "optional-fail: epoch recorded" || fail "optional-fail: epoch not recorded"
+CURL_SNAP_FAIL=0; export CURL_SNAP_FAIL
+
 if [ "${FAILS}" -eq 0 ]; then
     echo "PASS: all mulga-eks-k3s-recovery cases"
     exit 0