mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-23 20:25:56 +00:00
Add shadow release lifecycle rehearsal workflow
Install a published release (latest stable by default) with its signed install.sh, seed auth, a webhook and a PVE node through the API, upgrade with the installed /bin/update --version helper, then roll back with the documented /bin/update --version command. Each step checks /api/version and the binary version, /api/health, unit state, the seeded settings against fixed expectations and the pre-upgrade read-back, and data-dir survival, then reports a phase table in the step summary. The workflow is read-only, hosted-only, uploads nothing and no release job depends on it.
This commit is contained in:
courtmanr@gmail.com
@@ -140,6 +140,17 @@ and verifies the live service health and exact version. The privileged systemd
|
||||
smoke environment is digest-pinned because a floating container image would
|
||||
otherwise be an unreviewed code path inside the release gate.
|
||||
|
||||
`release-lifecycle-rehearsal.yml` is a daily shadow rehearsal of the systemd
|
||||
lifecycle between two published releases, in the same digest-pinned systemd
|
||||
container. It installs the older release with its signature-verified
|
||||
`install.sh --version`, seeds real settings through the API, upgrades with the
|
||||
installed `/bin/update --version <to>` helper and rolls back with the documented
|
||||
`/bin/update --version <from>`, checking version, health, unit state, settings
|
||||
and data-dir survival after each step. It is read-only, uploads nothing and gates
|
||||
nothing; results are in the job log and step summary. Run
|
||||
`scripts/release_lifecycle_rehearsal.sh --from <tag> --to <tag>` to reproduce
|
||||
it locally (`PULSE_REHEARSAL_ENGINE=podman` on hosts without Docker).
|
||||
|
||||
The shared `install-sh-smoke-body.yml` inherits its caller's token permissions;
|
||||
keep it free of workflow- or job-level permission overrides. Continuity calls
|
||||
that body directly with `contents: read`. The existing `install-sh-smoke.yml`
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
name: Release Lifecycle Rehearsal
|
||||
|
||||
# Shadow rehearsal of the systemd lifecycle users run between two PUBLISHED
|
||||
# releases: install FROM with its signed install.sh, seed real settings, upgrade
|
||||
# with /bin/update --version TO, then roll back with the documented
|
||||
# /bin/update --version FROM, asserting identity, health, unit state, settings
|
||||
# and data-dir survival after each step. It reports only. No release job
|
||||
# depends on it, and it uploads nothing.
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
from_version:
|
||||
description: 'Published tag to install first (e.g. v6.4.1). Empty = latest stable.'
|
||||
required: false
|
||||
type: string
|
||||
default: ''
|
||||
to_version:
|
||||
description: 'Published tag to upgrade to. Empty = newest published release or prerelease newer than from_version.'
|
||||
required: false
|
||||
type: string
|
||||
default: ''
|
||||
schedule:
|
||||
# Daily, well outside the 02:00-06:00 UTC unattended-update window that an
|
||||
# upgraded install may arm, so the timer cannot race the assertions.
|
||||
- cron: '23 13 * * *'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: release-lifecycle-rehearsal
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
rehearse:
|
||||
name: Install, upgrade and roll back on systemd
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 45
|
||||
steps:
|
||||
- name: Checkout rehearsal harness (for README key and scripts)
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Resolve published release pair
|
||||
id: pair
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
REPOSITORY: ${{ github.repository }}
|
||||
FROM_INPUT: ${{ inputs.from_version }}
|
||||
TO_INPUT: ${{ inputs.to_version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
releases_json="$(mktemp)"
|
||||
gh api --paginate "repos/${REPOSITORY}/releases?per_page=100" \
|
||||
| jq -s 'add // []' > "${releases_json}"
|
||||
latest_tag="$(gh api "repos/${REPOSITORY}/releases/latest" --jq '.tag_name')"
|
||||
resolved="$(python3 scripts/release_lifecycle_rehearsal_versions.py \
|
||||
--releases-json "${releases_json}" \
|
||||
--latest-tag "${latest_tag}" \
|
||||
--from "${FROM_INPUT}" \
|
||||
--to "${TO_INPUT}")"
|
||||
rm -f "${releases_json}"
|
||||
from_tag="$(sed -n 's/^from_tag=//p' <<<"${resolved}")"
|
||||
to_tag="$(sed -n 's/^to_tag=//p' <<<"${resolved}")"
|
||||
reason="$(sed -n 's/^reason=//p' <<<"${resolved}")"
|
||||
python3 scripts/write_github_output.py from_tag "${from_tag}"
|
||||
python3 scripts/write_github_output.py to_tag "${to_tag}"
|
||||
echo "Rehearsing ${from_tag} -> ${to_tag} -> ${from_tag} (${reason})"
|
||||
{
|
||||
echo "Release pair: \`${from_tag}\` -> \`${to_tag}\` (${reason})."
|
||||
echo
|
||||
} >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Rehearse install, upgrade and rollback
|
||||
env:
|
||||
FROM_TAG: ${{ steps.pair.outputs.from_tag }}
|
||||
TO_TAG: ${{ steps.pair.outputs.to_tag }}
|
||||
PULSE_REHEARSAL_REPO: ${{ github.repository }}
|
||||
PULSE_REHEARSAL_ENGINE: docker
|
||||
run: scripts/release_lifecycle_rehearsal.sh --from "${FROM_TAG}" --to "${TO_TAG}"
|
||||
@@ -842,6 +842,9 @@ release-latency optimization.
|
||||
98. `scripts/verify-github-release-integrity.sh`
|
||||
99. `scripts/verify-release-container-images.sh`
|
||||
100. `scripts/release_control/verify_release_container_images_test.py`
|
||||
101. `.github/workflows/release-lifecycle-rehearsal.yml`
|
||||
102. `scripts/release_lifecycle_rehearsal.sh`
|
||||
103. `scripts/release_lifecycle_rehearsal_versions.py`
|
||||
|
||||
## Shared Boundaries
|
||||
|
||||
@@ -6106,6 +6109,82 @@ the current assignment; a host dependency acquisition on the next launch is
|
||||
required before this contract's Go evidence can be produced. No passing Go
|
||||
suite, installed build or release acceptance is claimed here.
|
||||
|
||||
### Published-release lifecycle rehearsal (shadow)
|
||||
|
||||
`.github/workflows/release-lifecycle-rehearsal.yml` rehearses the systemd
|
||||
install, upgrade and rollback journey between two published releases. It is
|
||||
the shadow first step toward making install, upgrade and rollback a publication
|
||||
gate. It runs daily and on manual dispatch, holds only `contents: read`, runs
|
||||
on the hosted `ubuntu-24.04` image, uses no secrets and uploads no artifacts.
|
||||
No release workflow calls it or waits on it, so it reports without gating.
|
||||
|
||||
`scripts/release_lifecycle_rehearsal_versions.py` chooses the pair from
|
||||
published, non-draft release tags under SemVer precedence. The default FROM is
|
||||
the release GitHub advertises as latest and must be stable. The default TO is
|
||||
the newest published release or prerelease newer than FROM. When FROM was
|
||||
defaulted and nothing newer is published, the pair is the previous stable to
|
||||
the latest stable. An explicit pair must be published and move forward, so a
|
||||
same-version or missing-target run fails instead of passing.
|
||||
|
||||
`scripts/release_lifecycle_rehearsal.sh` runs every phase inside the same
|
||||
digest-pinned `jrei/systemd-debian:12` container as the install smoke.
|
||||
|
||||
1. Install FROM with that release's own `install.sh --version`, after the
|
||||
asset verifies against the README-pinned `pulse-installer` key and passes
|
||||
the server-installer identity checks.
|
||||
2. Assert `/api/version` and `/opt/pulse/bin/pulse --version` both report
|
||||
FROM, `/api/health` reports `healthy`, and `pulse.service` is active and
|
||||
enabled with no newly failed unit.
|
||||
3. Seed state through the real API. First-run security setup uses the
|
||||
bootstrap token from `pulse bootstrap-token`, then a disabled webhook with
|
||||
a secret header and a PVE node with fake token credentials are created. The
|
||||
read-back must show that auth is required, that an unauthenticated request
|
||||
gets 401, and that the API token and password both get 200. The webhook must
|
||||
keep its id, URL, method, service, disabled state and header key. The node
|
||||
must keep its host, token name, stored token secret (`hasToken`), no
|
||||
password and `verifySSL=false`. The `/etc/pulse` file list and checksums
|
||||
become the baseline.
|
||||
4. Upgrade with the installed helper, `/bin/update --version TO`, after
|
||||
checking that the helper is the Pulse server installer's. The helper
|
||||
downloads the latest published `install.sh`, verifies it with its embedded
|
||||
key and runs it.
|
||||
5. After the upgrade, and again after the documented rollback
|
||||
`/bin/update --version FROM`, assert the expected identity, health and unit
|
||||
state. The settings read-back must equal the baseline and meet the fixed
|
||||
expectations. No baseline file may be missing from `/etc/pulse`.
|
||||
`.encryption.key`, `nodes.enc` and `webhooks.enc` must be byte-identical.
|
||||
The API reports only `hasToken` and redacted header values, so byte
|
||||
identity under an unchanged key is what proves the exact seeded secrets
|
||||
survived. A release that legitimately rewrites those stores fails the
|
||||
rehearsal instead of passing unproven. The API token and password checks
|
||||
prove those exact credentials directly.
|
||||
|
||||
Rollback still runs when the upgrade phase fails. A phase also fails if
|
||||
`pulse-update.service` started during the run, because an unattended update
|
||||
would make the version assertions unattributable. Three D-Bus-activated host
|
||||
services that cannot run in the container (`systemd-hostnamed`,
|
||||
`systemd-timedated` and `systemd-localed`) are reported as warnings rather
|
||||
than failures. A unit drop-in sets `PULSE_TELEMETRY=false` so daily CI installs
|
||||
stay out of usage telemetry. It lives outside the installer's files and the
|
||||
data dir.
|
||||
|
||||
Scope: only published assets and the amd64 systemd path are exercised. Docker,
|
||||
Helm, Proxmox LXC creation, the in-app updater and a prerelease's own
|
||||
`install.sh` (the helper always fetches the latest published installer) are
|
||||
not covered. A passing run is not release admission.
|
||||
|
||||
Verification: `scripts/tests/test_release_lifecycle_rehearsal.py` covers pair
|
||||
resolution (SemVer ordering, draft exclusion, forward-only and published-only
|
||||
pairs, the previous-stable fallback), the fixed settings expectations (each lost
|
||||
or altered seeded field fails `--check-snapshot`, and a deleted file or a
|
||||
rewritten secret store fails `--compare-datadir`), the harness contract (the real updater
|
||||
invocation without bypass flags, the signed installer, the shared image digest,
|
||||
and per-phase identity, settings and data-dir checks) and the workflow trust
|
||||
shape. A local podman run (amd64 emulation) of `v6.4.1 -> v6.4.5 -> v6.4.1`
|
||||
passed all four phases. An earlier run correctly failed phase 1 on a new failed
|
||||
unit (`systemd-hostnamed`), which led to the container-only list above. The
|
||||
hosted Docker run is not yet claimed.
|
||||
|
||||
## Release-body updater ownership (30 September 2026)
|
||||
|
||||
Each executable `/bin/update --version` example in a published body must carry
|
||||
|
||||
@@ -4536,6 +4536,7 @@
|
||||
".github/workflows/recover-release-activation.yml",
|
||||
".github/workflows/release-convergence.yml",
|
||||
".github/workflows/release-dry-run.yml",
|
||||
".github/workflows/release-lifecycle-rehearsal.yml",
|
||||
".github/workflows/retry-release-convergence.yml",
|
||||
".github/workflows/update-demo-server.yml",
|
||||
".github/workflows/validate-release-assets.yml",
|
||||
@@ -4630,6 +4631,8 @@
|
||||
"scripts/release_control/secure_runtime_source_manifest_v7.json",
|
||||
"scripts/release_control/validate_artifact_release_line.py",
|
||||
"scripts/release_ldflags.sh",
|
||||
"scripts/release_lifecycle_rehearsal.sh",
|
||||
"scripts/release_lifecycle_rehearsal_versions.py",
|
||||
"scripts/release_update_key.go",
|
||||
"scripts/run-release-backend-tests.sh",
|
||||
"scripts/run-release-preflight.sh",
|
||||
@@ -4817,6 +4820,21 @@
|
||||
"scripts/release_control/verify_github_release_integrity_test.py"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "release-lifecycle-rehearsal",
|
||||
"label": "published-release install, upgrade and rollback rehearsal proof",
|
||||
"match_prefixes": [],
|
||||
"match_files": [
|
||||
".github/workflows/release-lifecycle-rehearsal.yml",
|
||||
"scripts/release_lifecycle_rehearsal.sh",
|
||||
"scripts/release_lifecycle_rehearsal_versions.py"
|
||||
],
|
||||
"allow_same_subsystem_tests": false,
|
||||
"test_prefixes": [],
|
||||
"exact_files": [
|
||||
"scripts/tests/test_release_lifecycle_rehearsal.py"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "release-candidate-manifest-runtime",
|
||||
"label": "immutable release candidate manifest proof",
|
||||
|
||||
Executable
+700
@@ -0,0 +1,700 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Release lifecycle rehearsal: install, upgrade and roll back Pulse between two
|
||||
# PUBLISHED releases on a real systemd host (a privileged systemd container).
|
||||
#
|
||||
# 1. install FROM with that release's signature-verified install.sh --version
|
||||
# 2. assert FROM identity, health and unit state
|
||||
# 3. seed persistent state through the real API (first-run security setup,
|
||||
# a webhook, a PVE node with fake credentials) and snapshot the data dir
|
||||
# 4. upgrade with the installed updater users run: /bin/update --version TO
|
||||
# 5. assert TO identity, health, unit state, seeded settings and data dir
|
||||
# 6. roll back with the documented command: /bin/update --version FROM
|
||||
# 7. assert FROM identity, health, unit state, seeded settings and data dir
|
||||
#
|
||||
# The result is a phase table on stdout and in $GITHUB_STEP_SUMMARY (when set).
|
||||
# Nothing is uploaded. Exit status is non-zero when any phase fails.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/release_lifecycle_rehearsal.sh --from vX.Y.Z --to vX.Y.Z
|
||||
#
|
||||
# Environment:
|
||||
# PULSE_REHEARSAL_REPO owner/repo for release assets (default rcourtman/Pulse)
|
||||
# PULSE_REHEARSAL_ENGINE docker (default) or podman
|
||||
# PULSE_REHEARSAL_WORKDIR scratch directory (default: mktemp -d)
|
||||
# PULSE_REHEARSAL_README README holding the pinned installer key
|
||||
# (default: README.md next to this script's repo)
|
||||
|
||||
# Programs passed to cexec are single-quoted on purpose: they expand inside
|
||||
# the container from -e variables. Mount options legitimately contain commas.
|
||||
# shellcheck disable=SC2016,SC2054
|
||||
|
||||
set -euo pipefail
|
||||
export LC_ALL=C
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
SYSTEMD_IMAGE="docker.io/jrei/systemd-debian:12@sha256:61d70dc3e574337bd9df794674a60ae73113460fff16ab41a2d234b4a11dcd98"
|
||||
TAG_PATTERN='^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$'
|
||||
API="http://127.0.0.1:7655"
|
||||
DATA_DIR="/etc/pulse"
|
||||
SEED_WEBHOOK_NAME="lifecycle-rehearsal-webhook"
|
||||
SEED_WEBHOOK_URL="https://example.com/pulse-lifecycle-rehearsal"
|
||||
# PVE add-node skips live cluster detection for 192.168.77.x hosts and
|
||||
# "test-" names, so the fake node is persisted without a reachable endpoint.
|
||||
SEED_NODE_NAME="test-lifecycle-rehearsal"
|
||||
SEED_NODE_HOST="https://192.168.77.10:8006"
|
||||
SEED_NODE_TOKEN_NAME="root@pam!rehearsal"
|
||||
SEED_WEBHOOK_HEADER="X-Rehearsal-Secret"
|
||||
SEED_ADMIN_USER="rehearsal-admin"
|
||||
|
||||
FROM_TAG=""
|
||||
TO_TAG=""
|
||||
CHECK_SNAPSHOT=""
|
||||
COMPARE_DATADIR=()
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--from) FROM_TAG="${2:-}"; shift 2 ;;
|
||||
--to) TO_TAG="${2:-}"; shift 2 ;;
|
||||
# Offline self-tests used by the contract tests: apply the fixed
|
||||
# settings expectations to a snapshot file, or compare two data-dir
|
||||
# manifests (path<TAB>sha256), then exit.
|
||||
--check-snapshot) CHECK_SNAPSHOT="${2:-}"; shift 2 ;;
|
||||
--compare-datadir) COMPARE_DATADIR=("${2:-}" "${3:-}"); shift 3 ;;
|
||||
-h|--help) sed -n '2,/^$/p' "${BASH_SOURCE[0]}"; exit 0 ;;
|
||||
*) echo "unknown argument: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ -z "$CHECK_SNAPSHOT" && ${#COMPARE_DATADIR[@]} -eq 0 ]]; then
|
||||
for tag in "$FROM_TAG" "$TO_TAG"; do
|
||||
if [[ ! "$tag" =~ $TAG_PATTERN ]]; then
|
||||
echo "::error::--from and --to must be Pulse release tags (vX.Y.Z[-pre]); got '${tag}'" >&2
|
||||
exit 2
|
||||
fi
|
||||
done
|
||||
if [[ "$FROM_TAG" == "$TO_TAG" ]]; then
|
||||
echo "::error::--from and --to must differ; a same-version run proves no upgrade" >&2
|
||||
exit 2
|
||||
fi
|
||||
fi
|
||||
|
||||
REPO="${PULSE_REHEARSAL_REPO:-rcourtman/Pulse}"
|
||||
if [[ ! "$REPO" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]]; then
|
||||
echo "::error::invalid PULSE_REHEARSAL_REPO '${REPO}'" >&2
|
||||
exit 2
|
||||
fi
|
||||
ENGINE="${PULSE_REHEARSAL_ENGINE:-docker}"
|
||||
case "$ENGINE" in
|
||||
docker|podman) ;;
|
||||
*) echo "::error::PULSE_REHEARSAL_ENGINE must be docker or podman" >&2; exit 2 ;;
|
||||
esac
|
||||
README_PATH="${PULSE_REHEARSAL_README:-${ROOT_DIR}/README.md}"
|
||||
WORK_DIR="${PULSE_REHEARSAL_WORKDIR:-$(mktemp -d)}"
|
||||
mkdir -p "${WORK_DIR}/assets" "${WORK_DIR}/state"
|
||||
CONTAINER="pulse-lifecycle-rehearsal-$$"
|
||||
SUMMARY_FILE="${GITHUB_STEP_SUMMARY:-}"
|
||||
|
||||
# Throwaway credentials for an ephemeral container. They never leave it.
|
||||
ADMIN_PASSWORD="$(od -An -N24 -tx1 /dev/urandom | tr -d ' \n')"
|
||||
API_TOKEN="$(od -An -N32 -tx1 /dev/urandom | tr -d ' \n')"
|
||||
NODE_TOKEN_VALUE="$(od -An -N16 -tx1 /dev/urandom | tr -d ' \n')"
|
||||
|
||||
PHASE_ROWS=()
|
||||
FAILURES=()
|
||||
CURRENT_FAILURE=""
|
||||
OVERALL_STATUS=0
|
||||
CONTAINER_STARTED=false
|
||||
BASELINE_FAILED_UNITS=""
|
||||
# D-Bus-activated host services that cannot run inside the test container
|
||||
# (no hostname/clock/locale ownership). Any other new failed unit, and every
|
||||
# pulse* unit, fails the phase.
|
||||
CONTAINER_TOLERATED_UNITS="systemd-hostnamed.service systemd-timedated.service systemd-localed.service"
|
||||
|
||||
log() { printf '[rehearsal] %s\n' "$*"; }
|
||||
note_failure() {
|
||||
CURRENT_FAILURE="${CURRENT_FAILURE:+${CURRENT_FAILURE}; }$*"
|
||||
echo "::error::$*"
|
||||
}
|
||||
|
||||
# Run a bash program inside the container. Data reaches the program only as
|
||||
# environment variables, never by interpolation into the program text.
|
||||
cexec() {
|
||||
local program="$1"
|
||||
shift
|
||||
local -a env_args=(-e PULSE_INSTALL_ALLOW_DOCKER=1)
|
||||
local pair
|
||||
for pair in "$@"; do
|
||||
env_args+=(-e "$pair")
|
||||
done
|
||||
"$ENGINE" exec "${env_args[@]}" "$CONTAINER" bash -c "$program"
|
||||
}
|
||||
|
||||
print_diagnostics() {
|
||||
[[ "$CONTAINER_STARTED" == "true" ]] || return 0
|
||||
echo "::group::Diagnostics: pulse journal (tail)"
|
||||
cexec 'journalctl -u pulse --no-pager | tail -n 150' || true
|
||||
echo "::endgroup::"
|
||||
echo "::group::Diagnostics: unit state"
|
||||
cexec 'systemctl status pulse --no-pager; systemctl list-units --state=failed --no-pager; systemctl list-timers --all --no-pager | grep -i pulse' || true
|
||||
echo "::endgroup::"
|
||||
echo "::group::Diagnostics: pulse-update journal"
|
||||
cexec 'journalctl -u pulse-update --no-pager | tail -n 60' || true
|
||||
echo "::endgroup::"
|
||||
local logfile
|
||||
for logfile in "${WORK_DIR}"/state/*.log; do
|
||||
[[ -f "$logfile" ]] || continue
|
||||
echo "::group::Diagnostics: $(basename "$logfile") (tail)"
|
||||
tail -n 80 "$logfile" || true
|
||||
echo "::endgroup::"
|
||||
done
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
local status=$?
|
||||
if [[ "$OVERALL_STATUS" -ne 0 || "$status" -ne 0 ]]; then
|
||||
print_diagnostics
|
||||
fi
|
||||
if [[ "$CONTAINER_STARTED" == "true" ]]; then
|
||||
"$ENGINE" rm -f "$CONTAINER" >/dev/null 2>&1 || true
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
write_summary() {
|
||||
local table
|
||||
table=$(
|
||||
echo "## Release lifecycle rehearsal"
|
||||
echo
|
||||
echo "\`${FROM_TAG}\` -> \`${TO_TAG}\` -> \`${FROM_TAG}\` on a systemd host (${SYSTEMD_IMAGE%%@*}). Shadow run, not a release gate."
|
||||
echo
|
||||
echo "| Phase | Expected | Reported version | Health | Settings | Data dir | Units | Result |"
|
||||
echo "| --- | --- | --- | --- | --- | --- | --- | --- |"
|
||||
local row
|
||||
for row in "${PHASE_ROWS[@]}"; do
|
||||
echo "$row"
|
||||
done
|
||||
if [[ ${#FAILURES[@]} -gt 0 ]]; then
|
||||
echo
|
||||
echo "### Failures"
|
||||
local failure
|
||||
for failure in "${FAILURES[@]}"; do
|
||||
echo "- ${failure}"
|
||||
done
|
||||
fi
|
||||
)
|
||||
echo
|
||||
echo "$table"
|
||||
if [[ -n "$SUMMARY_FILE" ]]; then
|
||||
printf '%s\n' "$table" >> "$SUMMARY_FILE"
|
||||
fi
|
||||
}
|
||||
|
||||
# record_phase NAME EXPECTED VERSION HEALTH SETTINGS DATADIR UNITS
|
||||
record_phase() {
|
||||
local result="pass"
|
||||
if [[ -n "$CURRENT_FAILURE" ]]; then
|
||||
result="FAIL"
|
||||
FAILURES+=("$1: ${CURRENT_FAILURE}")
|
||||
OVERALL_STATUS=1
|
||||
fi
|
||||
PHASE_ROWS+=("| $1 | \`$2\` | ${3:--} | ${4:--} | ${5:--} | ${6:--} | ${7:--} | ${result} |")
|
||||
CURRENT_FAILURE=""
|
||||
[[ "$result" == "pass" ]]
|
||||
}
|
||||
|
||||
abort_run() {
|
||||
write_summary
|
||||
exit 1
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Published installer, verified against the README-pinned key
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
fetch_and_verify_installer() {
|
||||
local tag="$1"
|
||||
local dest="${WORK_DIR}/assets"
|
||||
local base="https://github.com/${REPO}/releases/download/${tag}"
|
||||
local asset
|
||||
for asset in install.sh install.sh.sshsig; do
|
||||
if ! curl -fsSL --retry 5 --retry-delay 5 --retry-all-errors \
|
||||
-o "${dest}/${asset}" "${base}/${asset}"; then
|
||||
echo "::error::could not download ${base}/${asset}"
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
|
||||
local readme_key allowed_signers
|
||||
readme_key=$(grep -oE 'ssh-ed25519 [A-Za-z0-9+/=]+ pulse-installer' "$README_PATH" | head -1)
|
||||
if [[ -z "$readme_key" ]]; then
|
||||
echo "::error::Could not extract the pulse-installer key from ${README_PATH}"
|
||||
return 1
|
||||
fi
|
||||
allowed_signers="${WORK_DIR}/allowed_signers"
|
||||
printf 'pulse-installer %s\n' "$readme_key" > "$allowed_signers"
|
||||
if ! ssh-keygen -Y verify \
|
||||
-f "$allowed_signers" \
|
||||
-I pulse-installer \
|
||||
-n pulse-install \
|
||||
-s "${dest}/install.sh.sshsig" < "${dest}/install.sh"; then
|
||||
echo "::error::${tag} install.sh.sshsig does not verify against the README's pinned key"
|
||||
return 1
|
||||
fi
|
||||
if ! grep -qE '^# Pulse Installer Script' "${dest}/install.sh" \
|
||||
|| grep -q 'Pulse Unified Agent Installer' "${dest}/install.sh" \
|
||||
|| ! grep -qE '^[[:space:]]*--version\)' "${dest}/install.sh"; then
|
||||
echo "::error::${tag} install.sh is not the Pulse server installer with --version support"
|
||||
return 1
|
||||
fi
|
||||
log "${tag} install.sh signature verifies against the README-pinned key"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Container
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
start_container() {
|
||||
local -a run_args=(run -d --name "$CONTAINER" --privileged
|
||||
-v "${WORK_DIR}/assets:/rehearsal:ro")
|
||||
if [[ "$ENGINE" == "docker" ]]; then
|
||||
# GHA ubuntu-24.04 uses the cgroup v2 unified hierarchy; without
|
||||
# --cgroupns=host systemd PID 1 cannot mount the cgroup tree.
|
||||
run_args+=(--cgroupns=host -v /sys/fs/cgroup:/sys/fs/cgroup:rw
|
||||
--tmpfs /tmp:rw,size=2g --tmpfs /run --tmpfs /run/lock)
|
||||
else
|
||||
# Podman's systemd mode mounts /run, /run/lock and the cgroup tree.
|
||||
# The pinned image is amd64-only.
|
||||
run_args+=(--systemd=always --arch amd64 --tmpfs /tmp:rw,size=2g)
|
||||
fi
|
||||
CONTAINER_STARTED=true
|
||||
if ! "$ENGINE" "${run_args[@]}" "$SYSTEMD_IMAGE" >/dev/null; then
|
||||
echo "::error::${ENGINE} could not start ${SYSTEMD_IMAGE}"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local i state
|
||||
for i in $(seq 1 45); do
|
||||
if ! "$ENGINE" inspect -f '{{.State.Running}}' "$CONTAINER" 2>/dev/null | grep -q true; then
|
||||
"$ENGINE" logs "$CONTAINER" || true
|
||||
echo "::error::systemd container exited during boot"
|
||||
return 1
|
||||
fi
|
||||
state=$("$ENGINE" exec "$CONTAINER" systemctl is-system-running 2>/dev/null || true)
|
||||
if [[ "$state" == "running" || "$state" == "degraded" ]]; then
|
||||
break
|
||||
fi
|
||||
if [[ "$i" -eq 45 ]]; then
|
||||
echo "::error::systemd did not become ready inside the container (state: ${state:-unknown})"
|
||||
return 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
BASELINE_FAILED_UNITS=$(cexec 'systemctl list-units --state=failed --no-legend --plain | awk "{print \$1}" | sort')
|
||||
log "systemd is up (pre-existing failed units: ${BASELINE_FAILED_UNITS:-none})"
|
||||
if ! cexec 'export DEBIAN_FRONTEND=noninteractive; apt-get update -qq >/dev/null && apt-get install -y -qq curl ca-certificates jq >/dev/null'; then
|
||||
echo "::error::could not install curl/jq inside the container"
|
||||
return 1
|
||||
fi
|
||||
# Keep a daily CI install out of real usage telemetry. A unit drop-in is
|
||||
# outside the installer's ownership and the data dir, so it does not
|
||||
# change what install, upgrade or rollback do.
|
||||
cexec 'mkdir -p /etc/systemd/system/pulse.service.d && printf "[Service]\nEnvironment=PULSE_TELEMETRY=false\n" > /etc/systemd/system/pulse.service.d/50-rehearsal-no-telemetry.conf'
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Assertions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
PHASE_VERSION=""
|
||||
PHASE_HEALTH=""
|
||||
PHASE_SETTINGS=""
|
||||
PHASE_DATADIR=""
|
||||
PHASE_UNITS=""
|
||||
|
||||
reset_phase_cells() {
|
||||
PHASE_VERSION="-"; PHASE_HEALTH="-"; PHASE_SETTINGS="-"; PHASE_DATADIR="-"; PHASE_UNITS="-"
|
||||
}
|
||||
|
||||
# Identity, health and systemd state for the expected release.
|
||||
assert_runtime() {
|
||||
local expected_tag="$1"
|
||||
local expected="${expected_tag#v}"
|
||||
|
||||
local active="" i
|
||||
for i in $(seq 1 60); do
|
||||
active=$(cexec 'systemctl is-active pulse' 2>/dev/null || true)
|
||||
[[ "$active" == "active" ]] && break
|
||||
sleep 2
|
||||
done
|
||||
|
||||
local health_body health_status
|
||||
health_body=$(cexec 'curl -fsS --retry 30 --retry-delay 2 --retry-connrefused --retry-all-errors "$API/api/health"' "API=${API}" 2>/dev/null || true)
|
||||
health_status=$(jq -r '.status // empty' <<<"$health_body" 2>/dev/null || true)
|
||||
if [[ "$health_status" == "healthy" ]]; then
|
||||
PHASE_HEALTH="healthy"
|
||||
else
|
||||
PHASE_HEALTH="${health_status:-no response}"
|
||||
note_failure "/api/health did not report healthy (got '${PHASE_HEALTH}')"
|
||||
fi
|
||||
|
||||
local version_body reported binary_version
|
||||
version_body=$(cexec 'curl -fsS "$API/api/version"' "API=${API}" 2>/dev/null || true)
|
||||
reported=$(jq -r '.version // empty' <<<"$version_body" 2>/dev/null || true)
|
||||
PHASE_VERSION="${reported:-none}"
|
||||
if [[ "${reported#v}" != "$expected" ]]; then
|
||||
note_failure "/api/version reported '${reported:-none}', expected ${expected_tag}"
|
||||
fi
|
||||
binary_version=$(cexec '/opt/pulse/bin/pulse --version 2>/dev/null | grep -oE "v[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9.]+)?" | head -1' 2>/dev/null || true)
|
||||
if [[ "${binary_version#v}" != "$expected" ]]; then
|
||||
note_failure "/opt/pulse/bin/pulse --version reported '${binary_version:-none}', expected ${expected_tag}"
|
||||
fi
|
||||
|
||||
local enabled failed_now new_failed
|
||||
enabled=$(cexec 'systemctl is-enabled pulse' 2>/dev/null || true)
|
||||
failed_now=$(cexec 'systemctl list-units --state=failed --no-legend --plain | awk "{print \$1}" | sort' 2>/dev/null || true)
|
||||
new_failed=$(comm -13 <(printf '%s\n' "$BASELINE_FAILED_UNITS" | sed '/^$/d') \
|
||||
<(printf '%s\n' "$failed_now" | sed '/^$/d') | tr '\n' ' ')
|
||||
PHASE_UNITS="pulse ${active:-unknown}/${enabled:-unknown}"
|
||||
if [[ "$active" != "active" ]]; then
|
||||
note_failure "pulse.service is '${active:-unknown}', expected active"
|
||||
fi
|
||||
if [[ "$enabled" != "enabled" ]]; then
|
||||
note_failure "pulse.service is '${enabled:-unknown}', expected enabled"
|
||||
fi
|
||||
local unit tolerated="" unexpected=""
|
||||
for unit in $new_failed; do
|
||||
if [[ " ${CONTAINER_TOLERATED_UNITS} " == *" ${unit} "* ]]; then
|
||||
tolerated="${tolerated:+${tolerated} }${unit}"
|
||||
else
|
||||
unexpected="${unexpected:+${unexpected} }${unit}"
|
||||
fi
|
||||
done
|
||||
if [[ -n "$unexpected" ]]; then
|
||||
PHASE_UNITS="${PHASE_UNITS}, failed: ${unexpected}"
|
||||
note_failure "systemd units failed during the rehearsal: ${unexpected}"
|
||||
fi
|
||||
if [[ -n "$tolerated" ]]; then
|
||||
PHASE_UNITS="${PHASE_UNITS}, container-only failures ignored: ${tolerated}"
|
||||
echo "::warning::container-environment units failed and were not counted: ${tolerated}"
|
||||
fi
|
||||
|
||||
if ! cexec 'systemctl show -p Environment --value pulse | grep -q "PULSE_TELEMETRY=false"' 2>/dev/null; then
|
||||
echo "::warning::the rehearsal telemetry opt-out drop-in is no longer applied to pulse.service"
|
||||
fi
|
||||
|
||||
# An unattended update firing mid-rehearsal would invalidate the version
|
||||
# assertions, so prove the timer-driven updater never started.
|
||||
local auto_update_started
|
||||
auto_update_started=$(cexec 'systemctl show -p ExecMainStartTimestampMonotonic --value pulse-update.service 2>/dev/null || echo 0' 2>/dev/null || echo 0)
|
||||
if [[ -n "$auto_update_started" && "$auto_update_started" != "0" ]]; then
|
||||
note_failure "pulse-update.service ran during the rehearsal; results are not attributable to the manual updater"
|
||||
fi
|
||||
}
|
||||
|
||||
api_status() {
|
||||
# api_status METHOD PATH [auth: none|token|basic] [body]
|
||||
cexec 'args=(-sS -o /dev/null -w "%{http_code}" -X "$METHOD")
|
||||
case "$AUTH" in
|
||||
token) args+=(-H "X-API-Token: $TOKEN") ;;
|
||||
basic) args+=(-u "$BASIC") ;;
|
||||
esac
|
||||
curl "${args[@]}" "$API$REQ_PATH"' \
|
||||
"API=${API}" "METHOD=$1" "REQ_PATH=$2" "AUTH=${3:-none}" \
|
||||
"TOKEN=${API_TOKEN}" "BASIC=${SEED_ADMIN_USER}:${ADMIN_PASSWORD}"
|
||||
}
|
||||
|
||||
api_json() {
|
||||
# api_json METHOD PATH [body]; authenticates with the seeded API token.
|
||||
cexec 'if [[ -n "$BODY" ]]; then
|
||||
curl -fsS -X "$METHOD" -H "X-API-Token: $TOKEN" -H "Content-Type: application/json" --data "$BODY" "$API$REQ_PATH"
|
||||
else
|
||||
curl -fsS -X "$METHOD" -H "X-API-Token: $TOKEN" "$API$REQ_PATH"
|
||||
fi' "API=${API}" "METHOD=$1" "REQ_PATH=$2" "BODY=${3:-}" "TOKEN=${API_TOKEN}"
|
||||
}
|
||||
|
||||
# Canonical, comparable view of the seeded settings as the API returns them.
|
||||
settings_snapshot() {
|
||||
local webhooks nodes security unauth token basic
|
||||
webhooks=$(api_json GET /api/notifications/webhooks) || return 1
|
||||
nodes=$(api_json GET /api/config/nodes) || return 1
|
||||
security=$(cexec 'curl -fsS "$API/api/security/status"' "API=${API}") || return 1
|
||||
unauth=$(api_status GET /api/config/nodes none)
|
||||
token=$(api_status GET /api/config/nodes token)
|
||||
basic=$(api_status GET /api/config/nodes basic)
|
||||
jq -n -S \
|
||||
--argjson webhooks "$webhooks" \
|
||||
--argjson nodes "$nodes" \
|
||||
--argjson security "$security" \
|
||||
--arg unauth "$unauth" --arg token "$token" --arg basic "$basic" \
|
||||
--arg webhook_name "$SEED_WEBHOOK_NAME" --arg node_name "$SEED_NODE_NAME" '
|
||||
{
|
||||
auth: {
|
||||
requiresAuth: ($security.requiresAuth // null),
|
||||
unauthenticated: $unauth,
|
||||
api_token: $token,
|
||||
password: $basic
|
||||
},
|
||||
webhook: ([$webhooks | .. | objects | select(.name? == $webhook_name)
|
||||
| {id, name, url, method, service, enabled,
|
||||
header_keys: ((.headers // {}) | keys)}] | first),
|
||||
node: ([$nodes | .. | objects | select(.name? == $node_name)
|
||||
| {name, type, host, tokenName, hasToken, hasPassword, verifySSL}] | first)
|
||||
}'
|
||||
}
|
||||
|
||||
# Expected shape of a healthy snapshot (independent of the baseline, so a
|
||||
# baseline that silently lost data cannot make later phases pass).
|
||||
check_snapshot_shape() {
|
||||
local snapshot="$1"
|
||||
local problems
|
||||
problems=$(jq -r \
|
||||
--arg webhook_name "$SEED_WEBHOOK_NAME" --arg webhook_url "$SEED_WEBHOOK_URL" \
|
||||
--arg node_name "$SEED_NODE_NAME" --arg node_host "$SEED_NODE_HOST" \
|
||||
--arg node_token_name "$SEED_NODE_TOKEN_NAME" --arg header "$SEED_WEBHOOK_HEADER" '
|
||||
[ (if .auth.requiresAuth != true then "security status does not require auth" else empty end),
|
||||
(if .auth.unauthenticated != "401" then "unauthenticated request returned \(.auth.unauthenticated), expected 401" else empty end),
|
||||
(if .auth.api_token != "200" then "API token request returned \(.auth.api_token), expected 200" else empty end),
|
||||
(if .auth.password != "200" then "password request returned \(.auth.password), expected 200" else empty end),
|
||||
(if .webhook == null then "seeded webhook \($webhook_name) missing" else empty end),
|
||||
(if .webhook != null and .webhook.url != $webhook_url then "seeded webhook URL changed to \(.webhook.url)" else empty end),
|
||||
(if .webhook != null and .webhook.method != "POST" then "seeded webhook method changed to \(.webhook.method)" else empty end),
|
||||
(if .webhook != null and .webhook.service != "generic" then "seeded webhook service changed to \(.webhook.service)" else empty end),
|
||||
(if .webhook != null and .webhook.enabled != false then "seeded webhook enabled state changed to \(.webhook.enabled)" else empty end),
|
||||
(if .webhook != null and ((.webhook.id // "") == "") then "seeded webhook lost its id" else empty end),
|
||||
(if .webhook != null and .webhook.header_keys != [$header] then "seeded webhook headers changed to \(.webhook.header_keys)" else empty end),
|
||||
(if .node == null then "seeded node \($node_name) missing" else empty end),
|
||||
(if .node != null and .node.type != "pve" then "seeded node type changed to \(.node.type)" else empty end),
|
||||
(if .node != null and .node.host != $node_host then "seeded node host changed to \(.node.host)" else empty end),
|
||||
(if .node != null and .node.tokenName != $node_token_name then "seeded node token name changed to \(.node.tokenName)" else empty end),
|
||||
(if .node != null and .node.hasToken != true then "seeded node lost its API token secret" else empty end),
|
||||
(if .node != null and .node.hasPassword != false then "seeded node gained a password" else empty end),
|
||||
(if .node != null and .node.verifySSL != false then "seeded node verifySSL changed to \(.node.verifySSL)" else empty end)
|
||||
] | .[]' <<<"$snapshot") || problems="snapshot is not valid JSON"
|
||||
if [[ -n "$problems" ]]; then
|
||||
while IFS= read -r problem; do
|
||||
note_failure "settings: ${problem}"
|
||||
done <<<"$problems"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
datadir_manifest() {
|
||||
# path<TAB>sha256 for every regular file, excluding SQLite sidecars and
|
||||
# temp files whose presence depends on shutdown timing.
|
||||
cexec 'cd "$DATA_DIR" && find . -type f ! -name "*-wal" ! -name "*-shm" ! -name "*.tmp" ! -name "*.lock" -printf "%P\n" | LC_ALL=C sort | while IFS= read -r f; do printf "%s\t%s\n" "$f" "$(sha256sum "$f" | cut -d" " -f1)"; done' \
|
||||
"DATA_DIR=${DATA_DIR}"
|
||||
}
|
||||
|
||||
# Encrypted stores holding the seeded secrets (node token value, webhook
|
||||
# header value). The API only exposes hasToken and redacted header values, so
|
||||
# byte identity of these files under an unchanged key is the proof that the
|
||||
# exact secrets survived. A version that legitimately rewrites them fails the
|
||||
# rehearsal loudly rather than passing unproven.
|
||||
SECRET_STORES=".encryption.key nodes.enc webhooks.enc"
|
||||
|
||||
# compare_datadir BASELINE CURRENT LABEL: manifest comparison (pure, testable).
|
||||
compare_datadir() {
|
||||
local baseline="$1" current="$2" label="$3"
|
||||
local missing changed added store before after
|
||||
missing=$(comm -23 <(cut -f1 "$baseline") <(cut -f1 "$current") | sed '/^$/d')
|
||||
added=$(comm -13 <(cut -f1 "$baseline") <(cut -f1 "$current") | sed '/^$/d' | wc -l | tr -d ' ')
|
||||
changed=$(join -t $'\t' "$baseline" "$current" | awk -F'\t' '$2 != $3' | wc -l | tr -d ' ')
|
||||
|
||||
PHASE_DATADIR="$(wc -l < "$current" | tr -d ' ') files, ${changed} changed, ${added} new"
|
||||
log "${DATA_DIR} after ${label}: ${PHASE_DATADIR}"
|
||||
if [[ -n "$missing" ]]; then
|
||||
PHASE_DATADIR="${PHASE_DATADIR}, $(wc -l <<<"$missing" | tr -d ' ') missing"
|
||||
note_failure "files present before the upgrade are gone from ${DATA_DIR}: $(tr '\n' ' ' <<<"$missing")"
|
||||
fi
|
||||
for store in $SECRET_STORES; do
|
||||
before=$(awk -F'\t' -v f="$store" '$1 == f {print $2}' "$baseline")
|
||||
after=$(awk -F'\t' -v f="$store" '$1 == f {print $2}' "$current")
|
||||
if [[ -z "$before" || "$before" != "$after" ]]; then
|
||||
note_failure "${DATA_DIR}/${store} is missing or was rewritten; the seeded secrets are no longer provably intact"
|
||||
fi
|
||||
done
|
||||
if [[ "$changed" -gt 0 ]]; then
|
||||
log "files rewritten in place (informational unless listed above):"
|
||||
join -t $'\t' "$baseline" "$current" | awk -F'\t' '$2 != $3 {print " " $1}'
|
||||
fi
|
||||
}
|
||||
|
||||
check_datadir() {
|
||||
local baseline="${WORK_DIR}/state/datadir.baseline.tsv"
|
||||
local current="${WORK_DIR}/state/datadir.$1.tsv"
|
||||
datadir_manifest > "$current" || { note_failure "could not list ${DATA_DIR}"; PHASE_DATADIR="unreadable"; return 1; }
|
||||
compare_datadir "$baseline" "$current" "$1"
|
||||
}
|
||||
|
||||
check_settings() {
|
||||
local snapshot
|
||||
if ! snapshot=$(settings_snapshot); then
|
||||
note_failure "settings could not be read back through the API"
|
||||
PHASE_SETTINGS="unreadable"
|
||||
return 1
|
||||
fi
|
||||
printf '%s\n' "$snapshot" > "${WORK_DIR}/state/settings.$1.json"
|
||||
local ok=true
|
||||
check_snapshot_shape "$snapshot" || ok=false
|
||||
if ! diff -u "${WORK_DIR}/state/settings.baseline.json" "${WORK_DIR}/state/settings.$1.json"; then
|
||||
note_failure "seeded settings read back differently than before the upgrade (diff above)"
|
||||
ok=false
|
||||
fi
|
||||
if [[ "$ok" == "true" ]]; then
|
||||
PHASE_SETTINGS="auth, webhook, node intact"
|
||||
else
|
||||
PHASE_SETTINGS="drift"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phases
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
phase_install() {
|
||||
reset_phase_cells
|
||||
log "Phase 1: install ${FROM_TAG} with its published install.sh --version"
|
||||
# No TTY: install.sh's prompts take their defaults, as for curl | bash.
|
||||
if ! cexec 'bash /rehearsal/install.sh --version "$FROM_TAG"' "FROM_TAG=${FROM_TAG}" \
|
||||
2>&1 | tee "${WORK_DIR}/state/install-${FROM_TAG}.log"; then
|
||||
note_failure "install.sh --version ${FROM_TAG} exited non-zero"
|
||||
fi
|
||||
assert_runtime "$FROM_TAG"
|
||||
record_phase "1. install" "$FROM_TAG" "$PHASE_VERSION" "$PHASE_HEALTH" "-" "-" "$PHASE_UNITS"
|
||||
}
|
||||
|
||||
phase_seed() {
|
||||
reset_phase_cells
|
||||
log "Phase 2: seed first-run security, a webhook and a PVE node through the API"
|
||||
local token_output setup_token setup_response
|
||||
token_output=$(cexec 'runuser -u pulse -- env PULSE_DATA_DIR="$DATA_DIR" /opt/pulse/bin/pulse bootstrap-token' "DATA_DIR=${DATA_DIR}" 2>&1 || true)
|
||||
setup_token=$(grep -oE 'Token: [^[:space:]]+' <<<"$token_output" | head -1 | cut -d' ' -f2)
|
||||
if [[ -z "$setup_token" ]]; then
|
||||
note_failure "pulse bootstrap-token did not print a setup token"
|
||||
record_phase "2. seed" "$FROM_TAG" "-" "-" "-" "-" "-"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local setup_body
|
||||
setup_body=$(jq -cn --arg u "$SEED_ADMIN_USER" --arg p "$ADMIN_PASSWORD" --arg t "$API_TOKEN" \
|
||||
'{username: $u, password: $p, apiToken: $t}')
|
||||
if ! setup_response=$(cexec 'curl -fsS -X POST -H "Content-Type: application/json" -H "X-Setup-Token: $SETUP_TOKEN" --data "$BODY" "$API/api/security/quick-setup"' \
|
||||
"API=${API}" "SETUP_TOKEN=${setup_token}" "BODY=${setup_body}") \
|
||||
|| [[ "$(jq -r '.success // false' <<<"$setup_response" 2>/dev/null)" != "true" ]]; then
|
||||
note_failure "first-run security setup failed: ${setup_response:-no response}"
|
||||
fi
|
||||
|
||||
local webhook_body node_body
|
||||
webhook_body=$(jq -cn --arg n "$SEED_WEBHOOK_NAME" --arg u "$SEED_WEBHOOK_URL" \
|
||||
--arg hk "$SEED_WEBHOOK_HEADER" --arg hv "$NODE_TOKEN_VALUE" \
|
||||
'{name: $n, url: $u, method: "POST", service: "generic", enabled: false,
|
||||
headers: {($hk): $hv}}')
|
||||
if ! api_json POST /api/notifications/webhooks "$webhook_body" >/dev/null; then
|
||||
note_failure "creating the seed webhook failed"
|
||||
fi
|
||||
node_body=$(jq -cn --arg n "$SEED_NODE_NAME" --arg h "$SEED_NODE_HOST" --arg v "$NODE_TOKEN_VALUE" \
|
||||
--arg tn "$SEED_NODE_TOKEN_NAME" \
|
||||
'{type: "pve", name: $n, host: $h, tokenName: $tn, tokenValue: $v, verifySSL: false}')
|
||||
if ! api_json POST /api/config/nodes "$node_body" >/dev/null; then
|
||||
note_failure "creating the seed PVE node failed"
|
||||
fi
|
||||
|
||||
local snapshot=""
|
||||
if snapshot=$(settings_snapshot) && check_snapshot_shape "$snapshot"; then
|
||||
printf '%s\n' "$snapshot" > "${WORK_DIR}/state/settings.baseline.json"
|
||||
PHASE_SETTINGS="auth, webhook, node seeded"
|
||||
log "Seeded settings as read back through the API:"
|
||||
printf '%s\n' "$snapshot"
|
||||
else
|
||||
if [[ -n "$snapshot" ]]; then
|
||||
printf '%s\n' "$snapshot"
|
||||
fi
|
||||
note_failure "seeded settings did not read back through the API"
|
||||
PHASE_SETTINGS="not readable"
|
||||
fi
|
||||
|
||||
# Persistence is asynchronous for some stores; let writes settle before
|
||||
# the data dir becomes the upgrade baseline.
|
||||
sleep 5
|
||||
if datadir_manifest > "${WORK_DIR}/state/datadir.baseline.tsv" \
|
||||
&& grep -q $'^\\.encryption\\.key\t' "${WORK_DIR}/state/datadir.baseline.tsv"; then
|
||||
PHASE_DATADIR="$(wc -l < "${WORK_DIR}/state/datadir.baseline.tsv" | tr -d ' ') files recorded"
|
||||
log "${DATA_DIR} baseline:"
|
||||
cut -f1 "${WORK_DIR}/state/datadir.baseline.tsv" | sed 's/^/ /'
|
||||
else
|
||||
note_failure "could not record ${DATA_DIR} (or it has no .encryption.key)"
|
||||
PHASE_DATADIR="not recorded"
|
||||
fi
|
||||
record_phase "2. seed" "$FROM_TAG" "-" "-" "$PHASE_SETTINGS" "$PHASE_DATADIR" "-"
|
||||
}
|
||||
|
||||
run_updater() {
|
||||
local target="$1" label="$2"
|
||||
# The helper must be the one the Pulse server installer wrote, not a
|
||||
# community-scripts updater that ignores --version.
|
||||
if ! cexec 'test -x /bin/update && grep -q "^# Pulse update command" /bin/update'; then
|
||||
note_failure "/bin/update is missing or is not the Pulse server installer's update helper"
|
||||
return 1
|
||||
fi
|
||||
# /bin/update downloads the latest published install.sh, verifies it with
|
||||
# its embedded signer key and runs it with the given arguments.
|
||||
if ! cexec '/bin/update --version "$TARGET"' "TARGET=${target}" \
|
||||
2>&1 | tee "${WORK_DIR}/state/${label}-${target}.log"; then
|
||||
note_failure "/bin/update --version ${target} exited non-zero"
|
||||
fi
|
||||
}
|
||||
|
||||
phase_upgrade() {
|
||||
reset_phase_cells
|
||||
log "Phase 3: upgrade with /bin/update --version ${TO_TAG}"
|
||||
run_updater "$TO_TAG" upgrade || true
|
||||
assert_runtime "$TO_TAG"
|
||||
check_settings upgrade || true
|
||||
check_datadir upgrade || true
|
||||
record_phase "3. upgrade (/bin/update)" "$TO_TAG" "$PHASE_VERSION" "$PHASE_HEALTH" "$PHASE_SETTINGS" "$PHASE_DATADIR" "$PHASE_UNITS"
|
||||
}
|
||||
|
||||
phase_rollback() {
|
||||
reset_phase_cells
|
||||
log "Phase 4: roll back with /bin/update --version ${FROM_TAG}"
|
||||
run_updater "$FROM_TAG" rollback || true
|
||||
assert_runtime "$FROM_TAG"
|
||||
check_settings rollback || true
|
||||
check_datadir rollback || true
|
||||
record_phase "4. rollback (/bin/update)" "$FROM_TAG" "$PHASE_VERSION" "$PHASE_HEALTH" "$PHASE_SETTINGS" "$PHASE_DATADIR" "$PHASE_UNITS"
|
||||
}
|
||||
|
||||
main() {
|
||||
log "Rehearsing ${FROM_TAG} -> ${TO_TAG} -> ${FROM_TAG} from ${REPO} with ${ENGINE}"
|
||||
if ! fetch_and_verify_installer "$FROM_TAG"; then
|
||||
CURRENT_FAILURE="published install.sh for ${FROM_TAG} failed verification"
|
||||
record_phase "0. verify installer" "$FROM_TAG" "-" "-" "-" "-" "-" || abort_run
|
||||
fi
|
||||
if ! start_container; then
|
||||
CURRENT_FAILURE="systemd container did not start"
|
||||
record_phase "0. systemd host" "-" "-" "-" "-" "-" "-" || abort_run
|
||||
fi
|
||||
|
||||
phase_install || abort_run
|
||||
phase_seed || abort_run
|
||||
# Rollback runs even when the upgrade fails: it is the recovery path a
|
||||
# user reaches for in exactly that situation.
|
||||
phase_upgrade || true
|
||||
phase_rollback || true
|
||||
|
||||
write_summary
|
||||
return "$OVERALL_STATUS"
|
||||
}
|
||||
|
||||
if [[ -n "$CHECK_SNAPSHOT" ]]; then
|
||||
check_snapshot_shape "$(cat "$CHECK_SNAPSHOT")"
|
||||
exit $?
|
||||
fi
|
||||
if [[ ${#COMPARE_DATADIR[@]} -gt 0 ]]; then
|
||||
compare_datadir "${COMPARE_DATADIR[0]}" "${COMPARE_DATADIR[1]}" self-test
|
||||
[[ -z "$CURRENT_FAILURE" ]]
|
||||
exit $?
|
||||
fi
|
||||
|
||||
main
|
||||
+184
@@ -0,0 +1,184 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Choose the published release pair for the release lifecycle rehearsal.
|
||||
|
||||
The rehearsal installs FROM, upgrades to TO and rolls back to FROM. Both tags
|
||||
must name published (non-draft) Pulse server releases and TO must be strictly
|
||||
newer than FROM under SemVer precedence, so a shadow run can never report a
|
||||
same-version "upgrade" as proof.
|
||||
|
||||
Defaults:
|
||||
FROM = the release GitHub advertises as latest (must be stable).
|
||||
TO = the newest published release or prerelease newer than FROM.
|
||||
When FROM was defaulted and nothing newer exists yet (the common case right
|
||||
after a stable release), the pair becomes previous stable -> latest stable,
|
||||
which is the upgrade users are taking at that moment.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import functools
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
TAG_RE = re.compile(
|
||||
r"^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)"
|
||||
r"(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?$"
|
||||
)
|
||||
|
||||
|
||||
class ResolutionError(ValueError):
|
||||
"""The requested or default release pair cannot be rehearsed."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Selection:
|
||||
from_tag: str
|
||||
to_tag: str
|
||||
reason: str
|
||||
|
||||
|
||||
def parse_tag(tag: str) -> tuple[tuple[int, int, int], tuple[str, ...]]:
|
||||
match = TAG_RE.fullmatch(tag)
|
||||
if not match:
|
||||
raise ResolutionError(f"not a Pulse release tag: {tag!r}")
|
||||
core = (int(match.group(1)), int(match.group(2)), int(match.group(3)))
|
||||
pre = tuple(match.group(4).split(".")) if match.group(4) else ()
|
||||
return core, pre
|
||||
|
||||
|
||||
def is_stable(tag: str) -> bool:
|
||||
return not parse_tag(tag)[1]
|
||||
|
||||
|
||||
def _compare_identifiers(left: tuple[str, ...], right: tuple[str, ...]) -> int:
|
||||
# SemVer 2.0.0 section 11: a release outranks any prerelease of the same
|
||||
# core; numeric identifiers compare numerically and rank below
|
||||
# alphanumeric ones; a longer identifier list wins when all shared
|
||||
# identifiers are equal.
|
||||
if not left and not right:
|
||||
return 0
|
||||
if not left:
|
||||
return 1
|
||||
if not right:
|
||||
return -1
|
||||
for a, b in zip(left, right):
|
||||
if a == b:
|
||||
continue
|
||||
a_num, b_num = a.isdigit(), b.isdigit()
|
||||
if a_num and b_num:
|
||||
return -1 if int(a) < int(b) else 1
|
||||
if a_num != b_num:
|
||||
return -1 if a_num else 1
|
||||
return -1 if a < b else 1
|
||||
if len(left) == len(right):
|
||||
return 0
|
||||
return -1 if len(left) < len(right) else 1
|
||||
|
||||
|
||||
def compare_tags(left: str, right: str) -> int:
|
||||
left_core, left_pre = parse_tag(left)
|
||||
right_core, right_pre = parse_tag(right)
|
||||
if left_core != right_core:
|
||||
return -1 if left_core < right_core else 1
|
||||
return _compare_identifiers(left_pre, right_pre)
|
||||
|
||||
|
||||
def published_tags(releases: list[dict]) -> list[str]:
|
||||
tags: set[str] = set()
|
||||
for release in releases:
|
||||
if not isinstance(release, dict) or release.get("draft") is not False:
|
||||
continue
|
||||
tag = release.get("tag_name")
|
||||
if isinstance(tag, str) and TAG_RE.fullmatch(tag):
|
||||
tags.add(tag)
|
||||
return sorted(tags, key=functools.cmp_to_key(compare_tags))
|
||||
|
||||
|
||||
def resolve(
|
||||
releases: list[dict],
|
||||
latest_tag: str,
|
||||
requested_from: str = "",
|
||||
requested_to: str = "",
|
||||
) -> Selection:
|
||||
tags = published_tags(releases)
|
||||
published = set(tags)
|
||||
|
||||
for label, value in (("from_version", requested_from), ("to_version", requested_to)):
|
||||
if value:
|
||||
parse_tag(value)
|
||||
if value not in published:
|
||||
raise ResolutionError(f"{label} {value} is not a published Pulse release")
|
||||
|
||||
if requested_from:
|
||||
from_tag = requested_from
|
||||
else:
|
||||
if not latest_tag or latest_tag not in published or not is_stable(latest_tag):
|
||||
raise ResolutionError(
|
||||
f"latest advertised release {latest_tag!r} is not a published stable Pulse release"
|
||||
)
|
||||
from_tag = latest_tag
|
||||
|
||||
if requested_to:
|
||||
if compare_tags(requested_to, from_tag) <= 0:
|
||||
raise ResolutionError(
|
||||
f"to_version {requested_to} must be newer than from_version {from_tag}"
|
||||
)
|
||||
return Selection(from_tag, requested_to, "requested pair")
|
||||
|
||||
newer = [tag for tag in tags if compare_tags(tag, from_tag) > 0]
|
||||
if newer:
|
||||
reason = "requested from_version" if requested_from else "latest stable"
|
||||
return Selection(from_tag, newer[-1], f"{reason} -> newest published release")
|
||||
|
||||
if requested_from:
|
||||
raise ResolutionError(f"no published release is newer than from_version {from_tag}")
|
||||
|
||||
older_stable = [
|
||||
tag for tag in tags if is_stable(tag) and compare_tags(tag, from_tag) < 0
|
||||
]
|
||||
if not older_stable:
|
||||
raise ResolutionError(f"no stable release precedes latest stable {from_tag}")
|
||||
return Selection(
|
||||
older_stable[-1],
|
||||
from_tag,
|
||||
"nothing newer than latest stable is published; previous stable -> latest stable",
|
||||
)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
parser.add_argument("--releases-json", required=True, type=Path,
|
||||
help="JSON array of GitHub release objects (all pages)")
|
||||
parser.add_argument("--latest-tag", default="",
|
||||
help="tag_name of GET /repos/{repo}/releases/latest")
|
||||
parser.add_argument("--from", dest="from_version", default="")
|
||||
parser.add_argument("--to", dest="to_version", default="")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
try:
|
||||
releases = json.loads(args.releases_json.read_text(encoding="utf-8"))
|
||||
if not isinstance(releases, list):
|
||||
raise ResolutionError("releases JSON must be an array")
|
||||
selection = resolve(
|
||||
releases,
|
||||
args.latest_tag.strip(),
|
||||
args.from_version.strip(),
|
||||
args.to_version.strip(),
|
||||
)
|
||||
except (ResolutionError, json.JSONDecodeError, OSError) as error:
|
||||
print(f"release lifecycle rehearsal: {error}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print(f"from_tag={selection.from_tag}")
|
||||
print(f"to_tag={selection.to_tag}")
|
||||
print(f"reason={selection.reason}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,336 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Contract tests for the shadow release lifecycle rehearsal."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
RESOLVER_PATH = REPO_ROOT / "scripts" / "release_lifecycle_rehearsal_versions.py"
|
||||
HARNESS_PATH = REPO_ROOT / "scripts" / "release_lifecycle_rehearsal.sh"
|
||||
WORKFLOW_PATH = REPO_ROOT / ".github" / "workflows" / "release-lifecycle-rehearsal.yml"
|
||||
SMOKE_BODY_PATH = REPO_ROOT / ".github" / "workflows" / "install-sh-smoke-body.yml"
|
||||
|
||||
SPEC = importlib.util.spec_from_file_location("release_lifecycle_rehearsal_versions", RESOLVER_PATH)
|
||||
assert SPEC and SPEC.loader
|
||||
versions = importlib.util.module_from_spec(SPEC)
|
||||
sys.modules[SPEC.name] = versions
|
||||
SPEC.loader.exec_module(versions)
|
||||
|
||||
|
||||
def release(tag: str, *, draft: bool = False, prerelease: bool | None = None) -> dict:
|
||||
return {
|
||||
"tag_name": tag,
|
||||
"draft": draft,
|
||||
"prerelease": ("-" in tag) if prerelease is None else prerelease,
|
||||
}
|
||||
|
||||
|
||||
RELEASES = [
|
||||
release("v6.4.5"),
|
||||
release("helm-chart-6.4.5"),
|
||||
release("v6.4.5-rc.5"),
|
||||
release("v6.4.5-rc.10"),
|
||||
release("v6.4.5-beta.3"),
|
||||
release("v6.4.1"),
|
||||
release("v6.4.0"),
|
||||
release("v6.4.0-rc.9"),
|
||||
release("v6.4.6-rc.1", draft=True),
|
||||
]
|
||||
|
||||
|
||||
class SemverOrderingTest(unittest.TestCase):
|
||||
def test_release_outranks_its_prereleases(self) -> None:
|
||||
self.assertGreater(versions.compare_tags("v6.4.5", "v6.4.5-rc.10"), 0)
|
||||
|
||||
def test_numeric_identifiers_compare_numerically(self) -> None:
|
||||
self.assertGreater(versions.compare_tags("v6.4.5-rc.10", "v6.4.5-rc.5"), 0)
|
||||
|
||||
def test_alphanumeric_identifiers_compare_lexically(self) -> None:
|
||||
self.assertGreater(versions.compare_tags("v6.4.5-rc.1", "v6.4.5-beta.3"), 0)
|
||||
|
||||
def test_rejects_non_release_tags(self) -> None:
|
||||
for tag in ("6.4.5", "helm-chart-6.4.5", "v6.4", "v6.4.5;id", "v06.4.5"):
|
||||
with self.subTest(tag=tag), self.assertRaises(versions.ResolutionError):
|
||||
versions.parse_tag(tag)
|
||||
|
||||
|
||||
class ResolvePairTest(unittest.TestCase):
|
||||
def test_defaults_to_newer_prerelease_after_latest_stable(self) -> None:
|
||||
releases = RELEASES + [release("v6.4.6-rc.2"), release("v6.4.6-beta.1")]
|
||||
selection = versions.resolve(releases, "v6.4.5")
|
||||
self.assertEqual((selection.from_tag, selection.to_tag), ("v6.4.5", "v6.4.6-rc.2"))
|
||||
|
||||
def test_defaults_fall_back_to_previous_stable_when_nothing_is_newer(self) -> None:
|
||||
selection = versions.resolve(RELEASES, "v6.4.5")
|
||||
self.assertEqual((selection.from_tag, selection.to_tag), ("v6.4.1", "v6.4.5"))
|
||||
self.assertIn("previous stable", selection.reason)
|
||||
|
||||
def test_draft_releases_are_never_targets(self) -> None:
|
||||
selection = versions.resolve(RELEASES, "v6.4.5")
|
||||
self.assertNotEqual(selection.to_tag, "v6.4.6-rc.1")
|
||||
with self.assertRaises(versions.ResolutionError):
|
||||
versions.resolve(RELEASES, "v6.4.5", "v6.4.5", "v6.4.6-rc.1")
|
||||
|
||||
def test_explicit_from_selects_newest_newer_release(self) -> None:
|
||||
selection = versions.resolve(RELEASES, "v6.4.5", "v6.4.0")
|
||||
self.assertEqual((selection.from_tag, selection.to_tag), ("v6.4.0", "v6.4.5"))
|
||||
|
||||
def test_explicit_from_without_newer_release_fails_instead_of_passing(self) -> None:
|
||||
with self.assertRaises(versions.ResolutionError):
|
||||
versions.resolve(RELEASES, "v6.4.5", "v6.4.5")
|
||||
|
||||
def test_explicit_pair_must_move_forward(self) -> None:
|
||||
for from_tag, to_tag in (("v6.4.5", "v6.4.1"), ("v6.4.5", "v6.4.5")):
|
||||
with self.subTest(pair=(from_tag, to_tag)), self.assertRaises(versions.ResolutionError):
|
||||
versions.resolve(RELEASES, "v6.4.5", from_tag, to_tag)
|
||||
|
||||
def test_unpublished_or_malformed_requests_fail(self) -> None:
|
||||
for from_tag, to_tag in (("v6.3.0", ""), ("v6.4.1", "v6.9.9"), ("v6.4.1 ; id", "")):
|
||||
with self.subTest(pair=(from_tag, to_tag)), self.assertRaises(versions.ResolutionError):
|
||||
versions.resolve(RELEASES, "v6.4.5", from_tag, to_tag)
|
||||
|
||||
def test_latest_must_be_published_stable(self) -> None:
|
||||
for latest in ("", "v6.4.5-rc.5", "v9.9.9"):
|
||||
with self.subTest(latest=latest), self.assertRaises(versions.ResolutionError):
|
||||
versions.resolve(RELEASES, latest)
|
||||
|
||||
def test_cli_emits_key_value_pair(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = Path(tmp) / "releases.json"
|
||||
path.write_text(json.dumps(RELEASES), encoding="utf-8")
|
||||
result = subprocess.run(
|
||||
[sys.executable, str(RESOLVER_PATH), "--releases-json", str(path),
|
||||
"--latest-tag", "v6.4.5", "--from", "v6.4.0", "--to", "v6.4.1"],
|
||||
capture_output=True, text=True, check=False,
|
||||
)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertIn("from_tag=v6.4.0\n", result.stdout)
|
||||
self.assertIn("to_tag=v6.4.1\n", result.stdout)
|
||||
|
||||
|
||||
class HarnessContractTest(unittest.TestCase):
|
||||
harness = HARNESS_PATH.read_text(encoding="utf-8")
|
||||
|
||||
def test_upgrade_and_rollback_use_the_installed_update_helper(self) -> None:
|
||||
self.assertIn("cexec '/bin/update --version \"$TARGET\"'", self.harness)
|
||||
self.assertIn('run_updater "$TO_TAG" upgrade', self.harness)
|
||||
self.assertIn('run_updater "$FROM_TAG" rollback', self.harness)
|
||||
self.assertIn('grep -q "^# Pulse update command" /bin/update', self.harness)
|
||||
# The updater invocation carries no flags that would bypass the
|
||||
# behavior users get (archives, preflight skips, forced auto-update).
|
||||
updater_lines = [line for line in self.harness.splitlines() if "/bin/update --" in line]
|
||||
for line in updater_lines:
|
||||
for flag in ("--archive", "--skip-upgrade-preflight", "--disable-auto-updates"):
|
||||
self.assertNotIn(flag, line)
|
||||
|
||||
def test_initial_install_uses_signed_published_installer(self) -> None:
|
||||
for needle in (
|
||||
"releases/download/${tag}",
|
||||
"grep -oE 'ssh-ed25519 [A-Za-z0-9+/=]+ pulse-installer'",
|
||||
"-I pulse-installer",
|
||||
"-n pulse-install",
|
||||
"bash /rehearsal/install.sh --version \"$FROM_TAG\"",
|
||||
):
|
||||
self.assertIn(needle, self.harness)
|
||||
|
||||
def test_uses_the_same_pinned_systemd_image_as_install_smoke(self) -> None:
|
||||
digest = re.search(r"jrei/systemd-debian:12@sha256:[0-9a-f]{64}", self.harness)
|
||||
self.assertIsNotNone(digest)
|
||||
self.assertIn(digest.group(0), SMOKE_BODY_PATH.read_text(encoding="utf-8"))
|
||||
|
||||
def test_every_post_change_phase_checks_identity_health_settings_and_data(self) -> None:
|
||||
for phase in ("phase_upgrade", "phase_rollback"):
|
||||
body = self.harness.split(f"{phase}() {{", 1)[1].split("\n}\n", 1)[0]
|
||||
for check in ("assert_runtime", "check_settings", "check_datadir", "record_phase"):
|
||||
with self.subTest(phase=phase, check=check):
|
||||
self.assertIn(check, body)
|
||||
|
||||
def test_settings_are_checked_against_fixed_expectations_not_only_baseline(self) -> None:
|
||||
body = self.harness.split("check_settings() {", 1)[1].split("\n}\n", 1)[0]
|
||||
self.assertIn("check_snapshot_shape", body)
|
||||
self.assertIn("settings.baseline.json", body)
|
||||
|
||||
def test_harness_passes_shellcheck_syntax(self) -> None:
|
||||
result = subprocess.run(["bash", "-n", str(HARNESS_PATH)], capture_output=True, text=True)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
|
||||
def test_rejects_unsafe_tags_before_touching_a_container(self) -> None:
|
||||
for args in (["--from", "v6.4.1", "--to", "v6.4.1"],
|
||||
["--from", "v6.4.1;id", "--to", "v6.4.5"],
|
||||
["--from", "", "--to", "v6.4.5"]):
|
||||
with self.subTest(args=args):
|
||||
result = subprocess.run(
|
||||
["bash", str(HARNESS_PATH), *args],
|
||||
capture_output=True, text=True,
|
||||
env={"PATH": "/usr/bin:/bin", "PULSE_REHEARSAL_ENGINE": "none-such"},
|
||||
)
|
||||
self.assertEqual(result.returncode, 2, result.stdout + result.stderr)
|
||||
|
||||
|
||||
GOOD_SNAPSHOT = {
|
||||
"auth": {"api_token": "200", "password": "200", "requiresAuth": True, "unauthenticated": "401"},
|
||||
"node": {
|
||||
"hasPassword": False,
|
||||
"hasToken": True,
|
||||
"host": "https://192.168.77.10:8006",
|
||||
"name": "test-lifecycle-rehearsal",
|
||||
"tokenName": "root@pam!rehearsal",
|
||||
"type": "pve",
|
||||
"verifySSL": False,
|
||||
},
|
||||
"webhook": {
|
||||
"enabled": False,
|
||||
"header_keys": ["X-Rehearsal-Secret"],
|
||||
"id": "webhook-123",
|
||||
"method": "POST",
|
||||
"name": "lifecycle-rehearsal-webhook",
|
||||
"service": "generic",
|
||||
"url": "https://example.com/pulse-lifecycle-rehearsal",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def mutated(path: tuple[str, ...], value: object) -> dict:
|
||||
snapshot = json.loads(json.dumps(GOOD_SNAPSHOT))
|
||||
target = snapshot
|
||||
for key in path[:-1]:
|
||||
target = target[key]
|
||||
target[path[-1]] = value
|
||||
return snapshot
|
||||
|
||||
|
||||
class SnapshotExpectationTest(unittest.TestCase):
|
||||
"""Lost or altered settings must fail even when the baseline also lost them."""
|
||||
|
||||
def check(self, snapshot: object) -> subprocess.CompletedProcess:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = Path(tmp) / "snapshot.json"
|
||||
path.write_text(json.dumps(snapshot) if not isinstance(snapshot, str) else snapshot,
|
||||
encoding="utf-8")
|
||||
return subprocess.run(
|
||||
["bash", str(HARNESS_PATH), "--check-snapshot", str(path)],
|
||||
capture_output=True, text=True, check=False,
|
||||
env={"PATH": "/usr/bin:/bin:/usr/local/bin:/opt/homebrew/bin"},
|
||||
)
|
||||
|
||||
def test_intact_snapshot_passes(self) -> None:
|
||||
result = self.check(GOOD_SNAPSHOT)
|
||||
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
||||
|
||||
def test_lost_or_changed_settings_fail(self) -> None:
|
||||
cases = {
|
||||
"node token secret lost": (("node", "hasToken"), False),
|
||||
"node token name reset": (("node", "tokenName"), ""),
|
||||
"node TLS verification flipped": (("node", "verifySSL"), True),
|
||||
"node host rewritten": (("node", "host"), "https://192.168.77.11:8006"),
|
||||
"node missing": (("node",), None),
|
||||
"webhook missing": (("webhook",), None),
|
||||
"webhook header lost": (("webhook", "header_keys"), []),
|
||||
"webhook re-enabled": (("webhook", "enabled"), True),
|
||||
"webhook URL changed": (("webhook", "url"), "https://example.org/"),
|
||||
"auth no longer required": (("auth", "requiresAuth"), False),
|
||||
"anonymous access allowed": (("auth", "unauthenticated"), "200"),
|
||||
"API token rejected": (("auth", "api_token"), "401"),
|
||||
"password rejected": (("auth", "password"), "401"),
|
||||
}
|
||||
for label, (path, value) in cases.items():
|
||||
with self.subTest(case=label):
|
||||
result = self.check(mutated(path, value))
|
||||
self.assertEqual(result.returncode, 1, result.stdout + result.stderr)
|
||||
self.assertIn("::error::settings:", result.stdout)
|
||||
|
||||
def test_unparseable_snapshot_fails(self) -> None:
|
||||
self.assertEqual(self.check("not json").returncode, 1)
|
||||
|
||||
|
||||
BASELINE_MANIFEST = {
|
||||
".encryption.key": "a" * 64,
|
||||
".env": "b" * 64,
|
||||
"metrics.db": "c" * 64,
|
||||
"nodes.enc": "d" * 64,
|
||||
"system.json": "e" * 64,
|
||||
"webhooks.enc": "f" * 64,
|
||||
}
|
||||
|
||||
|
||||
class DataDirComparisonTest(unittest.TestCase):
|
||||
def compare(self, current: dict[str, str]) -> subprocess.CompletedProcess:
|
||||
def manifest(entries: dict[str, str]) -> str:
|
||||
return "".join(f"{path}\t{digest}\n" for path, digest in sorted(entries.items()))
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
baseline_path = Path(tmp) / "baseline.tsv"
|
||||
current_path = Path(tmp) / "current.tsv"
|
||||
baseline_path.write_text(manifest(BASELINE_MANIFEST), encoding="utf-8")
|
||||
current_path.write_text(manifest(current), encoding="utf-8")
|
||||
return subprocess.run(
|
||||
["bash", str(HARNESS_PATH), "--compare-datadir", str(baseline_path), str(current_path)],
|
||||
capture_output=True, text=True, check=False,
|
||||
env={"PATH": "/usr/bin:/bin:/usr/local/bin:/opt/homebrew/bin"},
|
||||
)
|
||||
|
||||
def test_rewritten_plain_files_and_new_files_pass(self) -> None:
|
||||
current = dict(BASELINE_MANIFEST, **{"system.json": "1" * 64, "metrics.db": "2" * 64,
|
||||
"new-store.db": "3" * 64})
|
||||
result = self.compare(current)
|
||||
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
||||
|
||||
def test_missing_files_and_rewritten_secret_stores_fail(self) -> None:
|
||||
cases = {
|
||||
"file deleted": {k: v for k, v in BASELINE_MANIFEST.items() if k != "metrics.db"},
|
||||
"encryption key replaced": dict(BASELINE_MANIFEST, **{".encryption.key": "0" * 64}),
|
||||
"node secrets rewritten": dict(BASELINE_MANIFEST, **{"nodes.enc": "0" * 64}),
|
||||
"webhook secrets rewritten": dict(BASELINE_MANIFEST, **{"webhooks.enc": "0" * 64}),
|
||||
"data dir emptied": {},
|
||||
}
|
||||
for label, current in cases.items():
|
||||
with self.subTest(case=label):
|
||||
result = self.compare(current)
|
||||
self.assertEqual(result.returncode, 1, result.stdout + result.stderr)
|
||||
self.assertIn("::error::", result.stdout)
|
||||
|
||||
|
||||
class WorkflowContractTest(unittest.TestCase):
|
||||
workflow = WORKFLOW_PATH.read_text(encoding="utf-8")
|
||||
|
||||
def test_triggers_are_dispatch_and_daily_schedule_only(self) -> None:
|
||||
self.assertIn("workflow_dispatch:", self.workflow)
|
||||
self.assertIn("from_version:", self.workflow)
|
||||
self.assertIn("to_version:", self.workflow)
|
||||
self.assertRegex(self.workflow, r"(?m)^ schedule:\n # [^\n]*\n(?: #[^\n]*\n)* - cron: '23 13 \* \* \*'$")
|
||||
for trigger in ("pull_request", "push:", "workflow_run", "workflow_call", "release:"):
|
||||
self.assertNotIn(trigger, self.workflow)
|
||||
|
||||
def test_read_only_hosted_and_report_only(self) -> None:
|
||||
self.assertRegex(self.workflow, r"(?m)^permissions:\n contents: read$")
|
||||
self.assertNotIn(": write", self.workflow)
|
||||
self.assertNotIn("secrets.", self.workflow)
|
||||
self.assertNotIn("self-hosted", self.workflow)
|
||||
self.assertIn("runs-on: ubuntu-24.04", self.workflow)
|
||||
self.assertIn("persist-credentials: false", self.workflow)
|
||||
self.assertNotIn("upload-artifact", self.workflow)
|
||||
|
||||
def test_dispatch_inputs_reach_shell_only_through_env(self) -> None:
|
||||
for line in self.workflow.splitlines():
|
||||
if "${{" in line and ("inputs." in line or "steps." in line):
|
||||
self.assertRegex(line.strip(), r"^[A-Z_]+: \$\{\{ [a-z_.]+ \}\}$")
|
||||
|
||||
def test_no_release_workflow_depends_on_the_rehearsal(self) -> None:
|
||||
for path in (REPO_ROOT / ".github" / "workflows").glob("*.yml"):
|
||||
if path == WORKFLOW_PATH:
|
||||
continue
|
||||
with self.subTest(workflow=path.name):
|
||||
self.assertNotIn("release-lifecycle-rehearsal", path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user