diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 4c398c5875..ffb2efe873 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -140,6 +140,17 @@ and verifies the live service health and exact version. The privileged systemd smoke environment is digest-pinned because a floating container image would otherwise be an unreviewed code path inside the release gate. +`release-lifecycle-rehearsal.yml` is a daily shadow rehearsal of the systemd +lifecycle between two published releases, in the same digest-pinned systemd +container. It installs the older release with its signature-verified +`install.sh --version`, seeds real settings through the API, upgrades with the +installed `/bin/update --version ` helper and rolls back with the documented +`/bin/update --version `, checking version, health, unit state, settings +and data-dir survival after each step. It is read-only, uploads nothing and gates +nothing; results are in the job log and step summary. Run +`scripts/release_lifecycle_rehearsal.sh --from --to ` to reproduce +it locally (`PULSE_REHEARSAL_ENGINE=podman` on hosts without Docker). + The shared `install-sh-smoke-body.yml` inherits its caller's token permissions; keep it free of workflow- or job-level permission overrides. Continuity calls that body directly with `contents: read`. The existing `install-sh-smoke.yml` diff --git a/.github/workflows/release-lifecycle-rehearsal.yml b/.github/workflows/release-lifecycle-rehearsal.yml new file mode 100644 index 0000000000..801c89e029 --- /dev/null +++ b/.github/workflows/release-lifecycle-rehearsal.yml @@ -0,0 +1,81 @@ +name: Release Lifecycle Rehearsal + +# Shadow rehearsal of the systemd lifecycle users run between two PUBLISHED +# releases: install FROM with its signed install.sh, seed real settings, upgrade +# with /bin/update --version TO, then roll back with the documented +# /bin/update --version FROM, asserting identity, health, unit state, settings +# and data-dir survival after each step. It reports only. No release job +# depends on it, and it uploads nothing. +on: + workflow_dispatch: + inputs: + from_version: + description: 'Published tag to install first (e.g. v6.4.1). Empty = latest stable.' + required: false + type: string + default: '' + to_version: + description: 'Published tag to upgrade to. Empty = newest published release or prerelease newer than from_version.' + required: false + type: string + default: '' + schedule: + # Daily, well outside the 02:00-06:00 UTC unattended-update window that an + # upgraded install may arm, so the timer cannot race the assertions. + - cron: '23 13 * * *' + +permissions: + contents: read + +concurrency: + group: release-lifecycle-rehearsal + cancel-in-progress: false + +jobs: + rehearse: + name: Install, upgrade and roll back on systemd + runs-on: ubuntu-24.04 + timeout-minutes: 45 + steps: + - name: Checkout rehearsal harness (for README key and scripts) + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Resolve published release pair + id: pair + env: + GH_TOKEN: ${{ github.token }} + REPOSITORY: ${{ github.repository }} + FROM_INPUT: ${{ inputs.from_version }} + TO_INPUT: ${{ inputs.to_version }} + run: | + set -euo pipefail + releases_json="$(mktemp)" + gh api --paginate "repos/${REPOSITORY}/releases?per_page=100" \ + | jq -s 'add // []' > "${releases_json}" + latest_tag="$(gh api "repos/${REPOSITORY}/releases/latest" --jq '.tag_name')" + resolved="$(python3 scripts/release_lifecycle_rehearsal_versions.py \ + --releases-json "${releases_json}" \ + --latest-tag "${latest_tag}" \ + --from "${FROM_INPUT}" \ + --to "${TO_INPUT}")" + rm -f "${releases_json}" + from_tag="$(sed -n 's/^from_tag=//p' <<<"${resolved}")" + to_tag="$(sed -n 's/^to_tag=//p' <<<"${resolved}")" + reason="$(sed -n 's/^reason=//p' <<<"${resolved}")" + python3 scripts/write_github_output.py from_tag "${from_tag}" + python3 scripts/write_github_output.py to_tag "${to_tag}" + echo "Rehearsing ${from_tag} -> ${to_tag} -> ${from_tag} (${reason})" + { + echo "Release pair: \`${from_tag}\` -> \`${to_tag}\` (${reason})." + echo + } >> "${GITHUB_STEP_SUMMARY}" + + - name: Rehearse install, upgrade and rollback + env: + FROM_TAG: ${{ steps.pair.outputs.from_tag }} + TO_TAG: ${{ steps.pair.outputs.to_tag }} + PULSE_REHEARSAL_REPO: ${{ github.repository }} + PULSE_REHEARSAL_ENGINE: docker + run: scripts/release_lifecycle_rehearsal.sh --from "${FROM_TAG}" --to "${TO_TAG}" diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index 82ca498eac..5cb31ea930 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -842,6 +842,9 @@ release-latency optimization. 98. `scripts/verify-github-release-integrity.sh` 99. `scripts/verify-release-container-images.sh` 100. `scripts/release_control/verify_release_container_images_test.py` +101. `.github/workflows/release-lifecycle-rehearsal.yml` +102. `scripts/release_lifecycle_rehearsal.sh` +103. `scripts/release_lifecycle_rehearsal_versions.py` ## Shared Boundaries @@ -6106,6 +6109,82 @@ the current assignment; a host dependency acquisition on the next launch is required before this contract's Go evidence can be produced. No passing Go suite, installed build or release acceptance is claimed here. +### Published-release lifecycle rehearsal (shadow) + +`.github/workflows/release-lifecycle-rehearsal.yml` rehearses the systemd +install, upgrade and rollback journey between two published releases. It is +the shadow first step toward making install, upgrade and rollback a publication +gate. It runs daily and on manual dispatch, holds only `contents: read`, runs +on the hosted `ubuntu-24.04` image, uses no secrets and uploads no artifacts. +No release workflow calls it or waits on it, so it reports without gating. + +`scripts/release_lifecycle_rehearsal_versions.py` chooses the pair from +published, non-draft release tags under SemVer precedence. The default FROM is +the release GitHub advertises as latest and must be stable. The default TO is +the newest published release or prerelease newer than FROM. When FROM was +defaulted and nothing newer is published, the pair is the previous stable to +the latest stable. An explicit pair must be published and move forward, so a +same-version or missing-target run fails instead of passing. + +`scripts/release_lifecycle_rehearsal.sh` runs every phase inside the same +digest-pinned `jrei/systemd-debian:12` container as the install smoke. + +1. Install FROM with that release's own `install.sh --version`, after the + asset verifies against the README-pinned `pulse-installer` key and passes + the server-installer identity checks. +2. Assert `/api/version` and `/opt/pulse/bin/pulse --version` both report + FROM, `/api/health` reports `healthy`, and `pulse.service` is active and + enabled with no newly failed unit. +3. Seed state through the real API. First-run security setup uses the + bootstrap token from `pulse bootstrap-token`, then a disabled webhook with + a secret header and a PVE node with fake token credentials are created. The + read-back must show that auth is required, that an unauthenticated request + gets 401, and that the API token and password both get 200. The webhook must + keep its id, URL, method, service, disabled state and header key. The node + must keep its host, token name, stored token secret (`hasToken`), no + password and `verifySSL=false`. The `/etc/pulse` file list and checksums + become the baseline. +4. Upgrade with the installed helper, `/bin/update --version TO`, after + checking that the helper is the Pulse server installer's. The helper + downloads the latest published `install.sh`, verifies it with its embedded + key and runs it. +5. After the upgrade, and again after the documented rollback + `/bin/update --version FROM`, assert the expected identity, health and unit + state. The settings read-back must equal the baseline and meet the fixed + expectations. No baseline file may be missing from `/etc/pulse`. + `.encryption.key`, `nodes.enc` and `webhooks.enc` must be byte-identical. + The API reports only `hasToken` and redacted header values, so byte + identity under an unchanged key is what proves the exact seeded secrets + survived. A release that legitimately rewrites those stores fails the + rehearsal instead of passing unproven. The API token and password checks + prove those exact credentials directly. + +Rollback still runs when the upgrade phase fails. A phase also fails if +`pulse-update.service` started during the run, because an unattended update +would make the version assertions unattributable. Three D-Bus-activated host +services that cannot run in the container (`systemd-hostnamed`, +`systemd-timedated` and `systemd-localed`) are reported as warnings rather +than failures. A unit drop-in sets `PULSE_TELEMETRY=false` so daily CI installs +stay out of usage telemetry. It lives outside the installer's files and the +data dir. + +Scope: only published assets and the amd64 systemd path are exercised. Docker, +Helm, Proxmox LXC creation, the in-app updater and a prerelease's own +`install.sh` (the helper always fetches the latest published installer) are +not covered. A passing run is not release admission. + +Verification: `scripts/tests/test_release_lifecycle_rehearsal.py` covers pair +resolution (SemVer ordering, draft exclusion, forward-only and published-only +pairs, the previous-stable fallback), the fixed settings expectations (each lost +or altered seeded field fails `--check-snapshot`, and a deleted file or a +rewritten secret store fails `--compare-datadir`), the harness contract (the real updater +invocation without bypass flags, the signed installer, the shared image digest, +and per-phase identity, settings and data-dir checks) and the workflow trust +shape. A local podman run (amd64 emulation) of `v6.4.1 -> v6.4.5 -> v6.4.1` +passed all four phases. An earlier run correctly failed phase 1 on a new failed +unit (`systemd-hostnamed`), which led to the container-only list above. The +hosted Docker run is not yet claimed. + ## Release-body updater ownership (30 September 2026) Each executable `/bin/update --version` example in a published body must carry diff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.json index feeb999fb2..b6cb3524f9 100644 --- a/docs/release-control/v6/internal/subsystems/registry.json +++ b/docs/release-control/v6/internal/subsystems/registry.json @@ -4536,6 +4536,7 @@ ".github/workflows/recover-release-activation.yml", ".github/workflows/release-convergence.yml", ".github/workflows/release-dry-run.yml", + ".github/workflows/release-lifecycle-rehearsal.yml", ".github/workflows/retry-release-convergence.yml", ".github/workflows/update-demo-server.yml", ".github/workflows/validate-release-assets.yml", @@ -4630,6 +4631,8 @@ "scripts/release_control/secure_runtime_source_manifest_v7.json", "scripts/release_control/validate_artifact_release_line.py", "scripts/release_ldflags.sh", + "scripts/release_lifecycle_rehearsal.sh", + "scripts/release_lifecycle_rehearsal_versions.py", "scripts/release_update_key.go", "scripts/run-release-backend-tests.sh", "scripts/run-release-preflight.sh", @@ -4817,6 +4820,21 @@ "scripts/release_control/verify_github_release_integrity_test.py" ] }, + { + "id": "release-lifecycle-rehearsal", + "label": "published-release install, upgrade and rollback rehearsal proof", + "match_prefixes": [], + "match_files": [ + ".github/workflows/release-lifecycle-rehearsal.yml", + "scripts/release_lifecycle_rehearsal.sh", + "scripts/release_lifecycle_rehearsal_versions.py" + ], + "allow_same_subsystem_tests": false, + "test_prefixes": [], + "exact_files": [ + "scripts/tests/test_release_lifecycle_rehearsal.py" + ] + }, { "id": "release-candidate-manifest-runtime", "label": "immutable release candidate manifest proof", diff --git a/scripts/release_lifecycle_rehearsal.sh b/scripts/release_lifecycle_rehearsal.sh new file mode 100755 index 0000000000..b9ef5e5b53 --- /dev/null +++ b/scripts/release_lifecycle_rehearsal.sh @@ -0,0 +1,700 @@ +#!/usr/bin/env bash +# +# Release lifecycle rehearsal: install, upgrade and roll back Pulse between two +# PUBLISHED releases on a real systemd host (a privileged systemd container). +# +# 1. install FROM with that release's signature-verified install.sh --version +# 2. assert FROM identity, health and unit state +# 3. seed persistent state through the real API (first-run security setup, +# a webhook, a PVE node with fake credentials) and snapshot the data dir +# 4. upgrade with the installed updater users run: /bin/update --version TO +# 5. assert TO identity, health, unit state, seeded settings and data dir +# 6. roll back with the documented command: /bin/update --version FROM +# 7. assert FROM identity, health, unit state, seeded settings and data dir +# +# The result is a phase table on stdout and in $GITHUB_STEP_SUMMARY (when set). +# Nothing is uploaded. Exit status is non-zero when any phase fails. +# +# Usage: +# scripts/release_lifecycle_rehearsal.sh --from vX.Y.Z --to vX.Y.Z +# +# Environment: +# PULSE_REHEARSAL_REPO owner/repo for release assets (default rcourtman/Pulse) +# PULSE_REHEARSAL_ENGINE docker (default) or podman +# PULSE_REHEARSAL_WORKDIR scratch directory (default: mktemp -d) +# PULSE_REHEARSAL_README README holding the pinned installer key +# (default: README.md next to this script's repo) + +# Programs passed to cexec are single-quoted on purpose: they expand inside +# the container from -e variables. Mount options legitimately contain commas. +# shellcheck disable=SC2016,SC2054 + +set -euo pipefail +export LC_ALL=C + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +SYSTEMD_IMAGE="docker.io/jrei/systemd-debian:12@sha256:61d70dc3e574337bd9df794674a60ae73113460fff16ab41a2d234b4a11dcd98" +TAG_PATTERN='^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$' +API="http://127.0.0.1:7655" +DATA_DIR="/etc/pulse" +SEED_WEBHOOK_NAME="lifecycle-rehearsal-webhook" +SEED_WEBHOOK_URL="https://example.com/pulse-lifecycle-rehearsal" +# PVE add-node skips live cluster detection for 192.168.77.x hosts and +# "test-" names, so the fake node is persisted without a reachable endpoint. +SEED_NODE_NAME="test-lifecycle-rehearsal" +SEED_NODE_HOST="https://192.168.77.10:8006" +SEED_NODE_TOKEN_NAME="root@pam!rehearsal" +SEED_WEBHOOK_HEADER="X-Rehearsal-Secret" +SEED_ADMIN_USER="rehearsal-admin" + +FROM_TAG="" +TO_TAG="" +CHECK_SNAPSHOT="" +COMPARE_DATADIR=() +while [[ $# -gt 0 ]]; do + case "$1" in + --from) FROM_TAG="${2:-}"; shift 2 ;; + --to) TO_TAG="${2:-}"; shift 2 ;; + # Offline self-tests used by the contract tests: apply the fixed + # settings expectations to a snapshot file, or compare two data-dir + # manifests (pathsha256), then exit. + --check-snapshot) CHECK_SNAPSHOT="${2:-}"; shift 2 ;; + --compare-datadir) COMPARE_DATADIR=("${2:-}" "${3:-}"); shift 3 ;; + -h|--help) sed -n '2,/^$/p' "${BASH_SOURCE[0]}"; exit 0 ;; + *) echo "unknown argument: $1" >&2; exit 2 ;; + esac +done + +if [[ -z "$CHECK_SNAPSHOT" && ${#COMPARE_DATADIR[@]} -eq 0 ]]; then + for tag in "$FROM_TAG" "$TO_TAG"; do + if [[ ! "$tag" =~ $TAG_PATTERN ]]; then + echo "::error::--from and --to must be Pulse release tags (vX.Y.Z[-pre]); got '${tag}'" >&2 + exit 2 + fi + done + if [[ "$FROM_TAG" == "$TO_TAG" ]]; then + echo "::error::--from and --to must differ; a same-version run proves no upgrade" >&2 + exit 2 + fi +fi + +REPO="${PULSE_REHEARSAL_REPO:-rcourtman/Pulse}" +if [[ ! "$REPO" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]]; then + echo "::error::invalid PULSE_REHEARSAL_REPO '${REPO}'" >&2 + exit 2 +fi +ENGINE="${PULSE_REHEARSAL_ENGINE:-docker}" +case "$ENGINE" in + docker|podman) ;; + *) echo "::error::PULSE_REHEARSAL_ENGINE must be docker or podman" >&2; exit 2 ;; +esac +README_PATH="${PULSE_REHEARSAL_README:-${ROOT_DIR}/README.md}" +WORK_DIR="${PULSE_REHEARSAL_WORKDIR:-$(mktemp -d)}" +mkdir -p "${WORK_DIR}/assets" "${WORK_DIR}/state" +CONTAINER="pulse-lifecycle-rehearsal-$$" +SUMMARY_FILE="${GITHUB_STEP_SUMMARY:-}" + +# Throwaway credentials for an ephemeral container. They never leave it. +ADMIN_PASSWORD="$(od -An -N24 -tx1 /dev/urandom | tr -d ' \n')" +API_TOKEN="$(od -An -N32 -tx1 /dev/urandom | tr -d ' \n')" +NODE_TOKEN_VALUE="$(od -An -N16 -tx1 /dev/urandom | tr -d ' \n')" + +PHASE_ROWS=() +FAILURES=() +CURRENT_FAILURE="" +OVERALL_STATUS=0 +CONTAINER_STARTED=false +BASELINE_FAILED_UNITS="" +# D-Bus-activated host services that cannot run inside the test container +# (no hostname/clock/locale ownership). Any other new failed unit, and every +# pulse* unit, fails the phase. +CONTAINER_TOLERATED_UNITS="systemd-hostnamed.service systemd-timedated.service systemd-localed.service" + +log() { printf '[rehearsal] %s\n' "$*"; } +note_failure() { + CURRENT_FAILURE="${CURRENT_FAILURE:+${CURRENT_FAILURE}; }$*" + echo "::error::$*" +} + +# Run a bash program inside the container. Data reaches the program only as +# environment variables, never by interpolation into the program text. +cexec() { + local program="$1" + shift + local -a env_args=(-e PULSE_INSTALL_ALLOW_DOCKER=1) + local pair + for pair in "$@"; do + env_args+=(-e "$pair") + done + "$ENGINE" exec "${env_args[@]}" "$CONTAINER" bash -c "$program" +} + +print_diagnostics() { + [[ "$CONTAINER_STARTED" == "true" ]] || return 0 + echo "::group::Diagnostics: pulse journal (tail)" + cexec 'journalctl -u pulse --no-pager | tail -n 150' || true + echo "::endgroup::" + echo "::group::Diagnostics: unit state" + cexec 'systemctl status pulse --no-pager; systemctl list-units --state=failed --no-pager; systemctl list-timers --all --no-pager | grep -i pulse' || true + echo "::endgroup::" + echo "::group::Diagnostics: pulse-update journal" + cexec 'journalctl -u pulse-update --no-pager | tail -n 60' || true + echo "::endgroup::" + local logfile + for logfile in "${WORK_DIR}"/state/*.log; do + [[ -f "$logfile" ]] || continue + echo "::group::Diagnostics: $(basename "$logfile") (tail)" + tail -n 80 "$logfile" || true + echo "::endgroup::" + done +} + +cleanup() { + local status=$? + if [[ "$OVERALL_STATUS" -ne 0 || "$status" -ne 0 ]]; then + print_diagnostics + fi + if [[ "$CONTAINER_STARTED" == "true" ]]; then + "$ENGINE" rm -f "$CONTAINER" >/dev/null 2>&1 || true + fi +} +trap cleanup EXIT + +write_summary() { + local table + table=$( + echo "## Release lifecycle rehearsal" + echo + echo "\`${FROM_TAG}\` -> \`${TO_TAG}\` -> \`${FROM_TAG}\` on a systemd host (${SYSTEMD_IMAGE%%@*}). Shadow run, not a release gate." + echo + echo "| Phase | Expected | Reported version | Health | Settings | Data dir | Units | Result |" + echo "| --- | --- | --- | --- | --- | --- | --- | --- |" + local row + for row in "${PHASE_ROWS[@]}"; do + echo "$row" + done + if [[ ${#FAILURES[@]} -gt 0 ]]; then + echo + echo "### Failures" + local failure + for failure in "${FAILURES[@]}"; do + echo "- ${failure}" + done + fi + ) + echo + echo "$table" + if [[ -n "$SUMMARY_FILE" ]]; then + printf '%s\n' "$table" >> "$SUMMARY_FILE" + fi +} + +# record_phase NAME EXPECTED VERSION HEALTH SETTINGS DATADIR UNITS +record_phase() { + local result="pass" + if [[ -n "$CURRENT_FAILURE" ]]; then + result="FAIL" + FAILURES+=("$1: ${CURRENT_FAILURE}") + OVERALL_STATUS=1 + fi + PHASE_ROWS+=("| $1 | \`$2\` | ${3:--} | ${4:--} | ${5:--} | ${6:--} | ${7:--} | ${result} |") + CURRENT_FAILURE="" + [[ "$result" == "pass" ]] +} + +abort_run() { + write_summary + exit 1 +} + +# --------------------------------------------------------------------------- +# Published installer, verified against the README-pinned key +# --------------------------------------------------------------------------- + +fetch_and_verify_installer() { + local tag="$1" + local dest="${WORK_DIR}/assets" + local base="https://github.com/${REPO}/releases/download/${tag}" + local asset + for asset in install.sh install.sh.sshsig; do + if ! curl -fsSL --retry 5 --retry-delay 5 --retry-all-errors \ + -o "${dest}/${asset}" "${base}/${asset}"; then + echo "::error::could not download ${base}/${asset}" + return 1 + fi + done + + local readme_key allowed_signers + readme_key=$(grep -oE 'ssh-ed25519 [A-Za-z0-9+/=]+ pulse-installer' "$README_PATH" | head -1) + if [[ -z "$readme_key" ]]; then + echo "::error::Could not extract the pulse-installer key from ${README_PATH}" + return 1 + fi + allowed_signers="${WORK_DIR}/allowed_signers" + printf 'pulse-installer %s\n' "$readme_key" > "$allowed_signers" + if ! ssh-keygen -Y verify \ + -f "$allowed_signers" \ + -I pulse-installer \ + -n pulse-install \ + -s "${dest}/install.sh.sshsig" < "${dest}/install.sh"; then + echo "::error::${tag} install.sh.sshsig does not verify against the README's pinned key" + return 1 + fi + if ! grep -qE '^# Pulse Installer Script' "${dest}/install.sh" \ + || grep -q 'Pulse Unified Agent Installer' "${dest}/install.sh" \ + || ! grep -qE '^[[:space:]]*--version\)' "${dest}/install.sh"; then + echo "::error::${tag} install.sh is not the Pulse server installer with --version support" + return 1 + fi + log "${tag} install.sh signature verifies against the README-pinned key" +} + +# --------------------------------------------------------------------------- +# Container +# --------------------------------------------------------------------------- + +start_container() { + local -a run_args=(run -d --name "$CONTAINER" --privileged + -v "${WORK_DIR}/assets:/rehearsal:ro") + if [[ "$ENGINE" == "docker" ]]; then + # GHA ubuntu-24.04 uses the cgroup v2 unified hierarchy; without + # --cgroupns=host systemd PID 1 cannot mount the cgroup tree. + run_args+=(--cgroupns=host -v /sys/fs/cgroup:/sys/fs/cgroup:rw + --tmpfs /tmp:rw,size=2g --tmpfs /run --tmpfs /run/lock) + else + # Podman's systemd mode mounts /run, /run/lock and the cgroup tree. + # The pinned image is amd64-only. + run_args+=(--systemd=always --arch amd64 --tmpfs /tmp:rw,size=2g) + fi + CONTAINER_STARTED=true + if ! "$ENGINE" "${run_args[@]}" "$SYSTEMD_IMAGE" >/dev/null; then + echo "::error::${ENGINE} could not start ${SYSTEMD_IMAGE}" + return 1 + fi + + local i state + for i in $(seq 1 45); do + if ! "$ENGINE" inspect -f '{{.State.Running}}' "$CONTAINER" 2>/dev/null | grep -q true; then + "$ENGINE" logs "$CONTAINER" || true + echo "::error::systemd container exited during boot" + return 1 + fi + state=$("$ENGINE" exec "$CONTAINER" systemctl is-system-running 2>/dev/null || true) + if [[ "$state" == "running" || "$state" == "degraded" ]]; then + break + fi + if [[ "$i" -eq 45 ]]; then + echo "::error::systemd did not become ready inside the container (state: ${state:-unknown})" + return 1 + fi + sleep 2 + done + BASELINE_FAILED_UNITS=$(cexec 'systemctl list-units --state=failed --no-legend --plain | awk "{print \$1}" | sort') + log "systemd is up (pre-existing failed units: ${BASELINE_FAILED_UNITS:-none})" + if ! cexec 'export DEBIAN_FRONTEND=noninteractive; apt-get update -qq >/dev/null && apt-get install -y -qq curl ca-certificates jq >/dev/null'; then + echo "::error::could not install curl/jq inside the container" + return 1 + fi + # Keep a daily CI install out of real usage telemetry. A unit drop-in is + # outside the installer's ownership and the data dir, so it does not + # change what install, upgrade or rollback do. + cexec 'mkdir -p /etc/systemd/system/pulse.service.d && printf "[Service]\nEnvironment=PULSE_TELEMETRY=false\n" > /etc/systemd/system/pulse.service.d/50-rehearsal-no-telemetry.conf' +} + +# --------------------------------------------------------------------------- +# Assertions +# --------------------------------------------------------------------------- + +PHASE_VERSION="" +PHASE_HEALTH="" +PHASE_SETTINGS="" +PHASE_DATADIR="" +PHASE_UNITS="" + +reset_phase_cells() { + PHASE_VERSION="-"; PHASE_HEALTH="-"; PHASE_SETTINGS="-"; PHASE_DATADIR="-"; PHASE_UNITS="-" +} + +# Identity, health and systemd state for the expected release. +assert_runtime() { + local expected_tag="$1" + local expected="${expected_tag#v}" + + local active="" i + for i in $(seq 1 60); do + active=$(cexec 'systemctl is-active pulse' 2>/dev/null || true) + [[ "$active" == "active" ]] && break + sleep 2 + done + + local health_body health_status + health_body=$(cexec 'curl -fsS --retry 30 --retry-delay 2 --retry-connrefused --retry-all-errors "$API/api/health"' "API=${API}" 2>/dev/null || true) + health_status=$(jq -r '.status // empty' <<<"$health_body" 2>/dev/null || true) + if [[ "$health_status" == "healthy" ]]; then + PHASE_HEALTH="healthy" + else + PHASE_HEALTH="${health_status:-no response}" + note_failure "/api/health did not report healthy (got '${PHASE_HEALTH}')" + fi + + local version_body reported binary_version + version_body=$(cexec 'curl -fsS "$API/api/version"' "API=${API}" 2>/dev/null || true) + reported=$(jq -r '.version // empty' <<<"$version_body" 2>/dev/null || true) + PHASE_VERSION="${reported:-none}" + if [[ "${reported#v}" != "$expected" ]]; then + note_failure "/api/version reported '${reported:-none}', expected ${expected_tag}" + fi + binary_version=$(cexec '/opt/pulse/bin/pulse --version 2>/dev/null | grep -oE "v[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9.]+)?" | head -1' 2>/dev/null || true) + if [[ "${binary_version#v}" != "$expected" ]]; then + note_failure "/opt/pulse/bin/pulse --version reported '${binary_version:-none}', expected ${expected_tag}" + fi + + local enabled failed_now new_failed + enabled=$(cexec 'systemctl is-enabled pulse' 2>/dev/null || true) + failed_now=$(cexec 'systemctl list-units --state=failed --no-legend --plain | awk "{print \$1}" | sort' 2>/dev/null || true) + new_failed=$(comm -13 <(printf '%s\n' "$BASELINE_FAILED_UNITS" | sed '/^$/d') \ + <(printf '%s\n' "$failed_now" | sed '/^$/d') | tr '\n' ' ') + PHASE_UNITS="pulse ${active:-unknown}/${enabled:-unknown}" + if [[ "$active" != "active" ]]; then + note_failure "pulse.service is '${active:-unknown}', expected active" + fi + if [[ "$enabled" != "enabled" ]]; then + note_failure "pulse.service is '${enabled:-unknown}', expected enabled" + fi + local unit tolerated="" unexpected="" + for unit in $new_failed; do + if [[ " ${CONTAINER_TOLERATED_UNITS} " == *" ${unit} "* ]]; then + tolerated="${tolerated:+${tolerated} }${unit}" + else + unexpected="${unexpected:+${unexpected} }${unit}" + fi + done + if [[ -n "$unexpected" ]]; then + PHASE_UNITS="${PHASE_UNITS}, failed: ${unexpected}" + note_failure "systemd units failed during the rehearsal: ${unexpected}" + fi + if [[ -n "$tolerated" ]]; then + PHASE_UNITS="${PHASE_UNITS}, container-only failures ignored: ${tolerated}" + echo "::warning::container-environment units failed and were not counted: ${tolerated}" + fi + + if ! cexec 'systemctl show -p Environment --value pulse | grep -q "PULSE_TELEMETRY=false"' 2>/dev/null; then + echo "::warning::the rehearsal telemetry opt-out drop-in is no longer applied to pulse.service" + fi + + # An unattended update firing mid-rehearsal would invalidate the version + # assertions, so prove the timer-driven updater never started. + local auto_update_started + auto_update_started=$(cexec 'systemctl show -p ExecMainStartTimestampMonotonic --value pulse-update.service 2>/dev/null || echo 0' 2>/dev/null || echo 0) + if [[ -n "$auto_update_started" && "$auto_update_started" != "0" ]]; then + note_failure "pulse-update.service ran during the rehearsal; results are not attributable to the manual updater" + fi +} + +api_status() { + # api_status METHOD PATH [auth: none|token|basic] [body] + cexec 'args=(-sS -o /dev/null -w "%{http_code}" -X "$METHOD") +case "$AUTH" in + token) args+=(-H "X-API-Token: $TOKEN") ;; + basic) args+=(-u "$BASIC") ;; +esac +curl "${args[@]}" "$API$REQ_PATH"' \ + "API=${API}" "METHOD=$1" "REQ_PATH=$2" "AUTH=${3:-none}" \ + "TOKEN=${API_TOKEN}" "BASIC=${SEED_ADMIN_USER}:${ADMIN_PASSWORD}" +} + +api_json() { + # api_json METHOD PATH [body]; authenticates with the seeded API token. + cexec 'if [[ -n "$BODY" ]]; then + curl -fsS -X "$METHOD" -H "X-API-Token: $TOKEN" -H "Content-Type: application/json" --data "$BODY" "$API$REQ_PATH" +else + curl -fsS -X "$METHOD" -H "X-API-Token: $TOKEN" "$API$REQ_PATH" +fi' "API=${API}" "METHOD=$1" "REQ_PATH=$2" "BODY=${3:-}" "TOKEN=${API_TOKEN}" +} + +# Canonical, comparable view of the seeded settings as the API returns them. +settings_snapshot() { + local webhooks nodes security unauth token basic + webhooks=$(api_json GET /api/notifications/webhooks) || return 1 + nodes=$(api_json GET /api/config/nodes) || return 1 + security=$(cexec 'curl -fsS "$API/api/security/status"' "API=${API}") || return 1 + unauth=$(api_status GET /api/config/nodes none) + token=$(api_status GET /api/config/nodes token) + basic=$(api_status GET /api/config/nodes basic) + jq -n -S \ + --argjson webhooks "$webhooks" \ + --argjson nodes "$nodes" \ + --argjson security "$security" \ + --arg unauth "$unauth" --arg token "$token" --arg basic "$basic" \ + --arg webhook_name "$SEED_WEBHOOK_NAME" --arg node_name "$SEED_NODE_NAME" ' + { + auth: { + requiresAuth: ($security.requiresAuth // null), + unauthenticated: $unauth, + api_token: $token, + password: $basic + }, + webhook: ([$webhooks | .. | objects | select(.name? == $webhook_name) + | {id, name, url, method, service, enabled, + header_keys: ((.headers // {}) | keys)}] | first), + node: ([$nodes | .. | objects | select(.name? == $node_name) + | {name, type, host, tokenName, hasToken, hasPassword, verifySSL}] | first) + }' +} + +# Expected shape of a healthy snapshot (independent of the baseline, so a +# baseline that silently lost data cannot make later phases pass). +check_snapshot_shape() { + local snapshot="$1" + local problems + problems=$(jq -r \ + --arg webhook_name "$SEED_WEBHOOK_NAME" --arg webhook_url "$SEED_WEBHOOK_URL" \ + --arg node_name "$SEED_NODE_NAME" --arg node_host "$SEED_NODE_HOST" \ + --arg node_token_name "$SEED_NODE_TOKEN_NAME" --arg header "$SEED_WEBHOOK_HEADER" ' + [ (if .auth.requiresAuth != true then "security status does not require auth" else empty end), + (if .auth.unauthenticated != "401" then "unauthenticated request returned \(.auth.unauthenticated), expected 401" else empty end), + (if .auth.api_token != "200" then "API token request returned \(.auth.api_token), expected 200" else empty end), + (if .auth.password != "200" then "password request returned \(.auth.password), expected 200" else empty end), + (if .webhook == null then "seeded webhook \($webhook_name) missing" else empty end), + (if .webhook != null and .webhook.url != $webhook_url then "seeded webhook URL changed to \(.webhook.url)" else empty end), + (if .webhook != null and .webhook.method != "POST" then "seeded webhook method changed to \(.webhook.method)" else empty end), + (if .webhook != null and .webhook.service != "generic" then "seeded webhook service changed to \(.webhook.service)" else empty end), + (if .webhook != null and .webhook.enabled != false then "seeded webhook enabled state changed to \(.webhook.enabled)" else empty end), + (if .webhook != null and ((.webhook.id // "") == "") then "seeded webhook lost its id" else empty end), + (if .webhook != null and .webhook.header_keys != [$header] then "seeded webhook headers changed to \(.webhook.header_keys)" else empty end), + (if .node == null then "seeded node \($node_name) missing" else empty end), + (if .node != null and .node.type != "pve" then "seeded node type changed to \(.node.type)" else empty end), + (if .node != null and .node.host != $node_host then "seeded node host changed to \(.node.host)" else empty end), + (if .node != null and .node.tokenName != $node_token_name then "seeded node token name changed to \(.node.tokenName)" else empty end), + (if .node != null and .node.hasToken != true then "seeded node lost its API token secret" else empty end), + (if .node != null and .node.hasPassword != false then "seeded node gained a password" else empty end), + (if .node != null and .node.verifySSL != false then "seeded node verifySSL changed to \(.node.verifySSL)" else empty end) + ] | .[]' <<<"$snapshot") || problems="snapshot is not valid JSON" + if [[ -n "$problems" ]]; then + while IFS= read -r problem; do + note_failure "settings: ${problem}" + done <<<"$problems" + return 1 + fi +} + +datadir_manifest() { + # pathsha256 for every regular file, excluding SQLite sidecars and + # temp files whose presence depends on shutdown timing. + cexec 'cd "$DATA_DIR" && find . -type f ! -name "*-wal" ! -name "*-shm" ! -name "*.tmp" ! -name "*.lock" -printf "%P\n" | LC_ALL=C sort | while IFS= read -r f; do printf "%s\t%s\n" "$f" "$(sha256sum "$f" | cut -d" " -f1)"; done' \ + "DATA_DIR=${DATA_DIR}" +} + +# Encrypted stores holding the seeded secrets (node token value, webhook +# header value). The API only exposes hasToken and redacted header values, so +# byte identity of these files under an unchanged key is the proof that the +# exact secrets survived. A version that legitimately rewrites them fails the +# rehearsal loudly rather than passing unproven. +SECRET_STORES=".encryption.key nodes.enc webhooks.enc" + +# compare_datadir BASELINE CURRENT LABEL: manifest comparison (pure, testable). +compare_datadir() { + local baseline="$1" current="$2" label="$3" + local missing changed added store before after + missing=$(comm -23 <(cut -f1 "$baseline") <(cut -f1 "$current") | sed '/^$/d') + added=$(comm -13 <(cut -f1 "$baseline") <(cut -f1 "$current") | sed '/^$/d' | wc -l | tr -d ' ') + changed=$(join -t $'\t' "$baseline" "$current" | awk -F'\t' '$2 != $3' | wc -l | tr -d ' ') + + PHASE_DATADIR="$(wc -l < "$current" | tr -d ' ') files, ${changed} changed, ${added} new" + log "${DATA_DIR} after ${label}: ${PHASE_DATADIR}" + if [[ -n "$missing" ]]; then + PHASE_DATADIR="${PHASE_DATADIR}, $(wc -l <<<"$missing" | tr -d ' ') missing" + note_failure "files present before the upgrade are gone from ${DATA_DIR}: $(tr '\n' ' ' <<<"$missing")" + fi + for store in $SECRET_STORES; do + before=$(awk -F'\t' -v f="$store" '$1 == f {print $2}' "$baseline") + after=$(awk -F'\t' -v f="$store" '$1 == f {print $2}' "$current") + if [[ -z "$before" || "$before" != "$after" ]]; then + note_failure "${DATA_DIR}/${store} is missing or was rewritten; the seeded secrets are no longer provably intact" + fi + done + if [[ "$changed" -gt 0 ]]; then + log "files rewritten in place (informational unless listed above):" + join -t $'\t' "$baseline" "$current" | awk -F'\t' '$2 != $3 {print " " $1}' + fi +} + +check_datadir() { + local baseline="${WORK_DIR}/state/datadir.baseline.tsv" + local current="${WORK_DIR}/state/datadir.$1.tsv" + datadir_manifest > "$current" || { note_failure "could not list ${DATA_DIR}"; PHASE_DATADIR="unreadable"; return 1; } + compare_datadir "$baseline" "$current" "$1" +} + +check_settings() { + local snapshot + if ! snapshot=$(settings_snapshot); then + note_failure "settings could not be read back through the API" + PHASE_SETTINGS="unreadable" + return 1 + fi + printf '%s\n' "$snapshot" > "${WORK_DIR}/state/settings.$1.json" + local ok=true + check_snapshot_shape "$snapshot" || ok=false + if ! diff -u "${WORK_DIR}/state/settings.baseline.json" "${WORK_DIR}/state/settings.$1.json"; then + note_failure "seeded settings read back differently than before the upgrade (diff above)" + ok=false + fi + if [[ "$ok" == "true" ]]; then + PHASE_SETTINGS="auth, webhook, node intact" + else + PHASE_SETTINGS="drift" + return 1 + fi +} + +# --------------------------------------------------------------------------- +# Phases +# --------------------------------------------------------------------------- + +phase_install() { + reset_phase_cells + log "Phase 1: install ${FROM_TAG} with its published install.sh --version" + # No TTY: install.sh's prompts take their defaults, as for curl | bash. + if ! cexec 'bash /rehearsal/install.sh --version "$FROM_TAG"' "FROM_TAG=${FROM_TAG}" \ + 2>&1 | tee "${WORK_DIR}/state/install-${FROM_TAG}.log"; then + note_failure "install.sh --version ${FROM_TAG} exited non-zero" + fi + assert_runtime "$FROM_TAG" + record_phase "1. install" "$FROM_TAG" "$PHASE_VERSION" "$PHASE_HEALTH" "-" "-" "$PHASE_UNITS" +} + +phase_seed() { + reset_phase_cells + log "Phase 2: seed first-run security, a webhook and a PVE node through the API" + local token_output setup_token setup_response + token_output=$(cexec 'runuser -u pulse -- env PULSE_DATA_DIR="$DATA_DIR" /opt/pulse/bin/pulse bootstrap-token' "DATA_DIR=${DATA_DIR}" 2>&1 || true) + setup_token=$(grep -oE 'Token: [^[:space:]]+' <<<"$token_output" | head -1 | cut -d' ' -f2) + if [[ -z "$setup_token" ]]; then + note_failure "pulse bootstrap-token did not print a setup token" + record_phase "2. seed" "$FROM_TAG" "-" "-" "-" "-" "-" + return 1 + fi + + local setup_body + setup_body=$(jq -cn --arg u "$SEED_ADMIN_USER" --arg p "$ADMIN_PASSWORD" --arg t "$API_TOKEN" \ + '{username: $u, password: $p, apiToken: $t}') + if ! setup_response=$(cexec 'curl -fsS -X POST -H "Content-Type: application/json" -H "X-Setup-Token: $SETUP_TOKEN" --data "$BODY" "$API/api/security/quick-setup"' \ + "API=${API}" "SETUP_TOKEN=${setup_token}" "BODY=${setup_body}") \ + || [[ "$(jq -r '.success // false' <<<"$setup_response" 2>/dev/null)" != "true" ]]; then + note_failure "first-run security setup failed: ${setup_response:-no response}" + fi + + local webhook_body node_body + webhook_body=$(jq -cn --arg n "$SEED_WEBHOOK_NAME" --arg u "$SEED_WEBHOOK_URL" \ + --arg hk "$SEED_WEBHOOK_HEADER" --arg hv "$NODE_TOKEN_VALUE" \ + '{name: $n, url: $u, method: "POST", service: "generic", enabled: false, + headers: {($hk): $hv}}') + if ! api_json POST /api/notifications/webhooks "$webhook_body" >/dev/null; then + note_failure "creating the seed webhook failed" + fi + node_body=$(jq -cn --arg n "$SEED_NODE_NAME" --arg h "$SEED_NODE_HOST" --arg v "$NODE_TOKEN_VALUE" \ + --arg tn "$SEED_NODE_TOKEN_NAME" \ + '{type: "pve", name: $n, host: $h, tokenName: $tn, tokenValue: $v, verifySSL: false}') + if ! api_json POST /api/config/nodes "$node_body" >/dev/null; then + note_failure "creating the seed PVE node failed" + fi + + local snapshot="" + if snapshot=$(settings_snapshot) && check_snapshot_shape "$snapshot"; then + printf '%s\n' "$snapshot" > "${WORK_DIR}/state/settings.baseline.json" + PHASE_SETTINGS="auth, webhook, node seeded" + log "Seeded settings as read back through the API:" + printf '%s\n' "$snapshot" + else + if [[ -n "$snapshot" ]]; then + printf '%s\n' "$snapshot" + fi + note_failure "seeded settings did not read back through the API" + PHASE_SETTINGS="not readable" + fi + + # Persistence is asynchronous for some stores; let writes settle before + # the data dir becomes the upgrade baseline. + sleep 5 + if datadir_manifest > "${WORK_DIR}/state/datadir.baseline.tsv" \ + && grep -q $'^\\.encryption\\.key\t' "${WORK_DIR}/state/datadir.baseline.tsv"; then + PHASE_DATADIR="$(wc -l < "${WORK_DIR}/state/datadir.baseline.tsv" | tr -d ' ') files recorded" + log "${DATA_DIR} baseline:" + cut -f1 "${WORK_DIR}/state/datadir.baseline.tsv" | sed 's/^/ /' + else + note_failure "could not record ${DATA_DIR} (or it has no .encryption.key)" + PHASE_DATADIR="not recorded" + fi + record_phase "2. seed" "$FROM_TAG" "-" "-" "$PHASE_SETTINGS" "$PHASE_DATADIR" "-" +} + +run_updater() { + local target="$1" label="$2" + # The helper must be the one the Pulse server installer wrote, not a + # community-scripts updater that ignores --version. + if ! cexec 'test -x /bin/update && grep -q "^# Pulse update command" /bin/update'; then + note_failure "/bin/update is missing or is not the Pulse server installer's update helper" + return 1 + fi + # /bin/update downloads the latest published install.sh, verifies it with + # its embedded signer key and runs it with the given arguments. + if ! cexec '/bin/update --version "$TARGET"' "TARGET=${target}" \ + 2>&1 | tee "${WORK_DIR}/state/${label}-${target}.log"; then + note_failure "/bin/update --version ${target} exited non-zero" + fi +} + +phase_upgrade() { + reset_phase_cells + log "Phase 3: upgrade with /bin/update --version ${TO_TAG}" + run_updater "$TO_TAG" upgrade || true + assert_runtime "$TO_TAG" + check_settings upgrade || true + check_datadir upgrade || true + record_phase "3. upgrade (/bin/update)" "$TO_TAG" "$PHASE_VERSION" "$PHASE_HEALTH" "$PHASE_SETTINGS" "$PHASE_DATADIR" "$PHASE_UNITS" +} + +phase_rollback() { + reset_phase_cells + log "Phase 4: roll back with /bin/update --version ${FROM_TAG}" + run_updater "$FROM_TAG" rollback || true + assert_runtime "$FROM_TAG" + check_settings rollback || true + check_datadir rollback || true + record_phase "4. rollback (/bin/update)" "$FROM_TAG" "$PHASE_VERSION" "$PHASE_HEALTH" "$PHASE_SETTINGS" "$PHASE_DATADIR" "$PHASE_UNITS" +} + +main() { + log "Rehearsing ${FROM_TAG} -> ${TO_TAG} -> ${FROM_TAG} from ${REPO} with ${ENGINE}" + if ! fetch_and_verify_installer "$FROM_TAG"; then + CURRENT_FAILURE="published install.sh for ${FROM_TAG} failed verification" + record_phase "0. verify installer" "$FROM_TAG" "-" "-" "-" "-" "-" || abort_run + fi + if ! start_container; then + CURRENT_FAILURE="systemd container did not start" + record_phase "0. systemd host" "-" "-" "-" "-" "-" "-" || abort_run + fi + + phase_install || abort_run + phase_seed || abort_run + # Rollback runs even when the upgrade fails: it is the recovery path a + # user reaches for in exactly that situation. + phase_upgrade || true + phase_rollback || true + + write_summary + return "$OVERALL_STATUS" +} + +if [[ -n "$CHECK_SNAPSHOT" ]]; then + check_snapshot_shape "$(cat "$CHECK_SNAPSHOT")" + exit $? +fi +if [[ ${#COMPARE_DATADIR[@]} -gt 0 ]]; then + compare_datadir "${COMPARE_DATADIR[0]}" "${COMPARE_DATADIR[1]}" self-test + [[ -z "$CURRENT_FAILURE" ]] + exit $? +fi + +main diff --git a/scripts/release_lifecycle_rehearsal_versions.py b/scripts/release_lifecycle_rehearsal_versions.py new file mode 100755 index 0000000000..b7071992ee --- /dev/null +++ b/scripts/release_lifecycle_rehearsal_versions.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +"""Choose the published release pair for the release lifecycle rehearsal. + +The rehearsal installs FROM, upgrades to TO and rolls back to FROM. Both tags +must name published (non-draft) Pulse server releases and TO must be strictly +newer than FROM under SemVer precedence, so a shadow run can never report a +same-version "upgrade" as proof. + +Defaults: + FROM = the release GitHub advertises as latest (must be stable). + TO = the newest published release or prerelease newer than FROM. +When FROM was defaulted and nothing newer exists yet (the common case right +after a stable release), the pair becomes previous stable -> latest stable, +which is the upgrade users are taking at that moment. +""" + +from __future__ import annotations + +import argparse +import functools +import json +import re +import sys +from dataclasses import dataclass +from pathlib import Path + + +TAG_RE = re.compile( + r"^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)" + r"(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?$" +) + + +class ResolutionError(ValueError): + """The requested or default release pair cannot be rehearsed.""" + + +@dataclass(frozen=True) +class Selection: + from_tag: str + to_tag: str + reason: str + + +def parse_tag(tag: str) -> tuple[tuple[int, int, int], tuple[str, ...]]: + match = TAG_RE.fullmatch(tag) + if not match: + raise ResolutionError(f"not a Pulse release tag: {tag!r}") + core = (int(match.group(1)), int(match.group(2)), int(match.group(3))) + pre = tuple(match.group(4).split(".")) if match.group(4) else () + return core, pre + + +def is_stable(tag: str) -> bool: + return not parse_tag(tag)[1] + + +def _compare_identifiers(left: tuple[str, ...], right: tuple[str, ...]) -> int: + # SemVer 2.0.0 section 11: a release outranks any prerelease of the same + # core; numeric identifiers compare numerically and rank below + # alphanumeric ones; a longer identifier list wins when all shared + # identifiers are equal. + if not left and not right: + return 0 + if not left: + return 1 + if not right: + return -1 + for a, b in zip(left, right): + if a == b: + continue + a_num, b_num = a.isdigit(), b.isdigit() + if a_num and b_num: + return -1 if int(a) < int(b) else 1 + if a_num != b_num: + return -1 if a_num else 1 + return -1 if a < b else 1 + if len(left) == len(right): + return 0 + return -1 if len(left) < len(right) else 1 + + +def compare_tags(left: str, right: str) -> int: + left_core, left_pre = parse_tag(left) + right_core, right_pre = parse_tag(right) + if left_core != right_core: + return -1 if left_core < right_core else 1 + return _compare_identifiers(left_pre, right_pre) + + +def published_tags(releases: list[dict]) -> list[str]: + tags: set[str] = set() + for release in releases: + if not isinstance(release, dict) or release.get("draft") is not False: + continue + tag = release.get("tag_name") + if isinstance(tag, str) and TAG_RE.fullmatch(tag): + tags.add(tag) + return sorted(tags, key=functools.cmp_to_key(compare_tags)) + + +def resolve( + releases: list[dict], + latest_tag: str, + requested_from: str = "", + requested_to: str = "", +) -> Selection: + tags = published_tags(releases) + published = set(tags) + + for label, value in (("from_version", requested_from), ("to_version", requested_to)): + if value: + parse_tag(value) + if value not in published: + raise ResolutionError(f"{label} {value} is not a published Pulse release") + + if requested_from: + from_tag = requested_from + else: + if not latest_tag or latest_tag not in published or not is_stable(latest_tag): + raise ResolutionError( + f"latest advertised release {latest_tag!r} is not a published stable Pulse release" + ) + from_tag = latest_tag + + if requested_to: + if compare_tags(requested_to, from_tag) <= 0: + raise ResolutionError( + f"to_version {requested_to} must be newer than from_version {from_tag}" + ) + return Selection(from_tag, requested_to, "requested pair") + + newer = [tag for tag in tags if compare_tags(tag, from_tag) > 0] + if newer: + reason = "requested from_version" if requested_from else "latest stable" + return Selection(from_tag, newer[-1], f"{reason} -> newest published release") + + if requested_from: + raise ResolutionError(f"no published release is newer than from_version {from_tag}") + + older_stable = [ + tag for tag in tags if is_stable(tag) and compare_tags(tag, from_tag) < 0 + ] + if not older_stable: + raise ResolutionError(f"no stable release precedes latest stable {from_tag}") + return Selection( + older_stable[-1], + from_tag, + "nothing newer than latest stable is published; previous stable -> latest stable", + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--releases-json", required=True, type=Path, + help="JSON array of GitHub release objects (all pages)") + parser.add_argument("--latest-tag", default="", + help="tag_name of GET /repos/{repo}/releases/latest") + parser.add_argument("--from", dest="from_version", default="") + parser.add_argument("--to", dest="to_version", default="") + args = parser.parse_args(argv) + + try: + releases = json.loads(args.releases_json.read_text(encoding="utf-8")) + if not isinstance(releases, list): + raise ResolutionError("releases JSON must be an array") + selection = resolve( + releases, + args.latest_tag.strip(), + args.from_version.strip(), + args.to_version.strip(), + ) + except (ResolutionError, json.JSONDecodeError, OSError) as error: + print(f"release lifecycle rehearsal: {error}", file=sys.stderr) + return 1 + + print(f"from_tag={selection.from_tag}") + print(f"to_tag={selection.to_tag}") + print(f"reason={selection.reason}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/tests/test_release_lifecycle_rehearsal.py b/scripts/tests/test_release_lifecycle_rehearsal.py new file mode 100644 index 0000000000..9af5eb30fe --- /dev/null +++ b/scripts/tests/test_release_lifecycle_rehearsal.py @@ -0,0 +1,336 @@ +#!/usr/bin/env python3 +"""Contract tests for the shadow release lifecycle rehearsal.""" + +from __future__ import annotations + +import importlib.util +import json +import re +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] +RESOLVER_PATH = REPO_ROOT / "scripts" / "release_lifecycle_rehearsal_versions.py" +HARNESS_PATH = REPO_ROOT / "scripts" / "release_lifecycle_rehearsal.sh" +WORKFLOW_PATH = REPO_ROOT / ".github" / "workflows" / "release-lifecycle-rehearsal.yml" +SMOKE_BODY_PATH = REPO_ROOT / ".github" / "workflows" / "install-sh-smoke-body.yml" + +SPEC = importlib.util.spec_from_file_location("release_lifecycle_rehearsal_versions", RESOLVER_PATH) +assert SPEC and SPEC.loader +versions = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = versions +SPEC.loader.exec_module(versions) + + +def release(tag: str, *, draft: bool = False, prerelease: bool | None = None) -> dict: + return { + "tag_name": tag, + "draft": draft, + "prerelease": ("-" in tag) if prerelease is None else prerelease, + } + + +RELEASES = [ + release("v6.4.5"), + release("helm-chart-6.4.5"), + release("v6.4.5-rc.5"), + release("v6.4.5-rc.10"), + release("v6.4.5-beta.3"), + release("v6.4.1"), + release("v6.4.0"), + release("v6.4.0-rc.9"), + release("v6.4.6-rc.1", draft=True), +] + + +class SemverOrderingTest(unittest.TestCase): + def test_release_outranks_its_prereleases(self) -> None: + self.assertGreater(versions.compare_tags("v6.4.5", "v6.4.5-rc.10"), 0) + + def test_numeric_identifiers_compare_numerically(self) -> None: + self.assertGreater(versions.compare_tags("v6.4.5-rc.10", "v6.4.5-rc.5"), 0) + + def test_alphanumeric_identifiers_compare_lexically(self) -> None: + self.assertGreater(versions.compare_tags("v6.4.5-rc.1", "v6.4.5-beta.3"), 0) + + def test_rejects_non_release_tags(self) -> None: + for tag in ("6.4.5", "helm-chart-6.4.5", "v6.4", "v6.4.5;id", "v06.4.5"): + with self.subTest(tag=tag), self.assertRaises(versions.ResolutionError): + versions.parse_tag(tag) + + +class ResolvePairTest(unittest.TestCase): + def test_defaults_to_newer_prerelease_after_latest_stable(self) -> None: + releases = RELEASES + [release("v6.4.6-rc.2"), release("v6.4.6-beta.1")] + selection = versions.resolve(releases, "v6.4.5") + self.assertEqual((selection.from_tag, selection.to_tag), ("v6.4.5", "v6.4.6-rc.2")) + + def test_defaults_fall_back_to_previous_stable_when_nothing_is_newer(self) -> None: + selection = versions.resolve(RELEASES, "v6.4.5") + self.assertEqual((selection.from_tag, selection.to_tag), ("v6.4.1", "v6.4.5")) + self.assertIn("previous stable", selection.reason) + + def test_draft_releases_are_never_targets(self) -> None: + selection = versions.resolve(RELEASES, "v6.4.5") + self.assertNotEqual(selection.to_tag, "v6.4.6-rc.1") + with self.assertRaises(versions.ResolutionError): + versions.resolve(RELEASES, "v6.4.5", "v6.4.5", "v6.4.6-rc.1") + + def test_explicit_from_selects_newest_newer_release(self) -> None: + selection = versions.resolve(RELEASES, "v6.4.5", "v6.4.0") + self.assertEqual((selection.from_tag, selection.to_tag), ("v6.4.0", "v6.4.5")) + + def test_explicit_from_without_newer_release_fails_instead_of_passing(self) -> None: + with self.assertRaises(versions.ResolutionError): + versions.resolve(RELEASES, "v6.4.5", "v6.4.5") + + def test_explicit_pair_must_move_forward(self) -> None: + for from_tag, to_tag in (("v6.4.5", "v6.4.1"), ("v6.4.5", "v6.4.5")): + with self.subTest(pair=(from_tag, to_tag)), self.assertRaises(versions.ResolutionError): + versions.resolve(RELEASES, "v6.4.5", from_tag, to_tag) + + def test_unpublished_or_malformed_requests_fail(self) -> None: + for from_tag, to_tag in (("v6.3.0", ""), ("v6.4.1", "v6.9.9"), ("v6.4.1 ; id", "")): + with self.subTest(pair=(from_tag, to_tag)), self.assertRaises(versions.ResolutionError): + versions.resolve(RELEASES, "v6.4.5", from_tag, to_tag) + + def test_latest_must_be_published_stable(self) -> None: + for latest in ("", "v6.4.5-rc.5", "v9.9.9"): + with self.subTest(latest=latest), self.assertRaises(versions.ResolutionError): + versions.resolve(RELEASES, latest) + + def test_cli_emits_key_value_pair(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "releases.json" + path.write_text(json.dumps(RELEASES), encoding="utf-8") + result = subprocess.run( + [sys.executable, str(RESOLVER_PATH), "--releases-json", str(path), + "--latest-tag", "v6.4.5", "--from", "v6.4.0", "--to", "v6.4.1"], + capture_output=True, text=True, check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("from_tag=v6.4.0\n", result.stdout) + self.assertIn("to_tag=v6.4.1\n", result.stdout) + + +class HarnessContractTest(unittest.TestCase): + harness = HARNESS_PATH.read_text(encoding="utf-8") + + def test_upgrade_and_rollback_use_the_installed_update_helper(self) -> None: + self.assertIn("cexec '/bin/update --version \"$TARGET\"'", self.harness) + self.assertIn('run_updater "$TO_TAG" upgrade', self.harness) + self.assertIn('run_updater "$FROM_TAG" rollback', self.harness) + self.assertIn('grep -q "^# Pulse update command" /bin/update', self.harness) + # The updater invocation carries no flags that would bypass the + # behavior users get (archives, preflight skips, forced auto-update). + updater_lines = [line for line in self.harness.splitlines() if "/bin/update --" in line] + for line in updater_lines: + for flag in ("--archive", "--skip-upgrade-preflight", "--disable-auto-updates"): + self.assertNotIn(flag, line) + + def test_initial_install_uses_signed_published_installer(self) -> None: + for needle in ( + "releases/download/${tag}", + "grep -oE 'ssh-ed25519 [A-Za-z0-9+/=]+ pulse-installer'", + "-I pulse-installer", + "-n pulse-install", + "bash /rehearsal/install.sh --version \"$FROM_TAG\"", + ): + self.assertIn(needle, self.harness) + + def test_uses_the_same_pinned_systemd_image_as_install_smoke(self) -> None: + digest = re.search(r"jrei/systemd-debian:12@sha256:[0-9a-f]{64}", self.harness) + self.assertIsNotNone(digest) + self.assertIn(digest.group(0), SMOKE_BODY_PATH.read_text(encoding="utf-8")) + + def test_every_post_change_phase_checks_identity_health_settings_and_data(self) -> None: + for phase in ("phase_upgrade", "phase_rollback"): + body = self.harness.split(f"{phase}() {{", 1)[1].split("\n}\n", 1)[0] + for check in ("assert_runtime", "check_settings", "check_datadir", "record_phase"): + with self.subTest(phase=phase, check=check): + self.assertIn(check, body) + + def test_settings_are_checked_against_fixed_expectations_not_only_baseline(self) -> None: + body = self.harness.split("check_settings() {", 1)[1].split("\n}\n", 1)[0] + self.assertIn("check_snapshot_shape", body) + self.assertIn("settings.baseline.json", body) + + def test_harness_passes_shellcheck_syntax(self) -> None: + result = subprocess.run(["bash", "-n", str(HARNESS_PATH)], capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_rejects_unsafe_tags_before_touching_a_container(self) -> None: + for args in (["--from", "v6.4.1", "--to", "v6.4.1"], + ["--from", "v6.4.1;id", "--to", "v6.4.5"], + ["--from", "", "--to", "v6.4.5"]): + with self.subTest(args=args): + result = subprocess.run( + ["bash", str(HARNESS_PATH), *args], + capture_output=True, text=True, + env={"PATH": "/usr/bin:/bin", "PULSE_REHEARSAL_ENGINE": "none-such"}, + ) + self.assertEqual(result.returncode, 2, result.stdout + result.stderr) + + +GOOD_SNAPSHOT = { + "auth": {"api_token": "200", "password": "200", "requiresAuth": True, "unauthenticated": "401"}, + "node": { + "hasPassword": False, + "hasToken": True, + "host": "https://192.168.77.10:8006", + "name": "test-lifecycle-rehearsal", + "tokenName": "root@pam!rehearsal", + "type": "pve", + "verifySSL": False, + }, + "webhook": { + "enabled": False, + "header_keys": ["X-Rehearsal-Secret"], + "id": "webhook-123", + "method": "POST", + "name": "lifecycle-rehearsal-webhook", + "service": "generic", + "url": "https://example.com/pulse-lifecycle-rehearsal", + }, +} + + +def mutated(path: tuple[str, ...], value: object) -> dict: + snapshot = json.loads(json.dumps(GOOD_SNAPSHOT)) + target = snapshot + for key in path[:-1]: + target = target[key] + target[path[-1]] = value + return snapshot + + +class SnapshotExpectationTest(unittest.TestCase): + """Lost or altered settings must fail even when the baseline also lost them.""" + + def check(self, snapshot: object) -> subprocess.CompletedProcess: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "snapshot.json" + path.write_text(json.dumps(snapshot) if not isinstance(snapshot, str) else snapshot, + encoding="utf-8") + return subprocess.run( + ["bash", str(HARNESS_PATH), "--check-snapshot", str(path)], + capture_output=True, text=True, check=False, + env={"PATH": "/usr/bin:/bin:/usr/local/bin:/opt/homebrew/bin"}, + ) + + def test_intact_snapshot_passes(self) -> None: + result = self.check(GOOD_SNAPSHOT) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_lost_or_changed_settings_fail(self) -> None: + cases = { + "node token secret lost": (("node", "hasToken"), False), + "node token name reset": (("node", "tokenName"), ""), + "node TLS verification flipped": (("node", "verifySSL"), True), + "node host rewritten": (("node", "host"), "https://192.168.77.11:8006"), + "node missing": (("node",), None), + "webhook missing": (("webhook",), None), + "webhook header lost": (("webhook", "header_keys"), []), + "webhook re-enabled": (("webhook", "enabled"), True), + "webhook URL changed": (("webhook", "url"), "https://example.org/"), + "auth no longer required": (("auth", "requiresAuth"), False), + "anonymous access allowed": (("auth", "unauthenticated"), "200"), + "API token rejected": (("auth", "api_token"), "401"), + "password rejected": (("auth", "password"), "401"), + } + for label, (path, value) in cases.items(): + with self.subTest(case=label): + result = self.check(mutated(path, value)) + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + self.assertIn("::error::settings:", result.stdout) + + def test_unparseable_snapshot_fails(self) -> None: + self.assertEqual(self.check("not json").returncode, 1) + + +BASELINE_MANIFEST = { + ".encryption.key": "a" * 64, + ".env": "b" * 64, + "metrics.db": "c" * 64, + "nodes.enc": "d" * 64, + "system.json": "e" * 64, + "webhooks.enc": "f" * 64, +} + + +class DataDirComparisonTest(unittest.TestCase): + def compare(self, current: dict[str, str]) -> subprocess.CompletedProcess: + def manifest(entries: dict[str, str]) -> str: + return "".join(f"{path}\t{digest}\n" for path, digest in sorted(entries.items())) + + with tempfile.TemporaryDirectory() as tmp: + baseline_path = Path(tmp) / "baseline.tsv" + current_path = Path(tmp) / "current.tsv" + baseline_path.write_text(manifest(BASELINE_MANIFEST), encoding="utf-8") + current_path.write_text(manifest(current), encoding="utf-8") + return subprocess.run( + ["bash", str(HARNESS_PATH), "--compare-datadir", str(baseline_path), str(current_path)], + capture_output=True, text=True, check=False, + env={"PATH": "/usr/bin:/bin:/usr/local/bin:/opt/homebrew/bin"}, + ) + + def test_rewritten_plain_files_and_new_files_pass(self) -> None: + current = dict(BASELINE_MANIFEST, **{"system.json": "1" * 64, "metrics.db": "2" * 64, + "new-store.db": "3" * 64}) + result = self.compare(current) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_missing_files_and_rewritten_secret_stores_fail(self) -> None: + cases = { + "file deleted": {k: v for k, v in BASELINE_MANIFEST.items() if k != "metrics.db"}, + "encryption key replaced": dict(BASELINE_MANIFEST, **{".encryption.key": "0" * 64}), + "node secrets rewritten": dict(BASELINE_MANIFEST, **{"nodes.enc": "0" * 64}), + "webhook secrets rewritten": dict(BASELINE_MANIFEST, **{"webhooks.enc": "0" * 64}), + "data dir emptied": {}, + } + for label, current in cases.items(): + with self.subTest(case=label): + result = self.compare(current) + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + self.assertIn("::error::", result.stdout) + + +class WorkflowContractTest(unittest.TestCase): + workflow = WORKFLOW_PATH.read_text(encoding="utf-8") + + def test_triggers_are_dispatch_and_daily_schedule_only(self) -> None: + self.assertIn("workflow_dispatch:", self.workflow) + self.assertIn("from_version:", self.workflow) + self.assertIn("to_version:", self.workflow) + self.assertRegex(self.workflow, r"(?m)^ schedule:\n # [^\n]*\n(?: #[^\n]*\n)* - cron: '23 13 \* \* \*'$") + for trigger in ("pull_request", "push:", "workflow_run", "workflow_call", "release:"): + self.assertNotIn(trigger, self.workflow) + + def test_read_only_hosted_and_report_only(self) -> None: + self.assertRegex(self.workflow, r"(?m)^permissions:\n contents: read$") + self.assertNotIn(": write", self.workflow) + self.assertNotIn("secrets.", self.workflow) + self.assertNotIn("self-hosted", self.workflow) + self.assertIn("runs-on: ubuntu-24.04", self.workflow) + self.assertIn("persist-credentials: false", self.workflow) + self.assertNotIn("upload-artifact", self.workflow) + + def test_dispatch_inputs_reach_shell_only_through_env(self) -> None: + for line in self.workflow.splitlines(): + if "${{" in line and ("inputs." in line or "steps." in line): + self.assertRegex(line.strip(), r"^[A-Z_]+: \$\{\{ [a-z_.]+ \}\}$") + + def test_no_release_workflow_depends_on_the_rehearsal(self) -> None: + for path in (REPO_ROOT / ".github" / "workflows").glob("*.yml"): + if path == WORKFLOW_PATH: + continue + with self.subTest(workflow=path.name): + self.assertNotIn("release-lifecycle-rehearsal", path.read_text(encoding="utf-8")) + + +if __name__ == "__main__": + unittest.main()