From 910418c3b2a165ef5cfb136b144e8ccd95b5d264 Mon Sep 17 00:00:00 2001 From: rcourtman Date: Thu, 9 Jul 2026 20:10:05 +0100 Subject: [PATCH] Make stable patch releases unattended --- .github/scripts/check-demo-reachability.sh | 101 ++++++++ .github/scripts/setup-demo-ssh.sh | 10 +- .github/workflows/create-release.yml | 152 ++++++++++-- .github/workflows/deploy-demo-server.yml | 16 +- .github/workflows/publish-docker.yml | 6 + .github/workflows/release-dry-run.yml | 13 + .github/workflows/update-demo-server.yml | 107 +++++---- .../HIGH_RISK_RELEASE_VERIFICATION_MATRIX.md | 35 +++ .../v6/internal/PRE_RELEASE_CHECKLIST.md | 11 +- .../v6/internal/RELEASE_PROMOTION_POLICY.md | 52 +++- .../v6/internal/SOURCE_OF_TRUTH.md | 19 +- ...atch-unattended-release-path-2026-07-09.md | 72 ++++++ docs/release-control/v6/internal/status.json | 20 ++ .../subsystems/deployment-installability.md | 46 +++- .../v6/internal/subsystems/registry.json | 6 + .../installtests/build_release_assets_test.go | 74 +++++- .../release_promotion_policy_test.py | 66 +++++- .../resolve_release_promotion.py | 150 +++++++++++- .../resolve_release_promotion_test.py | 97 +++++++- scripts/trigger-stable-patch.sh | 223 ++++++++++++++++++ 20 files changed, 1154 insertions(+), 122 deletions(-) create mode 100755 .github/scripts/check-demo-reachability.sh create mode 100644 docs/release-control/v6/internal/records/stable-patch-unattended-release-path-2026-07-09.md create mode 100755 scripts/trigger-stable-patch.sh diff --git a/.github/scripts/check-demo-reachability.sh b/.github/scripts/check-demo-reachability.sh new file mode 100755 index 000000000..7a0a2f223 --- /dev/null +++ b/.github/scripts/check-demo-reachability.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${DEMO_SERVER_HOST:?DEMO_SERVER_HOST is required}" + +MODE="${1:-check}" +TCP_PORT="${DEMO_SERVER_PORT:-22}" +TCP_ATTEMPTS="${DEMO_TCP_ATTEMPTS:-6}" +TCP_RETRY_SECONDS="${DEMO_TCP_RETRY_SECONDS:-5}" + +print_safe_status() { + local status_file + if ! command -v tailscale >/dev/null 2>&1; then + echo "Tailscale CLI is not available." + return 0 + fi + + status_file="$(mktemp)" + if ! tailscale status --json >"$status_file" 2>/dev/null; then + echo "Tailscale status JSON is unavailable." + rm -f "$status_file" + return 0 + fi + + python3 - "$DEMO_SERVER_HOST" "$status_file" <<'PY' || true +import json +import sys + +host = sys.argv[1] +try: + with open(sys.argv[2], encoding="utf-8") as status_file: + status = json.load(status_file) +except (json.JSONDecodeError, OSError): + print("Tailscale status JSON is unavailable.") + raise SystemExit(0) + +self_node = status.get("Self") or {} +self_ips = [ip for ip in self_node.get("TailscaleIPs") or [] if ":" not in ip] +target = None +for peer in (status.get("Peer") or {}).values(): + peer_ips = peer.get("TailscaleIPs") or [] + peer_dns = (peer.get("DNSName") or "").rstrip(".") + if host in peer_ips or host.rstrip(".") == peer_dns: + target = peer + break + +print(f"Tailscale backend: {status.get('BackendState', 'unknown')}") +print(f"Runner Tailscale IPv4: {self_ips[0] if self_ips else 'unavailable'}") +if target is None: + print("Demo peer is not present in the runner peer map yet.") +else: + print( + "Demo peer state: " + f"online={bool(target.get('Online'))} " + f"active={bool(target.get('Active'))} " + f"relay={target.get('Relay') or 'none'}" + ) +PY + rm -f "$status_file" +} + +diagnose() { + print_safe_status + if command -v tailscale >/dev/null 2>&1; then + tailscale ping --c 1 --timeout 5s "$DEMO_SERVER_HOST" || true + fi + nc -z -w 5 "$DEMO_SERVER_HOST" "$TCP_PORT" || true +} + +if [ "$MODE" = "diagnose" ]; then + diagnose + exit 0 +fi +if [ "$MODE" != "check" ]; then + echo "Usage: $0 [check|diagnose]" >&2 + exit 2 +fi + +print_safe_status + +if ! tailscale ping --c 3 --timeout 10s "$DEMO_SERVER_HOST"; then + echo "::error::Tailscale cannot reach the demo peer. Verify that the workflow tag is authorized to reach the demo host tag and that the peer is online." + diagnose + exit 1 +fi + +for attempt in $(seq 1 "$TCP_ATTEMPTS"); do + if nc -z -w 5 "$DEMO_SERVER_HOST" "$TCP_PORT"; then + echo "Demo SSH transport is reachable over Tailscale." + exit 0 + fi + + echo "Demo TCP/${TCP_PORT} is not reachable on attempt ${attempt}/${TCP_ATTEMPTS}." + if [ "$attempt" -lt "$TCP_ATTEMPTS" ]; then + sleep "$TCP_RETRY_SECONDS" + fi +done + +echo "::error::Tailscale reached the demo peer, but TCP/${TCP_PORT} remained closed. Verify sshd and the host firewall on tailscale0." +diagnose +exit 1 diff --git a/.github/scripts/setup-demo-ssh.sh b/.github/scripts/setup-demo-ssh.sh index e0929778b..e3cc0cad1 100644 --- a/.github/scripts/setup-demo-ssh.sh +++ b/.github/scripts/setup-demo-ssh.sh @@ -37,7 +37,8 @@ if is_ip_literal "$DEMO_SERVER_HOST"; then echo "Demo SSH host is an IP literal; skipping DNS resolution wait." fi -MAX_SSH_SETUP_ATTEMPTS=18 +MAX_SSH_SETUP_ATTEMPTS="${DEMO_SSH_SETUP_ATTEMPTS:-3}" +SSH_SETUP_RETRY_SECONDS="${DEMO_SSH_SETUP_RETRY_SECONDS:-5}" for attempt in $(seq 1 "$MAX_SSH_SETUP_ATTEMPTS"); do if [ "$host_needs_dns" = "true" ] && ! getent hosts "$DEMO_SERVER_HOST" >/dev/null 2>&1; then echo "Demo SSH host is not resolvable yet on attempt ${attempt}/${MAX_SSH_SETUP_ATTEMPTS}." @@ -51,14 +52,11 @@ for attempt in $(seq 1 "$MAX_SSH_SETUP_ATTEMPTS"); do fi if [ "$attempt" -lt "$MAX_SSH_SETUP_ATTEMPTS" ]; then - sleep 10 + sleep "$SSH_SETUP_RETRY_SECONDS" fi done -echo "::error::Timed out waiting for the demo SSH host to become reachable and return host keys after Tailscale setup." -if command -v tailscale >/dev/null 2>&1; then - tailscale status --peers=false || true -fi +echo "::error::Demo network preflight passed, but ssh-keyscan did not return host keys. Verify sshd host-key configuration on the target." if [ -s "$keyscan_error" ]; then sed 's/^/ssh-keyscan: /' "$keyscan_error" || true fi diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 13f99d970..c7662cf8f 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -61,6 +61,7 @@ concurrency: cancel-in-progress: false permissions: + actions: read contents: read jobs: @@ -81,6 +82,9 @@ jobs: v5_eos_date: ${{ steps.promotion.outputs.v5_eos_date }} hotfix_exception: ${{ steps.promotion.outputs.hotfix_exception }} hotfix_reason: ${{ steps.promotion.outputs.hotfix_reason }} + promotion_mode: ${{ steps.promotion.outputs.promotion_mode }} + is_stable_patch: ${{ steps.promotion.outputs.is_stable_patch }} + preflight_run_url: ${{ steps.preflight.outputs.run_url }} historical_asset_backfill_only: ${{ steps.extract.outputs.historical_asset_backfill_only }} steps: - name: Extract version @@ -205,6 +209,36 @@ jobs: echo "[OK] Promotion policy validated for ${TAG}" + - name: Require recent exact-SHA stable patch preflight + id: preflight + if: ${{ steps.extract.outputs.historical_asset_backfill_only != 'true' && steps.promotion.outputs.is_stable_patch == 'true' }} + env: + GH_TOKEN: ${{ github.token }} + VERSION: ${{ steps.extract.outputs.version }} + run: | + set -euo pipefail + CUTOFF="$(date -u -d '24 hours ago' '+%Y-%m-%dT%H:%M:%SZ')" + RUN="$(gh api "repos/${{ github.repository }}/actions/workflows/release-dry-run.yml/runs?event=workflow_dispatch&status=success&head_sha=${GITHUB_SHA}&per_page=20" \ + | jq -c \ + --arg sha "$GITHUB_SHA" \ + --arg title "Release Dry Run v${VERSION}" \ + --arg cutoff "$CUTOFF" \ + '[.workflow_runs[] | select( + .conclusion == "success" + and .head_sha == $sha + and .display_title == $title + and .created_at >= $cutoff + )] | sort_by(.created_at) | last // empty')" + + if [ -z "$RUN" ]; then + echo "::error::Stable patch release requires a successful Release Dry Run for v${VERSION} at exact commit ${GITHUB_SHA} within the last 24 hours. Run ./scripts/trigger-stable-patch.sh --dry-run ${VERSION}, wait for success, then dispatch once." + exit 1 + fi + + RUN_URL="$(jq -r '.html_url' <<<"$RUN")" + echo "run_url=${RUN_URL}" >> "$GITHUB_OUTPUT" + echo "[OK] Exact-SHA release preflight passed: ${RUN_URL}" + # Frontend checks run in parallel with backend tests frontend_checks: needs: prepare @@ -1044,30 +1078,6 @@ jobs: if: ${{ github.event.inputs.draft_only == 'true' }} run: 'echo "Draft-only mode: ${{ steps.create_release.outputs.release_url }}"' - - name: Trigger Docker image publish - if: ${{ github.event.inputs.draft_only != 'true' }} - continue-on-error: true - env: - GH_TOKEN: ${{ secrets.WORKFLOW_PAT }} - REQUIRED_BRANCH: ${{ needs.prepare.outputs.required_branch }} - run: | - gh workflow run publish-docker.yml --ref "${REQUIRED_BRANCH}" -f tag="${{ needs.prepare.outputs.tag }}" - echo "[OK] Docker publish workflow dispatched from ${REQUIRED_BRANCH}" - - - name: Trigger demo server update - if: ${{ github.event.inputs.draft_only != 'true' }} - continue-on-error: true - env: - GH_TOKEN: ${{ secrets.WORKFLOW_PAT }} - REQUIRED_BRANCH: ${{ needs.prepare.outputs.required_branch }} - run: | - if [ "${{ needs.prepare.outputs.is_prerelease }}" = "true" ]; then - echo "[OK] Prerelease public demo update skipped; post-GA demo target is stable only." - exit 0 - fi - gh workflow run update-demo-server.yml --ref "${REQUIRED_BRANCH}" -f tag="${{ needs.prepare.outputs.tag }}" -f target="stable" - echo "[OK] Demo server update dispatched for stable from ${REQUIRED_BRANCH}" - - name: Summary run: | echo "[SUCCESS] Release published!" @@ -1127,11 +1137,27 @@ jobs: echo "[SUCCESS] Historical release assets repaired" echo "Release: ${{ needs.prepare.outputs.tag }}" + publish_docker: + needs: + - prepare + - create_release + if: ${{ always() && needs.prepare.result == 'success' && needs.create_release.result == 'success' && needs.prepare.outputs.historical_asset_backfill_only != 'true' && github.event.inputs.draft_only != 'true' }} + permissions: + contents: read + packages: write + id-token: write + attestations: write + uses: ./.github/workflows/publish-docker.yml + secrets: inherit + with: + tag: ${{ needs.prepare.outputs.tag }} + validate_release_assets: needs: - prepare - create_release - if: ${{ always() && needs.prepare.result == 'success' && needs.create_release.result == 'success' && needs.prepare.outputs.historical_asset_backfill_only != 'true' }} + - publish_docker + if: ${{ always() && needs.prepare.result == 'success' && needs.create_release.result == 'success' && needs.prepare.outputs.historical_asset_backfill_only != 'true' && (github.event.inputs.draft_only == 'true' || needs.publish_docker.result == 'success') }} permissions: contents: write issues: write @@ -1174,6 +1200,20 @@ jobs: version: ${{ needs.prepare.outputs.version }} repository: ${{ github.repository }} + update_stable_demo: + needs: + - prepare + - validate_release_assets + if: ${{ always() && needs.prepare.result == 'success' && needs.validate_release_assets.result == 'success' && needs.prepare.outputs.historical_asset_backfill_only != 'true' && github.event.inputs.draft_only != 'true' && needs.prepare.outputs.is_prerelease != 'true' && startsWith(needs.prepare.outputs.version, '6.') }} + permissions: + contents: read + uses: ./.github/workflows/update-demo-server.yml + secrets: inherit + with: + tag: ${{ needs.prepare.outputs.tag }} + target: stable + verify_only: false + # Publish the Helm chart for this release. publish-helm-chart.yml also # listens for `release: published` events directly, but the create_release # publish step PATCHes a draft release to draft=false rather than creating @@ -1337,3 +1377,65 @@ jobs: -f r2_prefix="${r2_prefix}" \ -f allow_ga_prefix="${allow_ga_publish}" wait_for_workflow rcourtman/pulse-pro "Promote Paid Runtime Release" main "${promote_started_at}" "private Pro live promotion" 3600 + + release_verdict: + name: Definitive Release Verdict + needs: + - prepare + - create_release + - publish_docker + - validate_release_assets + - install_sh_smoke + - update_stable_demo + - publish_helm_chart + - promote_floating_tags + - publish_private_pro_runtime + if: ${{ always() && needs.prepare.result == 'success' && needs.prepare.outputs.historical_asset_backfill_only != 'true' }} + runs-on: ubuntu-24.04 + steps: + - name: Enforce terminal release outcomes + env: + DRAFT_ONLY: ${{ github.event.inputs.draft_only }} + VERSION: ${{ needs.prepare.outputs.version }} + IS_PRERELEASE: ${{ needs.prepare.outputs.is_prerelease }} + PREFLIGHT_RUN_URL: ${{ needs.prepare.outputs.preflight_run_url }} + CREATE_RESULT: ${{ needs.create_release.result }} + DOCKER_RESULT: ${{ needs.publish_docker.result }} + VALIDATE_RESULT: ${{ needs.validate_release_assets.result }} + INSTALL_RESULT: ${{ needs.install_sh_smoke.result }} + DEMO_RESULT: ${{ needs.update_stable_demo.result }} + HELM_RESULT: ${{ needs.publish_helm_chart.result }} + FLOATING_RESULT: ${{ needs.promote_floating_tags.result }} + PRIVATE_PRO_RESULT: ${{ needs.publish_private_pro_runtime.result }} + run: | + set -euo pipefail + require_result() { + local name="$1" + local actual="$2" + local expected="$3" + if [ "$actual" != "$expected" ]; then + echo "::error::${name} ended as ${actual}; expected ${expected}." + return 1 + fi + } + + require_result "release assembly" "$CREATE_RESULT" success + require_result "release asset validation" "$VALIDATE_RESULT" success + + if [ "${DRAFT_ONLY:-false}" != "true" ]; then + require_result "Docker publication" "$DOCKER_RESULT" success + require_result "install.sh smoke" "$INSTALL_RESULT" success + require_result "Helm publication" "$HELM_RESULT" success + require_result "floating-tag promotion" "$FLOATING_RESULT" success + if [[ "$VERSION" == 6.* ]]; then + require_result "private Pro publication" "$PRIVATE_PRO_RESULT" success + if [ "$IS_PRERELEASE" != "true" ]; then + require_result "stable demo deployment and verification" "$DEMO_RESULT" success + fi + fi + fi + + echo "Release verdict passed for v${VERSION}." + if [ -n "${PREFLIGHT_RUN_URL:-}" ]; then + echo "Stable patch preflight: ${PREFLIGHT_RUN_URL}" + fi diff --git a/.github/workflows/deploy-demo-server.yml b/.github/workflows/deploy-demo-server.yml index 9fef4bbed..eafbbac0a 100644 --- a/.github/workflows/deploy-demo-server.yml +++ b/.github/workflows/deploy-demo-server.yml @@ -133,11 +133,25 @@ jobs: -o pulse ./cmd/pulse/ - name: Tailscale - uses: tailscale/github-action@4e4c49acaa9818630ce0bd7a564372c17e33fb4d # v2 + id: tailscale + uses: tailscale/github-action@306e68a486fd2350f2bfc3b19fcd143891a4a2d8 # v4 with: oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }} oauth-secret: ${{ secrets.TS_OAUTH_SECRET }} tags: tag:infra + version: '1.94.2' + ping: ${{ secrets.DEMO_SERVER_HOST }} + + - name: Diagnose Tailscale setup failure + if: failure() && steps.tailscale.outcome == 'failure' + env: + DEMO_SERVER_HOST: ${{ secrets.DEMO_SERVER_HOST }} + run: bash .github/scripts/check-demo-reachability.sh diagnose + + - name: Verify demo network path + env: + DEMO_SERVER_HOST: ${{ secrets.DEMO_SERVER_HOST }} + run: bash .github/scripts/check-demo-reachability.sh - name: Setup SSH env: diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml index 16341579f..1eb41c9d1 100644 --- a/.github/workflows/publish-docker.yml +++ b/.github/workflows/publish-docker.yml @@ -4,6 +4,12 @@ run-name: Publish Docker Images ${{ inputs.tag }} # Triggered by create-release.yml after staging images pass tests. # Builds multi-arch images (amd64+arm64) from source and publishes to Docker Hub and GHCR. on: + workflow_call: + inputs: + tag: + description: 'Release tag (e.g., v4.34.0)' + required: true + type: string workflow_dispatch: inputs: tag: diff --git a/.github/workflows/release-dry-run.yml b/.github/workflows/release-dry-run.yml index 57d49f511..2bc186a2e 100644 --- a/.github/workflows/release-dry-run.yml +++ b/.github/workflows/release-dry-run.yml @@ -1,4 +1,5 @@ name: Release Dry Run +run-name: Release Dry Run v${{ inputs.version || 'scheduled' }} on: # Weekly drift watchdog: fires every Tuesday 07:00 UTC against the governed @@ -384,3 +385,15 @@ jobs: with: name: rc-to-ga-rehearsal-summary path: release-dry-run/rc-to-ga-rehearsal-summary.md + + demo_path_preflight: + name: Verify Current Stable Demo Path (No Mutation) + needs: dry-run + permissions: + contents: read + uses: ./.github/workflows/update-demo-server.yml + secrets: inherit + with: + tag: latest + target: stable + verify_only: true diff --git a/.github/workflows/update-demo-server.yml b/.github/workflows/update-demo-server.yml index 70d4e70e3..5b463eb47 100644 --- a/.github/workflows/update-demo-server.yml +++ b/.github/workflows/update-demo-server.yml @@ -1,8 +1,22 @@ name: Update Demo Server on: - release: - types: [published] + workflow_call: + inputs: + tag: + description: 'Stable release tag to deploy, or latest for verification-only checks' + required: true + type: string + target: + description: 'Demo target to deploy' + required: false + default: stable + type: string + verify_only: + description: 'Verify connectivity and the current stable demo without changing the host' + required: false + default: false + type: boolean workflow_dispatch: inputs: tag: @@ -17,6 +31,11 @@ on: options: - auto - stable + verify_only: + description: 'Verify connectivity and the current stable demo without changing the host' + required: false + default: false + type: boolean permissions: contents: read @@ -28,25 +47,32 @@ jobs: tag: ${{ steps.target.outputs.tag }} target: ${{ steps.target.outputs.target }} environment_name: ${{ steps.target.outputs.environment_name }} - skip: ${{ steps.target.outputs.skip || steps.latest.outputs.skip || 'false' }} + skip: ${{ steps.target.outputs.skip || 'false' }} steps: - name: Resolve target tag and demo environment id: target env: - EVENT_NAME: ${{ github.event_name }} INPUT_TAG: ${{ inputs.tag }} INPUT_TARGET: ${{ inputs.target }} - RELEASE_TAG: ${{ github.event.release.tag_name }} + VERIFY_ONLY: ${{ inputs.verify_only }} + GH_TOKEN: ${{ github.token }} run: | set -euo pipefail - if [ "$EVENT_NAME" = "workflow_dispatch" ]; then - TAG="$INPUT_TAG" - REQUESTED_TARGET="$INPUT_TARGET" - else - TAG="$RELEASE_TAG" - REQUESTED_TARGET="auto" + TAG="$INPUT_TAG" + REQUESTED_TARGET="$INPUT_TARGET" + if [ -z "$TAG" ]; then + echo "::error::A stable release tag is required." + exit 1 + fi + if [ "$TAG" = "latest" ]; then + if [ "${VERIFY_ONLY:-false}" != "true" ]; then + echo "::error::The latest alias is allowed only for verification-only checks." + exit 1 + fi + TAG="$(gh api "repos/${{ github.repository }}/releases/latest" --jq '.tag_name')" + echo "Resolved verification-only target to latest stable release ${TAG}." fi VERSION="${TAG#v}" @@ -118,32 +144,6 @@ jobs: echo "[OK] ${TAG} validated for governed demo deployment on ${REQUIRED_BRANCH}" - - name: Skip if not latest published release for target - id: latest - if: github.event_name == 'release' && steps.target.outputs.skip != 'true' - env: - GH_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - TAG="${{ steps.target.outputs.tag }}" - - LATEST=$(gh api "repos/${{ github.repository }}/releases/latest" --jq '.tag_name') - - echo "Target tag: $TAG" - echo "Latest published stable release: $LATEST" - - if [ -z "$LATEST" ] || [ "$LATEST" = "null" ]; then - echo "::error::Could not determine the latest published stable release for demo deployment." - exit 1 - fi - - if [ "$TAG" != "$LATEST" ]; then - echo "skip=true" >> "$GITHUB_OUTPUT" - echo "Release is not the latest published stable tag; skipping demo update." - else - echo "skip=false" >> "$GITHUB_OUTPUT" - fi - update-demo: needs: resolve if: needs.resolve.outputs.skip != 'true' @@ -163,6 +163,7 @@ jobs: echo "Tag: ${{ needs.resolve.outputs.tag }}" echo "Target: ${{ needs.resolve.outputs.target }}" echo "Environment: ${{ needs.resolve.outputs.environment_name }}" + echo "Verification only: ${{ inputs.verify_only }}" - name: Validate demo environment configuration env: @@ -185,11 +186,13 @@ jobs: fetch-tags: true - name: Set up Go + if: inputs.verify_only != true uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 with: go-version-file: go.mod - name: Wait for release assets + if: inputs.verify_only != true run: | set -euo pipefail TAG="${{ needs.resolve.outputs.tag }}" @@ -224,6 +227,7 @@ jobs: exit 1 - name: Materialize tagged installer + if: inputs.verify_only != true env: PULSE_UPDATE_SIGNING_PUBLIC_KEY: ${{ vars.PULSE_UPDATE_SIGNING_PUBLIC_KEY }} run: | @@ -240,11 +244,25 @@ jobs: chmod +x /tmp/pulse-install.sh - name: Tailscale - uses: tailscale/github-action@4e4c49acaa9818630ce0bd7a564372c17e33fb4d # v2 + id: tailscale + uses: tailscale/github-action@306e68a486fd2350f2bfc3b19fcd143891a4a2d8 # v4 with: oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }} oauth-secret: ${{ secrets.TS_OAUTH_SECRET }} tags: tag:infra + version: '1.94.2' + ping: ${{ secrets.DEMO_SERVER_HOST }} + + - name: Diagnose Tailscale setup failure + if: failure() && steps.tailscale.outcome == 'failure' + env: + DEMO_SERVER_HOST: ${{ secrets.DEMO_SERVER_HOST }} + run: bash .github/scripts/check-demo-reachability.sh diagnose + + - name: Verify demo network path + env: + DEMO_SERVER_HOST: ${{ secrets.DEMO_SERVER_HOST }} + run: bash .github/scripts/check-demo-reachability.sh - name: Setup SSH env: @@ -305,8 +323,14 @@ jobs: echo "skip_current=false" >> "$GITHUB_OUTPUT" fi + - name: Refuse mutation during verification-only checks + if: inputs.verify_only == true && steps.current.outputs.skip_current != 'true' + run: | + echo "::error::Verification-only check found a demo version mismatch; refusing to update the host." + exit 1 + - name: Prepare demo host storage - if: steps.current.outputs.skip_current != 'true' + if: inputs.verify_only != true && steps.current.outputs.skip_current != 'true' env: DEMO_SERVER_HOST: ${{ secrets.DEMO_SERVER_HOST }} DEMO_SERVER_USER: ${{ secrets.DEMO_SERVER_USER }} @@ -400,7 +424,7 @@ jobs: ssh -i ~/.ssh/id_ed25519 "$DEMO_SERVER_USER@$DEMO_SERVER_HOST" "bash -s -- $(printf '%q ' "$SERVICE_NAME")" <<<"$REMOTE_SCRIPT" - name: Upload tagged installer - if: steps.current.outputs.skip_current != 'true' + if: inputs.verify_only != true && steps.current.outputs.skip_current != 'true' env: DEMO_SERVER_HOST: ${{ secrets.DEMO_SERVER_HOST }} DEMO_SERVER_USER: ${{ secrets.DEMO_SERVER_USER }} @@ -409,7 +433,7 @@ jobs: scp -i ~/.ssh/id_ed25519 /tmp/pulse-install.sh "$DEMO_SERVER_USER@$DEMO_SERVER_HOST:/tmp/pulse-install.sh" - name: Update demo server - if: steps.current.outputs.skip_current != 'true' + if: inputs.verify_only != true && steps.current.outputs.skip_current != 'true' env: DEMO_SERVER_HOST: ${{ secrets.DEMO_SERVER_HOST }} DEMO_SERVER_USER: ${{ secrets.DEMO_SERVER_USER }} @@ -432,6 +456,7 @@ jobs: ssh -i ~/.ssh/id_ed25519 "$DEMO_SERVER_USER@$DEMO_SERVER_HOST" "bash -s -- $(printf '%q ' "$TAG" "$SERVICE_NAME")" <<<"$REMOTE_SCRIPT" - name: Restore demo runtime configuration + if: inputs.verify_only != true env: DEMO_SERVER_HOST: ${{ secrets.DEMO_SERVER_HOST }} DEMO_SERVER_USER: ${{ secrets.DEMO_SERVER_USER }} diff --git a/docs/release-control/v6/internal/HIGH_RISK_RELEASE_VERIFICATION_MATRIX.md b/docs/release-control/v6/internal/HIGH_RISK_RELEASE_VERIFICATION_MATRIX.md index 9b8e61e3c..2130164a9 100644 --- a/docs/release-control/v6/internal/HIGH_RISK_RELEASE_VERIFICATION_MATRIX.md +++ b/docs/release-control/v6/internal/HIGH_RISK_RELEASE_VERIFICATION_MATRIX.md @@ -876,6 +876,41 @@ Companion drill: can expose invoice/license data, consume verification, or mutate Stripe or license state before the canonical case decision allows it. +## Gate: `stable-patch-unattended-release-path` + +- Owner lanes: `L1` +- Risk covered: + A stable patch can publish while its demo deployment is detached, stale + release code can bypass preflight, or a private-network failure can consume + operator time and force manual SSH deployment after the public cut. +- Primary runtime surfaces: + `.github/workflows/create-release.yml` + `.github/workflows/release-dry-run.yml` + `.github/workflows/update-demo-server.yml` + `.github/scripts/check-demo-reachability.sh` + `scripts/trigger-stable-patch.sh` +- Automated proof: + `python3 -m unittest scripts.release_control.resolve_release_promotion_test scripts.release_control.release_promotion_policy_test` + `go test ./scripts/installtests -run 'Test(Demo|DeployDemo|UpdateDemo|Release)' -count=1` +- Manual scenario: + 1. Push the exact candidate commit to the governed stable branch. + 2. Dispatch `./scripts/trigger-stable-patch.sh --dry-run ` and do not + publish another release. + 3. Confirm the run passes `Verify Current Stable Demo Path (No Mutation)` on + a GitHub-hosted runner, including Tailscale ping, TCP/22, SSH host identity, + current stable version, frontend parity, public health, and browser smoke. + 4. Confirm the public demo version and health remain unchanged after the run. +- Pass when: + The exact-SHA dry run succeeds without host mutation, routine patch metadata + rejects the documented RC-required risk paths, and the publish DAG awaits + Docker, demo, public verification, and the definitive terminal verdict. +- Latest exercised record: + `docs/release-control/v6/internal/records/stable-patch-unattended-release-path-2026-07-09.md` +- Block release if: + The exact-SHA dry run is missing or older than 24 hours, the no-mutation demo + path fails, demo deployment is detached from the release DAG, or routine mode + can bypass a same-version RC or an RC-required runtime change. + ## Gate Ownership Rule Update these machine-visible gate states in `docs/release-control/v6/internal/status.json` diff --git a/docs/release-control/v6/internal/PRE_RELEASE_CHECKLIST.md b/docs/release-control/v6/internal/PRE_RELEASE_CHECKLIST.md index 74fa5305d..29ca2c577 100644 --- a/docs/release-control/v6/internal/PRE_RELEASE_CHECKLIST.md +++ b/docs/release-control/v6/internal/PRE_RELEASE_CHECKLIST.md @@ -37,12 +37,15 @@ Use this as the final gate before cutting a Pulse v6 pre-release. - High-risk release confidence now lives in `docs/release-control/v6/internal/HIGH_RISK_RELEASE_VERIFICATION_MATRIX.md` and should be cleared alongside this checklist. ## Promotion Policy +- [ ] For a routine stable patch, run `./scripts/trigger-stable-patch.sh --dry-run ` from the exact pushed candidate SHA, wait for the whole run including `Verify Current Stable Demo Path (No Mutation)` to pass, then run `./scripts/trigger-stable-patch.sh ` once. +- [ ] Confirm a routine stable patch has no same-version RC and no diff in the RC-required authentication/tenant, licensing/billing, persisted-data/migration, relay/mobile-trust, or installer/update/rollback boundaries. Otherwise use RC promotion or record the emergency hotfix reason. +- [ ] Treat `Definitive Release Verdict` as the release result. Do not accept a green asset-publish job while Docker publication, demo deployment, public browser verification, Helm/floating-tag promotion, or private Pro promotion is detached or incomplete. - [x] Record the previous stable tag and exact rollback pin command before publishing a new prerelease or stable release. - [ ] For any prerelease or stable publication, confirm the repo variable `PULSE_UPDATE_SIGNING_PUBLIC_KEY` is set to the intended active update signer public key and that the release workflows are consuming it alongside `PULSE_UPDATE_SIGNING_KEY`, so accidental trust-root rotation fails closed before publication. -- [x] For the GA/stable candidate, confirm the release pipeline has already been exercised on a real prerelease tag, not only linted or YAML-parsed. -- [x] For stable promotion, confirm the candidate commit has already shipped on `rc`. -- [x] For stable promotion, confirm the chosen `promoted_from_tag` is a prerelease that was actually published through the governed prerelease path, not an accidental git tag. -- [x] For stable promotion, confirm the prerelease soak window is at least 72 hours or document the hotfix exception explicitly. +- [x] For the first GA or an RC-required stable promotion, confirm the release pipeline has already been exercised on a real prerelease tag, not only linted or YAML-parsed. +- [x] For an RC-required stable promotion, confirm the candidate commit has already shipped on `rc`. +- [x] For an RC-required stable promotion, confirm the chosen `promoted_from_tag` is a prerelease that was actually published through the governed prerelease path, not an accidental git tag. +- [x] For an RC-required stable promotion, confirm the prerelease soak window is at least 72 hours or document the hotfix exception explicitly. - [x] For stable promotion, record the 2026-07-02 release-owner decision accepting the current-branch validation risk for the post-RC7 changes. - [x] For GA/stable promotion, confirm `V5_MAINTENANCE_SUPPORT_POLICY.md` is still the intended policy and replace any placeholder GA notice dates with the exact v6 GA date and exact v5 end-of-support date that will ship with the announcement. - [x] For GA/stable promotion, confirm the pushed governed release-branch copy of `.github/workflows/release-dry-run.yml` already accepts the governed stable rehearsal metadata envelope (`promoted_from_tag`, `rollback_version`, `ga_date`, `v5_eos_date`) through `workflow_dispatch`, because GitHub executes the selected remote ref and does not see local-only governance state. diff --git a/docs/release-control/v6/internal/RELEASE_PROMOTION_POLICY.md b/docs/release-control/v6/internal/RELEASE_PROMOTION_POLICY.md index 12a596bdc..9a1113400 100644 --- a/docs/release-control/v6/internal/RELEASE_PROMOTION_POLICY.md +++ b/docs/release-control/v6/internal/RELEASE_PROMOTION_POLICY.md @@ -119,8 +119,9 @@ Cloud, and self-hosted production users. ## Stable Promotion Rules -1. A stable tag must be promoted from a commit that has already been exercised - as a published prerelease. +1. A first stable release, a stable minor release, and every patch that crosses + one of the RC-required risk boundaries below must be promoted from a commit + that has already been exercised as a published prerelease. 2. A prerelease git tag counts as stable-promotion lineage only if that prerelease was actually published through the governed prerelease path; accidental or abandoned git tags do not satisfy the stable-promotion requirement. @@ -130,24 +131,26 @@ Cloud, and self-hosted production users. 4. Every stable promotion requires: - Applicable items in `PRE_RELEASE_CHECKLIST.md` complete. - Applicable entries in `HIGH_RISK_RELEASE_VERIFICATION_MATRIX.md` cleared. + - The previous stable rollback target and exact reinstall command recorded. +5. A first stable release or RC-required stable promotion additionally requires: - No known unresolved RC-era user-visible issues intended for the v6 GA scope remain open. Each one must be fixed in the candidate, proven invalid with evidence, or conservatively superseded with the original failure resolved or explicitly narrowed. - - The previous stable rollback target and exact reinstall command recorded. - A live release-pipeline exercise already completed for the promoted prerelease tag, not only YAML lint or static workflow validation. +6. The first v6 GA promotion additionally requires: - The locked 90-day v5 maintenance-only policy in `V5_MAINTENANCE_SUPPORT_POLICY.md` and the exact end-of-support notice ready to publish with the promotion. -5. Normal stable promotions require a minimum 72-hour prerelease soak after the - candidate is available to internal or staging-like users. -6. Hotfix exception: - - A shorter soak is allowed only for narrowly scoped fixes to active - customer harm. +7. RC-derived stable promotions require a minimum 72-hour prerelease soak after + the candidate is available to internal or staging-like users. +8. Hotfix exception: + - Bypassing an RC requirement or shortening an RC soak is allowed only for + narrowly scoped fixes to active customer harm. - The exception plus the rollback target and exact reinstall command must be recorded in the release notes or release ticket before promotion. -7. v6.0.0 owner-risk exception: +9. v6.0.0 owner-risk exception: - On 2026-07-02, after seven v6 release candidates, the release owner explicitly approved promoting the current `pulse/v6-release` branch with accumulated post-RC7 changes without RC8, another soak, or additional @@ -160,6 +163,37 @@ Cloud, and self-hosted production users. and retain the prior governed release-pipeline rehearsal evidence as automation lineage rather than claiming the post-RC7 changes were RC-tested. +## Routine Stable Patch Path + +1. A normal stable patch may omit a same-version RC only when all of these are + true: + - the rollback target is the latest preceding stable tag and the candidate + descends from it; + - no same-version RC tag already exists; + - the diff does not touch authentication/authorization/tenant isolation, + licensing/entitlement/billing authority, persisted data/schema/migration, + relay/mobile trust protocol, or installer/updater/rollback execution; + - the canonical stable release-notes packet exists; + - the mobile-impact gate either proves no mobile-facing change or records + current candidate evidence; and + - `Release Dry Run` passed for the exact candidate SHA within the previous + 24 hours. That run must include the no-mutation stable-demo path check. +2. `scripts/trigger-stable-patch.sh` is the standard operator entrypoint. Run + it once with `--dry-run`, monitor asynchronously, then run it once without + `--dry-run`. It derives rollback and release notes, refuses local-only or + dirty state, and supplies the workflow metadata without interactive prompts. +3. Creating a same-version RC or touching an RC-required path moves the patch + onto the RC promotion path. The resolver enforces that boundary. Do not use + the routine helper to relabel a risky patch as routine. +4. `--emergency-hotfix-reason` is the narrow escape hatch for active customer + harm. It does not remove the exact-SHA dry-run requirement, and the reason is + recorded in the release metadata. +5. The release workflow must await Docker publication, stable demo deployment, + public health/browser verification, install smoke, Helm publication, + floating-tag promotion, and private Pro promotion where applicable. The + terminal `Definitive Release Verdict` job is the one release result; an + asynchronously dispatched demo workflow is not release completion. + ## Rollout Rules 1. Default installs stay on `stable`. diff --git a/docs/release-control/v6/internal/SOURCE_OF_TRUTH.md b/docs/release-control/v6/internal/SOURCE_OF_TRUTH.md index a158c541f..b8605761a 100644 --- a/docs/release-control/v6/internal/SOURCE_OF_TRUTH.md +++ b/docs/release-control/v6/internal/SOURCE_OF_TRUTH.md @@ -443,6 +443,11 @@ Assertion design rules: Claims reduce overlap, but they do not isolate hooks, formatters, staged reads, or unrelated dirt. Parallel mutation should use separate worktrees so each agent sees one slice's git state at a time. +22. Do not publish a routine stable patch without a successful exact-SHA + `Release Dry Run` from the previous 24 hours. That run must prove the + current stable demo network/SSH/browser path without mutation, and the + publish workflow must await demo deployment plus definitive verification; + manual SSH deployment is not an acceptable release completion path. ## Locked Decisions @@ -468,9 +473,10 @@ Assertion design rules: and downgrade safety. 6. Cloud and MSP Stripe `price_*` IDs are operational fill-in items, not architectural blockers. -7. Stable or GA promotion for v6 must come from an exercised RC and stay - blocked until the RC-to-GA promotion gate is cleared and the published v5 - maintenance-policy notice is ready. For v6.0.0 only, the 2026-07-02 +7. First-stable, GA, minor-line, and risk-bearing stable promotion for v6 must + come from an exercised RC and stay blocked until the applicable promotion + gates are cleared. Routine post-GA stable patches may use the governed + no-RC path defined below. For v6.0.0 only, the 2026-07-02 release-owner risk acceptance allows the current post-RC7 `pulse/v6-release` branch to ship without RC8, another soak, or additional current-branch validation before GA; this is not validation evidence and not a standing @@ -509,6 +515,13 @@ Assertion design rules: issue intended for v6 is fixed in the candidate, proven invalid with evidence, or conservatively superseded with the original problem resolved or explicitly narrowed. +15. Routine stable patch releases after GA do not require a fabricated RC. + They may use the no-RC path only when the candidate descends from the latest + stable rollback target, no same-version RC exists, no governed high-risk + auth/tenant, licensing/billing, persisted-data/migration, relay/mobile + trust, or installer/update/rollback path changed, and the exact-SHA release + dry run passed. Those risk conditions require RC lineage unless an active + customer-harm emergency is recorded through the hotfix exception. ## TrueNAS Support Floor diff --git a/docs/release-control/v6/internal/records/stable-patch-unattended-release-path-2026-07-09.md b/docs/release-control/v6/internal/records/stable-patch-unattended-release-path-2026-07-09.md new file mode 100644 index 000000000..1a9191aa4 --- /dev/null +++ b/docs/release-control/v6/internal/records/stable-patch-unattended-release-path-2026-07-09.md @@ -0,0 +1,72 @@ +# Stable Patch Unattended Release Path - 2026-07-09 + +## Scope + +Establish the durable gate that a routine stable patch requires minutes of +operator attention, one exact-SHA preflight, one publish dispatch, awaited demo +deployment, and one definitive release verdict. + +## v6.0.5 Evidence + +- Three release runs (`29013413583`, `29016263854`, and `29019195340`) each + spent approximately 37 to 46 minutes before failing integration tests. The + exact candidate SHA had no mandatory release preflight, so the operator paid + that cost inside the public-release operation on every attempt. +- The successful public `v6.0.5` release pipeline run `29022145812` took + approximately two hours. Docker publication and demo deployment were + dispatched asynchronously with `continue-on-error`, so its green result was + not a definitive release verdict. +- Demo update run `29032637236` joined the business tailnet as an ephemeral + `tag:infra` node but used Tailscale `1.42.0`; all 18 `ssh-keyscan` attempts + then timed out without a Tailscale peer-propagation ping or TCP/22 diagnosis. +- The active business-tailnet policy was inspected on 2026-07-09. The OAuth + credential `github-actions-infra` is active with all scopes, `tag:infra` is + owned by `autogroup:admin`, and the ACL explicitly allows `tag:infra` + sources to reach `tag:infra` destinations on TCP 22 and 443. The demo host + is online at its governed Tailscale address and local TCP/22 succeeds. The + external OAuth/tag/ACL configuration is therefore not the failing boundary. +- The workflow exposed no peer-map, Tailscale ping, or TCP/22 evidence. The + operator had to infer a network failure from blind host-key retries, inspect + Tailscale policy separately, and deploy a signed installer over local SSH. +- `https://demo.pulserelay.pro/api/version` reported stable `6.0.5`, and + `/api/health` reported healthy before the workflow correction. + +## Avoidable Delay and Intervention Inventory + +1. Release tests ran after the operator crossed the publication boundary + instead of as a recent exact-SHA prerequisite. +2. The stable resolver required RC lineage for every normal patch, forcing + routine low-risk fixes through RC ceremony or a misleading hotfix exception. +3. The general release helper prompted interactively for metadata already + derivable from the repository and release packet. +4. Docker publication and demo deployment were detached follow-on dispatches; + the top-level result could not answer whether the release was operational. +5. The demo path used an obsolete Tailscale client without waiting for + eventually consistent peer propagation. +6. Eighteen blind SSH host-key retries consumed about six minutes while hiding + whether the failing layer was tailnet policy, peer visibility, TCP/22, or + `sshd`. +7. Demo recovery required manual SSH and a locally materialized signed + installer after the release workflow had already reported success. + +## Repository Correction + +- Both demo workflows use the pinned Tailscale GitHub Action v4 client and its + target `ping` readiness input. +- Shared diagnostics distinguish tailnet reachability from TCP/22 and SSH host + key failures without printing credentials. +- `Update Demo Server` supports an awaited `workflow_call` and a no-mutation + verification mode used by `Release Dry Run`. +- The release workflow awaits Docker publication and stable demo deployment, + then emits a terminal `Definitive Release Verdict`. +- Routine stable patch release resolution no longer fabricates RC ceremony, + but it fails closed for documented high-risk runtime changes, an existing + same-version RC, stale rollback lineage, or a missing exact-SHA dry run. +- `scripts/trigger-stable-patch.sh` is the noninteractive operator entrypoint. + +## Current Verdict + +Blocked pending one pushed exact-SHA `Release Dry Run` that exercises the new +no-mutation demo path on GitHub-hosted infrastructure. This record must be +updated with that run URL and the gate promoted to `passed` only after the run +and local public-demo checks both succeed. diff --git a/docs/release-control/v6/internal/status.json b/docs/release-control/v6/internal/status.json index a47c1bc70..f8af87669 100644 --- a/docs/release-control/v6/internal/status.json +++ b/docs/release-control/v6/internal/status.json @@ -6764,6 +6764,26 @@ } ] }, + { + "id": "stable-patch-unattended-release-path", + "summary": "Confirm routine stable patches require one recent exact-SHA dry run, one noninteractive publish dispatch, awaited Docker and demo deployment, and a definitive release verdict without manual SSH recovery.", + "owner": "project-owner", + "blocking_level": "release-ready", + "minimum_evidence_tier": "real-external-e2e", + "status": "blocked", + "verification_doc": "docs/release-control/v6/internal/HIGH_RISK_RELEASE_VERIFICATION_MATRIX.md", + "lane_ids": [ + "L1" + ], + "evidence": [ + { + "repo": "pulse", + "path": "docs/release-control/v6/internal/records/stable-patch-unattended-release-path-2026-07-09.md", + "kind": "file", + "evidence_tier": "local-rehearsal" + } + ] + }, { "id": "unified-agent-v5-upgrade-continuity", "summary": "Confirm a real v5-installed Pulse Unified Agent upgrades through candidate v6 RC assets into one canonical v6 agent identity without duplicate registration, stale fallback or legacy-scope breakage, or agent-count drift.", diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index cdeb5ed52..6be38561c 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -125,6 +125,9 @@ TLS floor in the dynamic config. 89. `scripts/release_asset_common.sh` 90. `scripts/backfill-release-assets.sh` 91. `.github/workflows/backfill-release-assets.yml` +92. `.github/scripts/check-demo-reachability.sh` +93. `.github/scripts/setup-demo-ssh.sh` +94. `scripts/trigger-stable-patch.sh` ## Shared Boundaries @@ -675,14 +678,17 @@ TLS floor in the dynamic config. `scripts/install-docker.sh` fallback from the final RC image tag to the stable `6.0.0` image tag in the same commit as `VERSION=6.0.0`. Stable patch releases after `6.0.0` stay on this same governed release - boundary but do not need a fabricated same-version RC tag when the release - owner is intentionally publishing a hotfix patch from the current stable - branch. In that case `resolve_release_promotion.py` may accept an omitted - `promoted_from_tag` only for a stable semver patch version with - `hotfix_exception=true`, a non-empty `hotfix_reason`, and - `rollback_version` set to the previous stable tag. Non-hotfix stable - promotions, and any first-GA or minor-line stable promotion, still require - explicit promoted prerelease lineage and soak proof. Stable patch release + boundary but do not need a fabricated same-version RC tag for a routine + patch. `resolve_release_promotion.py` owns the machine boundary: the + rollback target must be the latest preceding stable tag, the candidate must + descend from it, no same-version RC may already exist, and the diff may not + touch authentication/tenant isolation, licensing/billing authority, + persisted-data/schema migration, relay/mobile trust protocol, or + installer/updater/rollback execution. Those risk classes require exercised + RC lineage unless active customer harm is recorded with + `hotfix_exception=true` and a non-empty `hotfix_reason`. First-GA and minor + stable promotions still require explicit promoted prerelease lineage and + soak proof. Stable patch release packets must also enumerate every customer-visible support fix included in the cut, and the release-asset proof must pin the current packet to those runtime fixes so a patch that includes support work cannot ship as a @@ -1115,6 +1121,17 @@ the manual `trigger-release*.sh` entrypoints must all derive their governed release line from control-plane metadata before they touch public artifacts or deployment targets, rather than treating tag names or workflow triggers as enough proof on their own. +For routine stable patches, `scripts/trigger-stable-patch.sh` is the +noninteractive operator path. It derives the latest stable rollback, consumes +the canonical `docs/releases/RELEASE_NOTES_vX.Y.Z.md` packet, infers +`no-mobile-impact` only when no mobile-facing path changed, and dispatches one +dry-run or publish workflow. `create-release.yml` must independently require a +successful `workflow_dispatch` `Release Dry Run` for the exact candidate SHA +and version from the previous 24 hours, so the UI and alternate helpers cannot +bypass the preflight. That dry run must call `update-demo-server.yml` in +verification-only mode against the latest stable release. It must prove +Tailscale, SSH host identity, runtime version, frontend parity, public health, +and browser smoke without changing the host. That same release-validation boundary also owns draft-versus-published asset state. When `.github/workflows/create-release.yml` runs in `draft_only` mode, it must pass the real draft state into `.github/workflows/validate-release-assets.yml` @@ -1188,10 +1205,19 @@ Those same governed demo deploy/update workflows also own the runner-to-host network path. They must establish the canonical Tailscale connectivity step before SSH setup so stable or preview targets may stay on governed private hostnames or Tailscale IPs, rather than silently depending on public SSH -reachability from GitHub-hosted runners. After Tailscale setup, shared SSH +reachability from GitHub-hosted runners. The workflows must use the current +pinned Tailscale GitHub Action, its target `ping` readiness gate, and the shared +`.github/scripts/check-demo-reachability.sh` TCP/22 diagnostic before SSH key +capture. A successful tailnet join alone is not connectivity proof. After that +network preflight, shared SSH setup must wait for configured demo hostnames to resolve, accept configured IP literals without a DNS precheck, and then capture host keys with bounded -retries before any installer or binary copy runs; a one-shot `ssh-keyscan` +short retries before any installer or binary copy runs; a long `ssh-keyscan` +loop must not hide an ACL, peer-propagation, firewall, or sshd failure. +`create-release.yml` must call the update workflow as an awaited reusable job, +and its terminal `Definitive Release Verdict` must require stable demo runtime, +frontend, public health, and browser proof. An asynchronous dispatch or manual +SSH deployment is not release completion. A one-shot `ssh-keyscan` against a private demo target is not sufficient release or deploy proof. Those same workflows also own customer-visible browser truth for the public demo shell. Health checks and entry-asset parity are necessary but not diff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.json index e9804c581..8002b151d 100644 --- a/docs/release-control/v6/internal/subsystems/registry.json +++ b/docs/release-control/v6/internal/subsystems/registry.json @@ -3299,6 +3299,8 @@ "owned_files": [ ".dockerignore", ".github/ISSUE_TEMPLATE/v6_rc_feedback.yml", + ".github/scripts/check-demo-reachability.sh", + ".github/scripts/setup-demo-ssh.sh", ".github/workflows/backfill-release-assets.yml", ".github/workflows/create-release.yml", ".github/workflows/deploy-demo-server.yml", @@ -3375,6 +3377,7 @@ "scripts/toggle-mock.sh", "scripts/trigger-release-dry-run.sh", "scripts/trigger-release.sh", + "scripts/trigger-stable-patch.sh", "tests/integration/playwright.config.ts", "tests/integration/QUICK_START.md", "tests/integration/README.md", @@ -3453,6 +3456,8 @@ ], "match_files": [ ".github/ISSUE_TEMPLATE/v6_rc_feedback.yml", + ".github/scripts/check-demo-reachability.sh", + ".github/scripts/setup-demo-ssh.sh", ".github/workflows/backfill-release-assets.yml", ".github/workflows/create-release.yml", ".github/workflows/helm-pages.yml", @@ -3478,6 +3483,7 @@ "scripts/release_control/validate_artifact_release_line.py", "scripts/trigger-release-dry-run.sh", "scripts/trigger-release.sh", + "scripts/trigger-stable-patch.sh", "VERSION" ], "allow_same_subsystem_tests": false, diff --git a/scripts/installtests/build_release_assets_test.go b/scripts/installtests/build_release_assets_test.go index 9a9bf0121..9159b767f 100644 --- a/scripts/installtests/build_release_assets_test.go +++ b/scripts/installtests/build_release_assets_test.go @@ -155,7 +155,7 @@ func TestCreateReleaseUploadsPowerShellInstaller(t *testing.T) { `git push origin "refs/tags/${TAG}" --force`, `-F target_commitish="${HEAD_SHA}"`, `historical_asset_backfill_only=${HISTORICAL_ASSET_BACKFILL_ONLY}`, - `if: ${{ always() && needs.prepare.result == 'success' && needs.create_release.result == 'success' && needs.prepare.outputs.historical_asset_backfill_only != 'true' }}`, + `if: ${{ always() && needs.prepare.result == 'success' && needs.create_release.result == 'success' && needs.prepare.outputs.historical_asset_backfill_only != 'true' && (github.event.inputs.draft_only == 'true' || needs.publish_docker.result == 'success') }}`, `if: ${{ needs.prepare.outputs.historical_asset_backfill_only == 'true' }}`, `permissions:`, `issues: write`, @@ -947,6 +947,9 @@ func TestDeployDemoWorkflowFailsClosedForStableAndVerifiesFrontendParity(t *test ` - stable`, `Capture expected frontend entry asset`, `Verify target host identity`, + `uses: tailscale/github-action@306e68a486fd2350f2bfc3b19fcd143891a4a2d8 # v4`, + `ping: ${{ secrets.DEMO_SERVER_HOST }}`, + `bash .github/scripts/check-demo-reachability.sh`, `bash .github/scripts/setup-demo-ssh.sh`, `SERVICE_NAME="pulse"`, `Unsupported demo target: ${TARGET}`, @@ -980,10 +983,16 @@ func TestUpdateDemoWorkflowUsesGovernedNetworkPath(t *testing.T) { workflow := string(workflowBytes) required := []string{ `- name: Tailscale`, - `uses: tailscale/github-action@4e4c49acaa9818630ce0bd7a564372c17e33fb4d # v2`, + `uses: tailscale/github-action@306e68a486fd2350f2bfc3b19fcd143891a4a2d8 # v4`, `oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }}`, `oauth-secret: ${{ secrets.TS_OAUTH_SECRET }}`, `tags: tag:infra`, + `version: '1.94.2'`, + `ping: ${{ secrets.DEMO_SERVER_HOST }}`, + `bash .github/scripts/check-demo-reachability.sh`, + `workflow_call:`, + `verify_only:`, + `Refuse mutation during verification-only checks`, `uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0`, `go run ./scripts/release_update_key.go public-key-ssh`, `sed -i "s|^PINNED_RELEASE_SSH_PUBLIC_KEY=.*|PINNED_RELEASE_SSH_PUBLIC_KEY=\"${TRUSTED_SSH_PUBLIC_KEY}\"|" /tmp/pulse-install.sh`, @@ -1032,8 +1041,8 @@ func TestDemoSshSetupHelperHandlesIpLiteralTargets(t *testing.T) { `Demo SSH host is an IP literal; skipping DNS resolution wait.`, `[ "$host_needs_dns" = "true" ] && ! getent hosts "$DEMO_SERVER_HOST"`, `ssh-keyscan -T 10 -H "$DEMO_SERVER_HOST"`, - `MAX_SSH_SETUP_ATTEMPTS=18`, - `Timed out waiting for the demo SSH host to become reachable and return host keys after Tailscale setup.`, + `MAX_SSH_SETUP_ATTEMPTS="${DEMO_SSH_SETUP_ATTEMPTS:-3}"`, + `Demo network preflight passed, but ssh-keyscan did not return host keys.`, } for _, needle := range required { if !strings.Contains(helper, needle) { @@ -1082,6 +1091,63 @@ func TestDemoSshSetupHelperHandlesIpLiteralTargets(t *testing.T) { } } +func TestDemoReachabilityHelperSeparatesTailnetAndSshTransportProof(t *testing.T) { + helperBytes, err := os.ReadFile(repoFile(".github", "scripts", "check-demo-reachability.sh")) + if err != nil { + t.Fatalf("read demo reachability helper: %v", err) + } + helper := string(helperBytes) + for _, needle := range []string{ + `tailscale status --json`, + `tailscale ping --c 3 --timeout 10s "$DEMO_SERVER_HOST"`, + `nc -z -w 5 "$DEMO_SERVER_HOST" "$TCP_PORT"`, + `Demo peer is not present in the runner peer map yet.`, + `Verify sshd and the host firewall on tailscale0.`, + } { + if !strings.Contains(helper, needle) { + t.Fatalf("demo reachability helper missing diagnostic contract: %s", needle) + } + } + + tmpDir := t.TempDir() + fakeBin := filepath.Join(tmpDir, "bin") + if err := os.MkdirAll(fakeBin, 0o755); err != nil { + t.Fatalf("create fake bin: %v", err) + } + tailscaleScript := `#!/bin/sh +if [ "$1" = "status" ]; then + printf '%s\n' '{"BackendState":"Running","Self":{"TailscaleIPs":["100.100.100.1"]},"Peer":{"demo":{"TailscaleIPs":["100.109.163.95"],"Online":true,"Active":true,"Relay":"lhr"}}}' + exit 0 +fi +if [ "$1" = "ping" ]; then + echo 'pong from demo' + exit 0 +fi +exit 1 +` + if err := os.WriteFile(filepath.Join(fakeBin, "tailscale"), []byte(tailscaleScript), 0o755); err != nil { + t.Fatalf("write fake tailscale: %v", err) + } + if err := os.WriteFile(filepath.Join(fakeBin, "nc"), []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil { + t.Fatalf("write fake nc: %v", err) + } + + cmd := exec.Command("bash", repoFile(".github", "scripts", "check-demo-reachability.sh")) + cmd.Env = append(os.Environ(), + "DEMO_SERVER_HOST=100.109.163.95", + "PATH="+fakeBin+string(os.PathListSeparator)+os.Getenv("PATH"), + ) + output, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("demo reachability helper failed: %v\n%s", err, output) + } + for _, needle := range []string{"Tailscale backend: Running", "Demo peer state: online=True active=True relay=lhr", "Demo SSH transport is reachable over Tailscale."} { + if !strings.Contains(string(output), needle) { + t.Fatalf("demo reachability output missing %q: %s", needle, output) + } + } +} + func TestDemoPublicBrowserSmokeWaitsForVisibleLoginUI(t *testing.T) { scriptBytes, err := os.ReadFile(repoFile("scripts", "demo_public_browser_smoke.cjs")) if err != nil { diff --git a/scripts/release_control/release_promotion_policy_test.py b/scripts/release_control/release_promotion_policy_test.py index 8da1c710c..f626a61b8 100644 --- a/scripts/release_control/release_promotion_policy_test.py +++ b/scripts/release_control/release_promotion_policy_test.py @@ -506,7 +506,7 @@ class ReleasePromotionPolicyTest(unittest.TestCase): self.assertIn("default_output_path", recorder) self.assertIn("rollback_version is required for every release rehearsal and promotion", resolver) self.assertIn("Stable promotion requires promoted_from_tag", resolver) - self.assertIn("Stable patch hotfix releases may omit promoted_from_tag", resolver) + self.assertIn("Only governed stable patch releases may use the routine no-RC path.", resolver) self.assertIn("Stable v6.0.0 requires ga_date in YYYY-MM-DD form", resolver) self.assertIn("release_notes must include the exact ga_date", resolver) self.assertIn("check-workflow-dispatch-inputs.py", dry_run_trigger) @@ -527,6 +527,7 @@ class ReleasePromotionPolicyTest(unittest.TestCase): update_demo_workflow = read(".github/workflows/update-demo-server.yml") deploy_demo_workflow = read(".github/workflows/deploy-demo-server.yml") demo_ssh_helper = read(".github/scripts/setup-demo-ssh.sh") + demo_reachability_helper = read(".github/scripts/check-demo-reachability.sh") validation_workflow = read(".github/workflows/validate-release-assets.yml") helper = read("scripts/trigger-release.sh") renderer = read("scripts/release_control/render_release_body.py") @@ -558,8 +559,11 @@ class ReleasePromotionPolicyTest(unittest.TestCase): self.assertIn("resolve_release_promotion.py", content) self.assertIn("render_release_body.py", content) self.assertIn("build_promotion_metadata_section", renderer) - self.assertIn('gh workflow run publish-docker.yml --ref "${REQUIRED_BRANCH}"', content) - self.assertIn('gh workflow run update-demo-server.yml --ref "${REQUIRED_BRANCH}"', content) + self.assertIn("uses: ./.github/workflows/publish-docker.yml", content) + self.assertIn("uses: ./.github/workflows/update-demo-server.yml", content) + self.assertIn("Definitive Release Verdict", content) + self.assertNotIn('gh workflow run publish-docker.yml --ref "${REQUIRED_BRANCH}"', content) + self.assertNotIn('gh workflow run update-demo-server.yml --ref "${REQUIRED_BRANCH}"', content) self.assertIn("sanitize_release_notes", renderer) self.assertIn("Do not treat this as published", renderer) self.assertIn("_DRAFT.md", renderer) @@ -582,7 +586,7 @@ class ReleasePromotionPolicyTest(unittest.TestCase): self.assertIn('-F target_commitish="${HEAD_SHA}"', content) self.assertIn('historical_asset_backfill_only=${HISTORICAL_ASSET_BACKFILL_ONLY}', content) self.assertIn( - "if: ${{ always() && needs.prepare.result == 'success' && needs.create_release.result == 'success' && needs.prepare.outputs.historical_asset_backfill_only != 'true' }}", + "if: ${{ always() && needs.prepare.result == 'success' && needs.create_release.result == 'success' && needs.prepare.outputs.historical_asset_backfill_only != 'true' && (github.event.inputs.draft_only == 'true' || needs.publish_docker.result == 'success') }}", content, ) self.assertIn("if: ${{ needs.prepare.outputs.historical_asset_backfill_only == 'true' }}", content) @@ -614,12 +618,18 @@ class ReleasePromotionPolicyTest(unittest.TestCase): self.assertIn('sed -i "s|^PINNED_RELEASE_SSH_PUBLIC_KEY=.*|PINNED_RELEASE_SSH_PUBLIC_KEY=\\"${TRUSTED_SSH_PUBLIC_KEY}\\"|" /tmp/pulse-install.sh', update_demo_workflow) for demo_workflow in (update_demo_workflow, deploy_demo_workflow): self.assertIn("bash .github/scripts/setup-demo-ssh.sh", demo_workflow) - self.assertIn("MAX_SSH_SETUP_ATTEMPTS=18", demo_ssh_helper) + self.assertIn("bash .github/scripts/check-demo-reachability.sh", demo_workflow) + self.assertIn("ping: ${{ secrets.DEMO_SERVER_HOST }}", demo_workflow) + self.assertIn("tailscale/github-action@306e68a486fd2350f2bfc3b19fcd143891a4a2d8 # v4", demo_workflow) + self.assertIn('MAX_SSH_SETUP_ATTEMPTS="${DEMO_SSH_SETUP_ATTEMPTS:-3}"', demo_ssh_helper) self.assertIn("ipaddress.ip_address(sys.argv[1])", demo_ssh_helper) self.assertIn("host_needs_dns=false", demo_ssh_helper) self.assertIn('getent hosts "$DEMO_SERVER_HOST"', demo_ssh_helper) self.assertIn('ssh-keyscan -T 10 -H "$DEMO_SERVER_HOST"', demo_ssh_helper) - self.assertIn("Timed out waiting for the demo SSH host to become reachable and return host keys after Tailscale setup", demo_ssh_helper) + self.assertIn("Demo network preflight passed, but ssh-keyscan did not return host keys", demo_ssh_helper) + self.assertIn('tailscale ping --c 3 --timeout 10s "$DEMO_SERVER_HOST"', demo_reachability_helper) + self.assertIn('nc -z -w 5 "$DEMO_SERVER_HOST" "$TCP_PORT"', demo_reachability_helper) + self.assertIn("Demo peer is not present in the runner peer map yet.", demo_reachability_helper) self.assertIn("derive the OpenSSH installer trust key from `PULSE_UPDATE_SIGNING_PUBLIC_KEY`", normalize_ws(contract)) self.assertIn('SYFT_VERSION="1.42.4"', content) self.assertIn('SYFT_ARCHIVE="syft_${SYFT_VERSION}_linux_amd64.tar.gz"', content) @@ -773,6 +783,7 @@ class ReleasePromotionPolicyTest(unittest.TestCase): demo = read(".github/workflows/update-demo-server.yml") preview_deploy = read(".github/workflows/deploy-demo-server.yml") release_workflow = read(".github/workflows/create-release.yml") + dry_run_workflow = read(".github/workflows/release-dry-run.yml") helm = read(".github/workflows/publish-helm-chart.yml") helm_pages = read(".github/workflows/helm-pages.yml") artifact_validator = read("scripts/release_control/validate_artifact_release_line.py") @@ -803,14 +814,22 @@ class ReleasePromotionPolicyTest(unittest.TestCase): self.assertIn("demo-stable", demo) self.assertIn("Refusing prerelease tag", demo) self.assertIn("Prerelease demo updates are retired after v6 GA", demo) - self.assertIn("Latest published stable release", demo) + self.assertIn("The latest alias is allowed only for verification-only checks.", demo) + self.assertIn("Resolved verification-only target to latest stable release", demo) + self.assertNotIn("github.event_name == 'release'", demo) self.assertNotIn("preview-v6", demo) self.assertNotIn("demo-preview-v6", demo) self.assertNotIn('SERVICE_NAME="pulse-v6-preview"', demo) self.assertNotIn("Preview demo updates must not target the stable pulse service.", demo) - self.assertIn("tailscale/github-action@4e4c49acaa9818630ce0bd7a564372c17e33fb4d # v2", demo) + self.assertIn("workflow_call:", demo) + self.assertIn("verify_only:", demo) + self.assertIn("tag: latest", dry_run_workflow) + self.assertIn("verify_only: true", dry_run_workflow) + self.assertIn("Verify Current Stable Demo Path (No Mutation)", dry_run_workflow) + self.assertIn("tailscale/github-action@306e68a486fd2350f2bfc3b19fcd143891a4a2d8 # v4", demo) self.assertIn("oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }}", demo) self.assertIn("oauth-secret: ${{ secrets.TS_OAUTH_SECRET }}", demo) + self.assertIn("ping: ${{ secrets.DEMO_SERVER_HOST }}", demo) # The static 90-day TS_AUTHKEY was retired for the OAuth client # (0a9a29d63); the runner mints an ephemeral tagged node key per run. self.assertNotIn("TS_AUTHKEY", demo) @@ -839,6 +858,13 @@ class ReleasePromotionPolicyTest(unittest.TestCase): self.assertIn("extract_entry_asset()", demo) self.assertIn(r']*\bsrc=\"(/assets/index-[^\"]*\.js)\"', demo) self.assertIn("Public demo is serving $PUBLIC_ASSET but the target service is serving $REMOTE_ASSET.", demo) + self.assertIn("uses: ./.github/workflows/publish-docker.yml", release_workflow) + self.assertIn("uses: ./.github/workflows/update-demo-server.yml", release_workflow) + self.assertIn("Definitive Release Verdict", release_workflow) + self.assertIn("Require recent exact-SHA stable patch preflight", release_workflow) + self.assertIn("Release Dry Run v${VERSION}", release_workflow) + self.assertNotIn("gh workflow run update-demo-server.yml", release_workflow) + self.assertNotIn("gh workflow run publish-docker.yml", release_workflow) self.assertNotIn("preview-v6", preview_deploy) self.assertNotIn("demo-preview-v6", preview_deploy) self.assertNotIn('SERVICE_NAME="pulse-v6-preview"', preview_deploy) @@ -886,8 +912,9 @@ class ReleasePromotionPolicyTest(unittest.TestCase): self.assertIn("helm status pulse || true", helm_pages) self.assertIn("kubectl describe pods -A || true", helm_pages) self.assertIn("kubectl get events -A --sort-by=.lastTimestamp || kubectl get events -A || true", helm_pages) - self.assertIn("Prerelease public demo update skipped; post-GA demo target is stable only.", release_workflow) - self.assertIn('gh workflow run update-demo-server.yml --ref "${REQUIRED_BRANCH}" -f tag="${{ needs.prepare.outputs.tag }}" -f target="stable"', release_workflow) + self.assertIn("uses: ./.github/workflows/update-demo-server.yml", release_workflow) + self.assertIn("Definitive Release Verdict", release_workflow) + self.assertNotIn('gh workflow run update-demo-server.yml --ref "${REQUIRED_BRANCH}"', release_workflow) self.assertNotIn('TARGET="preview-v6"', release_workflow) self.assertIn("sync_chart_release_metadata.py", helm) self.assertIn("sync_chart_release_metadata.py", helm_pages) @@ -985,6 +1012,25 @@ class ReleasePromotionPolicyTest(unittest.TestCase): " BLESS_GOVERNANCE_FIXTURES=1 python3 -m unittest release_promotion_policy_test" ) + def test_routine_stable_patch_entrypoint_is_noninteractive_and_preflight_gated(self) -> None: + helper = read("scripts/trigger-stable-patch.sh") + policy = read("docs/release-control/v6/internal/RELEASE_PROMOTION_POLICY.md") + contract = read("docs/release-control/v6/internal/subsystems/deployment-installability.md") + + self.assertNotIn("read -r", helper) + self.assertNotIn("read -p", helper) + self.assertIn("--dry-run", helper) + self.assertIn("--derive-rollback-latest-stable", helper) + self.assertIn("docs/releases/RELEASE_NOTES_v${VERSION}.md", helper) + self.assertIn("Release Dry Run v${VERSION}", helper) + self.assertIn("timedelta(hours=24)", helper) + self.assertIn(".createdAt >= $cutoff", helper) + self.assertIn("gh workflow run create-release.yml", helper) + self.assertIn("gh workflow run \"$WORKFLOW\"", helper) + self.assertIn("Routine Stable Patch Path", policy) + self.assertIn("exact candidate SHA within the previous 24 hours", normalize_ws(policy)) + self.assertIn("An asynchronous dispatch or manual SSH deployment is not release completion.", normalize_ws(contract)) + if __name__ == "__main__": unittest.main() diff --git a/scripts/release_control/resolve_release_promotion.py b/scripts/release_control/resolve_release_promotion.py index 407ac195c..52b82efd4 100644 --- a/scripts/release_control/resolve_release_promotion.py +++ b/scripts/release_control/resolve_release_promotion.py @@ -4,6 +4,7 @@ from __future__ import annotations import argparse +import fnmatch import re import subprocess import time @@ -17,6 +18,63 @@ SEMVER_STABLE_RE = re.compile(r"^(\d+)\.(\d+)\.(\d+)$") SEMVER_STABLE_TAG_RE = re.compile(r"^v(\d+)\.(\d+)\.(\d+)$") SEMVER_PRERELEASE_RE = re.compile(r"-(?:[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)(?:\+[0-9A-Za-z.-]+)?$") +ROUTINE_PATCH_RC_REQUIRED_RULES: tuple[tuple[str, tuple[str, ...]], ...] = ( + ( + "authentication, authorization, or tenant isolation", + ( + "internal/api/auth*.go", + "internal/api/security*.go", + "internal/api/saml*.go", + "internal/api/sso*.go", + "internal/auth/**", + "internal/securityutil/**", + "pkg/auth/**", + ), + ), + ( + "licensing, entitlement, or billing authority", + ( + "internal/api/billing*.go", + "internal/api/license*.go", + "internal/entitlements/**", + "internal/licensing/**", + "pkg/licensing/**", + ), + ), + ( + "persisted data format, schema, or migration", + ( + "internal/database/**", + "internal/migrations/**", + "internal/storage/**", + "pkg/database/**", + "pkg/storage/**", + "**/migrations/**", + "**/*migration*.go", + ), + ), + ( + "relay or mobile trust protocol", + ( + "internal/api/cloud_handoff*.go", + "internal/api/magic_link*.go", + "internal/api/mobile*.go", + "internal/relay/**", + "pkg/relay/**", + ), + ), + ( + "installer, updater, or rollback execution", + ( + "install.sh", + "internal/updates/**", + "scripts/install.ps1", + "scripts/install.sh", + "scripts/pulse-auto-update.sh", + ), + ), +) + def normalize_tag(value: str) -> str: value = (value or "").strip() @@ -99,6 +157,40 @@ def list_stable_tags() -> list[str]: return [tag for tag in result.stdout.split() if SEMVER_STABLE_TAG_RE.match(tag)] +def list_same_version_rc_tags(version: str) -> list[str]: + result = subprocess.run( + ["git", "tag", "--list", f"v{version}-rc.*", "--sort=-version:refname"], + cwd=REPO_ROOT, + env=git_env(), + check=True, + capture_output=True, + text=True, + ) + return [tag for tag in result.stdout.splitlines() if tag.strip()] + + +def changed_paths_between(base_tag: str) -> list[str]: + result = subprocess.run( + ["git", "diff", "--name-only", f"{base_tag}..HEAD"], + cwd=REPO_ROOT, + env=git_env(), + check=True, + capture_output=True, + text=True, + ) + return [path for path in result.stdout.splitlines() if path.strip()] + + +def classify_routine_patch_risks(paths: list[str]) -> list[str]: + risks: list[str] = [] + for path in sorted(set(paths)): + for reason, patterns in ROUTINE_PATCH_RC_REQUIRED_RULES: + if any(fnmatch.fnmatchcase(path, pattern) for pattern in patterns): + risks.append(f"{path} ({reason})") + break + return risks + + def derive_latest_stable_rollback_tag(version: str, stable_tags: list[str]) -> str: base_match = re.match(r"^(\d+)\.(\d+)\.(\d+)", (version or "").strip()) if not base_match: @@ -131,6 +223,8 @@ def resolve_metadata( release_notes_input: str, derive_rollback_when_missing: bool = False, list_stable_tags_fn: Callable[[], list[str]] = list_stable_tags, + list_same_version_rc_tags_fn: Callable[[str], list[str]] = list_same_version_rc_tags, + changed_paths_fn: Callable[[str], list[str]] = changed_paths_between, tag_exists_fn: Callable[[str], bool] = tag_exists, tag_commit_fn: Callable[[str], str] = tag_commit, head_descends_from_fn: Callable[[str], bool] = head_descends_from, @@ -144,6 +238,8 @@ def resolve_metadata( hotfix_reason = normalize_whitespace(hotfix_reason_input) release_notes = release_notes_input or "" is_prerelease = is_prerelease_version(version) + stable_patch = is_stable_patch_version(version) + promotion_mode = "prerelease" if is_prerelease else "stable-rc-promotion" if not rollback_tag and derive_rollback_when_missing: rollback_tag = derive_latest_stable_rollback_tag(version, list_stable_tags_fn()) @@ -167,14 +263,56 @@ def resolve_metadata( else: promoted_from_tag = normalize_tag(promoted_from_tag_input) if not promoted_from_tag: - if is_stable_patch_version(version) and hotfix_exception: - if not hotfix_reason: - raise ValueError("hotfix_reason is required when hotfix_exception is true.") - else: + if not stable_patch: raise ValueError( "Stable promotion requires promoted_from_tag naming the prerelease being promoted. " - "Stable patch hotfix releases may omit promoted_from_tag only when hotfix_exception is true and hotfix_reason is set." + "Only governed stable patch releases may use the routine no-RC path." ) + + expected_rollback_tag = derive_latest_stable_rollback_tag( + version, + list_stable_tags_fn(), + ) + if rollback_tag != expected_rollback_tag: + raise ValueError( + f"Routine stable patch {tag} must roll back to the latest preceding stable tag " + f"{expected_rollback_tag}, got {rollback_tag}." + ) + + rollback_commit = tag_commit_fn(rollback_tag) + if not head_descends_from_fn(rollback_commit): + raise ValueError( + f"Routine stable patch {tag} must descend from rollback target {rollback_tag}." + ) + + same_version_rc_tags = list_same_version_rc_tags_fn(version) + routine_patch_risks = classify_routine_patch_risks( + changed_paths_fn(rollback_tag) + ) + if (same_version_rc_tags or routine_patch_risks) and not hotfix_exception: + reasons: list[str] = [] + if same_version_rc_tags: + reasons.append( + "same-version release candidates already exist: " + + ", ".join(same_version_rc_tags) + ) + if routine_patch_risks: + reasons.append( + "RC-required runtime changes: " + + "; ".join(routine_patch_risks) + ) + raise ValueError( + "Routine stable patch mode is not allowed because " + + " | ".join(reasons) + + ". Promote the exercised RC, or use hotfix_exception with a concrete emergency reason." + ) + + if hotfix_exception: + if not hotfix_reason: + raise ValueError("hotfix_reason is required when hotfix_exception is true.") + promotion_mode = "emergency-stable-patch" + else: + promotion_mode = "routine-stable-patch" else: if not re.match(rf"^v{re.escape(version)}-rc\.\d+$", promoted_from_tag): raise ValueError( @@ -227,6 +365,8 @@ def resolve_metadata( ) return { + "promotion_mode": promotion_mode, + "is_stable_patch": "true" if stable_patch else "false", "promoted_from_tag": promoted_from_tag, "rollback_tag": rollback_tag, "rollback_command": rollback_command, diff --git a/scripts/release_control/resolve_release_promotion_test.py b/scripts/release_control/resolve_release_promotion_test.py index 50731e621..9509564d0 100644 --- a/scripts/release_control/resolve_release_promotion_test.py +++ b/scripts/release_control/resolve_release_promotion_test.py @@ -194,7 +194,12 @@ class ResolveReleasePromotionTest(unittest.TestCase): hotfix_exception=True, hotfix_reason_input="Patch release for v6.0.1 agent upgrade recovery.", release_notes_input="", + list_stable_tags_fn=lambda: ["v6.0.1"], + list_same_version_rc_tags_fn=lambda version: ["v6.0.2-rc.1"], + changed_paths_fn=lambda tag: ["internal/api/auth.go"], tag_exists_fn=lambda tag: tag == "v6.0.1", + tag_commit_fn=lambda tag: "rollback-commit", + head_descends_from_fn=lambda commit: commit == "rollback-commit", ) self.assertEqual(metadata["promoted_from_tag"], "") @@ -206,9 +211,50 @@ class ResolveReleasePromotionTest(unittest.TestCase): "Patch release for v6.0.1 agent upgrade recovery.", ) self.assertEqual(metadata["soak_hours"], "") + self.assertEqual(metadata["promotion_mode"], "emergency-stable-patch") - def test_stable_patch_without_promoted_tag_requires_hotfix_exception(self) -> None: - with self.assertRaisesRegex(ValueError, "Stable promotion requires promoted_from_tag"): + def test_routine_stable_patch_can_omit_rc_ceremony(self) -> None: + metadata = resolver.resolve_metadata( + version="6.0.2", + promoted_from_tag_input="", + rollback_version_input="6.0.1", + ga_date_input="", + v5_eos_date_input="", + hotfix_exception=False, + hotfix_reason_input="", + release_notes_input="bounded customer fixes", + list_stable_tags_fn=lambda: ["v5.1.35", "v6.0.1"], + list_same_version_rc_tags_fn=lambda version: [], + changed_paths_fn=lambda tag: ["frontend-modern/src/features/settings/Settings.tsx"], + tag_exists_fn=lambda tag: tag == "v6.0.1", + tag_commit_fn=lambda tag: "rollback-commit", + head_descends_from_fn=lambda commit: commit == "rollback-commit", + ) + + self.assertEqual(metadata["promotion_mode"], "routine-stable-patch") + self.assertEqual(metadata["is_stable_patch"], "true") + self.assertEqual(metadata["rollback_tag"], "v6.0.1") + self.assertEqual(metadata["hotfix_exception"], "false") + + def test_routine_stable_patch_requires_latest_stable_rollback(self) -> None: + with self.assertRaisesRegex(ValueError, "latest preceding stable tag v6.0.1"): + resolver.resolve_metadata( + version="6.0.2", + promoted_from_tag_input="", + rollback_version_input="6.0.0", + ga_date_input="", + v5_eos_date_input="", + hotfix_exception=False, + hotfix_reason_input="", + release_notes_input="bounded customer fixes", + list_stable_tags_fn=lambda: ["v6.0.0", "v6.0.1"], + list_same_version_rc_tags_fn=lambda version: [], + changed_paths_fn=lambda tag: [], + tag_exists_fn=lambda tag: True, + ) + + def test_routine_stable_patch_requires_rc_for_risk_changes(self) -> None: + with self.assertRaisesRegex(ValueError, "RC-required runtime changes"): resolver.resolve_metadata( version="6.0.2", promoted_from_tag_input="", @@ -217,10 +263,48 @@ class ResolveReleasePromotionTest(unittest.TestCase): v5_eos_date_input="", hotfix_exception=False, hotfix_reason_input="", - release_notes_input="", - tag_exists_fn=lambda tag: tag == "v6.0.1", + release_notes_input="authentication correction", + list_stable_tags_fn=lambda: ["v6.0.1"], + list_same_version_rc_tags_fn=lambda version: [], + changed_paths_fn=lambda tag: ["internal/api/auth.go"], + tag_exists_fn=lambda tag: True, + tag_commit_fn=lambda tag: "rollback-commit", + head_descends_from_fn=lambda commit: True, ) + def test_routine_stable_patch_requires_rc_when_candidate_exists(self) -> None: + with self.assertRaisesRegex(ValueError, "same-version release candidates already exist"): + resolver.resolve_metadata( + version="6.0.2", + promoted_from_tag_input="", + rollback_version_input="6.0.1", + ga_date_input="", + v5_eos_date_input="", + hotfix_exception=False, + hotfix_reason_input="", + release_notes_input="bounded customer fixes", + list_stable_tags_fn=lambda: ["v6.0.1"], + list_same_version_rc_tags_fn=lambda version: ["v6.0.2-rc.1"], + changed_paths_fn=lambda tag: [], + tag_exists_fn=lambda tag: True, + tag_commit_fn=lambda tag: "rollback-commit", + head_descends_from_fn=lambda commit: True, + ) + + def test_routine_patch_risk_classifier_covers_governed_categories(self) -> None: + risks = resolver.classify_routine_patch_risks( + [ + "internal/api/auth.go", + "pkg/licensing/license.go", + "internal/storage/schema.go", + "internal/relay/client.go", + "internal/updates/apply.go", + "frontend-modern/src/App.tsx", + ] + ) + self.assertEqual(len(risks), 5) + self.assertFalse(any("frontend-modern/src/App.tsx" in risk for risk in risks)) + def test_stable_patch_hotfix_without_promoted_tag_requires_reason(self) -> None: with self.assertRaisesRegex(ValueError, "hotfix_reason is required"): resolver.resolve_metadata( @@ -232,7 +316,12 @@ class ResolveReleasePromotionTest(unittest.TestCase): hotfix_exception=True, hotfix_reason_input="", release_notes_input="", + list_stable_tags_fn=lambda: ["v6.0.1"], + list_same_version_rc_tags_fn=lambda version: [], + changed_paths_fn=lambda tag: [], tag_exists_fn=lambda tag: tag == "v6.0.1", + tag_commit_fn=lambda tag: "rollback-commit", + head_descends_from_fn=lambda commit: True, ) def test_stable_hotfix_requires_reason(self) -> None: diff --git a/scripts/trigger-stable-patch.sh b/scripts/trigger-stable-patch.sh new file mode 100755 index 000000000..fd3943a45 --- /dev/null +++ b/scripts/trigger-stable-patch.sh @@ -0,0 +1,223 @@ +#!/usr/bin/env bash +set -euo pipefail + +MODE="publish" +VERSION="" +MOBILE_RELEASE_DECISION="" +MOBILE_RELEASE_EVIDENCE="" +HOTFIX_REASON="" + +usage() { + cat <<'EOF' +Usage: scripts/trigger-stable-patch.sh [--dry-run] [options] [version] + +Runs the noninteractive stable patch preflight and dispatches exactly one +governed workflow. Run once with --dry-run, wait for that workflow to pass, +then run again without --dry-run to publish. + +Options: + --dry-run Dispatch Release Dry Run only. + --mobile-release-decision VALUE Override the inferred mobile decision. + --mobile-release-evidence VALUE Evidence for a mobile compatibility decision. + --emergency-hotfix-reason VALUE Bypass an RC-required risk with an explicit reason. + -h, --help Show this help. +EOF +} + +while [ "$#" -gt 0 ]; do + case "$1" in + --dry-run) + MODE="dry-run" + shift + ;; + --mobile-release-decision) + MOBILE_RELEASE_DECISION="${2:?--mobile-release-decision requires a value}" + shift 2 + ;; + --mobile-release-evidence) + MOBILE_RELEASE_EVIDENCE="${2:?--mobile-release-evidence requires a value}" + shift 2 + ;; + --emergency-hotfix-reason) + HOTFIX_REASON="${2:?--emergency-hotfix-reason requires a value}" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + --*) + echo "Unknown option: $1" >&2 + usage >&2 + exit 2 + ;; + *) + if [ -n "$VERSION" ]; then + echo "Only one version may be supplied." >&2 + exit 2 + fi + VERSION="$1" + shift + ;; + esac +done + +VERSION="${VERSION:-$(tr -d '\n' < VERSION)}" +if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[1-9][0-9]*$ ]]; then + echo "Stable patch version required, got: ${VERSION}" >&2 + exit 1 +fi + +FILE_VERSION="$(tr -d '\n' < VERSION)" +if [ "$FILE_VERSION" != "$VERSION" ]; then + echo "VERSION contains ${FILE_VERSION}; requested ${VERSION}." >&2 + exit 1 +fi + +if [ -n "$(git status --porcelain=v1)" ]; then + echo "The release worktree must be clean." >&2 + git status --short + exit 1 +fi + +CURRENT_BRANCH="$(git branch --show-current)" +REQUIRED_BRANCH="$(python3 scripts/release_control/control_plane.py --branch-for-version "$VERSION")" +if [ "$CURRENT_BRANCH" != "$REQUIRED_BRANCH" ]; then + echo "Version ${VERSION} must be released from ${REQUIRED_BRANCH}, not ${CURRENT_BRANCH}." >&2 + exit 1 +fi + +git fetch --quiet --prune origin "$REQUIRED_BRANCH" --tags +LOCAL_SHA="$(git rev-parse HEAD)" +REMOTE_SHA="$(git rev-parse "origin/${REQUIRED_BRANCH}")" +if [ "$LOCAL_SHA" != "$REMOTE_SHA" ]; then + echo "The exact release commit must already be pushed to origin/${REQUIRED_BRANCH}." >&2 + exit 1 +fi + +NOTES_FILE="docs/releases/RELEASE_NOTES_v${VERSION}.md" +if [ ! -s "$NOTES_FILE" ]; then + echo "Canonical release notes are required at ${NOTES_FILE}." >&2 + exit 1 +fi + +RESOLVER_ARGS=( + --version "$VERSION" + --derive-rollback-latest-stable + --release-notes-file "$NOTES_FILE" +) +HOTFIX_EXCEPTION="false" +if [ -n "$HOTFIX_REASON" ]; then + HOTFIX_EXCEPTION="true" + RESOLVER_ARGS+=(--hotfix-exception --hotfix-reason "$HOTFIX_REASON") +fi + +PROMOTION_METADATA="$(python3 scripts/release_control/resolve_release_promotion.py "${RESOLVER_ARGS[@]}")" +ROLLBACK_TAG="$(awk -F= '$1 == "rollback_tag" {print $2}' <<<"$PROMOTION_METADATA")" +if [ -z "$ROLLBACK_TAG" ]; then + echo "Release preflight did not resolve a rollback tag." >&2 + exit 1 +fi + +MOBILE_IMPACT_PATHS="$({ + git diff --name-only "${ROLLBACK_TAG}..HEAD" | rg '^(internal/(relay|mobile)/|pkg/relay/|internal/api/(cloud_handoff|magic_link|mobile)|tests/integration/.*(mobile|relay)|scripts/.*mobile)' +} || true)" +if [ -z "$MOBILE_RELEASE_DECISION" ]; then + if [ -n "$MOBILE_IMPACT_PATHS" ]; then + echo "Mobile-facing paths changed since ${ROLLBACK_TAG}:" >&2 + printf '%s\n' "$MOBILE_IMPACT_PATHS" >&2 + echo "Supply --mobile-release-decision and --mobile-release-evidence after completing the governed mobile check." >&2 + exit 1 + fi + MOBILE_RELEASE_DECISION="no-mobile-impact" + MOBILE_RELEASE_EVIDENCE="No mobile-facing paths changed between ${ROLLBACK_TAG} and ${LOCAL_SHA}." +fi + +python3 scripts/release_control/mobile_release_gate.py \ + --version "$VERSION" \ + --decision "$MOBILE_RELEASE_DECISION" \ + --evidence "$MOBILE_RELEASE_EVIDENCE" + +if [ "$MODE" = "dry-run" ]; then + WORKFLOW="release-dry-run.yml" + python3 scripts/check-workflow-dispatch-inputs.py \ + --workflow-path .github/workflows/release-dry-run.yml \ + --branch "$CURRENT_BRANCH" \ + --require version \ + --require promoted_from_tag \ + --require rollback_version \ + --require ga_date \ + --require v5_eos_date \ + --require hotfix_exception \ + --require hotfix_reason \ + --require note \ + --require mobile_release_decision \ + --require mobile_release_evidence + + gh workflow run "$WORKFLOW" \ + --ref "$CURRENT_BRANCH" \ + -f version="$VERSION" \ + -f promoted_from_tag="" \ + -f rollback_version="$ROLLBACK_TAG" \ + -f ga_date="" \ + -f v5_eos_date="" \ + -f hotfix_exception="$HOTFIX_EXCEPTION" \ + -f hotfix_reason="$HOTFIX_REASON" \ + -f note="Stable patch preflight for ${VERSION} at ${LOCAL_SHA}" \ + -f mobile_release_decision="$MOBILE_RELEASE_DECISION" \ + -f mobile_release_evidence="$MOBILE_RELEASE_EVIDENCE" +else + PREFLIGHT_CUTOFF="$(python3 - <<'PY' +from datetime import datetime, timedelta, timezone + +cutoff = datetime.now(timezone.utc) - timedelta(hours=24) +print(cutoff.strftime("%Y-%m-%dT%H:%M:%SZ")) +PY +)" + PREFLIGHT_RUN="$(gh run list \ + --workflow=release-dry-run.yml \ + --branch "$CURRENT_BRANCH" \ + --event workflow_dispatch \ + --limit 30 \ + --json displayTitle,headSha,conclusion,url,createdAt \ + | jq -c --arg sha "$LOCAL_SHA" --arg title "Release Dry Run v${VERSION}" --arg cutoff "$PREFLIGHT_CUTOFF" \ + '[.[] | select(.headSha == $sha and .displayTitle == $title and .conclusion == "success" and .createdAt >= $cutoff)] | sort_by(.createdAt) | last // empty')" + if [ -z "$PREFLIGHT_RUN" ]; then + echo "No successful exact-SHA Release Dry Run from the last 24 hours exists for v${VERSION}." >&2 + echo "Run $0 --dry-run ${VERSION}, wait for success, then rerun this command." >&2 + exit 1 + fi + + python3 scripts/check-workflow-dispatch-inputs.py \ + --workflow-path .github/workflows/create-release.yml \ + --branch "$CURRENT_BRANCH" \ + --require version \ + --require release_notes \ + --require promoted_from_tag \ + --require rollback_version \ + --require ga_date \ + --require v5_eos_date \ + --require hotfix_exception \ + --require hotfix_reason \ + --require draft_only \ + --require mobile_release_decision \ + --require mobile_release_evidence + + gh workflow run create-release.yml \ + --ref "$CURRENT_BRANCH" \ + -f version="$VERSION" \ + -f release_notes="$(<"$NOTES_FILE")" \ + -f promoted_from_tag="" \ + -f rollback_version="$ROLLBACK_TAG" \ + -f ga_date="" \ + -f v5_eos_date="" \ + -f hotfix_exception="$HOTFIX_EXCEPTION" \ + -f hotfix_reason="$HOTFIX_REASON" \ + -f draft_only=false \ + -f mobile_release_decision="$MOBILE_RELEASE_DECISION" \ + -f mobile_release_evidence="$MOBILE_RELEASE_EVIDENCE" + + echo "Accepted preflight: $(jq -r '.url' <<<"$PREFLIGHT_RUN")" +fi + +echo "Dispatched ${WORKFLOW:-create-release.yml} for v${VERSION} at ${LOCAL_SHA}."