diff --git a/.github/workflows/recover-demo-server.yml b/.github/workflows/recover-demo-server.yml index 7344cfbe81..1a712775fa 100644 --- a/.github/workflows/recover-demo-server.yml +++ b/.github/workflows/recover-demo-server.yml @@ -120,6 +120,35 @@ jobs: demo-runtime-recovery.json echo "mutated=$(jq -r '.mutated' demo-runtime-recovery.json)" >> "$GITHUB_OUTPUT" + - name: Capture bounded Pulse failure diagnostics + if: failure() + env: + DEMO_SERVER_HOST: ${{ secrets.DEMO_SERVER_HOST }} + DEMO_SERVER_USER: ${{ secrets.DEMO_SERVER_USER }} + run: | + set -euo pipefail + timeout --signal=TERM --kill-after=5s 45s \ + ssh -i ~/.ssh/id_ed25519 \ + -o IdentitiesOnly=yes \ + -o StrictHostKeyChecking=yes \ + -o UserKnownHostsFile=~/.ssh/known_hosts \ + "$DEMO_SERVER_USER@$DEMO_SERVER_HOST" \ + 'bash -s' > demo-runtime-diagnostics.txt <<'EOF' + set -uo pipefail + systemctl show pulse \ + --property=ActiveState,SubState,Result,MainPID,NRestarts,ExecMainCode,ExecMainStatus,StateChangeTimestamp \ + --no-pager + ps -C pulse -o pid=,ppid=,stat=,etime=,pcpu=,pmem=,rss=,vsz=,cmd= || true + sudo journalctl -u pulse --since '-30 minutes' --no-pager -o cat \ + | grep -Ei 'panic|fatal|error|warn|mock|metric|database|migration|listen|health|startup|started|failed|timeout' \ + | tail -200 \ + | sed -E \ + -e 's/(Bearer )[A-Za-z0-9._-]+/\1[REDACTED]/g' \ + -e 's/((token|secret|password|authorization|api[_-]?key)[^=: ]*[=: ][[:space:]]*)[^ ,"]+/\1[REDACTED]/Ig' \ + || true + EOF + cat demo-runtime-diagnostics.txt + - name: Verify public health run: curl --connect-timeout 5 --max-time 20 -fsS "$DEMO_PUBLIC_HEALTH_URL" @@ -179,6 +208,7 @@ jobs: path: | demo-runtime-recovery.json demo-runtime-rollback.json + demo-runtime-diagnostics.txt if-no-files-found: warn retention-days: 14 diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index 08e1e682d2..9c3ad72a29 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -2037,6 +2037,11 @@ when it carries an explicit `mockLargeEstateStartupReady` marker earned by that complete startup and browser proof. The eager-history and update-cohort source markers remain necessary but are not sufficient. Runtimes without the explicit proof marker use the governed 8-node, two-hour-seed legacy-bounded profile. +Failed stable-demo recovery must also retain bounded, redacted service evidence: +structured systemd state, bounded process metadata, and only startup- or +health-relevant recent journal lines. Diagnostics run after compensation-safe +recovery failure, carry a hard SSH timeout, and remain part of the existing +14-day recovery artifact rather than widening the recovery mutation boundary. The preceding `v6.4.0-rc.8` qualification attempt used exact source SHA `bac7e5d9526d76a6b4e34738511b07609dda80ed`. Release run `33128595650` diff --git a/scripts/release_control/release_promotion_policy_test.py b/scripts/release_control/release_promotion_policy_test.py index 37832d20ab..a45a120782 100644 --- a/scripts/release_control/release_promotion_policy_test.py +++ b/scripts/release_control/release_promotion_policy_test.py @@ -2088,6 +2088,9 @@ class ReleasePromotionPolicyTest(unittest.TestCase): self.assertIn("Verify public frontend parity", workflow) self.assertIn("./scripts/run_demo_public_browser_smoke.sh", workflow) self.assertIn("Compensate failed mutated recovery", workflow) + self.assertIn("Capture bounded Pulse failure diagnostics", workflow) + self.assertIn("demo-runtime-diagnostics.txt", workflow) + self.assertIn("journalctl -u pulse --since '-30 minutes'", workflow) self.assertIn("sudo systemctl stop pulse", workflow) self.assertIn("Retain bounded recovery evidence", workflow) self.assertNotIn("release-convergence.yml", workflow)