pulse/.github/workflows/release-convergence.yml

name: Release Convergence
run-name: Release convergence ${{ inputs.tag }} source ${{ inputs.source_release_run_id }}

# GitHub publication is an irreversible commit only after create-release.yml
# uploads and publicly verifies release-activation.json. This workflow is
# dispatched before that commit, waits for the marker, serializes every global
# mutable customer surface behind an explicit ref lease, and remains safely
# re-runnable until all surfaces converge.
on:
  workflow_dispatch:
    inputs:
      tag:
        description: "Exact committed GitHub release tag."
        required: true
        type: string
      version:
        description: "Exact release version without the leading v."
        required: true
        type: string
      prerelease:
        description: "Whether this release targets the RC channel."
        required: true
        type: boolean
      target_commitish:
        description: "Exact release source commit."
        required: true
        type: string
      release_id:
        description: "GitHub release database ID committed by create-release.yml."
        required: true
        type: string
      r2_prefix:
        description: "Signed private Pro packet prefix for v6 convergence."
        required: false
        default: ""
        type: string
      source_release_run_id:
        description: "create-release.yml run that staged this convergence."
        required: true
        type: string

permissions:
  contents: read

jobs:
  await_activation_commit:
    name: Await irreversible activation commit
    runs-on: ubuntu-24.04
    timeout-minutes: 360
    permissions:
      actions: read
      contents: read
    outputs:
      activation_owner_run_id: ${{ steps.marker.outputs.activation_owner_run_id }}
      activation_marker_sha256: ${{ steps.marker.outputs.activation_marker_sha256 }}
      server_image_digest: ${{ steps.marker.outputs.server_image_digest }}
      control_plane_image_digest: ${{ steps.marker.outputs.control_plane_image_digest }}
      helm_chart_digest: ${{ steps.marker.outputs.helm_chart_digest }}
    steps:
      - name: Checkout release integrity control
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      - name: Wait for verified public activation marker
        id: marker
        env:
          GH_TOKEN: ${{ github.token }}
          TAG: ${{ inputs.tag }}
          EXPECTED_COMMIT: ${{ inputs.target_commitish }}
          EXPECTED_RELEASE_ID: ${{ inputs.release_id }}
          EXPECTED_SOURCE_RUN_ID: ${{ inputs.source_release_run_id }}
          EXPECTED_CONVERGENCE_RUN_ID: ${{ github.run_id }}
          EXPECTED_R2_PREFIX: ${{ inputs.r2_prefix }}
        run: |
          set -euo pipefail
          marker_url="https://github.com/${{ github.repository }}/releases/download/${TAG}/release-activation.json"
          max_attempts=10500
          completed_propagation_attempts=0
          # Each public read already includes two two-second retries. Twenty
          # outer attempts allow roughly two minutes of post-upload edge
          # propagation without hiding a genuine publication failure.
          max_completed_propagation_attempts=20
          for attempt in $(seq 1 "${max_attempts}"); do
            release_state="$(
              gh api "repos/${{ github.repository }}/releases/tags/${TAG}" \
                --jq '[.id, .tag_name, .target_commitish, (.draft | tostring), (.immutable // false | tostring), (.published_at // "")] | @tsv' \
                2>/dev/null || true
            )"
            release_id="$(awk -F '\t' '{print $1}' <<<"${release_state}")"
            actual_tag="$(awk -F '\t' '{print $2}' <<<"${release_state}")"
            actual_commit="$(awk -F '\t' '{print $3}' <<<"${release_state}")"
            is_draft="$(awk -F '\t' '{print $4}' <<<"${release_state}")"
            is_immutable="$(awk -F '\t' '{print $5}' <<<"${release_state}")"
            published_at="$(awk -F '\t' '{print $6}' <<<"${release_state}")"
            marker="$(mktemp)"
            marker_downloaded=false
            if [ "${release_id}" = "${EXPECTED_RELEASE_ID}" ] && \
               [ "${actual_tag}" = "${TAG}" ] && \
               [ "${actual_commit}" = "${EXPECTED_COMMIT}" ] && \
               [ "${is_draft}" = "false" ] && \
               [ "${is_immutable}" = "true" ] && \
               [ -n "${published_at}" ] && \
               curl -fsSL --retry 2 --retry-delay 2 --retry-all-errors \
                 -o "${marker}" "${marker_url}"; then
              marker_downloaded=true
            fi
            if [ "${marker_downloaded}" = "true" ]; then
              # Verify the exact bytes this job will parse. The integrity
              # helper must not reacquire a parallel copy and then let this
              # unverified download drive customer-facing image pointers.
              if ! ./scripts/verify-github-release-integrity.sh \
                  "${TAG}" "${GITHUB_REPOSITORY}" \
                  "${EXPECTED_RELEASE_ID}" "${EXPECTED_COMMIT}" "${marker}"; then
                rm -f "${marker}"
                echo "::error::Immutable release attestation verification failed for ${TAG}; customer convergence is blocked."
                exit 1
              fi
              if ! jq -e \
                 --arg tag "${TAG}" \
                 --arg commit "${EXPECTED_COMMIT}" \
                 --arg release_id "${EXPECTED_RELEASE_ID}" \
                 --arg source_run_id "${EXPECTED_SOURCE_RUN_ID}" \
                 --arg r2_prefix "${EXPECTED_R2_PREFIX}" \
                 '.schema_version == 1 and .tag == $tag and .target_commitish == $commit and .release_id == $release_id and .source_release_run_id == $source_run_id and (.convergence_run_id | test("^[0-9]+$")) and .r2_prefix == $r2_prefix and (.server_image_digest | test("^sha256:[0-9a-f]{64}$")) and (.control_plane_image_digest | test("^sha256:[0-9a-f]{64}$")) and (.helm_chart_digest | test("^sha256:[0-9a-f]{64}$"))' \
                 "${marker}" >/dev/null; then
                rm -f "${marker}"
                echo "::error::Public activation marker for ${TAG} does not match the expected immutable release identity."
                exit 1
              fi
              activation_owner_run_id="$(jq -r '.convergence_run_id' "${marker}")"
              if [ "${activation_owner_run_id}" != "${EXPECTED_CONVERGENCE_RUN_ID}" ]; then
                original_status="$(
                  gh api \
                    "repos/${{ github.repository }}/actions/runs/${activation_owner_run_id}" \
                    --jq '.status' 2>/dev/null || true
                )"
                if [ "${original_status}" != "completed" ]; then
                  rm -f "${marker}"
                  echo "Original convergence owner ${activation_owner_run_id} is ${original_status:-unavailable}; successor adoption is not yet allowed."
                  sleep 2
                  continue
                fi
                echo "[OK] Adopting committed ${TAG} from completed convergence owner ${activation_owner_run_id}."
              fi
              activation_marker_sha256="$(sha256sum "${marker}" | awk '{print $1}')"
              server_image_digest="$(jq -r '.server_image_digest' "${marker}")"
              control_plane_image_digest="$(jq -r '.control_plane_image_digest' "${marker}")"
              helm_chart_digest="$(jq -r '.helm_chart_digest' "${marker}")"
              echo "activation_owner_run_id=${activation_owner_run_id}" >> "$GITHUB_OUTPUT"
              echo "activation_marker_sha256=${activation_marker_sha256}" >> "$GITHUB_OUTPUT"
              echo "server_image_digest=${server_image_digest}" >> "$GITHUB_OUTPUT"
              echo "control_plane_image_digest=${control_plane_image_digest}" >> "$GITHUB_OUTPUT"
              echo "helm_chart_digest=${helm_chart_digest}" >> "$GITHUB_OUTPUT"
              rm -f "${marker}"
              echo "[OK] ${TAG} crossed the immutable, attested activation commit point."
              exit 0
            fi
            rm -f "${marker}"
            source_state="$(
              gh run view "${EXPECTED_SOURCE_RUN_ID}" \
                --repo "${{ github.repository }}" \
                --json status,conclusion,url \
                --jq '[.status, (.conclusion // ""), .url] | @tsv' \
                2>/dev/null || true
            )"
            source_status="$(awk -F '\t' '{print $1}' <<<"${source_state}")"
            source_conclusion="$(awk -F '\t' '{print $2}' <<<"${source_state}")"
            source_url="$(awk -F '\t' '{print $3}' <<<"${source_state}")"
            if [ "${source_status}" = "completed" ]; then
              if [ "${source_conclusion}" != "success" ]; then
                echo "::error::Source release run completed without the exact activation marker: conclusion=${source_conclusion:-none} ${source_url}."
                exit 1
              fi
              marker_asset_state="$(
                gh api \
                  "repos/${{ github.repository }}/releases/${EXPECTED_RELEASE_ID}/assets?per_page=100" \
                  --paginate \
                  --jq '.[] | select(.name == "release-activation.json") | .state' \
                  2>/dev/null || true
              )"
              if [ "${marker_asset_state}" != "uploaded" ]; then
                echo "::error::Source release run completed successfully without an uploaded activation marker: ${source_url}."
                exit 1
              fi
              completed_propagation_attempts=$((completed_propagation_attempts + 1))
              if [ "${completed_propagation_attempts}" -gt "${max_completed_propagation_attempts}" ]; then
                echo "::error::Uploaded activation marker for ${TAG} did not become publicly readable after ${max_completed_propagation_attempts} propagation attempts."
                exit 1
              fi
              echo "Activation marker for ${TAG} is uploaded but not publicly readable yet (${completed_propagation_attempts}/${max_completed_propagation_attempts})."
              sleep 2
              continue
            fi
            echo "Activation marker for ${TAG} is not committed yet (${attempt}/${max_attempts})."
            sleep 2
          done
          echo "::error::Timed out waiting for the verified activation marker for ${TAG}."
          exit 1

  acquire_customer_promotion_lease:
    name: Acquire global customer-promotion lease
    needs: await_activation_commit
    runs-on: ubuntu-24.04
    timeout-minutes: 360
    permissions:
      actions: read
      contents: write
    outputs:
      lock_sha: ${{ steps.acquire.outputs.lock_sha }}
      superseded: ${{ steps.admit.outputs.superseded }}
      desired_tag: ${{ steps.admit.outputs.desired_tag }}
      owner_asset_name: ${{ steps.acquire.outputs.owner_asset_name }}
      owner_asset_sha256: ${{ steps.acquire.outputs.owner_asset_sha256 }}
      server_image_digest: ${{ needs.await_activation_commit.outputs.server_image_digest }}
      control_plane_image_digest: ${{ needs.await_activation_commit.outputs.control_plane_image_digest }}
      helm_chart_digest: ${{ needs.await_activation_commit.outputs.helm_chart_digest }}
    steps:
      - name: Checkout release control
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          fetch-depth: 1
          persist-credentials: true  # required: authenticated git writes

      - name: Acquire repository-ref lease
        id: acquire
        env:
          GH_TOKEN: ${{ github.token }}
          TAG: ${{ inputs.tag }}
          TARGET_COMMITISH: ${{ inputs.target_commitish }}
          RELEASE_ID: ${{ inputs.release_id }}
          SOURCE_RELEASE_RUN_ID: ${{ inputs.source_release_run_id }}
          R2_PREFIX: ${{ inputs.r2_prefix }}
          ACTIVATION_OWNER_RUN_ID: ${{ needs.await_activation_commit.outputs.activation_owner_run_id }}
          ACTIVATION_MARKER_SHA256: ${{ needs.await_activation_commit.outputs.activation_marker_sha256 }}
        run: |
          set -euo pipefail
          scripts/release_control/customer_promotion_lease.sh acquire "${TAG}"

      - name: Enforce committed-release monotonicity
        id: admit
        env:
          GH_TOKEN: ${{ github.token }}
          TAG: ${{ inputs.tag }}
          IS_PRERELEASE: ${{ inputs.prerelease }}
        run: |
          set -euo pipefail
          releases="$(mktemp)"
          marker="$(mktemp)"
          cleanup() { rm -f "${releases}" "${marker}"; }
          trap cleanup EXIT
          gh release list --limit 500 --json isDraft,isPrerelease,publishedAt,tagName > "${releases}"

          desired_tag=""
          while IFS= read -r candidate; do
            [ -n "${candidate}" ] || continue
            marker_url="https://github.com/${{ github.repository }}/releases/download/${candidate}/release-activation.json"
            if curl -fsSL --retry 2 --retry-delay 1 --retry-all-errors \
                 -o "${marker}" "${marker_url}" && \
               jq -e --arg tag "${candidate}" \
                 '.schema_version == 1 and .tag == $tag' "${marker}" >/dev/null; then
              desired_tag="${candidate}"
              break
            fi
          done < <(
            jq -r --argjson prerelease "${IS_PRERELEASE}" \
              '.[] | select(.isDraft == false and .isPrerelease == $prerelease and .publishedAt != null) | .tagName' \
              "${releases}" | sort -Vr
          )

          if [ -z "${desired_tag}" ]; then
            echo "::error::No committed ${IS_PRERELEASE} channel release was found."
            exit 1
          fi
          echo "desired_tag=${desired_tag}" >> "$GITHUB_OUTPUT"
          if [ "${desired_tag}" != "${TAG}" ]; then
            echo "superseded=true" >> "$GITHUB_OUTPUT"
            echo "[OK] ${TAG} is superseded by committed ${desired_tag}; no global customer pointer will move backward."
          else
            echo "superseded=false" >> "$GITHUB_OUTPUT"
            echo "[OK] ${TAG} is the monotonic ${IS_PRERELEASE} channel convergence target."
          fi

  promote_floating_tags:
    name: Converge Docker aliases
    needs: acquire_customer_promotion_lease
    if: ${{ needs.acquire_customer_promotion_lease.result == 'success' && needs.acquire_customer_promotion_lease.outputs.superseded != 'true' }}
    permissions:
      contents: read
      packages: write
    uses: ./.github/workflows/promote-floating-tags.yml
    secrets: inherit
    with:
      tag: ${{ inputs.tag }}
      prerelease: ${{ inputs.prerelease }}
      source_sha: ${{ inputs.target_commitish }}
      server_digest: ${{ needs.acquire_customer_promotion_lease.outputs.server_image_digest }}
      control_plane_digest: ${{ needs.acquire_customer_promotion_lease.outputs.control_plane_image_digest }}

  publish_helm_pages:
    name: Converge Helm Pages
    needs: acquire_customer_promotion_lease
    # Helm Pages is an additive version index, not a floating pointer. Every
    # committed chart is merged under the lease, even when a newer release has
    # already won admission for the rollback-prone customer surfaces.
    if: ${{ needs.acquire_customer_promotion_lease.result == 'success' }}
    permissions:
      actions: read
      contents: write
      packages: read
    uses: ./.github/workflows/helm-pages.yml
    secrets: inherit
    with:
      chart_version: ${{ inputs.version }}
      source_release_run_id: ${{ inputs.source_release_run_id }}
      target_commitish: ${{ inputs.target_commitish }}
      chart_digest: ${{ needs.acquire_customer_promotion_lease.outputs.helm_chart_digest }}

  promote_private_pro_runtime:
    name: Converge paid-runtime broker
    needs: acquire_customer_promotion_lease
    if: ${{ needs.acquire_customer_promotion_lease.result == 'success' && needs.acquire_customer_promotion_lease.outputs.superseded != 'true' && startsWith(inputs.version, '6.') }}
    permissions:
      contents: read
    uses: ./.github/workflows/promote-private-pro-runtime.yml
    secrets: inherit
    with:
      version: ${{ inputs.version }}
      tag: ${{ inputs.tag }}
      prerelease: ${{ inputs.prerelease }}
      r2_prefix: ${{ inputs.r2_prefix }}
      pulse_lease_sha: ${{ needs.acquire_customer_promotion_lease.outputs.lock_sha }}
      pulse_convergence_run_id: ${{ github.run_id }}
      pulse_owner_asset_name: ${{ needs.acquire_customer_promotion_lease.outputs.owner_asset_name }}
      pulse_owner_asset_sha256: ${{ needs.acquire_customer_promotion_lease.outputs.owner_asset_sha256 }}

  update_stable_demo:
    name: Converge stable demo
    needs: acquire_customer_promotion_lease
    if: ${{ needs.acquire_customer_promotion_lease.result == 'success' && needs.acquire_customer_promotion_lease.outputs.superseded != 'true' && inputs.prerelease != true && startsWith(inputs.version, '6.') }}
    permissions:
      contents: read
    uses: ./.github/workflows/update-demo-server.yml
    secrets: inherit
    with:
      tag: ${{ inputs.tag }}
      target: stable
      verify_only: false
      activation_convergence_run_id: ${{ github.run_id }}
      customer_promotion_lease_sha: ${{ needs.acquire_customer_promotion_lease.outputs.lock_sha }}
      convergence_owner_asset_name: ${{ needs.acquire_customer_promotion_lease.outputs.owner_asset_name }}
      convergence_owner_asset_sha256: ${{ needs.acquire_customer_promotion_lease.outputs.owner_asset_sha256 }}

  convergence_verdict:
    name: Customer Promotion Convergence Verdict
    needs:
      - acquire_customer_promotion_lease
      - promote_floating_tags
      - publish_helm_pages
      - promote_private_pro_runtime
      - update_stable_demo
    if: ${{ always() }}
    runs-on: ubuntu-24.04
    timeout-minutes: 10
    steps:
      - name: Require every applicable surface to converge
        env:
          VERSION: ${{ inputs.version }}
          IS_PRERELEASE: ${{ inputs.prerelease }}
          LEASE_RESULT: ${{ needs.acquire_customer_promotion_lease.result }}
          SUPERSEDED: ${{ needs.acquire_customer_promotion_lease.outputs.superseded }}
          DESIRED_TAG: ${{ needs.acquire_customer_promotion_lease.outputs.desired_tag }}
          FLOATING_RESULT: ${{ needs.promote_floating_tags.result }}
          HELM_PAGES_RESULT: ${{ needs.publish_helm_pages.result }}
          PRIVATE_PRO_RESULT: ${{ needs.promote_private_pro_runtime.result }}
          DEMO_RESULT: ${{ needs.update_stable_demo.result }}
        run: |
          set -euo pipefail
          require_success() {
            local surface="$1"
            local result="$2"
            if [ "${result}" != "success" ]; then
              echo "::error::${surface} convergence ended as ${result}; the committed release remains public and this convergence run must be retried."
              return 1
            fi
          }

          require_success "global customer-promotion lease" "${LEASE_RESULT}"
          require_success "Helm Pages" "${HELM_PAGES_RESULT}"
          if [ "${SUPERSEDED}" = "true" ]; then
            if [ "${FLOATING_RESULT}" != "skipped" ] || \
               [ "${PRIVATE_PRO_RESULT}" != "skipped" ] || \
               [ "${DEMO_RESULT}" != "skipped" ]; then
              echo "::error::Superseded convergence mutated or attempted a rollback-prone customer surface."
              exit 1
            fi
            echo "[OK] Added ${VERSION} to Helm Pages without moving rollback-prone pointers behind ${DESIRED_TAG}."
            exit 0
          fi
          require_success "Docker aliases" "${FLOATING_RESULT}"
          if [[ "${VERSION}" == 6.* ]]; then
            require_success "paid-runtime broker" "${PRIVATE_PRO_RESULT}"
            if [ "${IS_PRERELEASE}" != "true" ]; then
              require_success "stable demo" "${DEMO_RESULT}"
            fi
          fi
          echo "[OK] Every applicable customer surface converged for v${VERSION}."

  release_customer_promotion_lease:
    name: Release global customer-promotion lease
    needs:
      - acquire_customer_promotion_lease
      - convergence_verdict
    if: ${{ always() && needs.acquire_customer_promotion_lease.outputs.lock_sha != '' }}
    runs-on: ubuntu-24.04
    timeout-minutes: 10
    permissions:
      contents: write
    steps:
      - name: Checkout release control
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          fetch-depth: 1
          persist-credentials: true  # required: authenticated git writes

      - name: Release owned repository-ref lease
        env:
          GH_TOKEN: ${{ github.token }}
          LOCK_SHA: ${{ needs.acquire_customer_promotion_lease.outputs.lock_sha }}
        run: |
          set -euo pipefail
          scripts/release_control/customer_promotion_lease.sh release "${LOCK_SHA}"