diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index f91fd5cba..818c1768a 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -1591,29 +1591,39 @@ jobs: committed=false require_viable_convergence_owner() { - local owner_state - owner_state="$( - gh run view "${CONVERGENCE_RUN_ID}" \ - --repo "${{ github.repository }}" \ - --json event,status,conclusion,workflowName,displayTitle,url \ - --jq '[.event, .status, (.conclusion // ""), .workflowName, .displayTitle, .url] | @tsv' - )" - owner_event="$(awk -F '\t' '{print $1}' <<<"${owner_state}")" - owner_status="$(awk -F '\t' '{print $2}' <<<"${owner_state}")" - owner_conclusion="$(awk -F '\t' '{print $3}' <<<"${owner_state}")" - owner_workflow="$(awk -F '\t' '{print $4}' <<<"${owner_state}")" - owner_title="$(awk -F '\t' '{print $5}' <<<"${owner_state}")" - owner_url="$(awk -F '\t' '{print $6}' <<<"${owner_state}")" + local attempt owner_state owner_event owner_status owner_conclusion + local owner_workflow owner_title owner_url expected_title expected_title="Release convergence ${TAG} source ${GITHUB_RUN_ID}" - if [ "${owner_event}" != "workflow_dispatch" ] || \ - [ "${owner_workflow}" != "Release Convergence" ] || \ - [ "${owner_title}" != "${expected_title}" ] || \ - [ "${owner_status}" = "completed" ] || \ - [ -n "${owner_conclusion}" ]; then - echo "::error::Exact convergence owner ${CONVERGENCE_RUN_ID} is not viable for ${TAG}: status=${owner_status} conclusion=${owner_conclusion:-none} ${owner_url}." - return 1 - fi - echo "Verified viable convergence owner ${CONVERGENCE_RUN_ID}: ${owner_status} ${owner_url}." + for attempt in $(seq 1 12); do + owner_state="$( + gh run view "${CONVERGENCE_RUN_ID}" \ + --repo "${{ github.repository }}" \ + --json event,status,conclusion,workflowName,displayTitle,url \ + --jq '[.event, .status, (.conclusion // ""), .workflowName, .displayTitle, .url] | @tsv' + )" + owner_event="$(awk -F '\t' '{print $1}' <<<"${owner_state}")" + owner_status="$(awk -F '\t' '{print $2}' <<<"${owner_state}")" + owner_conclusion="$(awk -F '\t' '{print $3}' <<<"${owner_state}")" + owner_workflow="$(awk -F '\t' '{print $4}' <<<"${owner_state}")" + owner_title="$(awk -F '\t' '{print $5}' <<<"${owner_state}")" + owner_url="$(awk -F '\t' '{print $6}' <<<"${owner_state}")" + if [ "${owner_event}" = "workflow_dispatch" ] && \ + [ "${owner_workflow}" = "Release Convergence" ] && \ + [ "${owner_title}" = "${expected_title}" ] && \ + [ "${owner_status}" != "completed" ] && \ + [ -z "${owner_conclusion}" ]; then + echo "Verified viable convergence owner ${CONVERGENCE_RUN_ID}: ${owner_status} ${owner_url}." + return 0 + fi + if [ "${owner_status}" = "completed" ] || [ -n "${owner_conclusion}" ]; then + echo "::error::Exact convergence owner ${CONVERGENCE_RUN_ID} is terminal for ${TAG}: status=${owner_status} conclusion=${owner_conclusion:-none} ${owner_url}." + return 1 + fi + echo "Convergence owner ${CONVERGENCE_RUN_ID} metadata is not coherent yet (${attempt}/12); waiting for GitHub indexing." + sleep 2 + done + echo "::error::Exact convergence owner ${CONVERGENCE_RUN_ID} metadata did not converge for ${TAG}: event=${owner_event:-missing} workflow=${owner_workflow:-missing} title=${owner_title:-missing} status=${owner_status:-missing} ${owner_url:-}." + return 1 } quarantine_on_error() { diff --git a/.github/workflows/recover-release-activation.yml b/.github/workflows/recover-release-activation.yml index f92ccf4bb..2242aeb5c 100644 --- a/.github/workflows/recover-release-activation.yml +++ b/.github/workflows/recover-release-activation.yml @@ -257,29 +257,39 @@ jobs: committed=false require_viable_convergence_owner() { - local owner_state - owner_state="$( - gh run view "${CONVERGENCE_RUN_ID}" \ - --repo "${GITHUB_REPOSITORY}" \ - --json event,status,conclusion,workflowName,displayTitle,url \ - --jq '[.event, .status, (.conclusion // ""), .workflowName, .displayTitle, .url] | @tsv' - )" - owner_event=$(awk -F '\t' '{print $1}' <<<"${owner_state}") - owner_status=$(awk -F '\t' '{print $2}' <<<"${owner_state}") - owner_conclusion=$(awk -F '\t' '{print $3}' <<<"${owner_state}") - owner_workflow=$(awk -F '\t' '{print $4}' <<<"${owner_state}") - owner_title=$(awk -F '\t' '{print $5}' <<<"${owner_state}") - owner_url=$(awk -F '\t' '{print $6}' <<<"${owner_state}") + local attempt owner_state owner_event owner_status owner_conclusion + local owner_workflow owner_title owner_url expected_title expected_title="Release convergence ${TAG} source ${SOURCE_RELEASE_RUN_ID}" - if [ "${owner_event}" != "workflow_dispatch" ] || \ - [ "${owner_workflow}" != "Release Convergence" ] || \ - [ "${owner_title}" != "${expected_title}" ] || \ - [ "${owner_status}" = "completed" ] || \ - [ -n "${owner_conclusion}" ]; then - echo "::error::Exact recovery convergence owner ${CONVERGENCE_RUN_ID} is not viable for ${TAG}: status=${owner_status} conclusion=${owner_conclusion:-none} ${owner_url}." - return 1 - fi - echo "Verified viable recovery convergence owner ${CONVERGENCE_RUN_ID}: ${owner_status} ${owner_url}." + for attempt in $(seq 1 12); do + owner_state="$( + gh run view "${CONVERGENCE_RUN_ID}" \ + --repo "${GITHUB_REPOSITORY}" \ + --json event,status,conclusion,workflowName,displayTitle,url \ + --jq '[.event, .status, (.conclusion // ""), .workflowName, .displayTitle, .url] | @tsv' + )" + owner_event=$(awk -F '\t' '{print $1}' <<<"${owner_state}") + owner_status=$(awk -F '\t' '{print $2}' <<<"${owner_state}") + owner_conclusion=$(awk -F '\t' '{print $3}' <<<"${owner_state}") + owner_workflow=$(awk -F '\t' '{print $4}' <<<"${owner_state}") + owner_title=$(awk -F '\t' '{print $5}' <<<"${owner_state}") + owner_url=$(awk -F '\t' '{print $6}' <<<"${owner_state}") + if [ "${owner_event}" = "workflow_dispatch" ] && \ + [ "${owner_workflow}" = "Release Convergence" ] && \ + [ "${owner_title}" = "${expected_title}" ] && \ + [ "${owner_status}" != "completed" ] && \ + [ -z "${owner_conclusion}" ]; then + echo "Verified viable recovery convergence owner ${CONVERGENCE_RUN_ID}: ${owner_status} ${owner_url}." + return 0 + fi + if [ "${owner_status}" = "completed" ] || [ -n "${owner_conclusion}" ]; then + echo "::error::Exact recovery convergence owner ${CONVERGENCE_RUN_ID} is terminal for ${TAG}: status=${owner_status} conclusion=${owner_conclusion:-none} ${owner_url}." + return 1 + fi + echo "Recovery convergence owner ${CONVERGENCE_RUN_ID} metadata is not coherent yet (${attempt}/12); waiting for GitHub indexing." + sleep 2 + done + echo "::error::Exact recovery convergence owner ${CONVERGENCE_RUN_ID} metadata did not converge for ${TAG}: event=${owner_event:-missing} workflow=${owner_workflow:-missing} title=${owner_title:-missing} status=${owner_status:-missing} ${owner_url:-}." + return 1 } quarantine_on_error() { diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index 4b54c53f5..62ef6e75c 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -999,6 +999,10 @@ upgrade, update, release, or artifact-selection behavior. require the same draft release ID, tag, target commit, and absent activation marker. It then dispatches a fresh durable convergence owner and repeats the activation commit without rebuilding or replacing any candidate artifact. + Because a newly dispatched Actions run can be visible before its workflow + name and display title are coherently indexed, both normal and recovered + activation use a bounded metadata-propagation wait before rejecting the + convergence owner; a terminal owner still fails immediately. One immutable-readiness join must cover the staged release packet, staged install smoke, exact public Docker images, exact Helm OCI chart, and (for v6) the exact Pro image and signed packet. After that join, activation must diff --git a/scripts/installtests/build_release_assets_test.go b/scripts/installtests/build_release_assets_test.go index d1a26badc..1bc325b6b 100644 --- a/scripts/installtests/build_release_assets_test.go +++ b/scripts/installtests/build_release_assets_test.go @@ -2121,6 +2121,8 @@ func TestReleasePipelinePromotesOneImmutableCandidate(t *testing.T) { `failure outside the recoverable activation boundary`, `release-convergence.yml/dispatches`, `activation_recovery_run_id`, + `for attempt in $(seq 1 12)`, + `waiting for GitHub indexing`, `--repo "${GITHUB_REPOSITORY}"`, } { if !strings.Contains(recoveryWorkflow, needle) { diff --git a/scripts/release_control/release_promotion_policy_test.py b/scripts/release_control/release_promotion_policy_test.py index d22bec86e..a57a0f860 100644 --- a/scripts/release_control/release_promotion_policy_test.py +++ b/scripts/release_control/release_promotion_policy_test.py @@ -446,6 +446,8 @@ class ReleasePromotionPolicyTest(unittest.TestCase): self.assertIn("release-convergence.yml/dispatches", job) self.assertIn("return_run_details: true", job) self.assertIn("activation_recovery_run_id", job) + self.assertIn("for attempt in $(seq 1 12)", job) + self.assertIn("waiting for GitHub indexing", job) self.assertIn('--repo "${GITHUB_REPOSITORY}"', job) self.assertNotIn("build-release-candidate.yml", recovery) self.assertNotIn("scripts/build-release.sh", recovery) @@ -706,6 +708,8 @@ class ReleasePromotionPolicyTest(unittest.TestCase): self.assertIn("require_viable_convergence_owner()", activation) self.assertEqual(activation.count("require_viable_convergence_owner"), 3) self.assertIn('gh run view "${CONVERGENCE_RUN_ID}"', activation) + self.assertIn("for attempt in $(seq 1 12)", activation) + self.assertIn("waiting for GitHub indexing", activation) self.assertIn("--json event,status,conclusion,workflowName,displayTitle,url", activation) self.assertIn('expected_title="Release convergence ${TAG} source ${GITHUB_RUN_ID}"', activation) self.assertIn('[ "${owner_status}" = "completed" ]', activation)