feat: add release verification job with draft-then-promote flow

.github/workflows/release.yml

@@ -35,13 +35,114 @@ jobs:
         uses: docker/setup-qemu-action@v4
 
       - name: Build distribution tarballs
-        run: cd spinifex && make distro VERSION=${{ github.ref_name }}
+        env:
+          VERSION: ${{ github.ref_name }}
+        run: cd spinifex && make distro VERSION="$VERSION"
 
-      - name: Create GitHub Release
+      - name: Create Draft GitHub Release
         uses: softprops/action-gh-release@v2
         with:
+          draft: true
           files: |
             spinifex/dist/*.tar.gz
             spinifex/dist/*.sha256
             spinifex/scripts/setup.sh
           generate_release_notes: true
+
+      - name: Upload tarball for verify job
+        uses: actions/upload-artifact@v4
+        with:
+          name: release-assets
+          path: |
+            spinifex/dist/spinifex-${{ github.ref_name }}-linux-amd64.tar.gz
+            spinifex/scripts/setup.sh
+
+  verify:
+    needs: distro
+    runs-on: [self-hosted, ci-single]
+    timeout-minutes: 25
+    env:
+      TOFU_LOG: /tmp/release-verify-${{ github.run_id }}.log
+      TAG: ${{ github.ref_name }}
+      REPO: ${{ github.repository }}
+    steps:
+      - name: Clean workspace
+        run: rm -rf "$GITHUB_WORKSPACE"/*
+
+      - uses: actions/create-github-app-token@v3
+        id: app-token
+        with:
+          app-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          repositories: mulga
+
+      - uses: actions/checkout@v6
+        with:
+          repository: mulgadc/mulga
+          token: ${{ steps.app-token.outputs.token }}
+          sparse-checkout: scripts/tofu-cluster
+          path: mulga
+
+      - name: Download release assets
+        uses: actions/download-artifact@v4
+        with:
+          name: release-assets
+          path: release-assets
+
+      - name: Provision VM
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          tofu init -input=false >> "$TOFU_LOG" 2>&1
+          tofu workspace select -or-create env3 >> "$TOFU_LOG" 2>&1
+          tofu destroy -var-file=envs/env3.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true
+          ./scrub-env.sh env3 >> "$TOFU_LOG" 2>&1 || true
+          tofu apply -var-file=envs/env3.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
+
+      - name: Bootstrap via binary install
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          SETUP_SH="${GITHUB_WORKSPACE}/release-assets/spinifex/scripts/setup.sh"
+          TARBALL="${GITHUB_WORKSPACE}/release-assets/spinifex/dist/spinifex-${TAG}-linux-amd64.tar.gz"
+          ./bootstrap-install.sh env3 "$SETUP_SH" "$TARBALL"
+
+      - name: Run smoke test
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          INVENTORY=$(tofu output -json inventory)
+          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
+          SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR \
+            -i "$SSH_KEY" \
+            smoke-test-install.sh \
+            "tf-user@${WAN_IP}:/tmp/smoke-test.sh"
+          ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR \
+            -i "$SSH_KEY" "tf-user@${WAN_IP}" \
+            'bash /tmp/smoke-test.sh'
+
+      - name: Promote release
+        if: success()
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh release edit "$TAG" --repo "$REPO" --draft=false
+          echo "Release ${TAG} promoted to latest"
+
+      - name: Mark release failed
+        if: failure()
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          EXISTING_BODY=$(gh release view "$TAG" --repo "$REPO" --json body -q .body)
+          NOTES_FILE=$(mktemp)
+          printf 'Verification failed — see workflow run for details.\n\n%s' "$EXISTING_BODY" > "$NOTES_FILE"
+          gh release edit "$TAG" --repo "$REPO" --notes-file "$NOTES_FILE"
+          rm -f "$NOTES_FILE"
+
+      - name: Teardown
+        if: always()
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          tofu workspace select -or-create env3 >> "$TOFU_LOG" 2>&1
+          tofu destroy -var-file=envs/env3.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
+          ./scrub-env.sh env3 >> "$TOFU_LOG" 2>&1 || true