From e78d0dbac91fbbbee785fd0b5e31d9f502019cb5 Mon Sep 17 00:00:00 2001 From: Josh Siviour Date: Thu, 9 Jul 2026 22:20:48 +1000 Subject: [PATCH] fix(ci): upload full tofu apply/destroy log as an artifact after teardown (#489) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every tofu-based e2e workflow set TOFU_LOG to a /tmp path outside ARTIFACT_DIR. ci-fmt.sh's quiet_run helper appends all tofu init/apply/ destroy output to that file, but the Teardown step (which runs the destroy) always ran AFTER the existing upload-artifact step, so the full log — including destroy-time failures — was silently discarded on every run. Add an if: always() upload-artifact step after Teardown in each tofu-based job (e2e.yml, e2e-suites.yml, e2e-nightly.yml, e2e-ddil.yml) that uploads $TOFU_LOG directly. Since quiet_run appends both apply and destroy output to the same file, one upload after Teardown now captures everything, recoverable even when destroy itself fails. Shipping this log to Elasticsearch/OTLP is deferred to mulga-5n7fe, which will consume this artifact. --- .github/workflows/e2e-ddil.yml | 9 ++++++++ .github/workflows/e2e-nightly.yml | 9 ++++++++ .github/workflows/e2e-suites.yml | 36 +++++++++++++++++++++++++++++++ .github/workflows/e2e.yml | 18 ++++++++++++++++ 4 files changed, 72 insertions(+) diff --git a/.github/workflows/e2e-ddil.yml b/.github/workflows/e2e-ddil.yml index a276f63a5..847c1ad30 100644 --- a/.github/workflows/e2e-ddil.yml +++ b/.github/workflows/e2e-ddil.yml @@ -208,3 +208,12 @@ jobs: quiet_run "tofu destroy" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false quiet_run "scrub-env" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true ok "teardown complete" + + - name: Upload tofu log + if: always() + uses: actions/upload-artifact@v7 + with: + name: tofu-log-ddil-${{ github.run_id }} + path: ${{ env.TOFU_LOG }} + if-no-files-found: ignore + retention-days: 30 diff --git a/.github/workflows/e2e-nightly.yml b/.github/workflows/e2e-nightly.yml index 1cf5adf21..90a60a672 100644 --- a/.github/workflows/e2e-nightly.yml +++ b/.github/workflows/e2e-nightly.yml @@ -348,6 +348,15 @@ jobs: tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false >> "$TOFU_LOG" 2>&1 ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true + - name: Upload tofu log + if: always() + uses: actions/upload-artifact@v7 + with: + name: tofu-log-cell-${{ matrix.cell }}-${{ github.run_id }} + path: ${{ env.TOFU_LOG }} + if-no-files-found: ignore + retention-days: 14 + # Bare-metal single-node suite. Doesn't fit the tofu matrix shape, so it # runs as a sibling job via the reusable e2e-baremetal workflow. Gated like a # matrix cell ("cell 20"): every weeknight cron, or dispatch with 20 selected diff --git a/.github/workflows/e2e-suites.yml b/.github/workflows/e2e-suites.yml index 0214df8a3..b1570e4cf 100644 --- a/.github/workflows/e2e-suites.yml +++ b/.github/workflows/e2e-suites.yml @@ -267,6 +267,15 @@ jobs: "$VM":/tmp/start-net-capture.sh ssh "${SSH_OPTS[@]}" "$VM" "bash /tmp/start-net-capture.sh" || true + - name: Upload tofu log + if: always() + uses: actions/upload-artifact@v7 + with: + name: tofu-log-single-${{ github.run_id }} + path: ${{ env.TOFU_LOG }} + if-no-files-found: ignore + retention-days: 14 + single_suite: name: Single env · suite needs: single_provision @@ -467,6 +476,15 @@ jobs: quiet_run "scrub-env" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" ok "teardown complete" + - name: Upload tofu log + if: always() + uses: actions/upload-artifact@v7 + with: + name: tofu-log-single-td-${{ github.run_id }} + path: ${{ env.TOFU_LOG }} + if-no-files-found: ignore + retention-days: 14 + # --------------------------------------------------------------------------- # Multinode env # --------------------------------------------------------------------------- @@ -617,6 +635,15 @@ jobs: ssh "${SSH_OPTS[@]}" tf-user@"$ip" "bash /tmp/start-net-capture.sh" || true done + - name: Upload tofu log + if: always() + uses: actions/upload-artifact@v7 + with: + name: tofu-log-multi-${{ github.run_id }} + path: ${{ env.TOFU_LOG }} + if-no-files-found: ignore + retention-days: 14 + multi_suite: name: Multi env · suite needs: multi_provision @@ -789,6 +816,15 @@ jobs: quiet_run "scrub-env" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" ok "teardown complete" + - name: Upload tofu log + if: always() + uses: actions/upload-artifact@v7 + with: + name: tofu-log-multi-td-${{ github.run_id }} + path: ${{ env.TOFU_LOG }} + if-no-files-found: ignore + retention-days: 14 + e2e_suites_status: name: E2E suite blocks · summary if: always() diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index e750e55ea..cef2b6333 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -466,6 +466,15 @@ jobs: quiet_run "scrub-env" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" ok "teardown complete" + - name: Upload tofu log + if: always() + uses: actions/upload-artifact@v7 + with: + name: tofu-log-single-${{ github.run_id }} + path: ${{ env.TOFU_LOG }} + if-no-files-found: ignore + retention-days: 14 + - name: Fail job if any suite failed if: always() run: | @@ -881,6 +890,15 @@ jobs: quiet_run "scrub-env" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" ok "teardown complete" + - name: Upload tofu log + if: always() + uses: actions/upload-artifact@v7 + with: + name: tofu-log-multi-${{ github.run_id }} + path: ${{ env.TOFU_LOG }} + if-no-files-found: ignore + retention-days: 14 + - name: Fail job if any suite failed if: always() run: |