fix(ci): upload full tofu apply/destroy log as an artifact after teardown (#489)

Every tofu-based e2e workflow set TOFU_LOG to a /tmp path outside
ARTIFACT_DIR. ci-fmt.sh's quiet_run helper appends all tofu init/apply/
destroy output to that file, but the Teardown step (which runs the
destroy) always ran AFTER the existing upload-artifact step, so the
full log — including destroy-time failures — was silently discarded
on every run.

Add an if: always() upload-artifact step after Teardown in each
tofu-based job (e2e.yml, e2e-suites.yml, e2e-nightly.yml, e2e-ddil.yml)
that uploads $TOFU_LOG directly. Since quiet_run appends both apply and
destroy output to the same file, one upload after Teardown now
captures everything, recoverable even when destroy itself fails.

Shipping this log to Elasticsearch/OTLP is deferred to mulga-5n7fe,
which will consume this artifact.
This commit is contained in:
Josh Siviour
2026-07-09 22:20:48 +10:00
committed by GitHub
parent beee92b468
commit e78d0dbac9
4 changed files with 72 additions and 0 deletions
+9
View File
@@ -208,3 +208,12 @@ jobs:
quiet_run "tofu destroy" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
quiet_run "scrub-env" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
ok "teardown complete"
- name: Upload tofu log
if: always()
uses: actions/upload-artifact@v7
with:
name: tofu-log-ddil-${{ github.run_id }}
path: ${{ env.TOFU_LOG }}
if-no-files-found: ignore
retention-days: 30
+9
View File
@@ -348,6 +348,15 @@ jobs:
tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false >> "$TOFU_LOG" 2>&1
./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
- name: Upload tofu log
if: always()
uses: actions/upload-artifact@v7
with:
name: tofu-log-cell-${{ matrix.cell }}-${{ github.run_id }}
path: ${{ env.TOFU_LOG }}
if-no-files-found: ignore
retention-days: 14
# Bare-metal single-node suite. Doesn't fit the tofu matrix shape, so it
# runs as a sibling job via the reusable e2e-baremetal workflow. Gated like a
# matrix cell ("cell 20"): every weeknight cron, or dispatch with 20 selected
+36
View File
@@ -267,6 +267,15 @@ jobs:
"$VM":/tmp/start-net-capture.sh
ssh "${SSH_OPTS[@]}" "$VM" "bash /tmp/start-net-capture.sh" || true
- name: Upload tofu log
if: always()
uses: actions/upload-artifact@v7
with:
name: tofu-log-single-${{ github.run_id }}
path: ${{ env.TOFU_LOG }}
if-no-files-found: ignore
retention-days: 14
single_suite:
name: Single env · suite
needs: single_provision
@@ -467,6 +476,15 @@ jobs:
quiet_run "scrub-env" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
ok "teardown complete"
- name: Upload tofu log
if: always()
uses: actions/upload-artifact@v7
with:
name: tofu-log-single-td-${{ github.run_id }}
path: ${{ env.TOFU_LOG }}
if-no-files-found: ignore
retention-days: 14
# ---------------------------------------------------------------------------
# Multinode env
# ---------------------------------------------------------------------------
@@ -617,6 +635,15 @@ jobs:
ssh "${SSH_OPTS[@]}" tf-user@"$ip" "bash /tmp/start-net-capture.sh" || true
done
- name: Upload tofu log
if: always()
uses: actions/upload-artifact@v7
with:
name: tofu-log-multi-${{ github.run_id }}
path: ${{ env.TOFU_LOG }}
if-no-files-found: ignore
retention-days: 14
multi_suite:
name: Multi env · suite
needs: multi_provision
@@ -789,6 +816,15 @@ jobs:
quiet_run "scrub-env" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
ok "teardown complete"
- name: Upload tofu log
if: always()
uses: actions/upload-artifact@v7
with:
name: tofu-log-multi-td-${{ github.run_id }}
path: ${{ env.TOFU_LOG }}
if-no-files-found: ignore
retention-days: 14
e2e_suites_status:
name: E2E suite blocks · summary
if: always()
+18
View File
@@ -466,6 +466,15 @@ jobs:
quiet_run "scrub-env" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
ok "teardown complete"
- name: Upload tofu log
if: always()
uses: actions/upload-artifact@v7
with:
name: tofu-log-single-${{ github.run_id }}
path: ${{ env.TOFU_LOG }}
if-no-files-found: ignore
retention-days: 14
- name: Fail job if any suite failed
if: always()
run: |
@@ -881,6 +890,15 @@ jobs:
quiet_run "scrub-env" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
ok "teardown complete"
- name: Upload tofu log
if: always()
uses: actions/upload-artifact@v7
with:
name: tofu-log-multi-${{ github.run_id }}
path: ${{ env.TOFU_LOG }}
if-no-files-found: ignore
retention-days: 14
- name: Fail job if any suite failed
if: always()
run: |