From be966b20c44fa2776f9a1e3cd59b1cd4da44943d Mon Sep 17 00:00:00 2001 From: joshsiv-mulga Date: Mon, 18 May 2026 17:16:11 +1000 Subject: [PATCH] ci: colored phase banners + quiet-on-success tofu output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pretty up the live Actions log without flooding it. Three changes: 1. .github/scripts/ci-fmt.sh — sourceable bash helpers (banner, ok, bad, warn, note, quiet_run) using ANSI colors that the Actions UI renders. 2. e2e-go.yml + e2e-ddil.yml — tofu phases switch back to quiet redirect into $TOFU_LOG (the previous tee'd version regressed live noise vs the original silent redirect). Each phase now prints one colored ok/✗ line; on failure the tail of $TOFU_LOG dumps inside a single collapsed group, so the live log stays scannable but the failing tail is one click away. 3. junit-report render.py — compact summary: short status icons (✅/❌) in a single column, no separate err column, single-line failure list (truncated to 120 chars) instead of
blocks that wrap badly on narrow screens. Per-suite test runs still wrap in ::group:: with an inline cyan banner so pass output collapses and the suite name is obvious when expanded. --- .github/actions/junit-report/render.py | 47 ++++++----- .github/scripts/ci-fmt.sh | 63 +++++++++++++++ .github/workflows/e2e-ddil.yml | 54 +++++-------- .github/workflows/e2e-go.yml | 105 ++++++++++++------------- 4 files changed, 162 insertions(+), 107 deletions(-) create mode 100644 .github/scripts/ci-fmt.sh diff --git a/.github/actions/junit-report/render.py b/.github/actions/junit-report/render.py index e326cf8e3..24ab7146c 100644 --- a/.github/actions/junit-report/render.py +++ b/.github/actions/junit-report/render.py @@ -148,22 +148,28 @@ def main() -> int: grand_total = sum(f.tests for f in totals_by_file) grand_fail = sum(f.failures + f.errors for f in totals_by_file) grand_skip = sum(f.skipped for f in totals_by_file) + grand_pass = grand_total - grand_fail - grand_skip grand_time = sum(f.time for f in totals_by_file) - verdict = ":white_check_mark: pass" if grand_fail == 0 else f":x: {grand_fail} failed" - summary.write( - f"**{verdict}** · {grand_total} tests · " - f"{grand_skip} skipped · {grand_time:.1f}s total\n\n" - ) + if grand_fail == 0: + verdict = f"✅ **pass** — {grand_pass} / {grand_total}" + else: + verdict = f"❌ **{grand_fail} failed** — {grand_pass} / {grand_total} pass" + skip_part = f", {grand_skip} skipped" if grand_skip else "" + summary.write(f"{verdict}{skip_part}   ·   {grand_time:.1f}s\n\n") - summary.write("| Suite | Tests | Pass | Fail | Err | Skip | Time |\n") - summary.write("|---|---:|---:|---:|---:|---:|---:|\n") + # Compact table — single status column, shorter headers. Drops + # separate err column (errors fold into fail count) so narrow + # screens don't wrap. + summary.write("| | Suite | Pass | Fail | Skip | Time |\n") + summary.write("|:-:|---|---:|---:|---:|---:|\n") for f in totals_by_file: - passed = f.tests - f.failures - f.errors - f.skipped - icon = ":white_check_mark:" if (f.failures + f.errors) == 0 else ":x:" + fails = f.failures + f.errors + passed = f.tests - fails - f.skipped + icon = "✅" if fails == 0 else "❌" summary.write( - f"| {icon} `{_file_label(f.path)}` | {f.tests} | {passed} | " - f"{f.failures} | {f.errors} | {f.skipped} | {f.time:.1f}s |\n" + f"| {icon} | `{_file_label(f.path)}` | {passed} | {fails} | " + f"{f.skipped} | {f.time:.1f}s |\n" ) summary.write("\n") @@ -171,21 +177,26 @@ def main() -> int: c for f in totals_by_file for c in f.cases if c.status in ("fail", "error") ] if failed_cases: - summary.write("### Failures\n\n") + summary.write("---\n\n**Failures**\n\n") for c in failed_cases: header = f"{c.classname}.{c.name}" if c.classname else c.name - summary.write(f"
{header}\n\n") - if c.message: - summary.write(f"**{c.message}**\n\n") - if c.detail: - summary.write("```\n" + c.detail + "\n```\n\n") - summary.write("
\n\n") + # One line per failure: name + truncated message. Full + # detail lives in the test artifact log; deep nesting and + # long messages wrap badly on narrow screens. + msg_short = (c.message or "").splitlines()[0] if c.message else "" + if len(msg_short) > 120: + msg_short = msg_short[:117] + "…" + line = f"- ❌ `{header}`" + if msg_short: + line += f" — {msg_short}" + summary.write(line + "\n") # Workflow annotation — surfaces in run header + Files-changed view. # classname for Go tests is the package path; map to a file when we can. file_hint = c.classname.replace(".", "/") msg = c.message or f"{c.status}: {header}" print(f"::error file={file_hint},title={header}::{_esc(msg)}") + summary.write("\n") outputs.write(f"failures={grand_fail}\n") outputs.write(f"total={grand_total}\n") diff --git a/.github/scripts/ci-fmt.sh b/.github/scripts/ci-fmt.sh new file mode 100644 index 000000000..189b2af57 --- /dev/null +++ b/.github/scripts/ci-fmt.sh @@ -0,0 +1,63 @@ +# shellcheck shell=bash +# Source from a workflow `run:` block: +# source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" +# +# Provides colored banners + ::group::/::endgroup:: wrappers. ANSI escapes +# render in the GitHub Actions live log; the same lines also land in any +# artifact log (tee'd output) without breaking parsers because they only +# affect terminal display. +# +# Functions kept short and unindented on purpose — wide nested output wraps +# badly on narrow terminals. + +if [[ -n "${GITHUB_ACTIONS:-}" ]] || [[ -t 1 ]]; then + _R=$'\033[0m' + _B=$'\033[1m' # bold + _D=$'\033[2m' # dim + _RED=$'\033[31m' + _GRN=$'\033[32m' + _YEL=$'\033[33m' + _BLU=$'\033[34m' + _CYA=$'\033[36m' +else + _R="" _B="" _D="" _RED="" _GRN="" _YEL="" _BLU="" _CYA="" +fi + +# Single-line rule using box-drawing chars — narrow enough that GitHub's +# timestamp prefix + this rule still fit on an 80-col terminal. +# banner LABEL — bold cyan title with surrounding rule. Used at the start +# of an inherently multi-line phase (test run, artifact pull) so the live +# log has an obvious visual marker. +banner() { + printf '\n%s%s━━ %s ━━%s\n' "$_B" "$_CYA" "$*" "$_R" +} + +# Status indicators. Single char prefix only — keeps width on narrow logs. +ok() { printf '%s✓%s %s\n' "$_GRN" "$_R" "$*"; } +bad() { printf '%s✗%s %s\n' "$_RED" "$_R" "$*"; } +warn() { printf '%s!%s %s\n' "$_YEL" "$_R" "$*"; } +note() { printf '%s·%s %s\n' "$_D" "$_R" "$*"; } + +# group_open LABEL / group_close — thin wrapper around GitHub's collapse +# markers. No extra body printed; banner() is the visible inside-group +# heading when callers want one. +group_open() { echo "::group::$*"; } +group_close() { echo "::endgroup::"; } + +# quiet_run LABEL LOG_FILE -- CMD ARGS … +# Run CMD redirected to LOG_FILE (append). Print one colored ok/✗ line. +# On failure, dump the last 80 lines of LOG_FILE inside a collapsed group +# so the failing tail lands in the UI without surfacing the whole apply. +quiet_run() { + local label="$1" log="$2"; shift 2 + if "$@" >> "$log" 2>&1; then + ok "$label" + else + local rc=$? + bad "$label (rc=$rc) — last 80 lines of $(basename "$log"):" + group_open "tail $(basename "$log")" + tail -n 80 "$log" + group_close + return "$rc" + fi +} diff --git a/.github/workflows/e2e-ddil.yml b/.github/workflows/e2e-ddil.yml index 55aed5a8d..a7fde3c24 100644 --- a/.github/workflows/e2e-ddil.yml +++ b/.github/workflows/e2e-ddil.yml @@ -88,23 +88,15 @@ jobs: working-directory: mulga/scripts/tofu-cluster run: | set -euo pipefail + # shellcheck source=/dev/null + source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" export TF_VAR_git_branch="${GIT_BRANCH}" - # Each phase wrapped in ::group:: so the Actions UI collapses the - # noisy tofu output by default but the human can expand the - # failing phase without downloading $TOFU_LOG. - run() { - local label="$1"; shift - echo "::group::${label}" - "$@" 2>&1 | tee -a "$TOFU_LOG" - local rc=${PIPESTATUS[0]} - echo "::endgroup::" - return "$rc" - } - run "tofu init" tofu init -input=false - run "tofu workspace select" tofu workspace select -or-create "${SPINIFEX_CI_ENV}" - run "tofu destroy (pre-clean)" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true - run "scrub-env (pre-clean)" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true - run "tofu apply" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false + quiet_run "tofu init" "$TOFU_LOG" tofu init -input=false + quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}" + quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true + quiet_run "scrub-env (pre-clean)" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true + quiet_run "tofu apply" "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false + ok "VMs provisioned" - name: Bootstrap via Binary Install working-directory: mulga/scripts/tofu-cluster @@ -163,7 +155,9 @@ jobs: # plus machine-readable jsonfile + JUnit XML artifacts. The JUnit # file feeds the junit-report composite action below so failures # become summary table rows + workflow annotations. - echo "::group::gotestsum scenarios" + # shellcheck source=/dev/null + source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" + banner "DDIL scenarios" DDIL_NODES="${NODE_IPS}" \ DDIL_SSH_USER="${SSH_USER}" \ DDIL_SSH_KEY="${SSH_KEY}" \ @@ -177,7 +171,7 @@ jobs: --format testname \ -- -tags=e2e -timeout 45m ./tests/e2e/ddil/scenarios/... rc=$? - echo "::endgroup::" + if [[ $rc -eq 0 ]]; then ok "all scenarios passed"; else bad "scenarios failed (rc=$rc)"; fi exit $rc - name: Publish test summary @@ -203,16 +197,15 @@ jobs: LOG_DIR="${RUNNER_TEMP}/ddil-results/cluster-logs" mkdir -p "${LOG_DIR}" + # shellcheck source=/dev/null + source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" mapfile -t NODE_IPS < <(echo "$INVENTORY" | jq -r '.nodes[].wan_ip') for ip in "${NODE_IPS[@]}"; do - echo "::group::collect ${ip}" SSH="ssh -i ${SSH_KEY} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ${SSH_USER}@${ip}" ${SSH} 'sudo journalctl -u "spinifex-*" --no-pager -n 5000' > "${LOG_DIR}/${ip}-spinifex.log" 2>&1 ${SSH} 'ps auxw | grep -E "qemu-system|spinifex" | grep -v grep' > "${LOG_DIR}/${ip}-procs.txt" 2>&1 ${SSH} 'ls -la /var/lib/spinifex 2>/dev/null; ls -la /var/log/spinifex 2>/dev/null' > "${LOG_DIR}/${ip}-state.txt" 2>&1 - echo " spinifex.log: $(wc -l < "${LOG_DIR}/${ip}-spinifex.log") lines" - echo " procs.txt: $(wc -l < "${LOG_DIR}/${ip}-procs.txt") lines" - echo "::endgroup::" + ok "$ip: $(wc -l < "${LOG_DIR}/${ip}-spinifex.log") journal lines, $(wc -l < "${LOG_DIR}/${ip}-procs.txt") procs" done - name: Upload DDIL Results @@ -228,14 +221,9 @@ jobs: working-directory: mulga/scripts/tofu-cluster run: | set -euo pipefail - run() { - local label="$1"; shift - echo "::group::${label}" - "$@" 2>&1 | tee -a "$TOFU_LOG" - local rc=${PIPESTATUS[0]} - echo "::endgroup::" - return "$rc" - } - run "tofu workspace select" tofu workspace select -or-create "${SPINIFEX_CI_ENV}" - run "tofu destroy" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false - run "scrub-env" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true + # shellcheck source=/dev/null + source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" + quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}" + quiet_run "tofu destroy" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false + quiet_run "scrub-env" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true + ok "teardown complete" diff --git a/.github/workflows/e2e-go.yml b/.github/workflows/e2e-go.yml index 7a990e817..27fec4e10 100644 --- a/.github/workflows/e2e-go.yml +++ b/.github/workflows/e2e-go.yml @@ -85,23 +85,15 @@ jobs: working-directory: mulga/scripts/tofu-cluster run: | set -euo pipefail + # shellcheck source=/dev/null + source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" export TF_VAR_git_branch="${GIT_BRANCH}" - # Each phase wrapped in ::group:: so the Actions UI collapses - # the noisy tofu output by default but the human can expand the - # failing phase without downloading $TOFU_LOG. - run() { - local label="$1"; shift - echo "::group::${label}" - "$@" 2>&1 | tee -a "$TOFU_LOG" - local rc=${PIPESTATUS[0]} - echo "::endgroup::" - return "$rc" - } - run "tofu init" tofu init -input=false - run "tofu workspace select" tofu workspace select -or-create "${SPINIFEX_CI_ENV}" - run "tofu destroy (pre-clean)" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true - run "scrub-env (pre-clean)" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true - run "tofu apply" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false + quiet_run "tofu init" "$TOFU_LOG" tofu init -input=false + quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}" + quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true + quiet_run "scrub-env (pre-clean)" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true + quiet_run "tofu apply" "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false + ok "VM provisioned" - name: Bootstrap via Binary Install working-directory: mulga/scripts/tofu-cluster @@ -127,13 +119,18 @@ jobs: working-directory: spinifex/tests/e2e run: | set -euo pipefail + # shellcheck source=/dev/null + source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" # SUITES is in make-target form (e.g. "e2e-cert e2e-lb"); strip the # "e2e-" prefix to get package directory names for `go test -c`. SUITES_BUILD=$(echo "${SUITES}" | sed 's/e2e-//g') - echo "::group::go test -c (build ${SUITES_BUILD// /, })" + banner "build ${SUITES_BUILD// /, }" + group_open "go test -c" make e2e-build SUITES_BUILD="${SUITES_BUILD}" - ls -lh _bin/ - echo "::endgroup::" + group_close + for f in _bin/*.test; do + ok "$(basename "$f") $(stat -c '%s' "$f" | numfmt --to=iec)" + done - name: Install go-junit-report on runner run: | @@ -145,15 +142,13 @@ jobs: - name: Ship test binaries to VM run: | set -euo pipefail - echo "::group::tar-over-ssh: prep remote dir + stream binaries" - ssh -o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}" \ - tf-user@${{ steps.vm.outputs.wan_ip }} \ - "rm -rf \$HOME/spinifex-e2e-bin && mkdir -p \$HOME/spinifex-e2e-bin" - tar -C spinifex/tests/e2e/_bin -cf - . \ - | ssh -o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}" \ - tf-user@${{ steps.vm.outputs.wan_ip }} \ - "tar -xf - -C \$HOME/spinifex-e2e-bin" - echo "::endgroup::" + # shellcheck source=/dev/null + source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" + SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}") + VM=tf-user@${{ steps.vm.outputs.wan_ip }} + ssh "${SSH_OPTS[@]}" "$VM" "rm -rf \$HOME/spinifex-e2e-bin && mkdir -p \$HOME/spinifex-e2e-bin" + tar -C spinifex/tests/e2e/_bin -cf - . | ssh "${SSH_OPTS[@]}" "$VM" "tar -xf - -C \$HOME/spinifex-e2e-bin" + ok "shipped binaries to $VM" - name: Run Go E2E suites id: e2e @@ -161,17 +156,19 @@ jobs: set +e # SUITES are make-target names ("e2e-cert" → cert.test). Each # pre-built binary is self-contained (testdata //go:embed'd) so the - # VM needs no go toolchain, no source tree, no gotestsum. - # Each suite is wrapped in its own ::group:: so the Actions UI - # gives the human one collapsible block per suite. + # VM needs no go toolchain, no source tree, no gotestsum. Each + # suite gets its own ::group:: + cyan banner inside so the Actions + # UI collapses suite output by default; expand only what failed. SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g') ssh -o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}" \ tf-user@${{ steps.vm.outputs.wan_ip }} \ "set -u; mkdir -p '${ARTIFACT_DIR}'; \ - cd \$HOME/spinifex-e2e-bin && \ + cd \$HOME/spinifex-e2e-bin; \ + R=\$'\033[0m' B=\$'\033[1m' C=\$'\033[36m' G=\$'\033[32m' RED=\$'\033[31m'; \ FAIL=0; \ for SUITE in ${SUITES_RUN}; do \ echo \"::group::suite \$SUITE\"; \ + printf '\n%s%s━━ suite %s ━━%s\n' \"\$B\" \"\$C\" \"\$SUITE\" \"\$R\"; \ SPINIFEX_E2E=1 SPINIFEX_MODE=single \ ARTIFACT_DIR='${ARTIFACT_DIR}' \ ./\$SUITE.test -test.v -test.timeout=30m \ @@ -179,9 +176,12 @@ jobs: rc=\${PIPESTATUS[0]}; \ echo \"::endgroup::\"; \ if [ \$rc -ne 0 ]; then \ + printf '%s✗%s suite %s failed (exit %d)\n' \"\$RED\" \"\$R\" \"\$SUITE\" \"\$rc\"; \ echo \"::error::suite \$SUITE failed (exit \$rc)\"; \ echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; \ FAIL=1; \ + else \ + printf '%s✓%s suite %s passed\n' \"\$G\" \"\$R\" \"\$SUITE\"; \ fi; \ done; \ exit \$FAIL" @@ -192,22 +192,20 @@ jobs: - name: Collect artifacts from VM if: always() run: | + set +e + # shellcheck source=/dev/null + source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" mkdir -p "${ARTIFACT_DIR}" - echo "::group::tar-over-ssh: pull artifact dir" + SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}") + VM=tf-user@${{ steps.vm.outputs.wan_ip }} # `|| true` so a missing dir on an early failure doesn't mask the # original error. - ssh -o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}" \ - tf-user@${{ steps.vm.outputs.wan_ip }} \ - "tar -C '${ARTIFACT_DIR}' -cf - . 2>/dev/null || true" \ - | tar -C "${ARTIFACT_DIR}" -xf - || true - echo "::endgroup::" - echo "::group::journalctl snapshot (spinifex-*)" - ssh -o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}" \ - tf-user@${{ steps.vm.outputs.wan_ip }} \ - 'sudo journalctl -u "spinifex-*" --no-pager --since "60 min ago"' \ - > "${ARTIFACT_DIR}/spinifex-journal.log" || true - echo "wrote ${ARTIFACT_DIR}/spinifex-journal.log ($(wc -l < "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null || echo 0) lines)" - echo "::endgroup::" + ssh "${SSH_OPTS[@]}" "$VM" "tar -C '${ARTIFACT_DIR}' -cf - . 2>/dev/null || true" \ + | tar -C "${ARTIFACT_DIR}" -xf - 2>/dev/null + ok "artifacts pulled to ${ARTIFACT_DIR}" + ssh "${SSH_OPTS[@]}" "$VM" 'sudo journalctl -u "spinifex-*" --no-pager --since "60 min ago"' \ + > "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null + ok "journal: $(wc -l < "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null || echo 0) lines" - name: Convert test logs to JUnit XML if: always() @@ -243,17 +241,12 @@ jobs: working-directory: mulga/scripts/tofu-cluster run: | set -euo pipefail - run() { - local label="$1"; shift - echo "::group::${label}" - "$@" 2>&1 | tee -a "$TOFU_LOG" - local rc=${PIPESTATUS[0]} - echo "::endgroup::" - return "$rc" - } - run "tofu workspace select" tofu workspace select -or-create "${SPINIFEX_CI_ENV}" - run "tofu destroy" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false - run "scrub-env" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true + # shellcheck source=/dev/null + source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" + quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}" + quiet_run "tofu destroy" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false + quiet_run "scrub-env" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true + ok "teardown complete" - name: Fail job if any suite failed if: always()