spinifex/.github/scripts/suite-sets.sh

# shellcheck shell=bash
# shellcheck disable=SC2034  # every variable here is consumed by the sourcing workflow
# Source from a workflow `run:` block:
#   . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
#
# Which e2e suites each topology can run. Sourced by every workflow that drives
# suites (e2e.yml, e2e-suites.yml, e2e-nightly.yml) so the lists cannot drift
# apart — they previously disagreed three ways.
#
# Eligibility is a property of the suite, not of the workflow: a suite belongs
# to a topology when all of its assertions are reachable there. lb is
# multi-only for exactly that reason — on a single node every internet-facing
# subtest skips for want of a peer, so a single-node lb run reports green
# having exercised a fraction of the suite.

# Suites runnable against a one-node environment.
#
# partialblock gates merges because it earned it, not because it is new: on one
# node, same binary and workload, it fails against the pre-fix engine (1758 of
# 8192 halves reverted a generation) and passes against the fixed one (0 of
# 8192). A suite that has never been shown to go red for the reason it claims
# does not belong here.
# rds is single-only for the opposite reason to lb: v1 has no multi-AZ
# behaviour, so a second node adds no assertion. Its DB VMs are capped at four
# concurrent by the suite's own semaphore, which is what keeps it inside the
# job's 50-minute budget alongside the suites above.
# quota belongs to both topologies because the gate it asserts is gateway code
# that runs identically on either, and because a quota left switched on would
# cap every suite sharing the environment — so it must be proved to restore
# itself wherever it runs, not only on one node.
# storagefault runs last in both lists and belongs to both topologies, for
# different reasons. On one node it is RS(1,0), so freezing the only predastore
# is a total outage and the guest-survival assertion is unambiguous. On three it
# is RS(2,1) with degraded writes on, so two must be frozen to break the shard
# floor — and only there is a cross-node restart possible at all, which is the
# case that takes a volume over from a node holding writes the backend never
# received. It is last because it SIGSTOPs a cluster-wide
# service: anything sharing the environment would see the same outage and report
# a failure that says nothing about itself.
E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds quota storagefault"

# Suites runnable against a multi-node environment.
#
# instancerecovery is multi-only and needs three: recovery is inert below that,
# because there is no survivor to move a guest to that is not the node that
# failed. Like storagefault it takes a node away and freezes a cluster-wide
# service, so it runs last and owns its environment.
#
# lbrecovery is multi-only and needs three for two reasons rather than one: a
# survivor to move the load balancer to, and a node for its backends that is not
# the load balancer's, so taking that node away does not take the backends with
# it. It runs beside instancerecovery at the end because it also takes a node
# away and owns its environment while it does.
#
# kvquorum is multi-only and needs three, for the same reason as those two: a
# surviving majority. It is what proves the control plane's own state is
# replicated rather than sitting on one node, so it runs before the pair that
# take a node away — a cluster whose buckets are already wrong should report
# that, not report a recovery failing.
E2E_SUITES_MULTI="multinode lb cert quota kvquorum lbrecovery instancerecovery storagefault"

# grep -xE alternation form, for narrowing a requested set to the eligible one.
E2E_SUITES_SINGLE_RE="$(printf '%s' "${E2E_SUITES_SINGLE}" | tr ' ' '|')"
E2E_SUITES_MULTI_RE="$(printf '%s' "${E2E_SUITES_MULTI}" | tr ' ' '|')"

# What every e2e-nightly permutation cell must prove, deliberately narrower
# than the two sets above: those cells have a 35-minute budget and exist to show
# that every install / network / host-OS combination boots and serves. The
# service suites are not deferred to the PR gate — eks, ecs and rds each run
# nightly in their own dedicated cell, because each needs a guest AMI built
# first and a service failure should not mask whether a host OS boots. These
# are subsets, and e2e_suites_assert_subset below enforces that.
#
# storagefault has a dedicated cell for two reasons. It SIGSTOPs a cluster-wide
# service, so in a permutation cell every suite sharing that environment would
# report the outage as its own failure. And a full pass is ~63 minutes, nearly
# all of it deliberate waiting, which is too long to add to the PR gate — where
# it has never run, since the gate's default suite list does not name it.
#
# Its cell is three-node: the RS(2,1) shard floor and the cross-node takeover
# after a failed seal do not exist on one node, and they are the assertions
# worth a nightly.
E2E_SUITES_NIGHTLY_SINGLE="single cert iam"
#
# The multi-node nightly subset carries kvquorum's read-only half through
# `multinode`, which sweeps every KV bucket rather than the three the daemon
# owns. The node-loss half is too slow for a 35-minute cell and has its own.
E2E_SUITES_NIGHTLY_MULTI="multinode cert lb"

# How long a suite gets, for the same reason the lists above live here: the
# timeout was written out separately in each workflow and drifted, so rds ran
# 50m on the PR gate and 30m nightly and panicked there. A suite needing more
# than the default declares it once, here.
#   e2e_suite_timeout <suite>
e2e_suite_timeout() {
  case "$1" in
    # rds serialises its DB VMs behind a 3 GiB semaphore, so its wall clock is
    # what the queue costs, not what the tests do: single tests logged 11m, 22m
    # and 26m waiting for budget. 17 tests at that concurrency is over an hour
    # before anything goes wrong, and the first pass to run deep into the suite
    # still had three left at 60m. The loaded graceful-reboot case adds an
    # instance that holds its slot through a pgbench load as well as a create.
    rds) echo "105m" ;;
    # diskperf runs two 16 GiB fio profiles per repetition, each on a volume it
    # creates first. One pass is ~20 minutes on the bare-metal cell and a
    # baseline capture at SPINIFEX_DISKPERF_REPS=3 is three times that.
    diskperf) echo "180m" ;;
    # storagefault is mostly deliberate waiting: one test alone holds predastore
    # SIGSTOPped for seven minutes to prove a guest survives it. A full pass on
    # dev-prod is ~63 minutes, so the 30m default truncated it mid-suite.
    storagefault) echo "90m" ;;
    # instancerecovery is almost entirely waiting, and none of it is avoidable:
    # a dead owner has to be seen stale twice, a guest has to boot, and the
    # storage case waits for a pause measured at 4m47s and then holds four
    # minutes to prove nothing moved. Three tests, each taking a node away and
    # bringing it back, and the partition case adds a lease validity, a lease
    # TTL and a boot on top of everything the other two wait for.
    instancerecovery) echo "90m" ;;
    # lbrecovery pays for a load balancer twice: once to stand one up with two
    # backends placed around it, and once for the recovery itself — a stale
    # owner, a lease, a boot, HAProxy's own start, and its health checks against
    # the backends before it forwards anything. Then the node comes back and has
    # to be shown not to answer for an address it no longer holds.
    lbrecovery) echo "75m" ;;
    # kvquorum audits every bucket in seconds and then spends the rest of its
    # budget on a node stopping and starting: seven units down, a raft election,
    # and the whole service stack back up with every stream caught up.
    kvquorum) echo "40m" ;;
    *) echo "30m" ;;
  esac
}

# One matrix leg per suite, carrying the job budget that suite needs. The outer
# timeout must always be later than the `go test` one inside it: go test reports
# where it stopped, a cancelled job reports only "The operation was canceled".
#   e2e_suite_matrix <suite-list> <floor-minutes> [allowance-minutes]
e2e_suite_matrix() {
  local list="$1" floor="$2" allowance="${3:-15}" suite budget
  for suite in ${list}; do
    budget="$(e2e_suite_timeout "${suite}")"
    budget=$((${budget%m} + allowance))
    if [ "${budget}" -lt "${floor}" ]; then budget="${floor}"; fi
    printf '%s\t%s\n' "${suite}" "${budget}"
  done | jq -cRs 'split("\n")|map(select(length>0)|split("\t")|{suite:.[0],timeout:(.[1]|tonumber)})'
}

# Fail loudly when a narrowed list names a suite its topology cannot run. This
# is the guard that stops the lists drifting again: a suite added to a nightly
# subset without also being declared eligible above stops the job here rather
# than silently skipping half its assertions in CI.
#   e2e_suites_assert_subset <label> <subset> <eligible-set>
e2e_suites_assert_subset() {
  local label="$1" subset="$2" eligible="$3" suite found eligible_suite
  for suite in ${subset}; do
    found=0
    for eligible_suite in ${eligible}; do
      if [ "${suite}" = "${eligible_suite}" ]; then found=1; break; fi
    done
    if [ "${found}" -ne 1 ]; then
      echo "::error::${label} names suite '${suite}', which is not eligible for that topology (eligible: ${eligible}). Fix the list in .github/scripts/suite-sets.sh." >&2
      return 1
    fi
  done
  return 0
}