# shellcheck shell=bash # shellcheck disable=SC2034 # every variable here is consumed by the sourcing workflow # Source from a workflow `run:` block: # . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh" # # Which e2e suites each topology can run. Sourced by every workflow that drives # suites (e2e.yml, e2e-suites.yml, e2e-nightly.yml) so the lists cannot drift # apart — they previously disagreed three ways. # # Eligibility is a property of the suite, not of the workflow: a suite belongs # to a topology when all of its assertions are reachable there. lb is # multi-only for exactly that reason — on a single node every internet-facing # subtest skips for want of a peer, so a single-node lb run reports green # having exercised a fraction of the suite. # Suites runnable against a one-node environment. # # partialblock gates merges because it earned it, not because it is new: on one # node, same binary and workload, it fails against the pre-fix engine (1758 of # 8192 halves reverted a generation) and passes against the fixed one (0 of # 8192). A suite that has never been shown to go red for the reason it claims # does not belong here. # rds is single-only for the opposite reason to lb: v1 has no multi-AZ # behaviour, so a second node adds no assertion. Its DB VMs are capped at four # concurrent by the suite's own semaphore, which is what keeps it inside the # job's 50-minute budget alongside the suites above. # quota belongs to both topologies because the gate it asserts is gateway code # that runs identically on either, and because a quota left switched on would # cap every suite sharing the environment — so it must be proved to restore # itself wherever it runs, not only on one node. # storagefault runs last in both lists and belongs to both topologies, for # different reasons. On one node it is RS(1,0), so freezing the only predastore # is a total outage and the guest-survival assertion is unambiguous. On three it # is RS(2,1) with degraded writes on, so two must be frozen to break the shard # floor — and only there is a cross-node restart possible at all, which is the # case that takes a volume over from a node holding writes the backend never # received. It is last because it SIGSTOPs a cluster-wide # service: anything sharing the environment would see the same outage and report # a failure that says nothing about itself. E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds quota storagefault" # Suites runnable against a multi-node environment. # # instancerecovery is multi-only and needs three: recovery is inert below that, # because there is no survivor to move a guest to that is not the node that # failed. Like storagefault it takes a node away and freezes a cluster-wide # service, so it runs last and owns its environment. # # lbrecovery is multi-only and needs three for two reasons rather than one: a # survivor to move the load balancer to, and a node for its backends that is not # the load balancer's, so taking that node away does not take the backends with # it. It runs beside instancerecovery at the end because it also takes a node # away and owns its environment while it does. # # kvquorum is multi-only and needs three, for the same reason as those two: a # surviving majority. It is what proves the control plane's own state is # replicated rather than sitting on one node, so it runs before the pair that # take a node away — a cluster whose buckets are already wrong should report # that, not report a recovery failing. E2E_SUITES_MULTI="multinode lb cert quota kvquorum lbrecovery instancerecovery storagefault" # grep -xE alternation form, for narrowing a requested set to the eligible one. E2E_SUITES_SINGLE_RE="$(printf '%s' "${E2E_SUITES_SINGLE}" | tr ' ' '|')" E2E_SUITES_MULTI_RE="$(printf '%s' "${E2E_SUITES_MULTI}" | tr ' ' '|')" # What every e2e-nightly permutation cell must prove, deliberately narrower # than the two sets above: those cells have a 35-minute budget and exist to show # that every install / network / host-OS combination boots and serves. The # service suites are not deferred to the PR gate — eks, ecs and rds each run # nightly in their own dedicated cell, because each needs a guest AMI built # first and a service failure should not mask whether a host OS boots. These # are subsets, and e2e_suites_assert_subset below enforces that. # # storagefault has a dedicated cell for two reasons. It SIGSTOPs a cluster-wide # service, so in a permutation cell every suite sharing that environment would # report the outage as its own failure. And a full pass is ~63 minutes, nearly # all of it deliberate waiting, which is too long to add to the PR gate — where # it has never run, since the gate's default suite list does not name it. # # Its cell is three-node: the RS(2,1) shard floor and the cross-node takeover # after a failed seal do not exist on one node, and they are the assertions # worth a nightly. E2E_SUITES_NIGHTLY_SINGLE="single cert iam" # # The multi-node nightly subset carries kvquorum's read-only half through # `multinode`, which sweeps every KV bucket rather than the three the daemon # owns. The node-loss half is too slow for a 35-minute cell and has its own. E2E_SUITES_NIGHTLY_MULTI="multinode cert lb" # How long a suite gets, for the same reason the lists above live here: the # timeout was written out separately in each workflow and drifted, so rds ran # 50m on the PR gate and 30m nightly and panicked there. A suite needing more # than the default declares it once, here. # e2e_suite_timeout e2e_suite_timeout() { case "$1" in # rds serialises its DB VMs behind a 3 GiB semaphore, so its wall clock is # what the queue costs, not what the tests do: single tests logged 11m, 22m # and 26m waiting for budget. 17 tests at that concurrency is over an hour # before anything goes wrong, and the first pass to run deep into the suite # still had three left at 60m. The loaded graceful-reboot case adds an # instance that holds its slot through a pgbench load as well as a create. rds) echo "105m" ;; # diskperf runs two 16 GiB fio profiles per repetition, each on a volume it # creates first. One pass is ~20 minutes on the bare-metal cell and a # baseline capture at SPINIFEX_DISKPERF_REPS=3 is three times that. diskperf) echo "180m" ;; # storagefault is mostly deliberate waiting: one test alone holds predastore # SIGSTOPped for seven minutes to prove a guest survives it. A full pass on # dev-prod is ~63 minutes, so the 30m default truncated it mid-suite. storagefault) echo "90m" ;; # instancerecovery is almost entirely waiting, and none of it is avoidable: # a dead owner has to be seen stale twice, a guest has to boot, and the # storage case waits for a pause measured at 4m47s and then holds four # minutes to prove nothing moved. Three tests, each taking a node away and # bringing it back, and the partition case adds a lease validity, a lease # TTL and a boot on top of everything the other two wait for. instancerecovery) echo "90m" ;; # lbrecovery pays for a load balancer twice: once to stand one up with two # backends placed around it, and once for the recovery itself — a stale # owner, a lease, a boot, HAProxy's own start, and its health checks against # the backends before it forwards anything. Then the node comes back and has # to be shown not to answer for an address it no longer holds. lbrecovery) echo "75m" ;; # kvquorum audits every bucket in seconds and then spends the rest of its # budget on a node stopping and starting: seven units down, a raft election, # and the whole service stack back up with every stream caught up. kvquorum) echo "40m" ;; *) echo "30m" ;; esac } # One matrix leg per suite, carrying the job budget that suite needs. The outer # timeout must always be later than the `go test` one inside it: go test reports # where it stopped, a cancelled job reports only "The operation was canceled". # e2e_suite_matrix [allowance-minutes] e2e_suite_matrix() { local list="$1" floor="$2" allowance="${3:-15}" suite budget for suite in ${list}; do budget="$(e2e_suite_timeout "${suite}")" budget=$((${budget%m} + allowance)) if [ "${budget}" -lt "${floor}" ]; then budget="${floor}"; fi printf '%s\t%s\n' "${suite}" "${budget}" done | jq -cRs 'split("\n")|map(select(length>0)|split("\t")|{suite:.[0],timeout:(.[1]|tonumber)})' } # Fail loudly when a narrowed list names a suite its topology cannot run. This # is the guard that stops the lists drifting again: a suite added to a nightly # subset without also being declared eligible above stops the job here rather # than silently skipping half its assertions in CI. # e2e_suites_assert_subset