mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-15 09:01:31 +00:00
18aa67b0e9
* feat: add the e2e guest disk-performance gate Nothing measured guest disk performance in CI, which is why a quadratic index scan shipped in viperblock 35dd9f2 and survived three subsequent fixes to adjacent code. It is also why the two availability clauses at the end of the ebs-performance plan's Stage 0 have never been evaluated: a sustained random-write workload took the guest, and then the host, down, and nothing since has demonstrated that it no longer does. tests/e2e/diskperf/ launches an instance, attaches a dedicated blank volume per repetition, and drives a pinned fio profile over guest SSH. Two profiles: randwrite-4k at 4 jobs x 4 GiB, the shape the exit criterion names, and randrw-70-30-4k, the mixed workload the headline IOPS came from. The target is a raw device with direct=1 rather than a file on the boot disk, so guest page cache, filesystem allocation and journal traffic are out of the path. Two classes of assertion, kept apart because folding them together lets a run that left the host unreachable pass on a respectable IOPS figure. Availability is hard pass/fail and absolute, so it gates from the first run: the job completes inside its budget, a guest responsiveness probe answers throughout on a second SSH channel, and the host's own sshd answers throughout. The guest probe reads the root volume while the workload runs on a different one, so a pass is evidence of isolation between volumes rather than of the job merely being slow. The host probe is a TCP connect plus an SSH banner read, which needs no credential and still proves the kernel scheduled sshd. Throughput and latency are banded against a committed baseline: fail above 25% regression, warn above 10%, per-metric median across repetitions. baseline.json is committed and embedded, with every figure zeroed, and a zero reference records and warns rather than failing. Only the availability class gates until numbers are captured on the bare-metal cell; a throughput band nobody measured would be a fabricated gate. A fresh volume per repetition is the reset. Extent count grows monotonically, so reusing a volume makes the gate order-dependent and drifts its own numbers upward over time, slowly normalising the regression it exists to catch. Each job is its own guest invocation and fio's JSON is fetched by a separate command, so a wedged job loses only itself and reports as a budget breach rather than a parse failure. e2e-baremetal.yml gains suites, suite-env and cell inputs so one reusable workflow serves both the correctness cell and nightly cell 28, which queues behind cell 20 on the shared concurrency group. Cells 25-27 are the dedicated eks, ecs and rds cells. The per-suite Go test timeout is resolved from e2e_suite_timeout in suite-sets.sh, where diskperf declares its 180-minute budget beside rds's rather than in a second place. The gate's own arithmetic runs in preflight via test-harness: band edges, the unmeasured cases, per-stream metric selection, fio JSON parsing, and the probe accounting including a slow-but-successful answer counting as a breach. A gate whose comparison logic is untested is the failure mode the plan is otherwise full of. The extent-count ceiling named as a fourth availability assertion is not implemented: it needs viperblock.index.entries from S1.5, which does not exist. * feat: let a manual bare-metal run choose its e2e suites A dispatch run only sees inputs declared under workflow_dispatch, which had none, so suites and suite-env resolved empty and fell back to the correctness set. Capturing a diskperf baseline by hand was therefore unreachable without a workflow_call caller. * fix: build the bare-metal suites the run step will execute The build step packs pre-built binaries into the tarball and did not see the suite selection, so a selected suite was absent from the node unless it happened to be one of the three the build script hardcoded. * feat: commit the measured disk-performance baseline Medians of three repetitions on the bare-metal cell. The throughput and latency bands now gate instead of warning, so a 2x regression in either job fails the suite. randwrite-4k's write tail is two orders above the mixed job's. It is recorded as measured rather than zeroed: an ungated metric would miss the regression class the gate exists to catch.
147 lines
4.2 KiB
Go
147 lines
4.2 KiB
Go
//go:build e2e
|
|
|
|
package diskperf
|
|
|
|
import (
|
|
"errors"
|
|
"net"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// oneShot builds a probe that closes stop after its first attempt, so watch
|
|
// returns after exactly one sample without waiting out probeInterval.
|
|
func oneShot(name string, attempt func() error, stop chan struct{}) probe {
|
|
var once sync.Once
|
|
return probe{
|
|
Name: name,
|
|
Bound: time.Second,
|
|
Attempt: func() error {
|
|
defer once.Do(func() { close(stop) })
|
|
return attempt()
|
|
},
|
|
}
|
|
}
|
|
|
|
func TestWatchRecordsAHealthySample(t *testing.T) {
|
|
stop := make(chan struct{})
|
|
reports := watch([]probe{oneShot("healthy", func() error { return nil }, stop)}, stop)
|
|
|
|
if len(reports) != 1 {
|
|
t.Fatalf("watch returned %d reports, want 1", len(reports))
|
|
}
|
|
r := reports[0]
|
|
if !r.ok() {
|
|
t.Errorf("healthy probe not ok: %s (%v)", r.summary(), r.FirstFailure)
|
|
}
|
|
if r.Samples != 1 {
|
|
t.Errorf("samples = %d, want 1", r.Samples)
|
|
}
|
|
}
|
|
|
|
func TestWatchRecordsAFailedSample(t *testing.T) {
|
|
stop := make(chan struct{})
|
|
boom := errors.New("connection refused")
|
|
reports := watch([]probe{oneShot("broken", func() error { return boom }, stop)}, stop)
|
|
|
|
r := reports[0]
|
|
if r.ok() {
|
|
t.Fatalf("failing probe reported ok: %s", r.summary())
|
|
}
|
|
if r.Failures != 1 {
|
|
t.Errorf("failures = %d, want 1", r.Failures)
|
|
}
|
|
if !errors.Is(r.FirstFailure, boom) {
|
|
t.Errorf("first failure does not wrap the cause: %v", r.FirstFailure)
|
|
}
|
|
}
|
|
|
|
// A probe that answers, but too late, is a breach. Without this the gate would
|
|
// pass on a host that took a minute to accept a connection, which is the
|
|
// symptom it exists to catch.
|
|
func TestWatchTreatsASlowAnswerAsABreach(t *testing.T) {
|
|
stop := make(chan struct{})
|
|
p := oneShot("slow", func() error {
|
|
time.Sleep(50 * time.Millisecond)
|
|
return nil
|
|
}, stop)
|
|
p.Bound = time.Millisecond
|
|
|
|
r := watch([]probe{p}, stop)[0]
|
|
if r.ok() {
|
|
t.Fatalf("slow probe reported ok: %s", r.summary())
|
|
}
|
|
if !strings.Contains(r.FirstFailure.Error(), "over the") {
|
|
t.Errorf("breach is not reported as a timing one: %v", r.FirstFailure)
|
|
}
|
|
}
|
|
|
|
// A probe with no samples is not a pass. The distinction matters because a
|
|
// watch that never ran and one that ran cleanly otherwise look identical.
|
|
func TestProbeReportWithNoSamplesIsNotOK(t *testing.T) {
|
|
if (probeReport{Name: "never ran"}).ok() {
|
|
t.Error("a report with zero samples reported ok")
|
|
}
|
|
}
|
|
|
|
func TestReadSSHBannerAcceptsAnSSHServer(t *testing.T) {
|
|
addr := serveOnce(t, func(c net.Conn) { _, _ = c.Write([]byte("SSH-2.0-OpenSSH_9.6\r\n")) })
|
|
if err := readSSHBanner(addr, time.Second); err != nil {
|
|
t.Errorf("readSSHBanner: %v", err)
|
|
}
|
|
}
|
|
|
|
// Something listening on 22 that is not sshd does not prove the host can still
|
|
// schedule sshd, so the banner is checked rather than the connect alone.
|
|
func TestReadSSHBannerRejectsANonSSHPeer(t *testing.T) {
|
|
addr := serveOnce(t, func(c net.Conn) { _, _ = c.Write([]byte("HTTP/1.1 400\r\n")) })
|
|
if err := readSSHBanner(addr, time.Second); err == nil {
|
|
t.Error("readSSHBanner accepted a peer that sent no SSH identification string")
|
|
}
|
|
}
|
|
|
|
// A connection that is accepted and then never spoken on is exactly what a
|
|
// starved host looks like, and it must not read as healthy.
|
|
func TestReadSSHBannerRejectsASilentPeer(t *testing.T) {
|
|
addr := serveOnce(t, func(net.Conn) { time.Sleep(time.Second) })
|
|
if err := readSSHBanner(addr, 100*time.Millisecond); err == nil {
|
|
t.Error("readSSHBanner accepted a peer that never sent a banner")
|
|
}
|
|
}
|
|
|
|
func TestReadSSHBannerRejectsAClosedPort(t *testing.T) {
|
|
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
|
if err != nil {
|
|
t.Fatalf("listen: %v", err)
|
|
}
|
|
addr := ln.Addr().String()
|
|
_ = ln.Close()
|
|
|
|
if err := readSSHBanner(addr, 200*time.Millisecond); err == nil {
|
|
t.Error("readSSHBanner accepted a closed port")
|
|
}
|
|
}
|
|
|
|
// serveOnce starts a listener that hands one connection to handle, and returns
|
|
// its address. The listener is closed when the test ends.
|
|
func serveOnce(t *testing.T, handle func(net.Conn)) string {
|
|
t.Helper()
|
|
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
|
if err != nil {
|
|
t.Fatalf("listen: %v", err)
|
|
}
|
|
t.Cleanup(func() { _ = ln.Close() })
|
|
|
|
go func() {
|
|
conn, aerr := ln.Accept()
|
|
if aerr != nil {
|
|
return
|
|
}
|
|
defer conn.Close()
|
|
handle(conn)
|
|
}()
|
|
return ln.Addr().String()
|
|
}
|