spinifex/tests/e2e/storagegrowth/main_test.go

//go:build e2e

// Package storagegrowth drives guest workloads that make predastore's backing
// store grow faster than the data it holds, so an external step can measure the
// gap. Data should settle at ~1.5x its logical footprint — the RS 2+1 erasure
// floor (data=2 parity=1 yields three half-size shards, which is not 3x
// replication) — plus a small AES-GCM fragment-framing overhead. Anything above
// that, and especially anything that grows monotonically, is a leak.
//
// It exists to isolate the *unreferenced-live* leak, which only a real guest
// workload can produce: viperblock mints chunk objects under unique monotonic
// keys, so they are never overwritten. It simply stops referencing an old chunk
// and never issues a Delete. predastore is behaving correctly by its own lights
// — nobody told it the object is garbage — so it counts those bytes LIVE.
//
// That is a different fault from the *overwrite-orphan* leak, where a key is
// rewritten in place and the superseded extent is never tombstoned, so
// compaction can never find it. Keeping the two apart matters for attributing
// blame within a leak, and that attribution is why this package's own
// in-process check stops at a coarse pass/fail gate rather than a full
// breakdown. The two leaks are distinguished precisely by predastore's own
// live-versus-dead split: unreferenced-live bytes inflate live totals,
// overwrite-orphan bytes inflate dead totals. A host-side `du` reports one
// number that mixes them, which cannot separate the target from the confound
// — and this workload drives both at once, because the drain each pass
// forces also rewrites the checkpoint map in place. Splitting that mix
// requires a tool that reads predastore's live-versus-dead accounting
// directly; that tool does not exist in a standalone spinifex checkout, and
// depending on it here would break this suite's CI.
//
// What this package's own tests DO assert on, in-process, is the coarser
// signal a node-wide `du` delta across the workload can still catch: the
// combined live+dead footprint growing well beyond the RS-erasure floor
// (see assertNPassBackendByteDelta in storage_growth_test.go). A per-volume
// S3 listing of the volume's own chunk prefix — what this suite originally
// used, and what the external tool below also reads — turned out to be
// unusable against a live deployment: the internal chunk bucket is owned by
// a system account distinct from the tenant-scoped identity this harness
// authenticates as, and predastore's same-account bucket-ownership check has
// no cross-account bucket-policy escape hatch, so every such listing fails
// closed with AccessDenied regardless of IAM grants. `du` needs no S3
// credential at all, only SSH to the node, which is why it is what this
// package can actually run itself.
//
// The division of labour: this package drives the guest, runs its own
// coarse node-wide gate, and reports what it did (see npassWorkload); an
// external measurement step reads that record for the measured denominator
// and supplies the fine-grained live-versus-dead backend breakdown this
// package cannot produce on its own. Run standalone, the workload still
// asserts every guest-side invariant, and the coarse gate, that it can see
// on its own.
//
// This is its own package rather than living in single/ or multinode/:
// single/ shares one singleton instance across the whole package that a
// multi-pass, multi-MiB workload would wreck for sibling tests, and
// multinode/'s fixture hard-skips outside ModeMultinode — meaningless on
// the single-node topology this fault was observed on. It is also
// deliberately excluded from the default e2e run and from
// docs/service-interfaces.yaml: like gpu/, select-suites never selects it
// automatically, so it is driven only by its own `make e2e-storagegrowth`
// target. Nothing under tests/e2e/harness/ is modified by this suite —
// that path is an infra glob, and editing it forces the full CI matrix on
// every change.
package storagegrowth

import (
	"bufio"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"testing"

	"github.com/mulgadc/spinifex/tests/e2e/harness"
)

var (
	pkgFixOnce    sync.Once
	pkgFix        *Fixture
	pkgFixErr     error
	pkgSkipReason string
)

func TestMain(m *testing.M) {
	code := m.Run()
	if pkgFix != nil {
		if pkgFix.Harness != nil {
			// A leaked resource fails the run: the suite may have passed, but it
			// left state on the node that the next run will trip over.
			if err := pkgFix.Harness.Close(); err != nil {
				fmt.Fprintf(os.Stderr, "e2e teardown: %v\n", err)
				code = 1
			}
		}
		if pkgFix.TmpDir != "" {
			_ = os.RemoveAll(pkgFix.TmpDir)
		}
	}
	os.Exit(code)
}

// Fixture carries per-process state shared across storage-growth tests.
type Fixture struct {
	Env     *harness.Env
	AWS     *harness.AWSClient
	Harness *harness.Fixture // memoized Ensure* fixture; spans the whole process.
	TmpDir  string           // package-scoped scratch dir; survives every Test* in the package.

	// PoolMode mirrors single/'s detectPoolMode. external_mode="pool" is
	// required for this suite to run at all: EIP allocation is what makes
	// guest SSH reachable, and guest SSH is the only channel for measuring
	// the guest-side denominator. A nat-mode env has no EIP and no inbound
	// path, so GuestExec is unusable and the suite skips outright.
	PoolMode bool
}

// ArtifactDir returns the artifact directory for the *currently running*
// test. Derived per-call from the live t (not memoized on the singleton
// Fixture): the process fixture is built lazily during whichever test runs
// first, so a stored dir would freeze to that test's name.
func (f *Fixture) ArtifactDir(t *testing.T) string {
	t.Helper()
	return harness.ArtifactDir(t, f.Env)
}

// requireStorageGrowthFixture returns the package-scoped Fixture singleton,
// building it on first call. Skips the calling test when:
//   - SPINIFEX_E2E is unset
//   - SPINIFEX_MODE is not "single"
//   - external_mode is not "pool" (no EIP → no guest SSH → nothing to measure)
func requireStorageGrowthFixture(t *testing.T) *Fixture {
	t.Helper()
	pkgFixOnce.Do(func() {
		if os.Getenv("SPINIFEX_E2E") == "" {
			return
		}
		env := harness.LoadEnv(t)
		if env.Mode != harness.ModeSingle {
			pkgSkipReason = "storagegrowth suite requires SPINIFEX_MODE=single"
			return
		}
		// Guard against harness.NewAWSClient calling t.Fatal (which exits via
		// runtime.Goexit and corrupts the Once state for subsequent tests) when
		// no CA cert is resolvable. ResolveCACert uses the same candidate
		// paths NewAWSClient falls back to, so a failure here gives a clean
		// skip with an actionable message — but only when NewAWSClient would
		// actually take that path: SPINIFEX_AWS_INSECURE=1 makes it skip CA
		// resolution entirely (see harness/aws.go), which is how a
		// runner-resident scenario reaches a remote cluster that doesn't
		// have the spinifex cert on disk.
		if os.Getenv("SPINIFEX_AWS_INSECURE") != "1" {
			if _, err := harness.ResolveCACert(env); err != nil {
				pkgSkipReason = "no Spinifex CA cert found: " + err.Error() +
					" — provision a local node first (ansible-playbook ansible/playbooks/dev-reset.yml), " +
					"or target a remote cluster with SPINIFEX_AWS_INSECURE=1 (skips CA verification; see harness/aws.go)"
				return
			}
		}
		awsCli := harness.NewAWSClient(t, env)

		h, err := harness.NewProcessFixture(awsCli)
		if err != nil {
			pkgFixErr = err
			return
		}
		tmpDir, err := os.MkdirTemp("", "storagegrowth-pkgfix-*")
		if err != nil {
			pkgFixErr = err
			return
		}
		harness.EnsureDefaultSGOpen(t, awsCli)
		pkgFix = &Fixture{
			Env:      env,
			AWS:      awsCli,
			Harness:  h,
			TmpDir:   tmpDir,
			PoolMode: detectPoolMode(env),
		}
	})
	if pkgFixErr != nil {
		t.Fatalf("storagegrowth fixture init: %v", pkgFixErr)
	}
	if pkgFix == nil {
		if pkgSkipReason != "" {
			t.Skip(pkgSkipReason)
		}
		t.Skip("SPINIFEX_E2E unset")
	}
	if !pkgFix.PoolMode {
		t.Skip("storagegrowth suite requires external_mode=pool — a nat env has no EIP and cannot reach guest SSH, the only channel for the guest-side denominator")
	}
	return pkgFix
}

// detectPoolMode reads external_mode from spinifex.toml. Defaults to false
// (dev_networking). Copied from single/'s helper rather than imported —
// tests/e2e/harness/** is deliberately never touched by this suite (see
// package doc), and this predicate is package-local everywhere it already
// exists.
func detectPoolMode(env *harness.Env) bool {
	cfg := os.ExpandEnv("$HOME/spinifex/config/spinifex.toml")
	if env.ConfigDir != "" {
		cfg = filepath.Join(env.ConfigDir, "spinifex.toml")
	}
	f, err := os.Open(cfg)
	if err != nil {
		return false
	}
	defer f.Close()

	scanner := bufio.NewScanner(f)
	inNetwork := false
	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())
		if strings.HasPrefix(line, "[") {
			inNetwork = line == "[network]"
			continue
		}
		if !inNetwork || !strings.HasPrefix(line, "external_mode") {
			continue
		}
		// external_mode = "pool" — quoted value; only "pool" enables this suite.
		if _, rhs, ok := strings.Cut(line, "="); ok {
			val := strings.Trim(strings.TrimSpace(rhs), "\"'")
			return val == "pool"
		}
	}
	return false
}