diff --git a/.github/workflows/segscan-oracle.yml b/.github/workflows/segscan-oracle.yml new file mode 100644 index 000000000..e78845b8d --- /dev/null +++ b/.github/workflows/segscan-oracle.yml @@ -0,0 +1,91 @@ +# Segscan storage oracle — push-triggered (main/dev) + manual dispatch only, +# deliberately NOT wired into spinifex.yml's pull_request path. +# +# tests/integration/segscan_oracle_test.go needs the mulga umbrella repo's +# scripts/segscan checked out alongside spinifex (it is a separate Go module, +# `package main`, and cannot be imported — see spinifex/testutil/segscanoracle). +# spinifex.yml's jobs check out only spinifex/viperblock/predastore and run on +# every PR; adding a second-repo checkout there would make every spinifex PR +# depend on mulga main being healthy and pay that checkout cost on every run, +# for one storage-accounting oracle test. Mirrors e2e.yml's push-only trigger +# (mulga-siv-283 precedent): this never gates a PR before merge, matching how +# e2e.yml's own "front gate" job already works. +name: Segscan Storage Oracle + +on: + push: + branches: + - main + - dev + paths: + - "**/*.go" + - go.mod + - go.sum + - Makefile + - .github/workflows/segscan-oracle.yml + workflow_dispatch: + +permissions: + contents: read + +jobs: + segscan_oracle: + name: Segscan Oracle + runs-on: ubuntu-26.04 + + steps: + - name: Checkout Spinifex + uses: actions/checkout@v7 + with: + path: spinifex + fetch-depth: 2 + + - name: Resolve sibling refs + id: siblings + env: + BRANCH: ${{ github.head_ref || github.ref_name }} + run: bash .github/resolve-sibling-refs.sh + working-directory: spinifex + + - name: Checkout Viperblock + uses: actions/checkout@v7 + with: + repository: mulgadc/viperblock + ref: ${{ steps.siblings.outputs.viperblock }} + path: viperblock + + - name: Checkout Predastore + uses: actions/checkout@v7 + with: + repository: mulgadc/predastore + ref: ${{ steps.siblings.outputs.predastore }} + path: predastore + + - uses: actions/create-github-app-token@v3 + id: app-token + with: + client-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + repositories: mulga + + - name: Checkout Mulga (scripts/segscan only) + uses: actions/checkout@v7 + with: + repository: mulgadc/mulga + token: ${{ steps.app-token.outputs.token }} + sparse-checkout: scripts/segscan + path: mulga + + - name: Setup Go + uses: actions/setup-go@v7 + with: + cache-dependency-path: "spinifex/go.sum" + go-version-file: "spinifex/go.mod" + + - name: Setup Go Workspace + run: go work init ./spinifex ./viperblock ./predastore + + - name: Run Segscan Oracle Test + env: + SEGSCAN_DIR: ${{ github.workspace }}/mulga/scripts/segscan + run: make -C spinifex test-segscan-oracle diff --git a/Makefile b/Makefile index f5faef84a..c7a64a191 100644 --- a/Makefile +++ b/Makefile @@ -150,6 +150,15 @@ test-integration: @echo -e "\n....Running in-process integration tests...." $(_Q)LOG_IGNORE=1 go test -tags=integration -timeout 60s ./tests/integration/... $(_RACEQ) +# Segscan storage oracle: needs the mulga umbrella repo's scripts/segscan +# checked out alongside spinifex (see spinifex/testutil/segscanoracle), which +# is not the default local or CI layout, so this is a separate target from +# test-integration rather than folded into it. Skips itself when segscan's +# source isn't found. +test-segscan-oracle: + @echo -e "\n....Running segscan storage oracle test...." + $(_Q)LOG_IGNORE=1 go test -tags=integration,segscanoracle -timeout 120s ./tests/integration/... -run TestSegscanOracle $(_RACEQ) + # Validate docs/service-interfaces.yaml. Schema check + cross-reference # of services/suites/fixtures + on-disk path existence. manifest-check: @@ -333,7 +342,7 @@ distro-arm64: distro-clean: rm -rf dist/ -.PHONY: build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-images test-harness test-integration manifest-check manifest-lint manifest-lint-update \ +.PHONY: build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-images test-harness test-integration test-segscan-oracle manifest-check manifest-lint manifest-lint-update \ deploy reinstall clean \ install-system install-go install-aws quickinstall \ lint fix govulncheck nilaway \ diff --git a/spinifex/testutil/segscanoracle/segscanoracle.go b/spinifex/testutil/segscanoracle/segscanoracle.go new file mode 100644 index 000000000..7d50f5f04 --- /dev/null +++ b/spinifex/testutil/segscanoracle/segscanoracle.go @@ -0,0 +1,271 @@ +// Package segscanoracle is a storage-state oracle that runs the mulga +// umbrella repo's scripts/segscan against a copy of a real predastore node +// data dir, reporting live / dead-tombstoned / dead-orphan bytes per the +// on-disk .seg segments. It is the segscan-side counterpart to +// spinifex/testutil/vbscan (see that package's comment for the chunk/config +// oracle) — together they let integration tests assert real persisted +// storage state instead of only "the handler did not error". +// +// scripts/segscan is `package main` in its own Go module +// (github.com/mulgadc/mulga/scripts/segscan) and cannot be imported, so this +// package builds and execs it rather than calling into it directly. That +// module lives in the mulga umbrella repo, not in spinifex, so it is only +// available where a caller has arranged for it to be checked out alongside +// spinifex (see Locate) — tests using this oracle must tolerate it being +// absent and skip rather than fail. +package segscanoracle + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "os" + "os/exec" + "path/filepath" + "runtime" + "sync" + "testing" + "time" + + "github.com/mulgadc/spinifex/tests/fixtures/scratch" + "github.com/stretchr/testify/require" +) + +// segscanEnvVar lets a caller point directly at scripts/segscan's source +// dir, overriding the sibling-checkout guesses in Locate. CI sets this +// explicitly rather than relying on the guess (see the segscan-oracle +// workflow). +const segscanEnvVar = "SEGSCAN_DIR" + +// buildDirPrefix names the shared build's temp dir so a later run's +// scratch.SweepAbandoned can reclaim one stranded by a killed process, +// mirroring tests/fixtures/predastore's fixtureDirPrefix. +const buildDirPrefix = "segscanoracle-bin-" + +// Locate finds scripts/segscan's module source directory. SEGSCAN_DIR +// overrides when set; otherwise it tries the two layouts spinifex is +// checked out under: a plain monorepo clone (mulga/spinifex with sibling +// mulga/scripts/segscan) and CI's sparse-checkout sibling (workspace/spinifex +// with workspace/mulga/scripts/segscan). Returns an error — never panics or +// fails a test — so callers can decide to skip. +func Locate() (string, error) { + _, thisFile, _, ok := runtime.Caller(0) + if !ok { + return "", errors.New("segscanoracle: could not resolve source location for module-relative lookup") + } + // thisFile is /spinifex/testutil/segscanoracle/segscanoracle.go. + root := filepath.Dir(filepath.Dir(filepath.Dir(filepath.Dir(thisFile)))) + return resolveSegscanDir(os.Getenv(segscanEnvVar), root) +} + +// resolveSegscanDir is Locate's resolution order as a pure function of its +// inputs, so it can be unit-tested against fake roots/env values without +// touching this package's own on-disk location. +func resolveSegscanDir(envOverride, root string) (string, error) { + if envOverride != "" { + if hasGoMod(envOverride) { + return envOverride, nil + } + return "", fmt.Errorf("%s=%s has no go.mod", segscanEnvVar, envOverride) + } + + candidates := []string{ + filepath.Join(root, "..", "scripts", "segscan"), // monorepo: mulga/spinifex + mulga/scripts/segscan + filepath.Join(root, "..", "mulga", "scripts", "segscan"), // CI: workspace/spinifex + workspace/mulga/scripts/segscan + } + for _, c := range candidates { + if hasGoMod(c) { + return c, nil + } + } + return "", fmt.Errorf("scripts/segscan source not found (tried %v); set %s or check out the mulga umbrella repo alongside spinifex", candidates, segscanEnvVar) +} + +func hasGoMod(dir string) bool { + info, err := os.Stat(filepath.Join(dir, "go.mod")) + return err == nil && !info.IsDir() +} + +// Totals mirrors the subset of scripts/segscan's --json "totals" object this +// oracle asserts against. Field names/tags are load-bearing: if segscan ever +// renames or restructures these, json.Unmarshal silently zeroes the +// corresponding field here rather than erroring, and the test consuming this +// package's Report is what turns that into a loud, failing assertion (e.g. an +// unexpectedly-zero LivePhysical after a real write) — see this repo's +// segscan_oracle_test.go for that check. +type Totals struct { + LiveLogical int64 `json:"liveLogical"` + LivePhysical int64 `json:"livePhysical"` + DeadTombstoned int64 `json:"deadTombstonedReclaimable"` + DeadOrphan int64 `json:"deadOrphanUnreclaimable"` +} + +// Report mirrors the store-wide subset of scripts/segscan's --json output +// this oracle needs. Per-segment detail is intentionally omitted: nothing +// under tests/ currently needs it, and declaring it would just be more +// surface for the JSON shape to drift against unused. +type Report struct { + Totals Totals `json:"totals"` +} + +// buildResult caches build's outcome for the life of the test binary. A +// struct field, rather than a package-level `var ... error`, sidesteps +// golangci's errname check (which reads any package-level error variable as +// a mis-named sentinel error) — this is a memoized result, not a sentinel. +type buildResult struct { + bin string + err error +} + +var ( + buildMu sync.Mutex + buildDone bool + buildState buildResult +) + +// build compiles scripts/segscan once per test binary (subsequent calls +// reuse the same binary) and returns its path. Like testpredastore's shared +// daemon, the build output must outlive any individual test, so it lives +// under a swept os.MkdirTemp dir rather than t.TempDir(). +func build(t *testing.T, srcDir string) (string, error) { + t.Helper() + buildMu.Lock() + defer buildMu.Unlock() + if buildDone { + return buildState.bin, buildState.err + } + buildDone = true + + scratch.SweepAbandoned(os.TempDir(), buildDirPrefix, scratch.DefaultMaxAge) + + binDir, err := os.MkdirTemp("", buildDirPrefix+"*") //nolint:usetesting // shared across every test in this binary, outlives any one of them + if err != nil { + buildState.err = fmt.Errorf("create build dir: %w", err) + return "", buildState.err + } + bin := filepath.Join(binDir, "segscan") + + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, "go", "build", "-o", bin, ".") + cmd.Dir = srcDir + // GOWORK=off: scripts/segscan is its own module with its own go.mod/go.sum. + // An ancestor directory of srcDir may have an unrelated go.work (the mulga + // umbrella repo has one, and CI's checkout layout can produce another one + // that doesn't `use` scripts/segscan at all) — force single-module mode so + // this build never depends on which, if any, go.work Go finds by walking + // up from srcDir. + cmd.Env = append(os.Environ(), "GOWORK=off") + var stderr bytes.Buffer + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + buildState.err = fmt.Errorf("go build %s: %w\n%s", srcDir, err, stderr.String()) + return "", buildState.err + } + + buildState.bin = bin + return buildState.bin, nil +} + +// CopyNodeDir copies srcDir — a live predastore node data dir (db/, *.seg, +// *.idx, state.json) — into a fresh t.TempDir() and returns the copy's path. +// segscan opens the Badger index read-write and truncates its value log on +// open (see scripts/segscan's own --help text), so it must never run +// directly against a live store's data dir; this is the only sanctioned way +// to get a dir Run can be pointed at. +func CopyNodeDir(t *testing.T, srcDir string) string { + t.Helper() + dst := t.TempDir() + + err := filepath.WalkDir(srcDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + rel, err := filepath.Rel(srcDir, path) + if err != nil { + return err + } + target := filepath.Join(dst, rel) + if d.IsDir() { + return os.MkdirAll(target, 0755) //nolint:gosec // ephemeral test-only copy + } + return copyFile(path, target) + }) + require.NoError(t, err, "copy node dir %s", srcDir) + return dst +} + +func copyFile(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + + info, err := in.Stat() + if err != nil { + return err + } + out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, info.Mode().Perm()) + if err != nil { + return err + } + defer out.Close() + + _, err = io.Copy(out, in) + return err +} + +// Run builds scripts/segscan (once per test binary) and execs it against +// copiedDataDir with --json, returning the decoded report. copiedDataDir +// MUST already be a copy — see CopyNodeDir — never a live node's data dir. +// +// If scripts/segscan's source isn't available (Locate fails — the normal +// case outside the dedicated segscan-oracle CI job, which is the only place +// the mulga umbrella repo is checked out alongside spinifex), Run skips the +// test rather than failing it. A build or exec failure once the source is +// found is a real problem and fails the test. +func Run(t *testing.T, copiedDataDir string) *Report { + t.Helper() + + srcDir, err := Locate() + if err != nil { + t.Skipf("segscanoracle: %v", err) + } + + bin, err := build(t, srcDir) + if err != nil { + t.Fatalf("segscanoracle: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + var stdout, stderr bytes.Buffer + cmd := exec.CommandContext(ctx, bin, "--dir", copiedDataDir, "--json") + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + t.Fatalf("segscanoracle: %s --dir %s --json: %v\nstderr:\n%s", bin, copiedDataDir, err, stderr.String()) + } + + rep, err := decodeReport(stdout.Bytes()) + if err != nil { + t.Fatalf("segscanoracle: decode segscan JSON: %v\noutput:\n%s", err, stdout.String()) + } + return rep +} + +// decodeReport parses segscan --json's output. Pulled out of Run so the +// decode step — including the malformed-payload error path — is unit +// testable without execing the real binary. +func decodeReport(data []byte) (*Report, error) { + var rep Report + if err := json.Unmarshal(data, &rep); err != nil { + return nil, err + } + return &rep, nil +} diff --git a/spinifex/testutil/segscanoracle/segscanoracle_test.go b/spinifex/testutil/segscanoracle/segscanoracle_test.go new file mode 100644 index 000000000..ea88b25eb --- /dev/null +++ b/spinifex/testutil/segscanoracle/segscanoracle_test.go @@ -0,0 +1,169 @@ +package segscanoracle + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDecodeReport_Valid(t *testing.T) { + data := []byte(`{"totals":{"liveLogical":10,"livePhysical":20,"deadTombstonedReclaimable":30,"deadOrphanUnreclaimable":40}}`) + + rep, err := decodeReport(data) + require.NoError(t, err) + assert.Equal(t, int64(10), rep.Totals.LiveLogical) + assert.Equal(t, int64(20), rep.Totals.LivePhysical) + assert.Equal(t, int64(30), rep.Totals.DeadTombstoned) + assert.Equal(t, int64(40), rep.Totals.DeadOrphan) +} + +func TestDecodeReport_Malformed(t *testing.T) { + _, err := decodeReport([]byte(`{not valid json`)) + require.Error(t, err) +} + +func TestResolveSegscanDir(t *testing.T) { + t.Run("env override with go.mod", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "go.mod"), []byte("module fake\n"), 0644)) + + got, err := resolveSegscanDir(dir, "/does/not/matter") + require.NoError(t, err) + assert.Equal(t, dir, got) + }) + + t.Run("env override without go.mod", func(t *testing.T) { + dir := t.TempDir() + + _, err := resolveSegscanDir(dir, "/does/not/matter") + require.Error(t, err) + }) + + t.Run("monorepo layout candidate", func(t *testing.T) { + base := t.TempDir() + root := filepath.Join(base, "mulga", "spinifex") + segscanDir := filepath.Join(base, "mulga", "scripts", "segscan") + require.NoError(t, os.MkdirAll(segscanDir, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(segscanDir, "go.mod"), []byte("module fake\n"), 0644)) + + got, err := resolveSegscanDir("", root) + require.NoError(t, err) + assert.Equal(t, segscanDir, got) + }) + + t.Run("CI sparse-checkout sibling layout candidate", func(t *testing.T) { + base := t.TempDir() + root := filepath.Join(base, "workspace", "spinifex") + segscanDir := filepath.Join(base, "workspace", "mulga", "scripts", "segscan") + require.NoError(t, os.MkdirAll(segscanDir, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(segscanDir, "go.mod"), []byte("module fake\n"), 0644)) + + got, err := resolveSegscanDir("", root) + require.NoError(t, err) + assert.Equal(t, segscanDir, got) + }) + + t.Run("not found", func(t *testing.T) { + base := t.TempDir() + root := filepath.Join(base, "isolated", "spinifex") + require.NoError(t, os.MkdirAll(root, 0755)) + + _, err := resolveSegscanDir("", root) + require.Error(t, err) + }) +} + +func TestHasGoMod(t *testing.T) { + dir := t.TempDir() + assert.False(t, hasGoMod(dir), "empty dir has no go.mod") + + require.NoError(t, os.WriteFile(filepath.Join(dir, "go.mod"), []byte("module fake\n"), 0644)) + assert.True(t, hasGoMod(dir)) + + // A directory literally named "go.mod" must not count as the file. + dir2 := t.TempDir() + require.NoError(t, os.Mkdir(filepath.Join(dir2, "go.mod"), 0755)) + assert.False(t, hasGoMod(dir2)) +} + +func TestCopyNodeDir(t *testing.T) { + src := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(src, "db"), 0755)) + require.NoError(t, os.WriteFile(filepath.Join(src, "state.json"), []byte(`{"segNum":1}`), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(src, "db", "MANIFEST"), []byte("manifest-bytes"), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(src, "0000000000000001.seg"), []byte("segment-bytes"), 0644)) + + srcBefore := snapshotDir(t, src) + + dst := CopyNodeDir(t, src) + + assert.Equal(t, srcBefore, snapshotDir(t, src), "source dir must be untouched by the copy") + + dstState, err := os.ReadFile(filepath.Join(dst, "state.json")) + require.NoError(t, err) + assert.Equal(t, `{"segNum":1}`, string(dstState)) + + dstManifest, err := os.ReadFile(filepath.Join(dst, "db", "MANIFEST")) + require.NoError(t, err) + assert.Equal(t, "manifest-bytes", string(dstManifest)) + + dstSeg, err := os.ReadFile(filepath.Join(dst, "0000000000000001.seg")) + require.NoError(t, err) + assert.Equal(t, "segment-bytes", string(dstSeg)) +} + +// TestRun_WithFakeSegscan exercises build+exec+decode end to end against a +// throwaway stand-in module rather than the real scripts/segscan (which is +// only available in the mulga umbrella repo, not in this module) — it just +// needs to be a buildable Go program that prints a --json-shaped payload, to +// prove Run's plumbing (compiling SEGSCAN_DIR, execing it with --dir/--json, +// decoding the result) actually works. +func TestRun_WithFakeSegscan(t *testing.T) { + fakeSrc := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(fakeSrc, "go.mod"), []byte("module fakesegscan\n\ngo 1.21\n"), 0644)) + mainGo := `package main + +import "fmt" + +func main() { + fmt.Print(` + "`" + `{"totals":{"liveLogical":1,"livePhysical":2,"deadTombstonedReclaimable":3,"deadOrphanUnreclaimable":4}}` + "`" + `) +} +` + require.NoError(t, os.WriteFile(filepath.Join(fakeSrc, "main.go"), []byte(mainGo), 0644)) + + t.Setenv(segscanEnvVar, fakeSrc) + + rep := Run(t, t.TempDir()) + assert.Equal(t, int64(1), rep.Totals.LiveLogical) + assert.Equal(t, int64(2), rep.Totals.LivePhysical) + assert.Equal(t, int64(3), rep.Totals.DeadTombstoned) + assert.Equal(t, int64(4), rep.Totals.DeadOrphan) +} + +// snapshotDir maps every regular file under dir (relative path) to its +// content, for asserting a directory tree is byte-identical before/after an +// operation that must not have touched it. +func snapshotDir(t *testing.T, dir string) map[string]string { + t.Helper() + snap := make(map[string]string) + err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { + if err != nil || d.IsDir() { + return err + } + rel, err := filepath.Rel(dir, path) + if err != nil { + return err + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + snap[rel] = string(data) + return nil + }) + require.NoError(t, err) + return snap +} diff --git a/tests/fixtures/predastore/predastore.go b/tests/fixtures/predastore/predastore.go index 218f2bdc1..314841b89 100644 --- a/tests/fixtures/predastore/predastore.go +++ b/tests/fixtures/predastore/predastore.go @@ -72,6 +72,12 @@ type Fixture struct { Region string AccessKey string SecretKey string + // DataDir is the daemon's base_path: it contains store/node-N/ for each + // configured node (db/, *.seg, *.idx, state.json), the exact layout a + // segment-inspection tool like segscan expects. The daemon keeps writing + // here for the life of the test binary, so callers that read it directly + // (rather than through the S3 API) must copy it first. + DataDir string } // Package-level singleton: one predastore daemon per test binary (per Go @@ -245,6 +251,7 @@ account_id = "123456789012" Region: Region, AccessKey: AccessKey, SecretKey: SecretKey, + DataDir: testDir, } started = true t.Logf("predastore fixture started, test dir: %s", testDir) diff --git a/tests/integration/segscan_oracle_test.go b/tests/integration/segscan_oracle_test.go new file mode 100644 index 000000000..12ae49b5a --- /dev/null +++ b/tests/integration/segscan_oracle_test.go @@ -0,0 +1,107 @@ +//go:build integration && segscanoracle + +package integration + +import ( + "bytes" + "fmt" + "path/filepath" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/mulgadc/spinifex/spinifex/testutil/segscanoracle" + testpredastore "github.com/mulgadc/spinifex/tests/fixtures/predastore" + "github.com/stretchr/testify/require" +) + +// segscanOracleNode is the fixture node this test scans. The fixture's RS +// config is data=3/parity=2 across all 5 configured nodes (see +// tests/fixtures/predastore.Start's config), so every PutObject places +// exactly one shard — one live extent — on every node, node-1 included. +const segscanOracleNode = "node-1" + +// TestSegscanOracle_PutOverwriteDelete drives real S3 writes against the +// predastore fixture and asserts scripts/segscan's live / dead-tombstoned / +// dead-orphan byte accounting for one node's data dir against expectations +// derived independently of segscan's own decoder. +// +// "Independent" here means: not re-deriving segscan's expected byte totals +// by hand-parsing the same .seg/.idx on-disk format segscan parses (that +// would just be a second copy of the same decoder, prone to drifting from +// the real format in lockstep with segscan itself, proving nothing). Instead +// the expectations come from predastore's own documented Store contract +// (store/store.go's commitExtent and Delete both write a tombstone for any +// extent they supersede, in the same transaction): an overwrite or delete +// must move an extent's bytes from live to dead-tombstoned, byte for byte, +// and must never produce dead-orphan bytes. That is a real invariant of the +// production write path, not an assumption about segscan's own parsing, so +// this test still fails loudly if segscan's decode ever drifts from what +// predastore actually persists. +func TestSegscanOracle_PutOverwriteDelete(t *testing.T) { + fixture := testpredastore.Start(t) + cli := predastoreS3Client(t, fixture) + + bucket := fmt.Sprintf("segscan-oracle-%d", time.Now().UnixNano()) + _, err := cli.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String(bucket)}) + require.NoError(t, err, "create bucket %s", bucket) + t.Cleanup(func() { + _, _ = cli.DeleteBucket(&s3.DeleteBucketInput{Bucket: aws.String(bucket)}) + }) + + nodeDir := filepath.Join(fixture.DataDir, "store", segscanOracleNode) + const key = "segscan-oracle/object" + + scan := func() *segscanoracle.Report { + t.Helper() + // Give an async write a moment to settle before copying: PutObject's + // QUIC round trip to the shard nodes completes before it returns, but + // this leaves a small margin against a torn on-disk read racing the + // daemon's own fsync, rather than relying on that timing exactly. + time.Sleep(200 * time.Millisecond) + copyDir := segscanoracle.CopyNodeDir(t, nodeDir) + return segscanoracle.Run(t, copyDir) + } + + baseline := scan() + + // Put: a fresh object adds exactly one new live extent on this node, and + // must not touch dead-tombstoned or dead-orphan totals. + payloadA := bytes.Repeat([]byte("a"), 100<<10) // 100 KiB + _, err = cli.PutObject(&s3.PutObjectInput{Bucket: aws.String(bucket), Key: aws.String(key), Body: bytes.NewReader(payloadA)}) + require.NoError(t, err, "put %s", key) + + afterPut := scan() + require.Equal(t, baseline.Totals.DeadOrphan, afterPut.Totals.DeadOrphan, "put must not create dead-orphan bytes") + require.Equal(t, baseline.Totals.DeadTombstoned, afterPut.Totals.DeadTombstoned, "put must not create dead-tombstoned bytes (nothing superseded yet)") + require.Greater(t, afterPut.Totals.LivePhysical, baseline.Totals.LivePhysical, "put must grow live physical bytes") + liveDeltaA := afterPut.Totals.LivePhysical - baseline.Totals.LivePhysical + + // Overwrite with a different-sized payload: the old version's extent must + // become dead-tombstoned with exactly its prior live byte count -- not + // dropped, not double-counted, and not reclassified as orphan. + payloadA2 := bytes.Repeat([]byte("b"), 500<<10) // 500 KiB + _, err = cli.PutObject(&s3.PutObjectInput{Bucket: aws.String(bucket), Key: aws.String(key), Body: bytes.NewReader(payloadA2)}) + require.NoError(t, err, "overwrite %s", key) + + afterOverwrite := scan() + require.Equal(t, baseline.Totals.DeadOrphan, afterOverwrite.Totals.DeadOrphan, "overwrite must not create dead-orphan bytes") + require.Equal(t, liveDeltaA, afterOverwrite.Totals.DeadTombstoned-baseline.Totals.DeadTombstoned, + "overwrite must tombstone exactly the superseded version's live physical bytes") + require.Greater(t, afterOverwrite.Totals.LivePhysical, baseline.Totals.LivePhysical, "overwrite must leave the new version live") + liveDeltaA2 := afterOverwrite.Totals.LivePhysical - baseline.Totals.LivePhysical + + // Delete: the current version's extent must also become dead-tombstoned + // (predastore's Delete writes a tombstone too, same as an overwrite's + // supersede path) -- not dead-orphan -- and live totals return to baseline. + _, err = cli.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)}) + require.NoError(t, err, "delete %s", key) + + afterDelete := scan() + require.Equal(t, baseline.Totals.DeadOrphan, afterDelete.Totals.DeadOrphan, "delete must not create dead-orphan bytes") + require.Equal(t, liveDeltaA2, afterDelete.Totals.DeadTombstoned-afterOverwrite.Totals.DeadTombstoned, + "delete must tombstone exactly the live version's physical bytes") + require.Equal(t, baseline.Totals.LivePhysical, afterDelete.Totals.LivePhysical, "live physical bytes must return to baseline once the object is gone") + require.Equal(t, baseline.Totals.LiveLogical, afterDelete.Totals.LiveLogical, "live logical bytes must return to baseline once the object is gone") +}