feat: segscan storage oracle for the real-predastore integration tier (#657)

* feat: add segscan storage oracle for the real-predastore integration tier

Adds spinifex/testutil/segscanoracle, which copies a fixture node data dir
and execs the mulga umbrella repo's scripts/segscan (a separate Go module
that cannot be imported) to assert live/dead-tombstoned/dead-orphan byte
accounting against real predastore writes. Kept off the default PR path: a
new segscanoracle build tag excludes the test from make test-integration,
and a separate segscan-oracle.yml workflow (push to main/dev + manual
dispatch only, mirroring e2e.yml's non-gating trigger) checks out
scripts/segscan rather than adding that checkout cost to spinifex.yml.

* test: cover segscanoracle's pure logic to satisfy diff coverage

segscanoracle has no build tag and always compiles into the default
build, but nothing in that build exercised it, so every new line
counted as uncovered against the 60% diff-coverage gate. Adds unit
tests for JSON decoding (including the malformed-payload path), the
Locate resolution order, and the directory-copy routine, plus a
throwaway stand-in module (not the real scripts/segscan) to exercise
build+exec+decode end to end.

Pulls decodeReport out of Run and resolveSegscanDir out of Locate so
the pure logic is testable without a *testing.T or a real binary.

.github/workflows/segscan-oracle.yml

@@ -0,0 +1,91 @@
+# Segscan storage oracle — push-triggered (main/dev) + manual dispatch only,
+# deliberately NOT wired into spinifex.yml's pull_request path.
+#
+# tests/integration/segscan_oracle_test.go needs the mulga umbrella repo's
+# scripts/segscan checked out alongside spinifex (it is a separate Go module,
+# `package main`, and cannot be imported — see spinifex/testutil/segscanoracle).
+# spinifex.yml's jobs check out only spinifex/viperblock/predastore and run on
+# every PR; adding a second-repo checkout there would make every spinifex PR
+# depend on mulga main being healthy and pay that checkout cost on every run,
+# for one storage-accounting oracle test. Mirrors e2e.yml's push-only trigger
+# (mulga-siv-283 precedent): this never gates a PR before merge, matching how
+# e2e.yml's own "front gate" job already works.
+name: Segscan Storage Oracle
+
+on:
+  push:
+    branches:
+      - main
+      - dev
+    paths:
+      - "**/*.go"
+      - go.mod
+      - go.sum
+      - Makefile
+      - .github/workflows/segscan-oracle.yml
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  segscan_oracle:
+    name: Segscan Oracle
+    runs-on: ubuntu-26.04
+
+    steps:
+      - name: Checkout Spinifex
+        uses: actions/checkout@v7
+        with:
+          path: spinifex
+          fetch-depth: 2
+
+      - name: Resolve sibling refs
+        id: siblings
+        env:
+          BRANCH: ${{ github.head_ref || github.ref_name }}
+        run: bash .github/resolve-sibling-refs.sh
+        working-directory: spinifex
+
+      - name: Checkout Viperblock
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/viperblock
+          ref: ${{ steps.siblings.outputs.viperblock }}
+          path: viperblock
+
+      - name: Checkout Predastore
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/predastore
+          ref: ${{ steps.siblings.outputs.predastore }}
+          path: predastore
+
+      - uses: actions/create-github-app-token@v3
+        id: app-token
+        with:
+          client-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          repositories: mulga
+
+      - name: Checkout Mulga (scripts/segscan only)
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/mulga
+          token: ${{ steps.app-token.outputs.token }}
+          sparse-checkout: scripts/segscan
+          path: mulga
+
+      - name: Setup Go
+        uses: actions/setup-go@v7
+        with:
+          cache-dependency-path: "spinifex/go.sum"
+          go-version-file: "spinifex/go.mod"
+
+      - name: Setup Go Workspace
+        run: go work init ./spinifex ./viperblock ./predastore
+
+      - name: Run Segscan Oracle Test
+        env:
+          SEGSCAN_DIR: ${{ github.workspace }}/mulga/scripts/segscan
+        run: make -C spinifex test-segscan-oracle

Makefile

@@ -150,6 +150,15 @@ test-integration:
 	@echo -e "\n....Running in-process integration tests...."
 	$(_Q)LOG_IGNORE=1 go test -tags=integration -timeout 60s ./tests/integration/... $(_RACEQ)
 
+# Segscan storage oracle: needs the mulga umbrella repo's scripts/segscan
+# checked out alongside spinifex (see spinifex/testutil/segscanoracle), which
+# is not the default local or CI layout, so this is a separate target from
+# test-integration rather than folded into it. Skips itself when segscan's
+# source isn't found.
+test-segscan-oracle:
+	@echo -e "\n....Running segscan storage oracle test...."
+	$(_Q)LOG_IGNORE=1 go test -tags=integration,segscanoracle -timeout 120s ./tests/integration/... -run TestSegscanOracle $(_RACEQ)
+
 # Validate docs/service-interfaces.yaml. Schema check + cross-reference
 # of services/suites/fixtures + on-disk path existence.
 manifest-check:
@@ -333,7 +342,7 @@ distro-arm64:
 distro-clean:
 	rm -rf dist/
 
-.PHONY: build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-images test-harness test-integration manifest-check manifest-lint manifest-lint-update \
+.PHONY: build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-images test-harness test-integration test-segscan-oracle manifest-check manifest-lint manifest-lint-update \
 	deploy reinstall clean \
 	install-system install-go install-aws quickinstall \
 	lint fix govulncheck nilaway \

spinifex/testutil/segscanoracle/segscanoracle.go

@@ -0,0 +1,271 @@
+// Package segscanoracle is a storage-state oracle that runs the mulga
+// umbrella repo's scripts/segscan against a copy of a real predastore node
+// data dir, reporting live / dead-tombstoned / dead-orphan bytes per the
+// on-disk .seg segments. It is the segscan-side counterpart to
+// spinifex/testutil/vbscan (see that package's comment for the chunk/config
+// oracle) — together they let integration tests assert real persisted
+// storage state instead of only "the handler did not error".
+//
+// scripts/segscan is `package main` in its own Go module
+// (github.com/mulgadc/mulga/scripts/segscan) and cannot be imported, so this
+// package builds and execs it rather than calling into it directly. That
+// module lives in the mulga umbrella repo, not in spinifex, so it is only
+// available where a caller has arranged for it to be checked out alongside
+// spinifex (see Locate) — tests using this oracle must tolerate it being
+// absent and skip rather than fail.
+package segscanoracle
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/tests/fixtures/scratch"
+	"github.com/stretchr/testify/require"
+)
+
+// segscanEnvVar lets a caller point directly at scripts/segscan's source
+// dir, overriding the sibling-checkout guesses in Locate. CI sets this
+// explicitly rather than relying on the guess (see the segscan-oracle
+// workflow).
+const segscanEnvVar = "SEGSCAN_DIR"
+
+// buildDirPrefix names the shared build's temp dir so a later run's
+// scratch.SweepAbandoned can reclaim one stranded by a killed process,
+// mirroring tests/fixtures/predastore's fixtureDirPrefix.
+const buildDirPrefix = "segscanoracle-bin-"
+
+// Locate finds scripts/segscan's module source directory. SEGSCAN_DIR
+// overrides when set; otherwise it tries the two layouts spinifex is
+// checked out under: a plain monorepo clone (mulga/spinifex with sibling
+// mulga/scripts/segscan) and CI's sparse-checkout sibling (workspace/spinifex
+// with workspace/mulga/scripts/segscan). Returns an error — never panics or
+// fails a test — so callers can decide to skip.
+func Locate() (string, error) {
+	_, thisFile, _, ok := runtime.Caller(0)
+	if !ok {
+		return "", errors.New("segscanoracle: could not resolve source location for module-relative lookup")
+	}
+	// thisFile is <spinifex-root>/spinifex/testutil/segscanoracle/segscanoracle.go.
+	root := filepath.Dir(filepath.Dir(filepath.Dir(filepath.Dir(thisFile))))
+	return resolveSegscanDir(os.Getenv(segscanEnvVar), root)
+}
+
+// resolveSegscanDir is Locate's resolution order as a pure function of its
+// inputs, so it can be unit-tested against fake roots/env values without
+// touching this package's own on-disk location.
+func resolveSegscanDir(envOverride, root string) (string, error) {
+	if envOverride != "" {
+		if hasGoMod(envOverride) {
+			return envOverride, nil
+		}
+		return "", fmt.Errorf("%s=%s has no go.mod", segscanEnvVar, envOverride)
+	}
+
+	candidates := []string{
+		filepath.Join(root, "..", "scripts", "segscan"),          // monorepo: mulga/spinifex + mulga/scripts/segscan
+		filepath.Join(root, "..", "mulga", "scripts", "segscan"), // CI: workspace/spinifex + workspace/mulga/scripts/segscan
+	}
+	for _, c := range candidates {
+		if hasGoMod(c) {
+			return c, nil
+		}
+	}
+	return "", fmt.Errorf("scripts/segscan source not found (tried %v); set %s or check out the mulga umbrella repo alongside spinifex", candidates, segscanEnvVar)
+}
+
+func hasGoMod(dir string) bool {
+	info, err := os.Stat(filepath.Join(dir, "go.mod"))
+	return err == nil && !info.IsDir()
+}
+
+// Totals mirrors the subset of scripts/segscan's --json "totals" object this
+// oracle asserts against. Field names/tags are load-bearing: if segscan ever
+// renames or restructures these, json.Unmarshal silently zeroes the
+// corresponding field here rather than erroring, and the test consuming this
+// package's Report is what turns that into a loud, failing assertion (e.g. an
+// unexpectedly-zero LivePhysical after a real write) — see this repo's
+// segscan_oracle_test.go for that check.
+type Totals struct {
+	LiveLogical    int64 `json:"liveLogical"`
+	LivePhysical   int64 `json:"livePhysical"`
+	DeadTombstoned int64 `json:"deadTombstonedReclaimable"`
+	DeadOrphan     int64 `json:"deadOrphanUnreclaimable"`
+}
+
+// Report mirrors the store-wide subset of scripts/segscan's --json output
+// this oracle needs. Per-segment detail is intentionally omitted: nothing
+// under tests/ currently needs it, and declaring it would just be more
+// surface for the JSON shape to drift against unused.
+type Report struct {
+	Totals Totals `json:"totals"`
+}
+
+// buildResult caches build's outcome for the life of the test binary. A
+// struct field, rather than a package-level `var ... error`, sidesteps
+// golangci's errname check (which reads any package-level error variable as
+// a mis-named sentinel error) — this is a memoized result, not a sentinel.
+type buildResult struct {
+	bin string
+	err error
+}
+
+var (
+	buildMu    sync.Mutex
+	buildDone  bool
+	buildState buildResult
+)
+
+// build compiles scripts/segscan once per test binary (subsequent calls
+// reuse the same binary) and returns its path. Like testpredastore's shared
+// daemon, the build output must outlive any individual test, so it lives
+// under a swept os.MkdirTemp dir rather than t.TempDir().
+func build(t *testing.T, srcDir string) (string, error) {
+	t.Helper()
+	buildMu.Lock()
+	defer buildMu.Unlock()
+	if buildDone {
+		return buildState.bin, buildState.err
+	}
+	buildDone = true
+
+	scratch.SweepAbandoned(os.TempDir(), buildDirPrefix, scratch.DefaultMaxAge)
+
+	binDir, err := os.MkdirTemp("", buildDirPrefix+"*") //nolint:usetesting // shared across every test in this binary, outlives any one of them
+	if err != nil {
+		buildState.err = fmt.Errorf("create build dir: %w", err)
+		return "", buildState.err
+	}
+	bin := filepath.Join(binDir, "segscan")
+
+	ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
+	defer cancel()
+	cmd := exec.CommandContext(ctx, "go", "build", "-o", bin, ".")
+	cmd.Dir = srcDir
+	// GOWORK=off: scripts/segscan is its own module with its own go.mod/go.sum.
+	// An ancestor directory of srcDir may have an unrelated go.work (the mulga
+	// umbrella repo has one, and CI's checkout layout can produce another one
+	// that doesn't `use` scripts/segscan at all) — force single-module mode so
+	// this build never depends on which, if any, go.work Go finds by walking
+	// up from srcDir.
+	cmd.Env = append(os.Environ(), "GOWORK=off")
+	var stderr bytes.Buffer
+	cmd.Stderr = &stderr
+	if err := cmd.Run(); err != nil {
+		buildState.err = fmt.Errorf("go build %s: %w\n%s", srcDir, err, stderr.String())
+		return "", buildState.err
+	}
+
+	buildState.bin = bin
+	return buildState.bin, nil
+}
+
+// CopyNodeDir copies srcDir — a live predastore node data dir (db/, *.seg,
+// *.idx, state.json) — into a fresh t.TempDir() and returns the copy's path.
+// segscan opens the Badger index read-write and truncates its value log on
+// open (see scripts/segscan's own --help text), so it must never run
+// directly against a live store's data dir; this is the only sanctioned way
+// to get a dir Run can be pointed at.
+func CopyNodeDir(t *testing.T, srcDir string) string {
+	t.Helper()
+	dst := t.TempDir()
+
+	err := filepath.WalkDir(srcDir, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		rel, err := filepath.Rel(srcDir, path)
+		if err != nil {
+			return err
+		}
+		target := filepath.Join(dst, rel)
+		if d.IsDir() {
+			return os.MkdirAll(target, 0755) //nolint:gosec // ephemeral test-only copy
+		}
+		return copyFile(path, target)
+	})
+	require.NoError(t, err, "copy node dir %s", srcDir)
+	return dst
+}
+
+func copyFile(src, dst string) error {
+	in, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+
+	info, err := in.Stat()
+	if err != nil {
+		return err
+	}
+	out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, info.Mode().Perm())
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	_, err = io.Copy(out, in)
+	return err
+}
+
+// Run builds scripts/segscan (once per test binary) and execs it against
+// copiedDataDir with --json, returning the decoded report. copiedDataDir
+// MUST already be a copy — see CopyNodeDir — never a live node's data dir.
+//
+// If scripts/segscan's source isn't available (Locate fails — the normal
+// case outside the dedicated segscan-oracle CI job, which is the only place
+// the mulga umbrella repo is checked out alongside spinifex), Run skips the
+// test rather than failing it. A build or exec failure once the source is
+// found is a real problem and fails the test.
+func Run(t *testing.T, copiedDataDir string) *Report {
+	t.Helper()
+
+	srcDir, err := Locate()
+	if err != nil {
+		t.Skipf("segscanoracle: %v", err)
+	}
+
+	bin, err := build(t, srcDir)
+	if err != nil {
+		t.Fatalf("segscanoracle: %v", err)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	var stdout, stderr bytes.Buffer
+	cmd := exec.CommandContext(ctx, bin, "--dir", copiedDataDir, "--json")
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+	if err := cmd.Run(); err != nil {
+		t.Fatalf("segscanoracle: %s --dir %s --json: %v\nstderr:\n%s", bin, copiedDataDir, err, stderr.String())
+	}
+
+	rep, err := decodeReport(stdout.Bytes())
+	if err != nil {
+		t.Fatalf("segscanoracle: decode segscan JSON: %v\noutput:\n%s", err, stdout.String())
+	}
+	return rep
+}
+
+// decodeReport parses segscan --json's output. Pulled out of Run so the
+// decode step — including the malformed-payload error path — is unit
+// testable without execing the real binary.
+func decodeReport(data []byte) (*Report, error) {
+	var rep Report
+	if err := json.Unmarshal(data, &rep); err != nil {
+		return nil, err
+	}
+	return &rep, nil
+}

spinifex/testutil/segscanoracle/segscanoracle_test.go

@@ -0,0 +1,169 @@
+package segscanoracle
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDecodeReport_Valid(t *testing.T) {
+	data := []byte(`{"totals":{"liveLogical":10,"livePhysical":20,"deadTombstonedReclaimable":30,"deadOrphanUnreclaimable":40}}`)
+
+	rep, err := decodeReport(data)
+	require.NoError(t, err)
+	assert.Equal(t, int64(10), rep.Totals.LiveLogical)
+	assert.Equal(t, int64(20), rep.Totals.LivePhysical)
+	assert.Equal(t, int64(30), rep.Totals.DeadTombstoned)
+	assert.Equal(t, int64(40), rep.Totals.DeadOrphan)
+}
+
+func TestDecodeReport_Malformed(t *testing.T) {
+	_, err := decodeReport([]byte(`{not valid json`))
+	require.Error(t, err)
+}
+
+func TestResolveSegscanDir(t *testing.T) {
+	t.Run("env override with go.mod", func(t *testing.T) {
+		dir := t.TempDir()
+		require.NoError(t, os.WriteFile(filepath.Join(dir, "go.mod"), []byte("module fake\n"), 0644))
+
+		got, err := resolveSegscanDir(dir, "/does/not/matter")
+		require.NoError(t, err)
+		assert.Equal(t, dir, got)
+	})
+
+	t.Run("env override without go.mod", func(t *testing.T) {
+		dir := t.TempDir()
+
+		_, err := resolveSegscanDir(dir, "/does/not/matter")
+		require.Error(t, err)
+	})
+
+	t.Run("monorepo layout candidate", func(t *testing.T) {
+		base := t.TempDir()
+		root := filepath.Join(base, "mulga", "spinifex")
+		segscanDir := filepath.Join(base, "mulga", "scripts", "segscan")
+		require.NoError(t, os.MkdirAll(segscanDir, 0755))
+		require.NoError(t, os.WriteFile(filepath.Join(segscanDir, "go.mod"), []byte("module fake\n"), 0644))
+
+		got, err := resolveSegscanDir("", root)
+		require.NoError(t, err)
+		assert.Equal(t, segscanDir, got)
+	})
+
+	t.Run("CI sparse-checkout sibling layout candidate", func(t *testing.T) {
+		base := t.TempDir()
+		root := filepath.Join(base, "workspace", "spinifex")
+		segscanDir := filepath.Join(base, "workspace", "mulga", "scripts", "segscan")
+		require.NoError(t, os.MkdirAll(segscanDir, 0755))
+		require.NoError(t, os.WriteFile(filepath.Join(segscanDir, "go.mod"), []byte("module fake\n"), 0644))
+
+		got, err := resolveSegscanDir("", root)
+		require.NoError(t, err)
+		assert.Equal(t, segscanDir, got)
+	})
+
+	t.Run("not found", func(t *testing.T) {
+		base := t.TempDir()
+		root := filepath.Join(base, "isolated", "spinifex")
+		require.NoError(t, os.MkdirAll(root, 0755))
+
+		_, err := resolveSegscanDir("", root)
+		require.Error(t, err)
+	})
+}
+
+func TestHasGoMod(t *testing.T) {
+	dir := t.TempDir()
+	assert.False(t, hasGoMod(dir), "empty dir has no go.mod")
+
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "go.mod"), []byte("module fake\n"), 0644))
+	assert.True(t, hasGoMod(dir))
+
+	// A directory literally named "go.mod" must not count as the file.
+	dir2 := t.TempDir()
+	require.NoError(t, os.Mkdir(filepath.Join(dir2, "go.mod"), 0755))
+	assert.False(t, hasGoMod(dir2))
+}
+
+func TestCopyNodeDir(t *testing.T) {
+	src := t.TempDir()
+	require.NoError(t, os.MkdirAll(filepath.Join(src, "db"), 0755))
+	require.NoError(t, os.WriteFile(filepath.Join(src, "state.json"), []byte(`{"segNum":1}`), 0644))
+	require.NoError(t, os.WriteFile(filepath.Join(src, "db", "MANIFEST"), []byte("manifest-bytes"), 0644))
+	require.NoError(t, os.WriteFile(filepath.Join(src, "0000000000000001.seg"), []byte("segment-bytes"), 0644))
+
+	srcBefore := snapshotDir(t, src)
+
+	dst := CopyNodeDir(t, src)
+
+	assert.Equal(t, srcBefore, snapshotDir(t, src), "source dir must be untouched by the copy")
+
+	dstState, err := os.ReadFile(filepath.Join(dst, "state.json"))
+	require.NoError(t, err)
+	assert.Equal(t, `{"segNum":1}`, string(dstState))
+
+	dstManifest, err := os.ReadFile(filepath.Join(dst, "db", "MANIFEST"))
+	require.NoError(t, err)
+	assert.Equal(t, "manifest-bytes", string(dstManifest))
+
+	dstSeg, err := os.ReadFile(filepath.Join(dst, "0000000000000001.seg"))
+	require.NoError(t, err)
+	assert.Equal(t, "segment-bytes", string(dstSeg))
+}
+
+// TestRun_WithFakeSegscan exercises build+exec+decode end to end against a
+// throwaway stand-in module rather than the real scripts/segscan (which is
+// only available in the mulga umbrella repo, not in this module) — it just
+// needs to be a buildable Go program that prints a --json-shaped payload, to
+// prove Run's plumbing (compiling SEGSCAN_DIR, execing it with --dir/--json,
+// decoding the result) actually works.
+func TestRun_WithFakeSegscan(t *testing.T) {
+	fakeSrc := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(fakeSrc, "go.mod"), []byte("module fakesegscan\n\ngo 1.21\n"), 0644))
+	mainGo := `package main
+
+import "fmt"
+
+func main() {
+	fmt.Print(` + "`" + `{"totals":{"liveLogical":1,"livePhysical":2,"deadTombstonedReclaimable":3,"deadOrphanUnreclaimable":4}}` + "`" + `)
+}
+`
+	require.NoError(t, os.WriteFile(filepath.Join(fakeSrc, "main.go"), []byte(mainGo), 0644))
+
+	t.Setenv(segscanEnvVar, fakeSrc)
+
+	rep := Run(t, t.TempDir())
+	assert.Equal(t, int64(1), rep.Totals.LiveLogical)
+	assert.Equal(t, int64(2), rep.Totals.LivePhysical)
+	assert.Equal(t, int64(3), rep.Totals.DeadTombstoned)
+	assert.Equal(t, int64(4), rep.Totals.DeadOrphan)
+}
+
+// snapshotDir maps every regular file under dir (relative path) to its
+// content, for asserting a directory tree is byte-identical before/after an
+// operation that must not have touched it.
+func snapshotDir(t *testing.T, dir string) map[string]string {
+	t.Helper()
+	snap := make(map[string]string)
+	err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error {
+		if err != nil || d.IsDir() {
+			return err
+		}
+		rel, err := filepath.Rel(dir, path)
+		if err != nil {
+			return err
+		}
+		data, err := os.ReadFile(path)
+		if err != nil {
+			return err
+		}
+		snap[rel] = string(data)
+		return nil
+	})
+	require.NoError(t, err)
+	return snap
+}

tests/fixtures/predastore/predastore.go

@@ -72,6 +72,12 @@ type Fixture struct {
 	Region    string
 	AccessKey string
 	SecretKey string
+	// DataDir is the daemon's base_path: it contains store/node-N/ for each
+	// configured node (db/, *.seg, *.idx, state.json), the exact layout a
+	// segment-inspection tool like segscan expects. The daemon keeps writing
+	// here for the life of the test binary, so callers that read it directly
+	// (rather than through the S3 API) must copy it first.
+	DataDir string
 }
 
 // Package-level singleton: one predastore daemon per test binary (per Go
@@ -245,6 +251,7 @@ account_id = "123456789012"
 		Region:    Region,
 		AccessKey: AccessKey,
 		SecretKey: SecretKey,
+		DataDir:   testDir,
 	}
 	started = true
 	t.Logf("predastore fixture started, test dir: %s", testDir)

tests/integration/segscan_oracle_test.go

@@ -0,0 +1,107 @@
+//go:build integration && segscanoracle
+
+package integration
+
+import (
+	"bytes"
+	"fmt"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/mulgadc/spinifex/spinifex/testutil/segscanoracle"
+	testpredastore "github.com/mulgadc/spinifex/tests/fixtures/predastore"
+	"github.com/stretchr/testify/require"
+)
+
+// segscanOracleNode is the fixture node this test scans. The fixture's RS
+// config is data=3/parity=2 across all 5 configured nodes (see
+// tests/fixtures/predastore.Start's config), so every PutObject places
+// exactly one shard — one live extent — on every node, node-1 included.
+const segscanOracleNode = "node-1"
+
+// TestSegscanOracle_PutOverwriteDelete drives real S3 writes against the
+// predastore fixture and asserts scripts/segscan's live / dead-tombstoned /
+// dead-orphan byte accounting for one node's data dir against expectations
+// derived independently of segscan's own decoder.
+//
+// "Independent" here means: not re-deriving segscan's expected byte totals
+// by hand-parsing the same .seg/.idx on-disk format segscan parses (that
+// would just be a second copy of the same decoder, prone to drifting from
+// the real format in lockstep with segscan itself, proving nothing). Instead
+// the expectations come from predastore's own documented Store contract
+// (store/store.go's commitExtent and Delete both write a tombstone for any
+// extent they supersede, in the same transaction): an overwrite or delete
+// must move an extent's bytes from live to dead-tombstoned, byte for byte,
+// and must never produce dead-orphan bytes. That is a real invariant of the
+// production write path, not an assumption about segscan's own parsing, so
+// this test still fails loudly if segscan's decode ever drifts from what
+// predastore actually persists.
+func TestSegscanOracle_PutOverwriteDelete(t *testing.T) {
+	fixture := testpredastore.Start(t)
+	cli := predastoreS3Client(t, fixture)
+
+	bucket := fmt.Sprintf("segscan-oracle-%d", time.Now().UnixNano())
+	_, err := cli.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String(bucket)})
+	require.NoError(t, err, "create bucket %s", bucket)
+	t.Cleanup(func() {
+		_, _ = cli.DeleteBucket(&s3.DeleteBucketInput{Bucket: aws.String(bucket)})
+	})
+
+	nodeDir := filepath.Join(fixture.DataDir, "store", segscanOracleNode)
+	const key = "segscan-oracle/object"
+
+	scan := func() *segscanoracle.Report {
+		t.Helper()
+		// Give an async write a moment to settle before copying: PutObject's
+		// QUIC round trip to the shard nodes completes before it returns, but
+		// this leaves a small margin against a torn on-disk read racing the
+		// daemon's own fsync, rather than relying on that timing exactly.
+		time.Sleep(200 * time.Millisecond)
+		copyDir := segscanoracle.CopyNodeDir(t, nodeDir)
+		return segscanoracle.Run(t, copyDir)
+	}
+
+	baseline := scan()
+
+	// Put: a fresh object adds exactly one new live extent on this node, and
+	// must not touch dead-tombstoned or dead-orphan totals.
+	payloadA := bytes.Repeat([]byte("a"), 100<<10) // 100 KiB
+	_, err = cli.PutObject(&s3.PutObjectInput{Bucket: aws.String(bucket), Key: aws.String(key), Body: bytes.NewReader(payloadA)})
+	require.NoError(t, err, "put %s", key)
+
+	afterPut := scan()
+	require.Equal(t, baseline.Totals.DeadOrphan, afterPut.Totals.DeadOrphan, "put must not create dead-orphan bytes")
+	require.Equal(t, baseline.Totals.DeadTombstoned, afterPut.Totals.DeadTombstoned, "put must not create dead-tombstoned bytes (nothing superseded yet)")
+	require.Greater(t, afterPut.Totals.LivePhysical, baseline.Totals.LivePhysical, "put must grow live physical bytes")
+	liveDeltaA := afterPut.Totals.LivePhysical - baseline.Totals.LivePhysical
+
+	// Overwrite with a different-sized payload: the old version's extent must
+	// become dead-tombstoned with exactly its prior live byte count -- not
+	// dropped, not double-counted, and not reclassified as orphan.
+	payloadA2 := bytes.Repeat([]byte("b"), 500<<10) // 500 KiB
+	_, err = cli.PutObject(&s3.PutObjectInput{Bucket: aws.String(bucket), Key: aws.String(key), Body: bytes.NewReader(payloadA2)})
+	require.NoError(t, err, "overwrite %s", key)
+
+	afterOverwrite := scan()
+	require.Equal(t, baseline.Totals.DeadOrphan, afterOverwrite.Totals.DeadOrphan, "overwrite must not create dead-orphan bytes")
+	require.Equal(t, liveDeltaA, afterOverwrite.Totals.DeadTombstoned-baseline.Totals.DeadTombstoned,
+		"overwrite must tombstone exactly the superseded version's live physical bytes")
+	require.Greater(t, afterOverwrite.Totals.LivePhysical, baseline.Totals.LivePhysical, "overwrite must leave the new version live")
+	liveDeltaA2 := afterOverwrite.Totals.LivePhysical - baseline.Totals.LivePhysical
+
+	// Delete: the current version's extent must also become dead-tombstoned
+	// (predastore's Delete writes a tombstone too, same as an overwrite's
+	// supersede path) -- not dead-orphan -- and live totals return to baseline.
+	_, err = cli.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)})
+	require.NoError(t, err, "delete %s", key)
+
+	afterDelete := scan()
+	require.Equal(t, baseline.Totals.DeadOrphan, afterDelete.Totals.DeadOrphan, "delete must not create dead-orphan bytes")
+	require.Equal(t, liveDeltaA2, afterDelete.Totals.DeadTombstoned-afterOverwrite.Totals.DeadTombstoned,
+		"delete must tombstone exactly the live version's physical bytes")
+	require.Equal(t, baseline.Totals.LivePhysical, afterDelete.Totals.LivePhysical, "live physical bytes must return to baseline once the object is gone")
+	require.Equal(t, baseline.Totals.LiveLogical, afterDelete.Totals.LiveLogical, "live logical bytes must return to baseline once the object is gone")
+}