Files
spinifex/tests/integration/predastore_test.go
T
Bryn Mailer c2b80a9b93 feat: match new predastore config surface area (#591)
* match new predastore config surface area

* fix(migrate): carry existing installs onto the predastore v5 topology

The cluster topology refactor breaks an install three ways at once. The old
config carries a top-level host key, which the new schema binds to the [[host]]
table array, so TOML rejects the document and the daemon dies before it reads
any topology. Node state moved from distributed/{db,nodes}/node-N to a per-host
data directory keyed by a node id that is now unique across roles. And because
object metadata records which node holds each shard, the shard nodes had to
keep their ids, so the state replicas took new ones and every replica's
persisted raft configuration stopped listing it.

predastore.toml gains a 4 to 5 migration covering all three: it rewrites the
tables, relocates whichever node directories exist on this machine, and
recovers each relocated state replica onto the new member set. The rewrite is
line-oriented rather than a re-render from the template, so buckets, auth,
rate limits and iam settings an operator edited survive.

spinifex.toml gains a 3 to 4 migration renaming the predastore node_id to
host_id, which predastore-start.sh greps to select the host this machine runs.

The predastore migration lives in its own package rather than in migrate,
which every handler that stamps a KV bucket version imports; reaching the
cluster runtime from there would pull raft, badger and the QUIC client into all
of them. admin_upgrade.go blank-imports it, so only the binary that runs the
upgrade links it, and a test pins both registrations.

* test(predastore): cover the service runtime, gateway and shutdown paths

The service wrapper's new seams had no unit tests: buildRuntime, serve and
the errgroup Start runs the cluster and the S3 gateway under. CI's diff
coverage gate measured the PR at 34.6% against a 60% threshold as a result.

Cover every buildRuntime error wrap and its host selection, both ways the
gateway can fail serve, and the full lifecycle: Start brings the nodes and
the gateway up together, Shutdown drains both by cancelling the context
they share. Topologies are written per test as real config, with node ids
from a process-wide counter — the pipe transport's listener registry is
global, and this package's integration tests hold a fixture daemon on the
low ids for the life of the test binary.
2026-08-03 09:59:25 +10:00

131 lines
5.0 KiB
Go

//go:build integration
package integration
import (
"bytes"
"crypto/tls"
"errors"
"fmt"
"net/http"
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
testpredastore "github.com/mulgadc/spinifex/tests/fixtures/predastore"
"github.com/stretchr/testify/require"
)
// TestPredastoreObjectLifecycle ports
// tests/e2e/single/predastore_test.go's runPredastoreObjectLifecycle: it
// verifies predastore's user-visible object contract end to end — an object
// is readable and intact after PutObject, absent after DeleteObject (both
// GET and ListObjectsV2 stop returning it), and the store keeps serving
// fresh writes once the delete has landed — against the real, in-process
// predastore daemon (testpredastore.Start), not a tofu-provisioned cluster.
// Not t.Parallel(): testpredastore.Start calls t.Setenv on whichever test
// first starts the shared fixture daemon in this binary, and t.Setenv panics
// if combined with t.Parallel on that same test.
func TestPredastoreObjectLifecycle(t *testing.T) {
fixture := testpredastore.Start(t)
cli := predastoreS3Client(t, fixture)
bucket := fmt.Sprintf("predastore-lifecycle-%d", time.Now().UnixNano())
_, err := cli.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String(bucket)})
require.NoError(t, err, "create bucket %s", bucket)
t.Cleanup(func() {
_, _ = cli.DeleteBucket(&s3.DeleteBucketInput{Bucket: aws.String(bucket)})
})
const key = "lifecycle/object"
payload := bytes.Repeat([]byte("p"), 256<<10) // 256 KiB
// Write, then confirm the object is readable and byte-identical.
_, err = cli.PutObject(&s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
Body: bytes.NewReader(payload),
})
require.NoError(t, err, "put %s", key)
got, err := predastoreGetBytes(t, cli, bucket, key)
require.NoError(t, err, "get %s after put", key)
require.True(t, bytes.Equal(got, payload), "get %s after put returned %d bytes, want %d", key, len(got), len(payload))
// Delete, then confirm the object is gone from both GET and List.
_, err = cli.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)})
require.NoError(t, err, "delete %s", key)
_, err = cli.GetObject(&s3.GetObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)})
require.True(t, isPredastoreNoSuchKey(err), "get %s after delete: want NoSuchKey, got %v", key, err)
list, err := cli.ListObjectsV2(&s3.ListObjectsV2Input{Bucket: aws.String(bucket), Prefix: aws.String(key)})
require.NoError(t, err, "list after delete")
require.Lenf(t, list.Contents, 0, "list after delete returned %d objects, want 0", len(list.Contents))
// The store must keep serving once the delete has landed: a fresh write
// round-trips, proving the delete did not wedge the bucket.
const key2 = "lifecycle/after-delete"
t.Cleanup(func() {
_, _ = cli.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String(key2)})
})
_, err = cli.PutObject(&s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key2),
Body: bytes.NewReader(payload),
})
require.NoError(t, err, "put %s after delete", key2)
_, err = predastoreGetBytes(t, cli, bucket, key2)
require.NoError(t, err, "get %s after delete", key2)
}
// predastoreS3Client builds an aws-sdk-go v1 S3 client against the shared
// predastore fixture daemon. TLS verification is skipped: the fixture serves
// the S3 API under a self-signed certificate it generates for itself (see
// testpredastore.Start), which no trust store knows about.
func predastoreS3Client(t *testing.T, fixture *testpredastore.Fixture) *s3.S3 {
t.Helper()
httpClient := &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec // self-signed cert generated by this same fixture, localhost-only
},
}
sess, err := session.NewSession(&aws.Config{
Region: aws.String(fixture.Region),
Endpoint: aws.String("https://" + fixture.Host),
Credentials: credentials.NewStaticCredentials(fixture.AccessKey, fixture.SecretKey, ""),
S3ForcePathStyle: aws.Bool(true),
HTTPClient: httpClient,
})
require.NoError(t, err, "predastore s3 session")
return s3.New(sess)
}
// predastoreGetBytes fetches an object and returns its full body.
func predastoreGetBytes(t *testing.T, cli *s3.S3, bucket, key string) ([]byte, error) {
t.Helper()
out, err := cli.GetObject(&s3.GetObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)})
if err != nil {
return nil, err
}
defer func() { _ = out.Body.Close() }()
buf := new(bytes.Buffer)
if _, err := buf.ReadFrom(out.Body); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
// isPredastoreNoSuchKey reports whether err is the S3 NoSuchKey error
// predastore returns for a GET on a deleted or missing key.
func isPredastoreNoSuchKey(err error) bool {
var aerr awserr.Error
if errors.As(err, &aerr) {
return aerr.Code() == s3.ErrCodeNoSuchKey
}
return false
}