diff --git a/build/scripts/predastore-start.sh b/build/scripts/predastore-start.sh index 1c95dbc31..e83f5b2f6 100755 --- a/build/scripts/predastore-start.sh +++ b/build/scripts/predastore-start.sh @@ -5,7 +5,6 @@ CONF=/etc/spinifex/spinifex.toml BIND="0.0.0.0:8443" -BACKEND="filesystem" NODE_ID="0" if [ -f "$CONF" ]; then @@ -23,7 +22,6 @@ fi export SPINIFEX_PREDASTORE_HOST="${BIND%%:*}" export SPINIFEX_PREDASTORE_PORT="${BIND##*:}" -export SPINIFEX_PREDASTORE_BACKEND="$BACKEND" export SPINIFEX_PREDASTORE_NODE_ID="$NODE_ID" exec /usr/local/bin/spx service predastore start diff --git a/cmd/spinifex/cmd/service.go b/cmd/spinifex/cmd/service.go index 6cdc7cc3f..e63bb5240 100644 --- a/cmd/spinifex/cmd/service.go +++ b/cmd/spinifex/cmd/service.go @@ -22,7 +22,6 @@ import ( "os" "strings" - "github.com/mulgadc/predastore/s3" "github.com/mulgadc/spinifex/spinifex/config" "github.com/mulgadc/spinifex/spinifex/service" "github.com/mulgadc/spinifex/spinifex/services/awsgw" @@ -116,15 +115,6 @@ var predastoreStartCmd = &cobra.Command{ return } - backendType := viper.GetString("backend") - var backend s3.BackendType - - if backendType == "distributed" { - backend = s3.BackendDistributed - } else { - backend = s3.BackendFilesystem - } - nodeID := viper.GetInt("node-id") pprofEnabled := viper.GetBool("pprof") pprofOutput := viper.GetString("pprof-output") @@ -138,8 +128,7 @@ var predastoreStartCmd = &cobra.Command{ TlsCert: tlsCert, TlsKey: tlsKey, - Backend: backend, - NodeID: nodeID, + NodeID: nodeID, PprofEnabled: pprofEnabled, PprofOutputPath: pprofOutput, diff --git a/go.mod b/go.mod index abba016b2..0b2af0d0a 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/kdomanski/iso9660 v0.4.0 github.com/klauspost/cpuid/v2 v2.3.0 github.com/mdlayher/packet v1.1.2 - github.com/mulgadc/predastore v1.1.6 + github.com/mulgadc/predastore v1.2.0 github.com/mulgadc/viperblock v1.2.1 github.com/nats-io/nats-server/v2 v2.12.7 github.com/nats-io/nats.go v1.51.0 diff --git a/go.sum b/go.sum index ede74a9bf..404f887e1 100644 --- a/go.sum +++ b/go.sum @@ -31,44 +31,30 @@ github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ= github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk= -github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY= -github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI= -github.com/aws/aws-sdk-go-v2/config v1.32.2 h1:4liUsdEpUUPZs5WVapsJLx5NPmQhQdez7nYFcovrytk= -github.com/aws/aws-sdk-go-v2/config v1.32.2/go.mod h1:l0hs06IFz1eCT+jTacU/qZtC33nvcnLADAPL/XyrkZI= -github.com/aws/aws-sdk-go-v2/credentials v1.19.2 h1:qZry8VUyTK4VIo5aEdUcBjPZHL2v4FyQ3QEOaWcFLu4= -github.com/aws/aws-sdk-go-v2/credentials v1.19.2/go.mod h1:YUqm5a1/kBnoK+/NY5WEiMocZihKSo15/tJdmdXnM5g= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.14 h1:WZVR5DbDgxzA0BJeudId89Kmgy6DIU4ORpxwsVHz0qA= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.14/go.mod h1:Dadl9QO0kHgbrH1GRqGiZdYtW5w+IXXaBNCHTIaheM4= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ= -github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 h1:hlSuz394kV0vhv9drL5lhuEFbEOEP1VyQpy15qWh1Pk= -github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM= -github.com/aws/aws-sdk-go-v2/service/signin v1.0.2 h1:MxMBdKTYBjPQChlJhi4qlEueqB1p1KcbTEa7tD5aqPs= -github.com/aws/aws-sdk-go-v2/service/signin v1.0.2/go.mod h1:iS6EPmNeqCsGo+xQmXv0jIMjyYtQfnwg36zl2FwEouk= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.5 h1:ksUT5KtgpZd3SAiFJNJ0AFEJVva3gjBmN7eXUZjzUwQ= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.5/go.mod h1:av+ArJpoYf3pgyrj6tcehSFW+y9/QvAY8kMooR9bZCw= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.10 h1:GtsxyiF3Nd3JahRBJbxLCCdYW9ltGQYrFWg8XdkGDd8= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.10/go.mod h1:/j67Z5XBVDx8nZVp9EuFM9/BS5dvBznbqILGuu73hug= -github.com/aws/aws-sdk-go-v2/service/sts v1.41.2 h1:a5UTtD4mHBU3t0o6aHQZFJTNKVfxFWfPX7J0Lr7G+uY= -github.com/aws/aws-sdk-go-v2/service/sts v1.41.2/go.mod h1:6TxbXoDSgBQ225Qd8Q+MbxUxUh6TtNKwbRt/EPS9xso= -github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= -github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/aws-sdk-go-v2 v1.41.6 h1:1AX0AthnBQzMx1vbmir3Y4WsnJgiydmnJjiLu+LvXOg= +github.com/aws/aws-sdk-go-v2 v1.41.6/go.mod h1:dy0UzBIfwSeot4grGvY1AqFWN5zgziMmWGzysDnHFcQ= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.9 h1:adBsCIIpLbLmYnkQU+nAChU5yhVTvu5PerROm+/Kq2A= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.9/go.mod h1:uOYhgfgThm/ZyAuJGNQ5YgNyOlYfqnGpTHXvk3cpykg= +github.com/aws/aws-sdk-go-v2/credentials v1.19.15 h1:fyvgWTszojq8hEnMi8PPBTvZdTtEVmAVyo+NFLHBhH4= +github.com/aws/aws-sdk-go-v2/credentials v1.19.15/go.mod h1:gJiYyMOjNg8OEdRWOf3CrFQxM2a98qmrtjx1zuiQfB8= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.22 h1:GmLa5Kw1ESqtFpXsx5MmC84QWa/ZrLZvlJGa2y+4kcQ= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.22/go.mod h1:6sW9iWm9DK9YRpRGga/qzrzNLgKpT2cIxb7Vo2eNOp0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.22 h1:dY4kWZiSaXIzxnKlj17nHnBcXXBfac6UlsAx2qL6XrU= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.22/go.mod h1:KIpEUx0JuRZLO7U6cbV204cWAEco2iC3l061IxlwLtI= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.23 h1:FPXsW9+gMuIeKmz7j6ENWcWtBGTe1kH8r9thNt5Uxx4= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.23/go.mod h1:7J8iGMdRKk6lw2C+cMIphgAnT8uTwBwNOsGkyOCm80U= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.8 h1:HtOTYcbVcGABLOVuPYaIihj6IlkqubBwFj10K5fxRek= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.8/go.mod h1:VsK9abqQeGlzPgUr+isNWzPlK2vKe9INMLWnY65f5Xs= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.14 h1:xnvDEnw+pnj5mctWiYuFbigrEzSm35x7k4KS/ZkCANg= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.14/go.mod h1:yS5rNogD8e0Wu9+l3MUwr6eENBzEeGejvINpN5PAYfY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.22 h1:PUmZeJU6Y1Lbvt9WFuJ0ugUK2xn6hIWUBBbKuOWF30s= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.22/go.mod h1:nO6egFBoAaoXze24a2C0NjQCvdpk8OueRoYimvEB9jo= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.22 h1:SE+aQ4DEqG53RRCAIHlCf//B2ycxGH7jFkpnAh/kKPM= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.22/go.mod h1:ES3ynECd7fYeJIL6+oax+uIEljmfps0S70BaQzbMd/o= +github.com/aws/aws-sdk-go-v2/service/s3 v1.100.0 h1:7G26Sae6PMKn4kMcU5JzNfrm1YrKwyOhowXPYR2WiWY= +github.com/aws/aws-sdk-go-v2/service/s3 v1.100.0/go.mod h1:Fw9aqhJicIVee1VytBBjH+l+5ov6/PhbtIK/u3rt/ls= +github.com/aws/smithy-go v1.25.0 h1:Sz/XJ64rwuiKtB6j98nDIPyYrV1nVNJ4YU74gttcl5U= +github.com/aws/smithy-go v1.25.0/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -300,8 +286,8 @@ github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELU github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc= github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= -github.com/mulgadc/predastore v1.1.6 h1:67rF2wa5BXges5cVJlFqYurYXnlkYleF3XJ1RyB+ZXk= -github.com/mulgadc/predastore v1.1.6/go.mod h1:WO6lNV8oxsYZGN2BzHROyX42+raOLflvX7iiirvpteA= +github.com/mulgadc/predastore v1.2.0 h1:E4+ehAm2YEi2n5t4DmqguURMUXwTSDrK7792zVzw7JM= +github.com/mulgadc/predastore v1.2.0/go.mod h1:+NWROaNCtpPE9f36fUxCHwskW8CuHHEIvjsDQrwOB00= github.com/mulgadc/viperblock v1.2.1 h1:G3cSgh/yspZd7Wdg3C9QGqkqzYbEfYpusd6uMddSDmg= github.com/mulgadc/viperblock v1.2.1/go.mod h1:CisHy13Uam0gUg5dxV0rKryJF2yGbgfDv09FYApRaNk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= @@ -562,3 +548,5 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= +pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= diff --git a/spinifex/services/predastore/predastore.go b/spinifex/services/predastore/predastore.go index c3fdcc35e..e25d5f30b 100644 --- a/spinifex/services/predastore/predastore.go +++ b/spinifex/services/predastore/predastore.go @@ -11,9 +11,6 @@ import ( "github.com/mulgadc/predastore/s3" "github.com/mulgadc/spinifex/spinifex/utils" - - // Import backends to trigger their init() registration - _ "github.com/mulgadc/predastore/backend/filesystem" ) var serviceName = "predastore" @@ -28,8 +25,7 @@ type Config struct { TlsCert string TlsKey string - Backend s3.BackendType - NodeID int + NodeID int // Profiling PprofEnabled bool @@ -66,7 +62,6 @@ func (svc *Service) Start() (int, error) { s3.WithTLS(svc.Config.TlsCert, svc.Config.TlsKey), s3.WithBasePath(svc.Config.BasePath), s3.WithDebug(svc.Config.Debug), - s3.WithBackend(svc.Config.Backend), s3.WithNodeID(svc.Config.NodeID), s3.WithPprof(svc.Config.PprofEnabled, svc.Config.PprofOutputPath), ) diff --git a/spinifex/services/predastore/predastore_integration_test.go b/spinifex/services/predastore/predastore_integration_test.go index 7b3081c57..9ef7daa8f 100644 --- a/spinifex/services/predastore/predastore_integration_test.go +++ b/spinifex/services/predastore/predastore_integration_test.go @@ -132,19 +132,6 @@ func startPredastoreServer(t *testing.T) *Config { } sharedTestDir = testDir - // Create bucket directories - testBucketDir := filepath.Join(testDir, "test-bucket") - publicBucketDir := filepath.Join(testDir, "public-bucket") - - err = os.MkdirAll(testBucketDir, 0755) - if err != nil { - t.Fatalf("Failed to create test-bucket dir: %v", err) - } - err = os.MkdirAll(publicBucketDir, 0755) - if err != nil { - t.Fatalf("Failed to create public-bucket dir: %v", err) - } - // Create TLS cert directory certDir := filepath.Join(testDir, "certs") err = os.MkdirAll(certDir, 0755) @@ -160,7 +147,10 @@ func startPredastoreServer(t *testing.T) *Config { t.Fatalf("Failed to generate certificate: %v", err) } - // Create config file + // Create config file. Five storage nodes are declared so predastore's dev-mode + // path launches all QUIC servers locally as goroutines (server.go:629). Buckets + // are created via the S3 API after startup — config buckets are no longer + // surfaced by ListBuckets (predastore commit 0711e7d, account-scoped auth). configPath := filepath.Join(testDir, "predastore_test.toml") configContent := `version = "1.0" region = "us-east-1" @@ -170,29 +160,43 @@ debug = false disable_logging = false base_path = "` + testDir + `/" -[[buckets]] -name = "test-bucket" -region = "us-east-1" -type = "fs" -pathname = "` + testBucketDir + `" -public = false -encryption = "" +[rs] +data = 3 +parity = 2 -[[buckets]] -name = "public-bucket" -region = "us-east-1" -type = "fs" -pathname = "` + publicBucketDir + `" -public = true -encryption = "" +[[nodes]] +id = 1 +host = "127.0.0.1" +port = 19991 +path = "store/node-1/" + +[[nodes]] +id = 2 +host = "127.0.0.1" +port = 19992 +path = "store/node-2/" + +[[nodes]] +id = 3 +host = "127.0.0.1" +port = 19993 +path = "store/node-3/" + +[[nodes]] +id = 4 +host = "127.0.0.1" +port = 19994 +path = "store/node-4/" + +[[nodes]] +id = 5 +host = "127.0.0.1" +port = 19995 +path = "store/node-5/" [[auth]] access_key_id = "AKIAIOSFODNN7EXAMPLE" secret_access_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" -policy = [ - { bucket = "test-bucket", actions = ["s3:ListBucket", "s3:GetObject", "s3:PutObject", "s3:DeleteObject", "s3:ListAllMyBuckets"] }, - { bucket = "public-bucket", actions = ["s3:ListBucket", "s3:GetObject", "s3:PutObject", "s3:DeleteObject"] }, -] ` err = os.WriteFile(configPath, []byte(configContent), 0644) @@ -222,19 +226,19 @@ policy = [ t.Fatal("Failed to start predastore server") } + // Create the test buckets via the S3 API so they're registered in + // distributed globalState (config buckets aren't visible to ListBuckets). + setupClient := createS3Client(validAccessKey, validSecretKey) + for _, bucket := range []string{testBucket, publicBucket} { + if _, err := setupClient.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String(bucket)}); err != nil { + t.Fatalf("Failed to create bucket %s: %v", bucket, err) + } + } + serverStarted = true t.Logf("Predastore server started successfully") t.Logf("Test directory: %s", testDir) - t.Logf("Bucket directories created: %s, %s", testBucketDir, publicBucketDir) - - // Verify directories exist - if _, err := os.Stat(testBucketDir); os.IsNotExist(err) { - t.Fatalf("test-bucket directory does not exist: %s", testBucketDir) - } - if _, err := os.Stat(publicBucketDir); os.IsNotExist(err) { - t.Fatalf("public-bucket directory does not exist: %s", publicBucketDir) - } - t.Logf("Verified bucket directories exist") + t.Logf("Created buckets: %s, %s", testBucket, publicBucket) // NOTE: Cleanup will happen automatically when test process exits // Don't use t.Cleanup() as it runs after each test, not at the end