diff --git a/.github/workflows/spinifex.yml b/.github/workflows/spinifex.yml index bf772bdf4..26ee7fe86 100644 --- a/.github/workflows/spinifex.yml +++ b/.github/workflows/spinifex.yml @@ -76,6 +76,20 @@ jobs: - name: Setup Go Workspace run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle + # The EBS provider contract suite needs a real implementation on each + # side of the boundary: qemu-img/qemu-nbd/qemu-io back qemunbdd, which + # shares no code with viperblock, and nbdkit plus viperblock's plugin + # back viperblockd, the provider production actually uses. Without them + # the suite runs against MemoryProvider alone. nbdkit-plugin-dev carries + # the headers and nbdkit.pc that cgo needs to build that plugin, and + # libnbd-bin supplies the third-party NBD client (nbdinfo/nbdcopy) that + # judges the published export by something outside this codebase. + - name: Install conformance tools + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends qemu-utils nbdkit nbdkit-plugin-dev libnbd-bin + + - name: Build viperblock nbdkit plugin + run: make -C viperblock go_build_nbd + - name: Build Spinifex run: make -C spinifex build @@ -83,6 +97,9 @@ jobs: run: make -C spinifex manifest-check manifest-lint - name: Run Tests with Coverage + env: + SPINIFEX_REQUIRE_CONFORMANCE_TOOLS: "1" + VIPERBLOCK_NBDKIT_PLUGIN: ${{ github.workspace }}/viperblock/lib/nbdkit-viperblock-plugin.so run: make -C spinifex test-cover - name: Run Action Tests diff --git a/.golangci.yml b/.golangci.yml index 25c3d7fa0..68623143d 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -69,6 +69,18 @@ linters: - pkg: math/rand$ desc: Use math/rand/v2 instead, see https://go.dev/blog/randv2 + # The EC2 control plane reaches block storage through ebsprovider, not + # by importing the engine. viperblock.New starts a chunk uploader, so + # every construction is a writer against a volume viperblockd serves. + # Nothing under handlers/ may import viperblock, in any form, tests + # included. A types-only import counts: one became an engine import. + ec2-control-plane: + files: + - '**/spinifex/handlers/**' + deny: + - pkg: github.com/mulgadc/viperblock + desc: Use the ebsprovider.EBSProvider interface; the control plane must not construct or name viperblock types. + embeddedstructfieldcheck: forbid-mutex: true diff --git a/cmd/spinifex/cmd/admin.go b/cmd/spinifex/cmd/admin.go index 951661402..a40127a3f 100644 --- a/cmd/spinifex/cmd/admin.go +++ b/cmd/spinifex/cmd/admin.go @@ -30,6 +30,8 @@ import ( "github.com/aws/aws-sdk-go/service/iam" "github.com/mulgadc/spinifex/spinifex/admin" "github.com/mulgadc/spinifex/spinifex/config" + "github.com/mulgadc/spinifex/spinifex/ebsmetadata" + "github.com/mulgadc/spinifex/spinifex/ebsprovider" "github.com/mulgadc/spinifex/spinifex/formation" "github.com/mulgadc/spinifex/spinifex/gpu" handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc" @@ -38,9 +40,6 @@ import ( "github.com/mulgadc/spinifex/spinifex/network/host" "github.com/mulgadc/spinifex/spinifex/objectstore" "github.com/mulgadc/spinifex/spinifex/utils" - "github.com/mulgadc/viperblock/viperblock" - "github.com/mulgadc/viperblock/viperblock/backends/s3" - "github.com/mulgadc/viperblock/viperblock/v_utils" "github.com/nats-io/nats.go" "github.com/pterm/pterm" "github.com/spf13/cobra" @@ -198,6 +197,25 @@ takes effect immediately. Prompts for confirmation unless --yes is passed.`, Run: runimagesPromoteCmd, } +var volumesCmd = &cobra.Command{ + Use: "volumes", + Short: "Inspect block storage volumes", + Long: `Inspect the volumes the storage provider holds, independently of the EC2 API.`, +} + +var volumesOrphansCmd = &cobra.Command{ + Use: "orphans", + Short: "Report volumes the provider holds with no control-plane record", + Long: `List volumes the storage provider holds that the control plane has no record +of. Their blocks are consuming space but they have no API handle, so they +cannot be described or deleted through the EC2 API. + +This command only reports. It never deletes: the evidence that an orphan is +still wanted is the record that went missing, so removal is an operator +decision made per volume.`, + Run: runVolumesOrphansCmd, +} + var accountCmd = &cobra.Command{ Use: "account", Short: "Manage Spinifex accounts", @@ -296,6 +314,9 @@ func init() { imagesCmd.AddCommand(imagesRemoveCmd) imagesCmd.AddCommand(imagesPromoteCmd) + adminCmd.AddCommand(volumesCmd) + volumesCmd.AddCommand(volumesOrphansCmd) + adminCmd.AddCommand(accountCmd) accountCmd.AddCommand(accountCreateCmd) accountCmd.AddCommand(accountListCmd) @@ -403,6 +424,10 @@ func init() { const bytesPerGiB = 1024 * 1024 * 1024 +// imageImportTimeout bounds each provider call the import makes. Generous +// because the snapshot at the end runs over the whole imported volume. +const imageImportTimeout = 30 * time.Minute + // amiVolumeSizeGiB returns the smallest whole GiB that still holds sizeBytes. // // Rounding up is load-bearing. The image is copied into a root volume of @@ -430,16 +455,10 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) { var imageStat os.FileInfo var err error - cfgFile, _ := cmd.Flags().GetString("config") forceCmd, _ := cmd.Flags().GetBool("force") skipVerify, _ := cmd.Flags().GetBool("skip-verify") ostmpDir, _ := cmd.Flags().GetString("tmp-dir") - // Use default config path - if cfgFile == "" { - cfgFile = DefaultConfigFile() - } - //configDir, _ := cmd.Flags().GetString("config-dir") baseDir, _ := cmd.Flags().GetString("spinifex-dir") @@ -610,57 +629,46 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) { os.Exit(1) } - // Create the specified manifest file to describe the image/AMI - manifest := viperblock.VolumeConfig{} - - // Calculate the size - - manifest.AMIMetadata.Name = fmt.Sprintf("ami-%s-%s-%s", image.Distro, image.Version, image.Arch) - if amiNameOverride != "" { - manifest.AMIMetadata.Name = amiNameOverride - } + // Describe the image/AMI as the control-plane document it will become. volumeId := utils.GenerateResourceID("ami") - manifest.AMIMetadata.ImageID = volumeId + amiName := fmt.Sprintf("ami-%s-%s-%s", image.Distro, image.Version, image.Arch) + if amiNameOverride != "" { + amiName = amiNameOverride + } - manifest.AMIMetadata.Description = fmt.Sprintf("%s cloud image prepared for Spinifex", manifest.AMIMetadata.Name) - manifest.AMIMetadata.Architecture = image.Arch - manifest.AMIMetadata.PlatformDetails = image.Platform - manifest.AMIMetadata.CreationDate = time.Now() - manifest.AMIMetadata.RootDeviceType = "ebs" - manifest.AMIMetadata.Virtualization = "hvm" - manifest.AMIMetadata.ImageOwnerAlias = "system" - manifest.AMIMetadata.VolumeSizeGiB = amiVolumeSizeGiB(imageStat.Size()) - manifest.AMIMetadata.BootMode = image.BootMode - manifest.AMIMetadata.Distro = image.Distro - manifest.AMIMetadata.DistroFamily = utils.DistroFamily(image.Distro) + ami := ebsmetadata.AMI{ + ImageID: volumeId, + Name: amiName, + Description: fmt.Sprintf("%s cloud image prepared for Spinifex", amiName), + Architecture: image.Arch, + PlatformDetails: image.Platform, + CreationDate: time.Now(), + RootDeviceType: "ebs", + Virtualization: "hvm", + ImageOwnerAlias: "system", + VolumeSizeGiB: amiVolumeSizeGiB(imageStat.Size()), + SnapshotID: admin.SnapPrefix(volumeId), + BootMode: image.BootMode, + Distro: image.Distro, + DistroFamily: utils.DistroFamily(image.Distro), + } // Copy catalog-provided tags (e.g. spinifex:managed-by for system AMIs) // onto the imported AMI so the UI can filter them out. if len(image.Tags) > 0 { - manifest.AMIMetadata.Tags = make(map[string]string, len(image.Tags)) - maps.Copy(manifest.AMIMetadata.Tags, image.Tags) + ami.Tags = make(map[string]string, len(image.Tags)) + maps.Copy(ami.Tags, image.Tags) } - // Volume Data - manifest.VolumeMetadata.VolumeID = volumeId // TODO: Confirm if unique, e.g vol-, if ami- used - manifest.VolumeMetadata.VolumeName = manifest.AMIMetadata.Name - manifest.VolumeMetadata.TenantID = "system" - manifest.VolumeMetadata.SizeGiB = manifest.AMIMetadata.VolumeSizeGiB - manifest.VolumeMetadata.State = "available" - manifest.VolumeMetadata.AvailabilityZone = "" // TODO: Confirm - manifest.VolumeMetadata.CreatedAt = time.Now() - manifest.VolumeMetadata.VolumeType = "gp3" - manifest.VolumeMetadata.IOPS = 1000 - // Write the manifest to disk // Save as JSON - jsonData, err := json.Marshal(manifest) + jsonData, err := json.Marshal(ami) if err != nil { fmt.Fprintf(os.Stderr, "Could not marshal manifest: %v\n", err) os.Exit(1) } - manifestFilename := fmt.Sprintf("%s/%s.json", imagePath, manifest.AMIMetadata.Name) + manifestFilename := fmt.Sprintf("%s/%s.json", imagePath, ami.Name) // Write to file err = os.WriteFile(manifestFilename, jsonData, 0600) if err != nil { @@ -668,51 +676,15 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) { os.Exit(1) } - // Upload the image to S3 (predastore) - - appConfig, err := config.LoadConfig(cfgFile) + defer os.RemoveAll(tmpDir) + appConfig, nc, err := loadConfigAndConnect() if err != nil { - fmt.Println("Error loading config file:", err) - return - } - - s3Config := s3.S3Config{ - VolumeName: volumeId, - VolumeSize: utils.SafeInt64ToUint64(imageStat.Size()), - Bucket: appConfig.Nodes[appConfig.Node].Predastore.Bucket, - Region: appConfig.Nodes[appConfig.Node].Predastore.Region, - AccessKey: appConfig.Nodes[appConfig.Node].Predastore.AccessKey, - SecretKey: appConfig.Nodes[appConfig.Node].Predastore.SecretKey, - Host: appConfig.Nodes[appConfig.Node].Predastore.Host, - } - - mkey, err := utils.LoadViperblockMasterKey(appConfig.Nodes[appConfig.Node].Viperblock.EncryptionKeyFile) - if err != nil { - fmt.Fprintf(os.Stderr, "Could not load viperblock encryption key: %v\n", err) + fmt.Fprintf(os.Stderr, "Could not connect to the cluster: %v\n", err) os.Exit(1) } - - // Import is an interactive command whose output is a progress bar, so the - // volume's routine mount chatter is noise here: a fresh import has no prior - // state, making the "no state found" / 404 lines expected rather than - // notable. Scoping the logger to this VB (New copies it onto the backend - // too) keeps it off the process-wide default. Errors still surface, both - // through this logger and as the returned error the caller prints. - vbConfig := viperblock.VB{ - VolumeName: volumeId, - VolumeSize: utils.SafeInt64ToUint64(imageStat.Size()), - BaseDir: tmpDir, - Cache: viperblock.Cache{ - Config: viperblock.CacheConfig{ - Size: 0, - }, - }, - VolumeConfig: manifest, - MasterKey: mkey, - EncryptionEnabled: mkey != nil, - Logger: slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError})), - } + defer nc.Close() + node := appConfig.Nodes[appConfig.Node] // Bake the deployment CA into the image trust store so a stock cloud image // trusts the gateway from first boot. Best-effort: never fails the import. @@ -721,32 +693,31 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) { // command, which would collapse the path to a relative config/ca.pem. bakeCACertIntoImage(extractedImagePath, filepath.Join(baseDir, "config", "ca.pem")) - // Render the flush bar here rather than inside viperblock, which stays a - // pure storage library. The bar is built lazily on the first update so it is - // never drawn if the import fails before any bytes flush; viperblock - // throttles the callback to ≤101 invocations, so the human-readable title - // renders without the per-block render-frequency regression. - var flushBar *pterm.ProgressbarPrinter - var flushUpdate func(current uint64) - progress := func(current, total uint64) { - if flushBar == nil { - flushBar, flushUpdate = utils.NewByteProgressBar("Flushing image to storage", total) - } - flushUpdate(current) - } - - err = v_utils.ImportDiskImage(&s3Config, &vbConfig, extractedImagePath, progress) - - if flushBar != nil { - _, _ = flushBar.Stop() - } - + fmt.Println("Writing image to storage ...") + provider := ebsprovider.NewNATSProvider(nc, imageImportTimeout) + err = admin.ImportImage(context.Background(), provider, admin.ImportOpts{ + VolumeID: volumeId, + NodeID: appConfig.Node, + SizeBytes: utils.SafeUint64ToInt64(ami.VolumeSizeGiB * bytesPerGiB), + AvailabilityZone: node.AZ, + SourcePath: extractedImagePath, + Snapshot: true, + Progress: os.Stdout, + }) if err != nil { - fmt.Fprintf(os.Stderr, "Could not import image to predastore: %v\n", err) + fmt.Fprintf(os.Stderr, "Could not import image: %v\n", err) os.Exit(1) } - defer os.RemoveAll(tmpDir) + // The document is what DescribeImages enumerates, so it is written last: + // until it exists the AMI is not launchable, and an import that failed + // partway leaves no half-registered image behind. + ami.State = "available" + metaStore := objectstore.NewS3ObjectStoreFromConfig(node.Predastore.Host, node.Predastore.Region, node.Predastore.AccessKey, node.Predastore.SecretKey) + if err := ebsmetadata.NewStore(metaStore, node.Predastore.Bucket).PutAMI(context.Background(), ami); err != nil { + fmt.Fprintf(os.Stderr, "Imported %s but could not register it: %v\n", volumeId, err) + os.Exit(1) + } fmt.Printf("✅ Image import complete. Image-ID (AMI): %s\n", volumeId) } @@ -809,6 +780,49 @@ func bakeCACertIntoImage(imagePath, caCertPath string) { } } +func runVolumesOrphansCmd(_ *cobra.Command, _ []string) { + appConfig, nc, err := loadConfigAndConnect() + if err != nil { + fmt.Fprintf(os.Stderr, "Could not connect to the cluster: %v\n", err) + os.Exit(1) + } + defer nc.Close() + + node := appConfig.Nodes[appConfig.Node] + store := objectstore.NewS3ObjectStoreFromConfig( + node.Predastore.Host, + node.Predastore.Region, + node.Predastore.AccessKey, + node.Predastore.SecretKey, + ) + + orphans, err := admin.FindOrphanVolumes( + context.Background(), + ebsprovider.NewNATSProvider(nc, imageImportTimeout), + ebsmetadata.NewStore(store, node.Predastore.Bucket), + ) + if err != nil { + fmt.Fprintln(os.Stderr, "Orphan scan failed:", err) + os.Exit(1) + } + if len(orphans) == 0 { + fmt.Println("No orphaned volumes: every volume the provider holds has a control-plane record.") + return + } + + fmt.Printf("%d orphaned volume(s) — held by the provider, unknown to the control plane:\n\n", len(orphans)) + for _, orphan := range orphans { + fmt.Printf(" %s\n", orphan.VolumeID) + fmt.Printf(" handle: %s\n", orphan.Handle) + if orphan.Derived { + fmt.Println(" note: derived volume; its base volume is also unknown") + } + } + fmt.Println() + fmt.Println("Nothing was deleted. Each of these holds data that cannot be reached") + fmt.Println("through the EC2 API; removing one is a per-volume operator decision.") +} + func runimagesRemoveCmd(cmd *cobra.Command, args []string) { imageID, _ := cmd.Flags().GetString("image-id") force, _ := cmd.Flags().GetBool("force") diff --git a/cmd/spinifex/cmd/admin_ochre.go b/cmd/spinifex/cmd/admin_ochre.go index 7325333ef..00512a812 100644 --- a/cmd/spinifex/cmd/admin_ochre.go +++ b/cmd/spinifex/cmd/admin_ochre.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "io" - "log/slog" "os" "os/exec" "path" @@ -15,16 +14,13 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" - "github.com/mulgadc/bluebottle/pkg/masterkey" "github.com/mulgadc/spinifex/spinifex/admin" "github.com/mulgadc/spinifex/spinifex/config" + "github.com/mulgadc/spinifex/spinifex/ebsprovider" gateway_bedrock "github.com/mulgadc/spinifex/spinifex/gateway/bedrock" handlers_ec2_snapshot "github.com/mulgadc/spinifex/spinifex/handlers/ec2/snapshot" "github.com/mulgadc/spinifex/spinifex/objectstore" "github.com/mulgadc/spinifex/spinifex/utils" - "github.com/mulgadc/viperblock/viperblock" - vbs3 "github.com/mulgadc/viperblock/viperblock/backends/s3" - "github.com/mulgadc/viperblock/viperblock/v_utils" "github.com/nats-io/nats.go/jetstream" "github.com/pterm/pterm" "github.com/spf13/cobra" @@ -307,63 +303,6 @@ func buildWeightsImage(srcDir, imagePath string, contentBytes int64) error { return mkfsExt4Runner(srcDir, imagePath) } -// snapshotImportedWeightsVolume snapshots a viperblock volume that -// v_utils.ImportDiskImage just wrote and closed -- never attached, never -// nbdkit-served, so there is no live checkpoint to load. This mirrors the -// offline snapshot sequence handlers/ec2/image/service_impl.go's -// snapshotStoppedVolume uses for a stopped instance's root volume: reopen -// read-only, load the numbered checkpoint Close() wrote, then CreateSnapshot. -// -// az names the zone recorded on the snapshot for DescribeSnapshots; it is not -// used to resolve the clone. -func snapshotImportedWeightsVolume(s3Config *vbs3.S3Config, volumeID string, volumeSize uint64, walDir, az string, mkey *masterkey.Key) (string, error) { - vbConfig := viperblock.VB{ - VolumeName: volumeID, - VolumeSize: volumeSize, - BaseDir: walDir, - Cache: viperblock.Cache{Config: viperblock.CacheConfig{Size: 0}}, - MasterKey: mkey, - EncryptionEnabled: mkey != nil, - Logger: slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError})), - } - - vb, err := viperblock.New(&vbConfig, "s3", *s3Config) - if err != nil { - return "", fmt.Errorf("new viperblock: %w", err) - } - defer func() { - vb.StopChunkUploader() - vb.StopWALSyncer() - }() - - if err := vb.Backend.Init(); err != nil { - return "", fmt.Errorf("backend init: %w", err) - } - if err := vb.LoadState(); err != nil { - return "", fmt.Errorf("load state: %w", err) - } - if err := vb.LoadBlockState(); err != nil { - return "", fmt.Errorf("load block state: %w", err) - } - defer func() { - if err := vb.RemoveLocalFiles(); err != nil { - slog.Warn("snapshotImportedWeightsVolume: failed to remove local files", "volumeId", volumeID, "err", err) - } - }() - - snapshotID := admin.SnapPrefix(volumeID) - if _, err := vb.CreateSnapshot(snapshotID); err != nil { - return "", fmt.Errorf("create snapshot: %w", err) - } - - store := objectstore.NewS3ObjectStoreFromConfig(s3Config.Host, s3Config.Region, s3Config.AccessKey, s3Config.SecretKey) - if err := registerWeightsSnapshot(store, s3Config.Bucket, snapshotID, volumeID, volumeSize, az, mkey != nil); err != nil { - return "", err - } - - return snapshotID, nil -} - // registerWeightsSnapshot writes the EC2 control plane's half of the snapshot // prefix. CreateSnapshot above writes only viperblock's half -- the block // checkpoint and config.json -- but CreateVolume resolves a SnapshotId through @@ -485,12 +424,11 @@ func runStageWeights(ctx context.Context, store objectstore.ObjectStore, weights } // materializeWeightsVolume is the real weightsMaterializer: it packages -// downloadDir into a filesystem image, imports it into a new viperblock -// volume backed by node's predastore, and snapshots it. This is the -// expensive, environment-dependent half of stage -- it shells out to -// mkfs.ext4 and talks to a live S3-shaped backend -- so it is exercised live -// (see the plan's Testing section) rather than in unit tests. -func materializeWeightsVolume(node config.Config, downloadDir string, contentBytes int64, mkey *masterkey.Key) (string, error) { +// downloadDir into a filesystem image, imports it into a new volume through +// the EBS provider, and snapshots it. This is the expensive, +// environment-dependent half of stage -- it shells out to mkfs.ext4 and needs +// a live provider -- so it is exercised live rather than in unit tests. +func materializeWeightsVolume(ctx context.Context, provider ebsprovider.EBSProvider, nodeID string, node config.Config, downloadDir string, contentBytes int64) (string, error) { tmpDir := filepath.Dir(downloadDir) volumeId := utils.GenerateResourceID("vol") imagePath := filepath.Join(tmpDir, volumeId+".img") @@ -505,74 +443,32 @@ func materializeWeightsVolume(node config.Config, downloadDir string, contentByt return "", fmt.Errorf("could not stat image: %w", err) } - // Round the volume up to a whole GiB rather than using the raw image size. - // viperblock rejects a tail write whose block overruns VolumeSize, so an - // unaligned size fails at the last block. This also keeps the byte size and - // the manifest's SizeGiB describing the same volume. + // Round the volume up to a whole GiB rather than using the raw image size, + // so the byte size and the recorded SizeGiB describe the same volume. sizeGiB := amiVolumeSizeGiB(imageStat.Size()) volumeBytes := sizeGiB * bytesPerGiB - s3Config := vbs3.S3Config{ - VolumeName: volumeId, - VolumeSize: volumeBytes, - Bucket: node.Predastore.Bucket, - Region: node.Predastore.Region, - AccessKey: node.Predastore.AccessKey, - SecretKey: node.Predastore.SecretKey, - Host: node.Predastore.Host, - } - - // AMIMetadata is deliberately left zero-valued: a weights volume is not - // bootable and must never be registered as a launchable AMI. Leaving it - // unset also means ImportDiskImage does not attempt its own automatic - // snapshot -- that happens explicitly below, after the volume is closed. - manifest := viperblock.VolumeConfig{} - manifest.VolumeMetadata.VolumeID = volumeId - manifest.VolumeMetadata.VolumeName = volumeId - manifest.VolumeMetadata.TenantID = "system" - manifest.VolumeMetadata.SizeGiB = sizeGiB - manifest.VolumeMetadata.State = "available" - manifest.VolumeMetadata.AvailabilityZone = node.Predastore.Region - manifest.VolumeMetadata.CreatedAt = time.Now() - manifest.VolumeMetadata.VolumeType = "gp3" - manifest.VolumeMetadata.IOPS = 1000 - - vbConfig := viperblock.VB{ - VolumeName: volumeId, - VolumeSize: volumeBytes, - BaseDir: tmpDir, - Cache: viperblock.Cache{Config: viperblock.CacheConfig{Size: 0}}, - VolumeConfig: manifest, - MasterKey: mkey, - EncryptionEnabled: mkey != nil, - Logger: slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError})), - } - - var flushBar *pterm.ProgressbarPrinter - var flushUpdate func(current uint64) - progress := func(current, total uint64) { - if flushBar == nil { - flushBar, flushUpdate = utils.NewByteProgressBar("Flushing weights to storage", total) - } - flushUpdate(current) - } - // Name the volume in the failure path: a part-written import leaves blocks // in predastore, and without the ID an operator cannot find them to reclaim. - if err := v_utils.ImportDiskImage(&s3Config, &vbConfig, imagePath, progress); err != nil { - if flushBar != nil { - _, _ = flushBar.Stop() - } + fmt.Println("Writing weights to storage ...") + if err := admin.ImportImage(ctx, provider, admin.ImportOpts{ + VolumeID: volumeId, + NodeID: nodeID, + SizeBytes: utils.SafeUint64ToInt64(volumeBytes), + AvailabilityZone: node.AZ, + SourcePath: imagePath, + Snapshot: true, + Progress: os.Stdout, + }); err != nil { return "", fmt.Errorf("could not import weights volume %s: %w", volumeId, err) } - if flushBar != nil { - _, _ = flushBar.Stop() - } - fmt.Println("Snapshotting weights volume ...") - snapshotID, err := snapshotImportedWeightsVolume(&s3Config, volumeId, volumeBytes, tmpDir, node.AZ, mkey) - if err != nil { - return "", fmt.Errorf("could not snapshot weights volume: %w", err) + // The provider wrote viperblock's half of the snapshot prefix; this writes + // the EC2 control plane's, without which a launch cannot resolve it. + snapshotID := admin.SnapPrefix(volumeId) + store := objectstore.NewS3ObjectStoreFromConfig(node.Predastore.Host, node.Predastore.Region, node.Predastore.AccessKey, node.Predastore.SecretKey) + if err := registerWeightsSnapshot(store, node.Predastore.Bucket, snapshotID, volumeId, volumeBytes, node.AZ, node.Viperblock.EncryptionKeyFile != ""); err != nil { + return "", err } return snapshotID, nil } @@ -629,15 +525,9 @@ func runOchreWeightsStage(cmd *cobra.Command, _ []string) { weightsStore := gateway_bedrock.NewWeightsStore(js, len(appConfig.Nodes)) store := objectstore.NewS3ObjectStoreFromConfig(node.Predastore.Host, node.Predastore.Region, node.Predastore.AccessKey, node.Predastore.SecretKey) - mkey, err := utils.LoadViperblockMasterKey(node.Viperblock.EncryptionKeyFile) - if err != nil { - fmt.Fprintf(os.Stderr, "Could not load viperblock encryption key: %v\n", err) - ochreExit(1) - return - } - - materialize := func(_ context.Context, downloadDir string, contentBytes int64) (string, error) { - return materializeWeightsVolume(node, downloadDir, contentBytes, mkey) + provider := ebsprovider.NewNATSProvider(nc, imageImportTimeout) + materialize := func(ctx context.Context, downloadDir string, contentBytes int64) (string, error) { + return materializeWeightsVolume(ctx, provider, appConfig.Node, node, downloadDir, contentBytes) } checkSnapshotLive := newWeightsSnapshotChecker(store, node.Predastore.Bucket) diff --git a/cmd/spinifex/cmd/service.go b/cmd/spinifex/cmd/service.go index 52b7b6136..9132fd78b 100644 --- a/cmd/spinifex/cmd/service.go +++ b/cmd/spinifex/cmd/service.go @@ -18,6 +18,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/services/nats" "github.com/mulgadc/spinifex/spinifex/services/northstar" "github.com/mulgadc/spinifex/spinifex/services/predastore" + "github.com/mulgadc/spinifex/spinifex/services/qemunbdd" "github.com/mulgadc/spinifex/spinifex/services/qmpcollector" "github.com/mulgadc/spinifex/spinifex/services/spinifexui" "github.com/mulgadc/spinifex/spinifex/services/viperblockd" @@ -373,7 +374,9 @@ var viperblockStartCmd = &cobra.Command{ encryptionKeyFile = envKey } - defer initTelemetry("viperblockd", false)() + debug := viper.GetBool("viperblock-debug") + + defer initTelemetry("viperblockd", debug)() service, err := service.New("viperblock", &viperblockd.Config{ NatsHost: nodeConfig.NATS.Host, @@ -390,6 +393,7 @@ var viperblockStartCmd = &cobra.Command{ ShardWAL: shardWAL, GCEnabled: gcEnabled, EncryptionKeyFile: encryptionKeyFile, + Debug: debug, }) if err != nil { @@ -439,6 +443,111 @@ var viperblockStatusCmd = &cobra.Command{ }, } +var qemunbdCmd = &cobra.Command{ + Use: "qemunbd", + Short: "Manage the qemunbd service", +} + +// Repeat for qemunbd. It answers the same ebs.provider.v1.* subjects as +// viperblock, so the two must never run against the same NATS cluster +// together. +var qemunbdStartCmd = &cobra.Command{ + Use: "start", + Short: "Start the qemunbd service", + Run: func(cmd *cobra.Command, args []string) { + fmt.Println("Starting qemunbd service...") + + cfgFile := viper.GetString("config") + + if cfgFile == "" { + fmt.Println("Config file is not set") + return + } + + fmt.Println("Loading config from:", cfgFile) + + clusterConfig, err := config.LoadConfig(cfgFile) + if err != nil { + fmt.Println("Error loading config file:", err) + return + } + nodeConfig := clusterConfig.Nodes[clusterConfig.Node] + + natsHost := viper.GetString("nats-host") + + if natsHost != "" { + fmt.Println("Overwriting natsHost:", natsHost) + nodeConfig.NATS.Host = natsHost + } + + baseDir := viper.GetString("base-dir") + if baseDir != "" { + fmt.Println("Overwriting base-dir:", baseDir) + nodeConfig.Predastore.BaseDir = baseDir + } + + // Apply changes back to cluster config + clusterConfig.Nodes[clusterConfig.Node] = nodeConfig + + debug := viper.GetBool("qemunbd-debug") + + defer initTelemetry("qemunbdd", debug)() + + service, err := service.New("qemunbd", &qemunbdd.Config{ + NatsHost: nodeConfig.NATS.Host, + NatsToken: nodeConfig.NATS.ACL.Token, + NatsCACert: nodeConfig.NATS.CACert, + BaseDir: nodeConfig.Predastore.BaseDir, + NodeName: clusterConfig.Node, + Debug: debug, + }) + + if err != nil { + fmt.Println("Error starting qemunbd service:", err) + return + } + + _, err = service.Start() + + if err != nil { + fmt.Println("Error starting qemunbd service:", err) + return + } + + fmt.Println("qemunbd service started") + }, +} + +var qemunbdStopCmd = &cobra.Command{ + Use: "stop", + Short: "Stop the qemunbd service", + Run: func(cmd *cobra.Command, args []string) { + fmt.Println("Stopping qemunbd service...") + + service, err := service.New("qemunbd", &qemunbdd.Config{}) + + if err != nil { + fmt.Println("Error stopping qemunbd service:", err) + return + } + + if err = service.Stop(); err != nil { + fmt.Println("Error stopping qemunbd service:", err) + os.Exit(1) + } + + fmt.Println("qemunbd service stopped") + }, +} + +var qemunbdStatusCmd = &cobra.Command{ + Use: "status", + Short: "Get status of the qemunbd service", + Run: func(cmd *cobra.Command, args []string) { + fmt.Println("qemunbd service status: ...") + }, +} + // Repeat for nats. var natsStartCmd = &cobra.Command{ Use: "start", @@ -1300,10 +1409,24 @@ func init() { viper.BindEnv("viperblock-encryption-key-file", "SPINIFEX_VIPERBLOCK_ENCRYPTION_KEY_FILE") viper.BindPFlag("viperblock-encryption-key-file", viperblockCmd.PersistentFlags().Lookup("encryption-key-file")) + viperblockCmd.PersistentFlags().Bool("debug", false, "Viperblock (EBS) debug logging") + viper.BindEnv("viperblock-debug", "SPINIFEX_VIPERBLOCK_DEBUG") + viper.BindPFlag("viperblock-debug", viperblockCmd.PersistentFlags().Lookup("debug")) + viperblockCmd.AddCommand(viperblockStartCmd) viperblockCmd.AddCommand(viperblockStopCmd) viperblockCmd.AddCommand(viperblockStatusCmd) + serviceCmd.AddCommand(qemunbdCmd) + + qemunbdCmd.PersistentFlags().Bool("debug", false, "qemunbd (EBS) debug logging") + viper.BindEnv("qemunbd-debug", "SPINIFEX_QEMUNBD_DEBUG") + viper.BindPFlag("qemunbd-debug", qemunbdCmd.PersistentFlags().Lookup("debug")) + + qemunbdCmd.AddCommand(qemunbdStartCmd) + qemunbdCmd.AddCommand(qemunbdStopCmd) + qemunbdCmd.AddCommand(qemunbdStatusCmd) + // Nats serviceCmd.AddCommand(natsCmd) diff --git a/cmd/spinifex/cmd/templates/spinifex.toml b/cmd/spinifex/cmd/templates/spinifex.toml index 5a586858e..56501481c 100644 --- a/cmd/spinifex/cmd/templates/spinifex.toml +++ b/cmd/spinifex/cmd/templates/spinifex.toml @@ -115,6 +115,9 @@ gc_enabled = true encryption_key_file = "{{.EncryptionKeyFile}}" {{- end}} +[nodes.{{.Node}}.ebs] +provider = "viperblockd" + [nodes.{{.Node}}.vpcd] ovn_nb_addr = "{{.OVNNBAddr}}" ovn_sb_addr = "{{.OVNSBAddr}}" diff --git a/go.mod b/go.mod index bdaf3ef79..278e34831 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/mulgadc/bluebottle v1.16.1-0.20260812012137-683e88e465e3 github.com/mulgadc/northstar v1.16.1-0.20260812012604-5d7a622ad059 github.com/mulgadc/predastore v1.16.1-0.20260812012531-12c90bd2e14a - github.com/mulgadc/viperblock v1.16.1-0.20260812012547-bb1225320f0c + github.com/mulgadc/viperblock v1.16.1-0.20260812033921-fc825c93eff6 github.com/nats-io/nats-server/v2 v2.14.4 github.com/nats-io/nats.go v1.52.0 github.com/opencontainers/runtime-spec v1.3.0 diff --git a/go.sum b/go.sum index 12efc4780..d47227e28 100644 --- a/go.sum +++ b/go.sum @@ -1327,6 +1327,8 @@ github.com/mulgadc/predastore v1.16.1-0.20260812012531-12c90bd2e14a h1:XSvL4FucG github.com/mulgadc/predastore v1.16.1-0.20260812012531-12c90bd2e14a/go.mod h1:S5vcni7R3VYi3nccelMagKbWdX58WtbKf0I1R/7MmNI= github.com/mulgadc/viperblock v1.16.1-0.20260812012547-bb1225320f0c h1:5PCU4696Gelc+4X4K+E+EzNVsyoLLWEiF/TcsIaAGi4= github.com/mulgadc/viperblock v1.16.1-0.20260812012547-bb1225320f0c/go.mod h1:8NT5u+udY1CsoKz/JAPY2SSMcQ98Omi+42r8BkoA4FY= +github.com/mulgadc/viperblock v1.16.1-0.20260812033921-fc825c93eff6 h1:QfdcaqnrDDzsO4neJNX7BdL8vzyga/WPn4A/04QZcVw= +github.com/mulgadc/viperblock v1.16.1-0.20260812033921-fc825c93eff6/go.mod h1:8NT5u+udY1CsoKz/JAPY2SSMcQ98Omi+42r8BkoA4FY= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= diff --git a/spinifex/admin/image_import.go b/spinifex/admin/image_import.go new file mode 100644 index 000000000..ed3dffef9 --- /dev/null +++ b/spinifex/admin/image_import.go @@ -0,0 +1,161 @@ +package admin + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "os" + "os/exec" + "strconv" + + "github.com/mulgadc/spinifex/spinifex/ebsprovider" + "github.com/mulgadc/spinifex/spinifex/utils" +) + +// ImportOpts describes one raw disk image to import into a new volume. +type ImportOpts struct { + // VolumeID names the volume to create. For an AMI import this is the + // image ID, so the snapshot lands at SnapPrefix(imageID). + VolumeID string + // NodeID is the node whose provider serves the NBD export written to. It + // must be the node this process runs on: the export is local to it. + NodeID string + // SizeBytes is the volume's capacity, which the caller rounds to a whole + // GiB. It must be at least the source image's size. + SizeBytes int64 + // AvailabilityZone is recorded on the volume. + AvailabilityZone string + // SourcePath is the raw disk image on local disk. + SourcePath string + // Snapshot creates SnapPrefix(VolumeID) once the write completes, so + // launches can clone the image instead of copying it block by block. + Snapshot bool + // Progress receives the importer's own output (a percentage counter). + // Nil discards it. + Progress io.Writer +} + +// ImportImage writes a local raw disk image into a new provider volume over +// NBD, then optionally snapshots it. Everything here goes through the provider +// contract, so an import works against any conforming provider rather than +// only the one whose storage library this process happens to link. +// +// A failed import deliberately leaves the volume behind: its blocks are +// already in the object store, and deleting it would take the ID an operator +// needs to reclaim them with it. +func ImportImage(ctx context.Context, provider ebsprovider.EBSProvider, opts ImportOpts) error { + if opts.VolumeID == "" || opts.NodeID == "" || opts.SourcePath == "" { + return errors.New("import: volume ID, node ID and source path are all required") + } + if opts.SizeBytes <= 0 { + return fmt.Errorf("import: invalid volume size %d", opts.SizeBytes) + } + + if _, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), + VolumeID: opts.VolumeID, + CapacityRange: ebsprovider.CapacityRange{RequiredBytes: opts.SizeBytes}, + AvailabilityZone: opts.AvailabilityZone, + }); err != nil { + return fmt.Errorf("import: create volume %s: %w", opts.VolumeID, err) + } + + published, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), + VolumeID: opts.VolumeID, + NodeID: opts.NodeID, + }) + if err != nil { + return fmt.Errorf("import: publish volume %s: %w", opts.VolumeID, err) + } + + writeErr := writeImage(ctx, opts.SourcePath, published.NBDURI, opts.Progress) + + // Unpublish regardless: a volume left published cannot be snapshotted or + // attached, and on the failure path it would strand the export too. + if err := provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), + VolumeID: opts.VolumeID, + NodeID: opts.NodeID, + }); err != nil { + if writeErr != nil { + return fmt.Errorf("import: write image: %w (unpublish also failed: %w)", writeErr, err) + } + return fmt.Errorf("import: unpublish volume %s: %w", opts.VolumeID, err) + } + if writeErr != nil { + return fmt.Errorf("import: write image to %s: %w", opts.VolumeID, writeErr) + } + + if opts.Snapshot { + if _, err := provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{ + Versioned: ebsprovider.NewVersioned(), + SnapshotID: SnapPrefix(opts.VolumeID), + VolumeID: opts.VolumeID, + }); err != nil { + return fmt.Errorf("import: snapshot volume %s: %w", opts.VolumeID, err) + } + } + + return nil +} + +// writeImage is the bulk copy step, indirected so a test can drive the verb +// sequence around it without a live NBD export to write to. +var writeImage = writeImageToNBD + +// writeImageToNBD copies a raw image onto a published export with qemu-img, +// which speaks NBD as a target and is already a runtime dependency. Passing +// the export as the target rather than a file is what keeps the bulk data off +// NATS, whose request size the provider contract caps far below an image. +func writeImageToNBD(ctx context.Context, sourcePath, nbdURI string, progress io.Writer) error { + target, err := qemuNBDTarget(nbdURI) + if err != nil { + return err + } + + // -n keeps qemu-img from trying to create a target that already exists, + // and --target-is-zero lets it skip the source's holes: the volume was + // created moments ago and has never been written. + cmd := exec.CommandContext(ctx, "qemu-img", "convert", + "-f", "raw", "-O", "raw", "-n", "--target-is-zero", "-p", + sourcePath, target) + if progress != nil { + cmd.Stdout = progress + } + var stderr []byte + stderrPipe, err := cmd.StderrPipe() + if err != nil { + return fmt.Errorf("qemu-img stderr: %w", err) + } + if err := cmd.Start(); err != nil { + return fmt.Errorf("start qemu-img: %w", err) + } + stderr, _ = io.ReadAll(stderrPipe) + if err := cmd.Wait(); err != nil { + return fmt.Errorf("qemu-img convert: %w: %s", err, stderr) + } + return nil +} + +// qemuNBDTarget renders a provider NBD URI in the form qemu-img accepts. The +// contract's unix form is not qemu's, so it is rebuilt rather than passed on. +func qemuNBDTarget(nbdURI string) (string, error) { + serverType, socketPath, host, port, err := utils.ParseNBDURI(nbdURI) + if err != nil { + return "", fmt.Errorf("parse NBD URI %q: %w", nbdURI, err) + } + switch serverType { + case "unix": + if _, err := os.Stat(socketPath); err != nil { + return "", fmt.Errorf("NBD socket %s: %w", socketPath, err) + } + return "nbd+unix:///?socket=" + socketPath, nil + case "inet": + return "nbd://" + net.JoinHostPort(host, strconv.Itoa(port)), nil + default: + return "", fmt.Errorf("unsupported NBD server type %q in %q", serverType, nbdURI) + } +} diff --git a/spinifex/admin/image_import_test.go b/spinifex/admin/image_import_test.go new file mode 100644 index 000000000..b4fe77f00 --- /dev/null +++ b/spinifex/admin/image_import_test.go @@ -0,0 +1,172 @@ +package admin + +import ( + "context" + "errors" + "io" + "os" + "testing" + + "github.com/mulgadc/spinifex/spinifex/ebsprovider" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// recordingProvider wraps MemoryProvider to record the verb sequence and the +// arguments the import passes, so a test can assert the order and the IDs +// rather than only the end state. +type recordingProvider struct { + *ebsprovider.MemoryProvider + + calls []string + publishes []ebsprovider.PublishVolumeRequest + snapshots []ebsprovider.CreateSnapshotRequest + publishURI string +} + +func newRecordingProvider() *recordingProvider { + return &recordingProvider{MemoryProvider: ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{CrashConsistentSnapshot: true}), publishURI: "nbd:unix:/tmp/does-not-matter.sock"} +} + +func (p *recordingProvider) CreateVolume(ctx context.Context, req ebsprovider.CreateVolumeRequest) (*ebsprovider.Volume, error) { + p.calls = append(p.calls, "create") + return p.MemoryProvider.CreateVolume(ctx, req) +} + +func (p *recordingProvider) PublishVolume(ctx context.Context, req ebsprovider.PublishVolumeRequest) (*ebsprovider.PublishedVolume, error) { + p.calls = append(p.calls, "publish") + p.publishes = append(p.publishes, req) + if _, err := p.MemoryProvider.PublishVolume(ctx, req); err != nil { + return nil, err + } + return &ebsprovider.PublishedVolume{VolumeID: req.VolumeID, NodeID: req.NodeID, NBDURI: p.publishURI}, nil +} + +func (p *recordingProvider) UnpublishVolume(ctx context.Context, req ebsprovider.UnpublishVolumeRequest) error { + p.calls = append(p.calls, "unpublish") + return p.MemoryProvider.UnpublishVolume(ctx, req) +} + +func (p *recordingProvider) CreateSnapshot(ctx context.Context, req ebsprovider.CreateSnapshotRequest) (*ebsprovider.Snapshot, error) { + p.calls = append(p.calls, "snapshot") + p.snapshots = append(p.snapshots, req) + return p.MemoryProvider.CreateSnapshot(ctx, req) +} + +// stubWriteImage swaps the bulk copy for one that records what it was handed +// and returns err, restoring the real writer when the test ends. +func stubWriteImage(t *testing.T, err error, seen *string) { + t.Helper() + original := writeImage + writeImage = func(_ context.Context, _, nbdURI string, _ io.Writer) error { + if seen != nil { + *seen = nbdURI + } + return err + } + t.Cleanup(func() { writeImage = original }) +} + +// TestImportImage_VerbSequenceAndSnapshotID is the assertion the old +// config.json read-back stood for: the snapshot the import creates must be the +// one the AMI document will name, and both derive from SnapPrefix. A snapshot +// under any other ID leaves the registered AMI pointing at nothing. +func TestImportImage_VerbSequenceAndSnapshotID(t *testing.T) { + var wroteTo string + stubWriteImage(t, nil, &wroteTo) + provider := newRecordingProvider() + + err := ImportImage(context.Background(), provider, ImportOpts{ + VolumeID: "ami-import001", + NodeID: "node-a", + SizeBytes: 8 << 30, + AvailabilityZone: "ap-southeast-2a", + SourcePath: "/dev/null", + Snapshot: true, + }) + require.NoError(t, err) + + assert.Equal(t, []string{"create", "publish", "unpublish", "snapshot"}, provider.calls, + "the volume must be unpublished before it is snapshotted") + require.Len(t, provider.snapshots, 1) + assert.Equal(t, SnapPrefix("ami-import001"), provider.snapshots[0].SnapshotID) + assert.Equal(t, "ami-import001", provider.snapshots[0].VolumeID) + require.Len(t, provider.publishes, 1) + assert.Equal(t, "node-a", provider.publishes[0].NodeID) + assert.Equal(t, provider.publishURI, wroteTo, "the copy must target the URI the provider published") +} + +// TestImportImage_WriteFailureStillUnpublishes covers the failure path: a +// volume left published can be neither attached nor snapshotted, so the +// export must be released even when the copy that needed it failed. +func TestImportImage_WriteFailureStillUnpublishes(t *testing.T) { + stubWriteImage(t, errors.New("copy exploded"), nil) + provider := newRecordingProvider() + + err := ImportImage(context.Background(), provider, ImportOpts{ + VolumeID: "ami-import002", + NodeID: "node-a", + SizeBytes: 1 << 30, + SourcePath: "/dev/null", + Snapshot: true, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "copy exploded") + + assert.Equal(t, []string{"create", "publish", "unpublish"}, provider.calls, + "a failed copy must not be snapshotted, and must still be unpublished") +} + +// TestImportImage_NoSnapshotWhenNotAsked covers the weights-style import that +// wants the volume but not an AMI snapshot. +func TestImportImage_NoSnapshotWhenNotAsked(t *testing.T) { + stubWriteImage(t, nil, nil) + provider := newRecordingProvider() + + require.NoError(t, ImportImage(context.Background(), provider, ImportOpts{ + VolumeID: "vol-import003", + NodeID: "node-a", + SizeBytes: 1 << 30, + SourcePath: "/dev/null", + })) + assert.Equal(t, []string{"create", "publish", "unpublish"}, provider.calls) + assert.Empty(t, provider.snapshots) +} + +// TestImportImage_RejectsIncompleteOpts keeps a caller from creating a volume +// it then cannot address, which would strand blocks under a name nobody holds. +func TestImportImage_RejectsIncompleteOpts(t *testing.T) { + stubWriteImage(t, nil, nil) + + for name, opts := range map[string]ImportOpts{ + "no volume ID": {NodeID: "node-a", SizeBytes: 1, SourcePath: "/dev/null"}, + "no node ID": {VolumeID: "vol-a", SizeBytes: 1, SourcePath: "/dev/null"}, + "no source": {VolumeID: "vol-a", NodeID: "node-a", SizeBytes: 1}, + "no size": {VolumeID: "vol-a", NodeID: "node-a", SourcePath: "/dev/null"}, + } { + t.Run(name, func(t *testing.T) { + provider := newRecordingProvider() + require.Error(t, ImportImage(context.Background(), provider, opts)) + assert.Empty(t, provider.calls, "nothing may be created before the options are checked") + }) + } +} + +// TestQemuNBDTarget_RewritesUnixForm locks the translation: the contract's +// unix URI is not a form qemu accepts, so passing it through unchanged would +// fail at the copy rather than here. +func TestQemuNBDTarget_RewritesUnixForm(t *testing.T) { + socket := t.TempDir() + "/export.sock" + require.NoError(t, os.WriteFile(socket, nil, 0o600)) + + got, err := qemuNBDTarget("nbd:unix:" + socket) + require.NoError(t, err) + assert.Equal(t, "nbd+unix:///?socket="+socket, got) + + got, err = qemuNBDTarget("nbd://10.0.0.5:10809") + require.NoError(t, err) + assert.Equal(t, "nbd://10.0.0.5:10809", got) + + _, err = qemuNBDTarget("nbd:unix:" + t.TempDir() + "/absent.sock") + require.Error(t, err, "a socket that is not there must fail before qemu-img is run") +} diff --git a/spinifex/admin/images_promote.go b/spinifex/admin/images_promote.go index 36f85b03e..3ada9f239 100644 --- a/spinifex/admin/images_promote.go +++ b/spinifex/admin/images_promote.go @@ -1,21 +1,16 @@ package admin import ( - "bytes" "context" - "encoding/json" "errors" "fmt" "log/slog" "strings" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/s3" "github.com/mulgadc/spinifex/spinifex/awserrors" - handlers_ec2_image "github.com/mulgadc/spinifex/spinifex/handlers/ec2/image" + "github.com/mulgadc/spinifex/spinifex/ebsmetadata" "github.com/mulgadc/spinifex/spinifex/objectstore" "github.com/mulgadc/spinifex/spinifex/utils" - "github.com/mulgadc/viperblock/viperblock" ) // SystemOwnerAlias is the fixed owner alias written to AMI config on promotion. @@ -49,13 +44,13 @@ func PromoteSystemImage(store objectstore.ObjectStore, bucket string, opts Promo return nil, errors.New(awserrors.ErrorInvalidAMIIDMalformed) } - meta, err := readAMIConfig(store, bucket, opts.ImageID) + meta, err := readAMI(store, bucket, opts.ImageID) switch { case err == nil: // ok case objectstore.IsNoSuchKeyError(err): return nil, errors.New(awserrors.ErrorInvalidAMIIDNotFound) - case errors.Is(err, handlers_ec2_image.ErrCorruptAMIConfig): + case errors.Is(err, ebsmetadata.ErrCorruptDocument): return nil, errors.New(awserrors.ErrorInvalidAMIIDNotFound) default: slog.Error("PromoteSystemImage: read AMI config", "imageId", opts.ImageID, "err", err) @@ -69,8 +64,8 @@ func PromoteSystemImage(store objectstore.ObjectStore, bucket string, opts Promo prev := meta.ImageOwnerAlias meta.ImageOwnerAlias = SystemOwnerAlias - if err := writeAMIConfig(store, bucket, opts.ImageID, meta); err != nil { - slog.Error("PromoteSystemImage: write AMI config", "imageId", opts.ImageID, "err", err) + if err := ebsmetadata.NewStore(store, bucket).PutAMI(context.Background(), meta); err != nil { + slog.Error("PromoteSystemImage: write AMI document", "imageId", opts.ImageID, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } @@ -78,39 +73,18 @@ func PromoteSystemImage(store objectstore.ObjectStore, bucket string, opts Promo return &PromoteImageResult{PreviousOwner: prev}, nil } -// writeAMIConfig persists updated AMIMetadata to {imageID}/config.json, -// preserving the VBState wrapper used by GetAMIConfig. -func writeAMIConfig(store objectstore.ObjectStore, bucket, imageID string, meta viperblock.AMIMetadata) error { - state := viperblock.VBState{ - VolumeConfig: viperblock.VolumeConfig{ - AMIMetadata: meta, - }, - } - data, err := json.Marshal(state) - if err != nil { - return err - } - _, err = store.PutObject(context.Background(), &s3.PutObjectInput{ - Bucket: aws.String(bucket), - Key: aws.String(imageID + "/config.json"), - Body: bytes.NewReader(data), - ContentType: aws.String("application/json"), - }) - return err -} - -// GetAMIMetadata reads and returns the AMIMetadata for the given image ID. -// Returns ErrorInvalidAMIIDNotFound for missing or corrupt configs. -func GetAMIMetadata(store objectstore.ObjectStore, bucket, imageID string) (viperblock.AMIMetadata, error) { - meta, err := readAMIConfig(store, bucket, imageID) +// GetAMIMetadata reads and returns the control-plane document for the given +// image ID. Returns ErrorInvalidAMIIDNotFound for missing or corrupt documents. +func GetAMIMetadata(store objectstore.ObjectStore, bucket, imageID string) (ebsmetadata.AMI, error) { + meta, err := readAMI(store, bucket, imageID) switch { case err == nil: return meta, nil case objectstore.IsNoSuchKeyError(err): - return viperblock.AMIMetadata{}, errors.New(awserrors.ErrorInvalidAMIIDNotFound) - case errors.Is(err, handlers_ec2_image.ErrCorruptAMIConfig): - return viperblock.AMIMetadata{}, errors.New(awserrors.ErrorInvalidAMIIDNotFound) + return ebsmetadata.AMI{}, errors.New(awserrors.ErrorInvalidAMIIDNotFound) + case errors.Is(err, ebsmetadata.ErrCorruptDocument): + return ebsmetadata.AMI{}, errors.New(awserrors.ErrorInvalidAMIIDNotFound) default: - return viperblock.AMIMetadata{}, errors.New(awserrors.ErrorServerInternal) + return ebsmetadata.AMI{}, errors.New(awserrors.ErrorServerInternal) } } diff --git a/spinifex/admin/images_promote_test.go b/spinifex/admin/images_promote_test.go index d6b0c1e67..626e98b7d 100644 --- a/spinifex/admin/images_promote_test.go +++ b/spinifex/admin/images_promote_test.go @@ -1,11 +1,8 @@ package admin import ( - "bytes" "testing" - "github.com/aws/aws-sdk-go/aws" - awss3 "github.com/aws/aws-sdk-go/service/s3" "github.com/mulgadc/spinifex/spinifex/awserrors" "github.com/mulgadc/spinifex/spinifex/objectstore" "github.com/stretchr/testify/assert" @@ -21,8 +18,8 @@ func TestPromoteSystemImage_HappyPath(t *testing.T) { require.NoError(t, err) assert.Equal(t, testRemoveAccountID, result.PreviousOwner) - // Verify the persisted config now carries the system alias. - meta, err := readAMIConfig(store, testRemoveBucket, id) + // Verify the persisted document now carries the system alias. + meta, err := readAMI(store, testRemoveBucket, id) require.NoError(t, err) assert.Equal(t, SystemOwnerAlias, meta.ImageOwnerAlias) // Other fields must be preserved. @@ -71,14 +68,9 @@ func TestPromoteSystemImage_InvalidPrefix_Malformed(t *testing.T) { func TestPromoteSystemImage_CorruptConfig_NotFound(t *testing.T) { store := objectstore.NewMemoryObjectStore() const id = "ami-corrupt-promote" - _, err := store.PutObject(t.Context(), &awss3.PutObjectInput{ - Bucket: aws.String(testRemoveBucket), - Key: aws.String(id + "/config.json"), - Body: bytes.NewReader([]byte("{not valid json")), - }) - require.NoError(t, err) + putCorruptAMI(t, store, id) - _, err = PromoteSystemImage(store, testRemoveBucket, PromoteImageOpts{ImageID: id}) + _, err := PromoteSystemImage(store, testRemoveBucket, PromoteImageOpts{ImageID: id}) require.Error(t, err) assert.Equal(t, awserrors.ErrorInvalidAMIIDNotFound, err.Error()) } @@ -105,14 +97,9 @@ func TestGetAMIMetadata_MissingConfig_NotFound(t *testing.T) { func TestGetAMIMetadata_CorruptConfig_NotFound(t *testing.T) { store := objectstore.NewMemoryObjectStore() const id = "ami-corrupt-meta" - _, err := store.PutObject(t.Context(), &awss3.PutObjectInput{ - Bucket: aws.String(testRemoveBucket), - Key: aws.String(id + "/config.json"), - Body: bytes.NewReader([]byte("not json {")), - }) - require.NoError(t, err) + putCorruptAMI(t, store, id) - _, err = GetAMIMetadata(store, testRemoveBucket, id) + _, err := GetAMIMetadata(store, testRemoveBucket, id) require.Error(t, err) assert.Equal(t, awserrors.ErrorInvalidAMIIDNotFound, err.Error()) } diff --git a/spinifex/admin/images_remove.go b/spinifex/admin/images_remove.go index 7c0e72efb..d55d916e3 100644 --- a/spinifex/admin/images_remove.go +++ b/spinifex/admin/images_remove.go @@ -2,10 +2,8 @@ package admin import ( "context" - "encoding/json" "errors" "fmt" - "io" "log/slog" "strings" "time" @@ -13,11 +11,10 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" "github.com/mulgadc/spinifex/spinifex/awserrors" - handlers_ec2_image "github.com/mulgadc/spinifex/spinifex/handlers/ec2/image" + "github.com/mulgadc/spinifex/spinifex/ebsmetadata" handlers_ec2_snapshot "github.com/mulgadc/spinifex/spinifex/handlers/ec2/snapshot" "github.com/mulgadc/spinifex/spinifex/objectstore" "github.com/mulgadc/spinifex/spinifex/utils" - "github.com/mulgadc/viperblock/viperblock" ) // RemoveImageOpts configures RemoveSystemImage. @@ -59,8 +56,8 @@ type RemovePreview struct { Name string Owner string Created time.Time - ConfigPresent bool // false when ami-/config.json is missing - ConfigCorrupt bool // true when config.json exists but is undecodable + ConfigPresent bool // false when the AMI's ebsmetadata document is missing + ConfigCorrupt bool // true when the document exists but is undecodable IsSystemOwned bool // ImageOwnerAlias is set and not an account ID AMIObjectCount int @@ -87,7 +84,7 @@ func PreviewRemoveSystemImage(store objectstore.ObjectStore, bucket, imageID str preview := &RemovePreview{ImageID: imageID} - meta, configErr := readAMIConfig(store, bucket, imageID) + meta, configErr := readAMI(store, bucket, imageID) switch { case configErr == nil: preview.ConfigPresent = true @@ -97,7 +94,7 @@ func PreviewRemoveSystemImage(store objectstore.ObjectStore, bucket, imageID str preview.IsSystemOwned = meta.ImageOwnerAlias != "" && !utils.IsAccountID(meta.ImageOwnerAlias) case objectstore.IsNoSuchKeyError(configErr): // Config absent — salvage candidate. Leave ConfigPresent=false. - case errors.Is(configErr, handlers_ec2_image.ErrCorruptAMIConfig): + case errors.Is(configErr, ebsmetadata.ErrCorruptDocument): preview.ConfigCorrupt = true default: return nil, fmt.Errorf("preview: read AMI config: %w", configErr) @@ -170,43 +167,32 @@ func FindAMIDependents(store objectstore.ObjectStore, bucket, imageID string) (D volSnapRefs[s] = true } - // Pass 2: volumes (vol-*/config.json) whose SnapshotID matches. - for _, p := range prefixes { - if !strings.HasPrefix(p, "vol-") { - continue - } - volID := strings.TrimSuffix(p, "/") - cfg, err := readVolumeConfig(store, bucket, volID) - if err != nil { - if objectstore.IsNoSuchKeyError(err) || errors.Is(err, errCorruptVolumeConfig) { - continue - } - return Dependents{}, fmt.Errorf("read volume %s: %w", volID, err) - } - if volSnapRefs[cfg.VolumeMetadata.SnapshotID] { - deps.Volumes = append(deps.Volumes, volID) + // Pass 2: volumes whose SnapshotID matches. The ebsmetadata documents are + // the whole fleet — a volume with blocks but no document does not exist as + // far as the control plane is concerned — and they carry SnapshotID + // already, so no second read per volume is needed. + volDocs, err := ebsmetadata.NewStore(store, bucket).ListVolumes(context.Background()) + if err != nil { + return Dependents{}, fmt.Errorf("list ebsmetadata volumes: %w", err) + } + for _, v := range volDocs { + if volSnapRefs[v.SnapshotID] { + deps.Volumes = append(deps.Volumes, v.VolumeID) } } - // Pass 3: AMIs (ami-*/config.json) whose SnapshotID is a derived snap. - // Skip the target AMI itself. - for _, p := range prefixes { - if !strings.HasPrefix(p, "ami-") { + // Pass 3: AMIs whose SnapshotID is a derived snap, skipping the target AMI + // itself. + docAMIs, err := ebsmetadata.NewStore(store, bucket).ListAMIs(context.Background()) + if err != nil { + return Dependents{}, fmt.Errorf("list ebsmetadata AMIs: %w", err) + } + for _, a := range docAMIs { + if a.ImageID == imageID { continue } - otherAMI := strings.TrimSuffix(p, "/") - if otherAMI == imageID { - continue - } - meta, err := readAMIConfig(store, bucket, otherAMI) - if err != nil { - if objectstore.IsNoSuchKeyError(err) || errors.Is(err, handlers_ec2_image.ErrCorruptAMIConfig) { - continue - } - return Dependents{}, fmt.Errorf("read AMI %s: %w", otherAMI, err) - } - if derived[meta.SnapshotID] { - deps.AMIs = append(deps.AMIs, otherAMI) + if derived[a.SnapshotID] { + deps.AMIs = append(deps.AMIs, a.ImageID) } } @@ -221,9 +207,9 @@ func RemoveSystemImage(store objectstore.ObjectStore, bucket string, opts Remove return nil, errors.New(awserrors.ErrorInvalidAMIIDMalformed) } - meta, configErr := readAMIConfig(store, bucket, opts.ImageID) + meta, configErr := readAMI(store, bucket, opts.ImageID) configMissing := objectstore.IsNoSuchKeyError(configErr) - configCorrupt := errors.Is(configErr, handlers_ec2_image.ErrCorruptAMIConfig) + configCorrupt := errors.Is(configErr, ebsmetadata.ErrCorruptDocument) switch { case configErr == nil: // fine @@ -256,18 +242,18 @@ func RemoveSystemImage(store objectstore.ObjectStore, bucket string, opts Remove result := &RemoveImageResult{} - // Step 1: drop config.json first — the barrier that hides the AMI from - // DescribeImages so no new launches can land on the blocks we're deleting. - if configErr == nil || (opts.Force && !configMissing) { - n, b, err := deletePrefix(store, bucket, opts.ImageID+"/config.json") - if err != nil { - return nil, fmt.Errorf("delete config: %w", err) - } - result.ObjectsDeleted += n - result.BytesDeleted += b + // Step 1: drop the ebsmetadata document first — the barrier that hides the + // AMI from DescribeImages, so no new launch can land on blocks step 2 is + // about to delete. Fatal, unlike the blocks: an AMI still advertised after + // its blocks are gone is worse than one whose blocks outlive it. + // An already-absent document is the barrier being satisfied, not a failure: + // this is exactly the state of an orphaned image, whose blocks are the only + // thing left to reclaim and the whole reason --force exists. + if err := ebsmetadata.NewStore(store, bucket).DeleteAMI(context.Background(), opts.ImageID); err != nil && !objectstore.IsNoSuchKeyError(err) { + return nil, fmt.Errorf("delete ebsmetadata document: %w", err) } - // Step 2: drop the rest of ami-/ (chunks, WAL, checkpoints). + // Step 2: drop ami-/ (chunks, WAL, checkpoints). n, b, err := deletePrefix(store, bucket, opts.ImageID+"/") if err != nil { return nil, fmt.Errorf("delete ami prefix: %w", err) @@ -304,62 +290,10 @@ func (e *DependentError) Error() string { e.ImageID, len(e.Dependents.Volumes), len(e.Dependents.Snapshots), len(e.Dependents.AMIs)) } -// readAMIConfig reads ami-/config.json and returns the AMIMetadata. -// Mirrors ImageServiceImpl.GetAMIConfig but operates package-locally so the -// admin tooling doesn't require an ImageServiceImpl (which carries NATS). -func readAMIConfig(store objectstore.ObjectStore, bucket, imageID string) (viperblock.AMIMetadata, error) { - key := imageID + "/config.json" - res, err := store.GetObject(context.Background(), &s3.GetObjectInput{ - Bucket: aws.String(bucket), - Key: aws.String(key), - }) - if err != nil { - return viperblock.AMIMetadata{}, err - } - defer res.Body.Close() - - body, err := io.ReadAll(res.Body) - if err != nil { - return viperblock.AMIMetadata{}, err - } - - var state viperblock.VBState - if err := json.Unmarshal(viperblock.StateBody(body), &state); err != nil { - return viperblock.AMIMetadata{}, fmt.Errorf("%w: %s: %w", handlers_ec2_image.ErrCorruptAMIConfig, key, err) - } - return state.VolumeConfig.AMIMetadata, nil -} - -// errCorruptVolumeConfig distinguishes an unparse-able config (walk continues) -// from a transport error (walk fails closed to prevent deleting live blocks). -var errCorruptVolumeConfig = errors.New("corrupt volume config") - -// readVolumeConfig reads vol-/config.json into VolumeConfig. -type volumeConfigWrapper struct { - VolumeConfig viperblock.VolumeConfig `json:"VolumeConfig"` -} - -func readVolumeConfig(store objectstore.ObjectStore, bucket, volumeID string) (*viperblock.VolumeConfig, error) { - key := volumeID + "/config.json" - res, err := store.GetObject(context.Background(), &s3.GetObjectInput{ - Bucket: aws.String(bucket), - Key: aws.String(key), - }) - if err != nil { - return nil, err - } - defer res.Body.Close() - - body, err := io.ReadAll(res.Body) - if err != nil { - return nil, err - } - - var w volumeConfigWrapper - if err := json.Unmarshal(viperblock.StateBody(body), &w); err != nil { - return nil, fmt.Errorf("%w: %s: %w", errCorruptVolumeConfig, key, err) - } - return &w.VolumeConfig, nil +// readAMI resolves an AMI's control-plane document. It exists so the admin +// tooling can read an AMI without an ImageServiceImpl, which carries NATS. +func readAMI(store objectstore.ObjectStore, bucket, imageID string) (ebsmetadata.AMI, error) { + return ebsmetadata.NewStore(store, bucket).GetAMI(context.Background(), imageID) } // listCommonPrefixes returns the top-level "directory" prefixes in the bucket diff --git a/spinifex/admin/images_remove_test.go b/spinifex/admin/images_remove_test.go index 8f69cbb10..ffb9df7c1 100644 --- a/spinifex/admin/images_remove_test.go +++ b/spinifex/admin/images_remove_test.go @@ -2,15 +2,15 @@ package admin import ( "bytes" - "encoding/json" + "context" "testing" "github.com/aws/aws-sdk-go/aws" awss3 "github.com/aws/aws-sdk-go/service/s3" "github.com/mulgadc/spinifex/spinifex/awserrors" + "github.com/mulgadc/spinifex/spinifex/ebsmetadata" handlers_ec2_snapshot "github.com/mulgadc/spinifex/spinifex/handlers/ec2/snapshot" "github.com/mulgadc/spinifex/spinifex/objectstore" - "github.com/mulgadc/viperblock/viperblock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -20,27 +20,30 @@ const ( testRemoveAccountID = "000000000001" ) -// putAMI writes an ami-/config.json with the given owner. Owner "" produces -// a corrupt config (empty ImageOwnerAlias is treated as corrupt by readers). +// putAMI registers an AMI the only way the control plane knows one: as an +// ebsmetadata document. It writes nothing under ami-/, so an object count +// taken over that prefix sees blocks alone. func putAMI(t *testing.T, store *objectstore.MemoryObjectStore, imageID, name, owner, snapshotID string) { t.Helper() - state := viperblock.VBState{ - VolumeConfig: viperblock.VolumeConfig{ - AMIMetadata: viperblock.AMIMetadata{ - ImageID: imageID, - Name: name, - ImageOwnerAlias: owner, - SnapshotID: snapshotID, - VolumeSizeGiB: 8, - }, - }, - } - data, err := json.Marshal(state) + require.NoError(t, ebsmetadata.NewStore(store, testRemoveBucket).PutAMI(t.Context(), ebsmetadata.AMI{ + ImageID: imageID, + Name: name, + ImageOwnerAlias: owner, + SnapshotID: snapshotID, + VolumeSizeGiB: 8, + })) +} + +// putCorruptAMI writes an undecodable document at the AMI's key, the state the +// salvage path exists for: present, so not not-found, but unreadable. +func putCorruptAMI(t *testing.T, store *objectstore.MemoryObjectStore, imageID string) { + t.Helper() + key, err := ebsmetadata.AMIKey(imageID) require.NoError(t, err) _, err = store.PutObject(t.Context(), &awss3.PutObjectInput{ Bucket: aws.String(testRemoveBucket), - Key: aws.String(imageID + "/config.json"), - Body: bytes.NewReader(data), + Key: aws.String(key), + Body: bytes.NewReader([]byte("{not valid json")), }) require.NoError(t, err) } @@ -86,26 +89,14 @@ func putSnapMetadata(t *testing.T, store *objectstore.MemoryObjectStore, snapID, require.NoError(t, handlers_ec2_snapshot.WriteSnapshotConfig(store, testRemoveBucket, snapID, cfg)) } -// putVolume writes vol-/config.json with the given SnapshotID reference. +// putVolume registers a volume document carrying the given SnapshotID. func putVolume(t *testing.T, store *objectstore.MemoryObjectStore, volID, snapshotID string) { t.Helper() - wrapper := volumeConfigWrapper{ - VolumeConfig: viperblock.VolumeConfig{ - VolumeMetadata: viperblock.VolumeMetadata{ - VolumeID: volID, - SnapshotID: snapshotID, - SizeGiB: 8, - }, - }, - } - data, err := json.Marshal(wrapper) - require.NoError(t, err) - _, err = store.PutObject(t.Context(), &awss3.PutObjectInput{ - Bucket: aws.String(testRemoveBucket), - Key: aws.String(volID + "/config.json"), - Body: bytes.NewReader(data), - }) - require.NoError(t, err) + require.NoError(t, ebsmetadata.NewStore(store, testRemoveBucket).PutVolume(t.Context(), ebsmetadata.Volume{ + VolumeID: volID, + SnapshotID: snapshotID, + CapacityGiB: 8, + })) } func TestRemoveSystemImage_HappyPath(t *testing.T) { @@ -120,8 +111,8 @@ func TestRemoveSystemImage_HappyPath(t *testing.T) { assert.True(t, preview.ConfigPresent) assert.True(t, preview.IsSystemOwned) assert.Equal(t, "system", preview.Owner) - // config.json + 3 chunks = 4 ami-prefix objects. - assert.Equal(t, 4, preview.AMIObjectCount) + // Only the 3 chunks: the document lives outside the ami-/ prefix. + assert.Equal(t, 3, preview.AMIObjectCount) assert.Equal(t, 2, preview.SnapObjectCount) assert.True(t, preview.Dependents.Empty()) @@ -132,6 +123,39 @@ func TestRemoveSystemImage_HappyPath(t *testing.T) { assert.Equal(t, 0, store.Count()) } +// strictDeleteStore reports a missing key on delete, the way predastore does. +// The in-memory store is silently tolerant and AWS S3 answers 204, so neither +// reproduces the backend this actually runs against. +type strictDeleteStore struct { + *objectstore.MemoryObjectStore +} + +var _ objectstore.ObjectStore = (*strictDeleteStore)(nil) + +func (s *strictDeleteStore) DeleteObject(ctx context.Context, input *awss3.DeleteObjectInput) (*awss3.DeleteObjectOutput, error) { + key := aws.StringValue(input.Key) + if _, err := s.HeadObject(ctx, &awss3.HeadObjectInput{Bucket: input.Bucket, Key: input.Key}); err != nil { + return nil, &objectstore.NoSuchKeyError{Key: key} + } + return s.MemoryObjectStore.DeleteObject(ctx, input) +} + +// TestRemoveSystemImage_OrphanedBlocksAreReclaimable covers the state the +// orphan report points operators at: blocks in the object store with no +// document. A live env19 run failed here on NoSuchKey, so the one documented +// way to reclaim an orphan was broken by the very thing that defines one. +func TestRemoveSystemImage_OrphanedBlocksAreReclaimable(t *testing.T) { + memory := objectstore.NewMemoryObjectStore() + store := &strictDeleteStore{MemoryObjectStore: memory} + const id = "ami-orphaned0001" + putAMIBlocks(t, memory, id, 3, 128) + + res, err := RemoveSystemImage(store, testRemoveBucket, RemoveImageOpts{ImageID: id, Force: true}) + require.NoError(t, err, "an image whose document is already gone must still have its blocks reclaimable") + assert.Equal(t, 3, res.ObjectsDeleted) + assert.Equal(t, 0, memory.Count(), "the stranded blocks must actually be released") +} + func TestRemoveSystemImage_AccountOwned_Refused(t *testing.T) { store := objectstore.NewMemoryObjectStore() const id = "ami-acct-001" @@ -173,14 +197,9 @@ func TestRemoveSystemImage_MissingConfig_NotFound(t *testing.T) { func TestRemoveSystemImage_CorruptConfig_NotFound(t *testing.T) { store := objectstore.NewMemoryObjectStore() const id = "ami-corrupt" - _, err := store.PutObject(t.Context(), &awss3.PutObjectInput{ - Bucket: aws.String(testRemoveBucket), - Key: aws.String(id + "/config.json"), - Body: bytes.NewReader([]byte("{not valid json")), - }) - require.NoError(t, err) + putCorruptAMI(t, store, id) - _, err = RemoveSystemImage(store, testRemoveBucket, RemoveImageOpts{ImageID: id}) + _, err := RemoveSystemImage(store, testRemoveBucket, RemoveImageOpts{ImageID: id}) require.Error(t, err) assert.Equal(t, awserrors.ErrorInvalidAMIIDNotFound, err.Error()) } @@ -245,16 +264,11 @@ func TestRemoveSystemImage_Force_OverridesDependents(t *testing.T) { res, err := RemoveSystemImage(store, testRemoveBucket, RemoveImageOpts{ImageID: id, Force: true}) require.NoError(t, err) assert.Positive(t, res.ObjectsDeleted) - // vol-orphan/config.json remains; the AMI is gone. - _, err = store.GetObject(t.Context(), &awss3.GetObjectInput{ - Bucket: aws.String(testRemoveBucket), - Key: aws.String(id + "/config.json"), - }) + // The dependent volume's document remains; the AMI's is gone. + metadata := ebsmetadata.NewStore(store, testRemoveBucket) + _, err = metadata.GetAMI(t.Context(), id) require.Error(t, err) - _, err = store.GetObject(t.Context(), &awss3.GetObjectInput{ - Bucket: aws.String(testRemoveBucket), - Key: aws.String("vol-orphan/config.json"), - }) + _, err = metadata.GetVolume(t.Context(), "vol-orphan") require.NoError(t, err) } @@ -280,23 +294,19 @@ func TestRemoveSystemImage_Salvage_MissingConfig_ForceCleans(t *testing.T) { func TestRemoveSystemImage_Salvage_CorruptConfig_ForceCleans(t *testing.T) { store := objectstore.NewMemoryObjectStore() const id = "ami-salvage-2" - _, err := store.PutObject(t.Context(), &awss3.PutObjectInput{ - Bucket: aws.String(testRemoveBucket), - Key: aws.String(id + "/config.json"), - Body: bytes.NewReader([]byte("garbage")), - }) - require.NoError(t, err) + putCorruptAMI(t, store, id) putAMIBlocks(t, store, id, 1, 8) // Without --force. - _, err = RemoveSystemImage(store, testRemoveBucket, RemoveImageOpts{ImageID: id}) + _, err := RemoveSystemImage(store, testRemoveBucket, RemoveImageOpts{ImageID: id}) require.Error(t, err) assert.Equal(t, awserrors.ErrorInvalidAMIIDNotFound, err.Error()) - // With --force: corrupt config + blocks all go. + // With --force: the corrupt document and the blocks all go. Only the + // block is counted; the document is deleted by key, not by prefix walk. res, err := RemoveSystemImage(store, testRemoveBucket, RemoveImageOpts{ImageID: id, Force: true}) require.NoError(t, err) - assert.Equal(t, 2, res.ObjectsDeleted) + assert.Equal(t, 1, res.ObjectsDeleted) assert.Equal(t, 0, store.Count()) } @@ -344,12 +354,7 @@ func TestPreviewRemoveSystemImage_Salvage(t *testing.T) { func TestPreviewRemoveSystemImage_CorruptConfig(t *testing.T) { store := objectstore.NewMemoryObjectStore() const id = "ami-corrupt-preview" - _, err := store.PutObject(t.Context(), &awss3.PutObjectInput{ - Bucket: aws.String(testRemoveBucket), - Key: aws.String(id + "/config.json"), - Body: bytes.NewReader([]byte("{nope")), - }) - require.NoError(t, err) + putCorruptAMI(t, store, id) preview, err := PreviewRemoveSystemImage(store, testRemoveBucket, id) require.NoError(t, err) diff --git a/spinifex/admin/orphan_volumes.go b/spinifex/admin/orphan_volumes.go new file mode 100644 index 000000000..43fdc0ced --- /dev/null +++ b/spinifex/admin/orphan_volumes.go @@ -0,0 +1,117 @@ +package admin + +import ( + "context" + "fmt" + "sort" + "strings" + + "github.com/mulgadc/spinifex/spinifex/ebsmetadata" + "github.com/mulgadc/spinifex/spinifex/ebsprovider" +) + +// efiVolumeSuffix marks a volume derived from another one. An EFI variable +// store is created alongside its root volume and deliberately gets no +// control-plane document, so it is only an orphan when its base is one too. +const efiVolumeSuffix = "-efi" + +// OrphanVolume is a volume the provider holds that the control plane has no +// document for. Its blocks are consuming space with no API handle, so it can +// be neither described nor deleted through EC2. +type OrphanVolume struct { + VolumeID string + Handle string + + // Derived marks a volume named after another one, such as an EFI variable + // store. It is reported only when its base volume is also undocumented, + // which means the pair was stranded together. + Derived bool +} + +// FindOrphanVolumes reports what storage holds that the control plane cannot +// see. It reads both sides independently: the provider for what exists, the +// metadata store for what is known, and returns the difference. +// +// It never deletes. An undocumented volume is the case where deletion is most +// tempting and most dangerous, because the evidence that it is still wanted is +// exactly the document that went missing. +func FindOrphanVolumes(ctx context.Context, provider ebsprovider.EBSProvider, metadata *ebsmetadata.Store) ([]OrphanVolume, error) { + if provider == nil || metadata == nil { + return nil, fmt.Errorf("orphan scan: provider and metadata store are both required") + } + + capabilities, err := provider.GetCapabilities(ctx, ebsprovider.GetCapabilitiesRequest{Versioned: ebsprovider.NewVersioned()}) + if err != nil { + return nil, fmt.Errorf("orphan scan: read provider capabilities: %w", err) + } + if !capabilities.Capabilities.VolumeEnumeration { + return nil, fmt.Errorf("orphan scan: %w: provider does not enumerate volumes", ebsprovider.ErrUnsupportedCapability) + } + + // Strict listing on purpose: a document this cannot decode must abort the + // scan, never be treated as absent. Reporting a volume as orphaned because + // its document was unreadable would point an operator at live data. + documented, err := metadata.ListVolumes(ctx) + if err != nil { + return nil, fmt.Errorf("orphan scan: list control-plane volumes: %w", err) + } + known := make(map[string]bool, len(documented)) + for _, volume := range documented { + known[volume.VolumeID] = true + } + + // An AMI's blocks live in a provider volume named after the image, and it + // is recorded as an AMI document rather than a volume one. Without this + // every registered image reads as an orphan. + images, err := metadata.ListAMIsStrict(ctx) + if err != nil { + return nil, fmt.Errorf("orphan scan: list control-plane images: %w", err) + } + for _, image := range images { + known[image.ImageID] = true + } + + held, err := listProviderVolumes(ctx, provider) + if err != nil { + return nil, err + } + + var orphans []OrphanVolume + for _, ref := range held { + if known[ref.ID] { + continue + } + base, derived := strings.CutSuffix(ref.ID, efiVolumeSuffix) + if derived && known[base] { + continue + } + orphans = append(orphans, OrphanVolume{VolumeID: ref.ID, Handle: ref.Handle, Derived: derived}) + } + sort.Slice(orphans, func(i, j int) bool { return orphans[i].VolumeID < orphans[j].VolumeID }) + return orphans, nil +} + +// listProviderVolumes drains every page the provider offers. A partial walk +// would under-report orphans, which is the one direction this must not fail +// in: a missed orphan is a volume nobody ever goes looking for again. +func listProviderVolumes(ctx context.Context, provider ebsprovider.EBSProvider) ([]ebsprovider.VolumeRef, error) { + var refs []ebsprovider.VolumeRef + var token string + for { + resp, err := provider.ListVolumes(ctx, ebsprovider.ListVolumesRequest{ + Versioned: ebsprovider.NewVersioned(), + StartingToken: token, + }) + if err != nil { + return nil, fmt.Errorf("orphan scan: list provider volumes: %w", err) + } + refs = append(refs, resp.Volumes...) + if resp.NextToken == "" { + return refs, nil + } + if resp.NextToken == token { + return nil, fmt.Errorf("orphan scan: provider returned a non-advancing page token %q", token) + } + token = resp.NextToken + } +} diff --git a/spinifex/admin/orphan_volumes_test.go b/spinifex/admin/orphan_volumes_test.go new file mode 100644 index 000000000..b43c63791 --- /dev/null +++ b/spinifex/admin/orphan_volumes_test.go @@ -0,0 +1,190 @@ +package admin + +import ( + "bytes" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/mulgadc/spinifex/spinifex/ebsmetadata" + "github.com/mulgadc/spinifex/spinifex/ebsprovider" + "github.com/mulgadc/spinifex/spinifex/objectstore" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const orphanTestBucket = "test-bucket" + +func newOrphanScanFixture(t *testing.T) (*ebsprovider.MemoryProvider, *ebsmetadata.Store) { + t.Helper() + provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{VolumeEnumeration: true, VolumeSeeding: true}) + return provider, ebsmetadata.NewStore(objectstore.NewMemoryObjectStore(), orphanTestBucket) +} + +// createProviderVolume gives the provider blocks without telling the control +// plane, which is what a volume looks like after its document is lost. +func createProviderVolume(t *testing.T, provider *ebsprovider.MemoryProvider, volumeID string) { + t.Helper() + _, err := provider.CreateVolume(t.Context(), ebsprovider.CreateVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), + VolumeID: volumeID, + CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}, + }) + require.NoError(t, err) +} + +func documentVolume(t *testing.T, metadata *ebsmetadata.Store, volumeID string) { + t.Helper() + require.NoError(t, metadata.PutVolume(t.Context(), ebsmetadata.Volume{ + VolumeID: volumeID, TenantID: "123456789012", CapacityGiB: 1, State: "available", + })) +} + +// putRawVolumeDocument writes bytes straight to a volume's document key, +// so a test can plant one the store cannot decode. +func putRawVolumeDocument(t *testing.T, store objectstore.ObjectStore, volumeID string, body []byte) { + t.Helper() + key, err := ebsmetadata.VolumeKey(volumeID) + require.NoError(t, err) + _, err = store.PutObject(t.Context(), &s3.PutObjectInput{ + Bucket: aws.String(orphanTestBucket), + Key: aws.String(key), + Body: bytes.NewReader(body), + }) + require.NoError(t, err) +} + +// putRawAMIDocument writes bytes straight to an AMI's document key. +func putRawAMIDocument(t *testing.T, store objectstore.ObjectStore, imageID string, body []byte) { + t.Helper() + key, err := ebsmetadata.AMIKey(imageID) + require.NoError(t, err) + _, err = store.PutObject(t.Context(), &s3.PutObjectInput{ + Bucket: aws.String(orphanTestBucket), + Key: aws.String(key), + Body: bytes.NewReader(body), + }) + require.NoError(t, err) +} + +// TestFindOrphanVolumes_FindsAVolumeWhoseDocumentIsGone is the reason the +// enumeration verb exists. Deleting the document is exactly the failure the +// consolidated boundary made unrecoverable: the volume vanishes from +// DescribeVolumes and from the leak reaper, both of which read documents. +func TestFindOrphanVolumes_FindsAVolumeWhoseDocumentIsGone(t *testing.T) { + provider, metadata := newOrphanScanFixture(t) + createProviderVolume(t, provider, "vol-documented000") + documentVolume(t, metadata, "vol-documented000") + createProviderVolume(t, provider, "vol-stranded00000") + + orphans, err := FindOrphanVolumes(t.Context(), provider, metadata) + require.NoError(t, err) + + require.Len(t, orphans, 1, "the scan must report exactly the volume with no document") + assert.Equal(t, "vol-stranded00000", orphans[0].VolumeID, + "a volume the provider holds with no control-plane document is unreachable through EC2 and must be surfaced") + assert.False(t, orphans[0].Derived) +} + +// TestFindOrphanVolumes_NeverDeletes locks ADR-0005 §3 for this tool: an +// orphan carries data, and the evidence it is still wanted is the very +// document that is missing. The scan may only report. +func TestFindOrphanVolumes_NeverDeletes(t *testing.T) { + provider, metadata := newOrphanScanFixture(t) + createProviderVolume(t, provider, "vol-stranded00000") + + orphans, err := FindOrphanVolumes(t.Context(), provider, metadata) + require.NoError(t, err) + require.Len(t, orphans, 1) + + got, err := provider.GetVolume(t.Context(), ebsprovider.GetVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-stranded00000", + }) + require.NoErrorf(t, err, "ADR-0005 §3: the orphan scan must never delete the volume it reports") + assert.Equal(t, "vol-stranded00000", got.ID) +} + +// An EFI variable store is created without a document by design, so it is only +// an orphan when the root volume it belongs to is also undocumented. +func TestFindOrphanVolumes_DerivedVolumeExcusedByItsBase(t *testing.T) { + provider, metadata := newOrphanScanFixture(t) + createProviderVolume(t, provider, "vol-live00000000") + createProviderVolume(t, provider, "vol-live00000000-efi") + documentVolume(t, metadata, "vol-live00000000") + + orphans, err := FindOrphanVolumes(t.Context(), provider, metadata) + require.NoError(t, err) + assert.Empty(t, orphans, "an EFI store whose root volume is documented is expected, not orphaned") +} + +func TestFindOrphanVolumes_DerivedVolumeReportedWhenItsBaseIsGone(t *testing.T) { + provider, metadata := newOrphanScanFixture(t) + createProviderVolume(t, provider, "vol-gone00000000") + createProviderVolume(t, provider, "vol-gone00000000-efi") + + orphans, err := FindOrphanVolumes(t.Context(), provider, metadata) + require.NoError(t, err) + + require.Len(t, orphans, 2, "a stranded pair must both be reported; the EFI store's lifecycle owner is gone too") + assert.Equal(t, "vol-gone00000000", orphans[0].VolumeID) + assert.Equal(t, "vol-gone00000000-efi", orphans[1].VolumeID) + assert.True(t, orphans[1].Derived) +} + +// An AMI's blocks live in a provider volume named after the image, recorded as +// an AMI document rather than a volume one. A live env19 run reported every +// registered image as an orphan before this was handled. +func TestFindOrphanVolumes_AMIBackedVolumeIsNotAnOrphan(t *testing.T) { + provider, metadata := newOrphanScanFixture(t) + createProviderVolume(t, provider, "ami-dd7f063caded1ff82") + require.NoError(t, metadata.PutAMI(t.Context(), ebsmetadata.AMI{ + ImageID: "ami-dd7f063caded1ff82", Name: "ubuntu", State: "available", + })) + + orphans, err := FindOrphanVolumes(t.Context(), provider, metadata) + require.NoError(t, err) + assert.Empty(t, orphans, "a registered AMI's backing volume is documented as an image, not as an orphan") +} + +// An undecodable AMI document must abort the scan for the same reason an +// undecodable volume document does: absence and unreadability are different +// answers, and only one of them means orphaned. +func TestFindOrphanVolumes_UndecodableAMIDocumentAbortsTheScan(t *testing.T) { + provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{VolumeEnumeration: true}) + store := objectstore.NewMemoryObjectStore() + metadata := ebsmetadata.NewStore(store, orphanTestBucket) + createProviderVolume(t, provider, "ami-corrupted000") + putRawAMIDocument(t, store, "ami-corrupted000", []byte("{not json")) + + _, err := FindOrphanVolumes(t.Context(), provider, metadata) + require.Error(t, err) +} + +// An unreadable document must abort the scan, never read as absent: reporting +// a volume as orphaned because its document failed to decode would point an +// operator at live data. +func TestFindOrphanVolumes_UndecodableDocumentAbortsTheScan(t *testing.T) { + provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{VolumeEnumeration: true}) + store := objectstore.NewMemoryObjectStore() + metadata := ebsmetadata.NewStore(store, orphanTestBucket) + createProviderVolume(t, provider, "vol-corruptdoc00") + putRawVolumeDocument(t, store, "vol-corruptdoc00", []byte("{not json")) + + _, err := FindOrphanVolumes(t.Context(), provider, metadata) + require.Error(t, err, "an undecodable document must fail the scan, not be treated as a missing one") +} + +func TestFindOrphanVolumes_RequiresEnumerationCapability(t *testing.T) { + provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{}) + metadata := ebsmetadata.NewStore(objectstore.NewMemoryObjectStore(), orphanTestBucket) + + _, err := FindOrphanVolumes(t.Context(), provider, metadata) + require.ErrorIs(t, err, ebsprovider.ErrUnsupportedCapability) +} + +func TestFindOrphanVolumes_EmptyProviderReportsNothing(t *testing.T) { + provider, metadata := newOrphanScanFixture(t) + orphans, err := FindOrphanVolumes(t.Context(), provider, metadata) + require.NoError(t, err) + assert.Empty(t, orphans) +} diff --git a/spinifex/config/config.go b/spinifex/config/config.go index fb60eb464..0031c2c2d 100644 --- a/spinifex/config/config.go +++ b/spinifex/config/config.go @@ -100,6 +100,7 @@ type Config struct { NATS NATSConfig `json:"NATS" mapstructure:"nats"` Predastore PredastoreConfig `json:"Predastore" mapstructure:"predastore"` Viperblock ViperblockConfig `json:"Viperblock" mapstructure:"viperblock"` + EBS EBSConfig `json:"EBS" mapstructure:"ebs"` AWSGW AWSGWConfig `json:"AWSGW" mapstructure:"awsgw"` VPCD VPCDConfig `json:"VPCD" mapstructure:"vpcd"` Northstar NorthstarConfig `json:"Northstar" mapstructure:"northstar"` @@ -136,6 +137,35 @@ type ViperblockConfig struct { GCEnabled *bool `json:"GCEnabled" mapstructure:"gc_enabled"` } +// EBS provider selectors. Viperblockd routes EBS calls to the viperblockd +// daemon and qemunbd to the qemunbdd daemon, both over the versioned +// ebs.provider.v1.* NATS contract. Embedded named the removed in-process +// engine and survives solely so a config that still selects it can be +// rejected by name. +const ( + EBSProviderEmbedded = "embedded" + EBSProviderViperblockd = "viperblockd" + EBSProviderQEMUNBD = "qemunbd" +) + +// EBSConfig selects which provider backs EBS. Not nested under +// ViperblockConfig: it names the provider boundary, not one provider's +// settings, so a second provider never needs a rename. +type EBSConfig struct { + // Provider is "viperblockd" or "qemunbd" and may be left unset. Volumes + // are persisted in ebsmetadata. + Provider string `json:"Provider" mapstructure:"provider"` +} + +// ResolvedProvider normalizes an empty Provider to EBSProviderViperblockd, +// the default provider. +func (c EBSConfig) ResolvedProvider() string { + if c.Provider == "" { + return EBSProviderViperblockd + } + return c.Provider +} + // VPCDConfig holds the VPC daemon (vpcd) configuration. type VPCDConfig struct { OVNNBAddr string `json:"OVNNBAddr" mapstructure:"ovn_nb_addr"` // OVN Northbound DB address; comma-separated list for a RAFT cluster (e.g., "tcp:127.0.0.1:6641" or "tcp:ip1:6641,tcp:ip2:6641,tcp:ip3:6641") @@ -414,10 +444,20 @@ func validateClusterConfig(cc *ClusterConfig) error { if viper.IsSet("network.external_dhcp") { return fmt.Errorf("config: [network] external_dhcp is no longer supported; remove the key (static WAN-pool allocation only)") } - for nodeName := range cc.Nodes { + for nodeName, nodeCfg := range cc.Nodes { if viper.IsSet("nodes." + nodeName + ".vpcd.dhcp_bind_bridge") { return fmt.Errorf("config: [nodes.%s.vpcd] dhcp_bind_bridge is no longer supported; remove the key (vpcd no longer runs a DHCP client)", nodeName) } + switch nodeCfg.EBS.Provider { + case "", EBSProviderViperblockd, EBSProviderQEMUNBD: + case EBSProviderEmbedded: + // Refuse rather than silently upgrade: the embedded engine is gone, + // and starting anyway would migrate this node's volumes to + // ebsmetadata one-way without the operator ever asking for it. + return fmt.Errorf("config: [nodes.%s.ebs] provider=%q has been removed; set provider = %q or remove the key (this is a one-way switch: volumes move to ebsmetadata)", nodeName, EBSProviderEmbedded, EBSProviderViperblockd) + default: + return fmt.Errorf("config: [nodes.%s.ebs] provider=%q unsupported; use %q or %q, or remove the key", nodeName, nodeCfg.EBS.Provider, EBSProviderViperblockd, EBSProviderQEMUNBD) + } } if len(cc.Network.NATExemptCIDRs) > 0 && cc.Network.ExternalMode != "nat" { diff --git a/spinifex/config/config_test.go b/spinifex/config/config_test.go index ab9a97219..a91ceffde 100644 --- a/spinifex/config/config_test.go +++ b/spinifex/config/config_test.go @@ -944,3 +944,95 @@ ovn_nb_addr = "tcp:127.0.0.1:6641" assert.Contains(t, err.Error(), `dhcp_mac is only valid with source="dhcp"`) }) } + +func TestEBSConfig_ResolvedProvider(t *testing.T) { + cases := []struct { + name string + in string + want string + }{ + {"empty defaults to viperblockd", "", EBSProviderViperblockd}, + {"embedded passes through unresolved", EBSProviderEmbedded, EBSProviderEmbedded}, + {"viperblockd passes through", EBSProviderViperblockd, EBSProviderViperblockd}, + {"qemunbd passes through", EBSProviderQEMUNBD, EBSProviderQEMUNBD}, + {"unknown value passes through unresolved", "bogus", "bogus"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cfg := EBSConfig{Provider: tc.in} + assert.Equal(t, tc.want, cfg.ResolvedProvider()) + }) + } +} + +func TestLoadConfig_EBSProvider_Valid(t *testing.T) { + base := ` +node = "n1" + +[nodes.n1] +region = "us-east-1" +%s +` + cases := []struct { + name string + section string + wantResolved string + }{ + {"unset defaults to viperblockd", "", EBSProviderViperblockd}, + {"viperblockd explicit", "[nodes.n1.ebs]\nprovider = \"viperblockd\"\n", EBSProviderViperblockd}, + {"qemunbd explicit", "[nodes.n1.ebs]\nprovider = \"qemunbd\"\n", EBSProviderQEMUNBD}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + resetViper(t) + path := filepath.Join(t.TempDir(), "spinifex.toml") + require.NoError(t, os.WriteFile(path, fmt.Appendf(nil, base, tc.section), 0600)) + + cfg, err := LoadConfig(path) + require.NoError(t, err) + assert.Equal(t, tc.wantResolved, cfg.Nodes["n1"].EBS.ResolvedProvider()) + }) + } +} + +// A deployed spinifex.toml that still names the removed embedded provider must +// fail loudly at load: starting anyway would migrate the node's volumes to +// ebsmetadata one-way without the operator having asked for it. +func TestLoadConfig_EBSProvider_EmbeddedRejected(t *testing.T) { + resetViper(t) + path := filepath.Join(t.TempDir(), "spinifex.toml") + toml := ` +node = "n1" + +[nodes.n1] +region = "us-east-1" + +[nodes.n1.ebs] +provider = "embedded" +` + require.NoError(t, os.WriteFile(path, []byte(toml), 0600)) + + _, err := LoadConfig(path) + require.Error(t, err) + assert.Contains(t, err.Error(), `provider="embedded" has been removed`) + assert.Contains(t, err.Error(), `viperblockd`) +} + +func TestLoadConfig_EBSProvider_UnknownRejected(t *testing.T) { + resetViper(t) + path := filepath.Join(t.TempDir(), "spinifex.toml") + toml := ` +node = "n1" + +[nodes.n1] +region = "us-east-1" + +[nodes.n1.ebs] +provider = "bogus" +` + require.NoError(t, os.WriteFile(path, []byte(toml), 0600)) + + _, err := LoadConfig(path) + require.Error(t, err) + assert.Contains(t, err.Error(), `provider="bogus"`) +} diff --git a/spinifex/daemon/daemon.go b/spinifex/daemon/daemon.go index e7bafaf08..4ece5cc7f 100644 --- a/spinifex/daemon/daemon.go +++ b/spinifex/daemon/daemon.go @@ -32,6 +32,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/admin" "github.com/mulgadc/spinifex/spinifex/awserrors" "github.com/mulgadc/spinifex/spinifex/config" + "github.com/mulgadc/spinifex/spinifex/ebsprovider" "github.com/mulgadc/spinifex/spinifex/gpu" handlers_acm "github.com/mulgadc/spinifex/spinifex/handlers/acm" handlers_bedrock "github.com/mulgadc/spinifex/spinifex/handlers/bedrock" @@ -124,19 +125,21 @@ var _ handlers_ec2_igw.GatePublisher = (*handlers_ec2_routetable.RouteTableServi // Daemon represents the main daemon service. type Daemon struct { - node string - clusterConfig *config.ClusterConfig - config *config.Config - natsConn *nats.Conn - resourceMgr *ResourceManager - instanceService *handlers_ec2_instance.InstanceServiceImpl - dnsWriter *handlers_dns.Writer - dnsReconciler *handlers_dns.Reconciler - dnsBaseDomain string - dnsInternalDomain string - keyService *handlers_ec2_key.KeyServiceImpl - imageService *handlers_ec2_image.ImageServiceImpl - volumeService *handlers_ec2_volume.VolumeServiceImpl + node string + clusterConfig *config.ClusterConfig + config *config.Config + natsConn *nats.Conn + resourceMgr *ResourceManager + instanceService *handlers_ec2_instance.InstanceServiceImpl + dnsWriter *handlers_dns.Writer + dnsReconciler *handlers_dns.Reconciler + dnsBaseDomain string + dnsInternalDomain string + keyService *handlers_ec2_key.KeyServiceImpl + imageService *handlers_ec2_image.ImageServiceImpl + volumeService *handlers_ec2_volume.VolumeServiceImpl + // ebsProvider is the sole EBS backend, set once during startup. + ebsProvider ebsprovider.EBSProvider accountService *handlers_ec2_account.AccountSettingsServiceImpl snapshotService *handlers_ec2_snapshot.SnapshotServiceImpl tagsService *handlers_ec2_tags.TagsServiceImpl @@ -1435,6 +1438,9 @@ func (d *Daemon) startCluster() error { d.snapshotService = snap.svc d.volumeService = handlers_ec2_volume.NewVolumeServiceImpl(d.config, d.natsConn, snap.kv) + if err := d.configureEBSProvider(); err != nil { + return fmt.Errorf("configure EBS provider: %w", err) + } d.tagsService = handlers_ec2_tags.NewTagsServiceImpl(d.config) d.eigwService, err = initServiceWithRetry("EIGW service", func() (*handlers_ec2_eigw.EgressOnlyIGWServiceImpl, error) { @@ -1887,6 +1893,9 @@ func (d *Daemon) startCluster() error { if eniRec := d.newENIReconciler(); eniRec != nil { reapers = append(reapers, eniRec) } + if eniOrphan := d.newENIOrphanReaper(); eniOrphan != nil { + reapers = append(reapers, eniOrphan) + } if gpuRec := d.newGPUPoolReconciler(); gpuRec != nil { reapers = append(reapers, gpuRec) } @@ -1920,6 +1929,50 @@ func (d *Daemon) startCluster() error { return nil } +// ebsProviderRequestTimeout bounds each ebs.provider.v1.* NATS request/reply. +// Passed explicitly rather than relying on NATSProvider's own 30s default so +// the daemon's behavior doesn't silently drift if that default ever changes. +const ebsProviderRequestTimeout = 30 * time.Second + +// ebsProviderProbeTimeout bounds the startup reachability check. Short because +// a missing responder must be reported promptly, and the check is advisory. +const ebsProviderProbeTimeout = 2 * time.Second + +// configureEBSProvider wires a single NATSProvider into every EBS-adjacent +// service. The control plane never constructs a provider implementation +// itself, so this always runs regardless of which daemon answers on the wire. +func (d *Daemon) configureEBSProvider() error { + provider := ebsprovider.NewNATSProvider(d.natsConn, ebsProviderRequestTimeout) + d.ebsProvider = provider + d.instanceService.SetEBSProvider(provider) + d.imageService.SetEBSProvider(provider) + d.snapshotService.SetEBSProvider(provider) + d.volumeService.SetEBSProvider(provider) + slog.Info("EBS provider path active", "provider", d.config.EBS.ResolvedProvider()) + d.probeEBSProvider(provider) + return nil +} + +// probeEBSProvider reports whether anything is serving the provider contract. +// Advisory rather than fatal: this daemon and viperblockd start independently, +// so an unanswered probe here can simply mean viperblockd has not come up yet. +func (d *Daemon) probeEBSProvider(provider ebsprovider.EBSProvider) { + // Own base context rather than the daemon's: the probe is bounded well + // below any shutdown deadline, and startCluster runs before d.ctx is set + // on some construction paths. + ctx, cancel := context.WithTimeout(context.Background(), ebsProviderProbeTimeout) + defer cancel() + + capabilities, err := provider.GetCapabilities(ctx, ebsprovider.GetCapabilitiesRequest{Versioned: ebsprovider.NewVersioned()}) + if err != nil { + slog.Warn("EBS provider did not answer the capability probe; EBS calls will fail until it does", + "provider", d.config.EBS.ResolvedProvider(), "err", err) + return + } + slog.Info("EBS provider reachable", "provider", d.config.EBS.ResolvedProvider(), + "capabilities", capabilities.Capabilities) +} + // clusterSweepLease gates the GC's cluster-wide reapers so exactly one node runs // them. The RDS reconciler's lease is that election — cluster-singular, held // continuously, and re-evaluated on its own ticker — so being its holder is the diff --git a/spinifex/daemon/daemon_handlers_image.go b/spinifex/daemon/daemon_handlers_image.go index b03ce4935..aec4bcafb 100644 --- a/spinifex/daemon/daemon_handlers_image.go +++ b/spinifex/daemon/daemon_handlers_image.go @@ -27,6 +27,13 @@ func (d *Daemon) handleSpinifexPromoteImage(msg *nats.Msg) { handleNATSRequest(promoteImage)(msg) } +// stoppedInstanceOwnerElector is implemented by state stores that can +// atomically mutate a stopped instance record. Checked via assertion so +// vm.StateStore itself need not widen for this one call site. +type stoppedInstanceOwnerElector interface { + UpdateStoppedInstance(id string, mutate func(*vm.VM)) (*vm.VM, error) +} + // handleEC2CreateImage is a stateful handler that extracts instance context // (root volume ID, source AMI, running state) before delegating to the image service. func (d *Daemon) handleEC2CreateImage(msg *nats.Msg) { @@ -88,6 +95,40 @@ func (d *Daemon) handleEC2CreateImage(msg *nats.Msg) { respondWithError(msg, awserrors.ErrorInvalidInstanceIDNotFound) return } + + // The KV is cluster-shared, so every node's lookup above succeeds; + // only LastNode picks a single owner. Non-owners decline like the + // miss above — Gather only short-circuits on success, so this can't + // preempt the owner's reply, it just stops a duplicate AMI/snapshot. + if stopped.LastNode == "" { + // Pre-LastNode record from an older build. Elect an owner via + // CAS so exactly one node wins, instead of every node + // proceeding (the leak) or every node declining forever. + elector, canElect := d.stateStore.(stoppedInstanceOwnerElector) + if !canElect { + slog.Error("CreateImage: state store cannot elect owner for legacy stopped instance", "instanceId", instanceID) + respondWithError(msg, awserrors.ErrorInvalidInstanceIDNotFound) + return + } + elected, uerr := elector.UpdateStoppedInstance(instanceID, func(v *vm.VM) { + if v.LastNode == "" { + v.LastNode = d.node + } + }) + if uerr != nil || elected == nil || elected.LastNode == "" { + slog.Warn("CreateImage: failed to elect owner for legacy stopped instance", "instanceId", instanceID, "err", uerr) + respondWithError(msg, awserrors.ErrorInvalidInstanceIDNotFound) + return + } + stopped.LastNode = elected.LastNode + } + if stopped.LastNode != d.node { + slog.Info("CreateImage: declining, instance owned by another node", + "instanceId", instanceID, "lastNode", stopped.LastNode) + respondWithError(msg, awserrors.ErrorInvalidInstanceIDNotFound) + return + } + instance = stopped status = stopped.Status if stopped.Instance != nil { diff --git a/spinifex/daemon/daemon_handlers_image_test.go b/spinifex/daemon/daemon_handlers_image_test.go new file mode 100644 index 000000000..30d994fd2 --- /dev/null +++ b/spinifex/daemon/daemon_handlers_image_test.go @@ -0,0 +1,169 @@ +package daemon + +import ( + "encoding/json" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mulgadc/spinifex/spinifex/awserrors" + "github.com/mulgadc/spinifex/spinifex/vm" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Tests in this file cover handleEC2CreateImage's stopped-instance LastNode +// gate. The image service always fails downstream here, so "not NotFound" +// proves a node passed the gate, and NotFound proves it declined first. + +// seedStoppedInstance writes a stopped VM directly to the daemon's shared KV +// so handleEC2CreateImage's vmMgr lookup misses and falls through to the +// LastNode-gated KV branch. +func seedStoppedInstance(t *testing.T, daemon *Daemon, id, lastNode string) { + t.Helper() + stoppedVM := &vm.VM{ + ID: id, + Status: vm.StateStopped, + AccountID: testAccountID, + LastNode: lastNode, + Instance: &ec2.Instance{ + InstanceId: aws.String(id), + ImageId: aws.String("ami-source-stopped"), + BlockDeviceMappings: []*ec2.InstanceBlockDeviceMapping{ + { + Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-" + id)}, + }, + }, + }, + } + err := daemon.jsManager.WriteStoppedInstance(id, stoppedVM) + require.NoError(t, err) + t.Cleanup(func() { _ = daemon.jsManager.DeleteStoppedInstance(id) }) +} + +// TestHandleEC2CreateImage_StoppedInstance_NonOwnerDeclines checks that a +// node which is not the stopped instance's LastNode declines exactly like +// an unknown instance ID, instead of proceeding into the image service. +func TestHandleEC2CreateImage_StoppedInstance_NonOwnerDeclines(t *testing.T) { + natsURL := sharedJSNATSURL + daemon := createFullTestDaemonWithJetStream(t, natsURL) + // createTestDaemon fixes the cluster config's own node at "node-1". + seedStoppedInstance(t, daemon, "i-stopped-nonowner", "node-2") + + sub, err := daemon.natsConn.Subscribe("ec2.CreateImage", daemon.handleEC2CreateImage) + require.NoError(t, err) + defer sub.Unsubscribe() + + input := &ec2.CreateImageInput{ + InstanceId: aws.String("i-stopped-nonowner"), + Name: aws.String("efi-verify-stopped-image"), + } + reqData, _ := json.Marshal(input) + reply, err := natsRequest(daemon.natsConn, "ec2.CreateImage", reqData, 5*time.Second) + require.NoError(t, err) + + var errResp map[string]any + require.NoError(t, json.Unmarshal(reply.Data, &errResp)) + assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, errResp["Code"], + "node-1 is not LastNode (node-2) — it must decline instead of creating a duplicate AMI/snapshot") +} + +// TestHandleEC2CreateImage_StoppedInstance_OwnerProceeds checks that the +// node recorded as LastNode is let through the ownership gate into the +// image service, rather than declining with NotFound. +func TestHandleEC2CreateImage_StoppedInstance_OwnerProceeds(t *testing.T) { + natsURL := sharedJSNATSURL + daemon := createFullTestDaemonWithJetStream(t, natsURL) + seedStoppedInstance(t, daemon, "i-stopped-owner", daemon.node) + + sub, err := daemon.natsConn.Subscribe("ec2.CreateImage", daemon.handleEC2CreateImage) + require.NoError(t, err) + defer sub.Unsubscribe() + + input := &ec2.CreateImageInput{ + InstanceId: aws.String("i-stopped-owner"), + Name: aws.String("efi-verify-stopped-image"), + } + reqData, _ := json.Marshal(input) + reply, err := natsRequest(daemon.natsConn, "ec2.CreateImage", reqData, 5*time.Second) + require.NoError(t, err) + + var errResp map[string]any + require.NoError(t, json.Unmarshal(reply.Data, &errResp)) + assert.NotEqual(t, awserrors.ErrorInvalidInstanceIDNotFound, errResp["Code"], + "node-1 is LastNode — it must be allowed to proceed into the image service") +} + +// TestHandleEC2CreateImage_RunningInstance_LastNodeIgnored checks the +// running-instance path stays governed solely by vmMgr's local-map filter — +// a stale VM.LastNode left over from a prior stop must not cause a decline. +func TestHandleEC2CreateImage_RunningInstance_LastNodeIgnored(t *testing.T) { + natsURL := sharedNATSURL + daemon := createFullTestDaemon(t, natsURL) + + daemon.vmMgr.Insert(&vm.VM{ + ID: "i-running-stale-lastnode", + Status: vm.StateRunning, + AccountID: testAccountID, + LastNode: "node-99", // stale value from a previous stop; must be ignored + Instance: &ec2.Instance{ + InstanceId: aws.String("i-running-stale-lastnode"), + ImageId: aws.String("ami-source"), + BlockDeviceMappings: []*ec2.InstanceBlockDeviceMapping{ + { + Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-running123")}, + }, + }, + }, + }) + + sub, err := daemon.natsConn.Subscribe("ec2.CreateImage", daemon.handleEC2CreateImage) + require.NoError(t, err) + defer sub.Unsubscribe() + + input := &ec2.CreateImageInput{ + InstanceId: aws.String("i-running-stale-lastnode"), + Name: aws.String("my-image"), + } + reqData, _ := json.Marshal(input) + reply, err := natsRequest(daemon.natsConn, "ec2.CreateImage", reqData, 5*time.Second) + require.NoError(t, err) + + var errResp map[string]any + require.NoError(t, json.Unmarshal(reply.Data, &errResp)) + assert.NotEqual(t, awserrors.ErrorInvalidInstanceIDNotFound, errResp["Code"], + "running instance found locally in vmMgr must proceed regardless of a stale VM.LastNode") +} + +// TestHandleEC2CreateImage_StoppedInstance_EmptyLastNodeElectsSelf checks a +// pre-LastNode legacy record: the requesting node self-elects via CAS +// instead of proceeding unconditionally or declining forever. +func TestHandleEC2CreateImage_StoppedInstance_EmptyLastNodeElectsSelf(t *testing.T) { + natsURL := sharedJSNATSURL + daemon := createFullTestDaemonWithJetStream(t, natsURL) + seedStoppedInstance(t, daemon, "i-stopped-legacy", "") // no LastNode, as an older build would leave it + + sub, err := daemon.natsConn.Subscribe("ec2.CreateImage", daemon.handleEC2CreateImage) + require.NoError(t, err) + defer sub.Unsubscribe() + + input := &ec2.CreateImageInput{ + InstanceId: aws.String("i-stopped-legacy"), + Name: aws.String("efi-verify-stopped-image"), + } + reqData, _ := json.Marshal(input) + reply, err := natsRequest(daemon.natsConn, "ec2.CreateImage", reqData, 5*time.Second) + require.NoError(t, err) + + var errResp map[string]any + require.NoError(t, json.Unmarshal(reply.Data, &errResp)) + assert.NotEqual(t, awserrors.ErrorInvalidInstanceIDNotFound, errResp["Code"], + "the requesting node must self-elect and proceed rather than decline forever") + + updated, err := daemon.jsManager.LoadStoppedInstance("i-stopped-legacy") + require.NoError(t, err) + require.NotNil(t, updated) + assert.Equal(t, daemon.node, updated.LastNode, + "self-election must persist LastNode in the KV so later requests see a resolved owner") +} diff --git a/spinifex/daemon/daemon_handlers_test.go b/spinifex/daemon/daemon_handlers_test.go index 962fcdd5d..6d99cf7e4 100644 --- a/spinifex/daemon/daemon_handlers_test.go +++ b/spinifex/daemon/daemon_handlers_test.go @@ -18,6 +18,7 @@ import ( awss3 "github.com/aws/aws-sdk-go/service/s3" "github.com/mulgadc/spinifex/spinifex/awserrors" + "github.com/mulgadc/spinifex/spinifex/ebsmetadata" handlers_ec2_account "github.com/mulgadc/spinifex/spinifex/handlers/ec2/account" handlers_ec2_eigw "github.com/mulgadc/spinifex/spinifex/handlers/ec2/eigw" handlers_ec2_eip "github.com/mulgadc/spinifex/spinifex/handlers/ec2/eip" @@ -35,9 +36,9 @@ import ( "github.com/mulgadc/spinifex/spinifex/objectstore" "github.com/mulgadc/spinifex/spinifex/qmp" "github.com/mulgadc/spinifex/spinifex/testutil" + "github.com/mulgadc/spinifex/spinifex/testutil/ebsfake" "github.com/mulgadc/spinifex/spinifex/types" "github.com/mulgadc/spinifex/spinifex/vm" - "github.com/mulgadc/viperblock/viperblock" "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" @@ -69,6 +70,7 @@ func createFullTestDaemonWithStore(t *testing.T, natsURL string) (*Daemon, *obje daemon.volumeService = handlers_ec2_volume.NewVolumeServiceImplWithStore(cfg, memStore, daemon.natsConn) daemon.snapshotService = handlers_ec2_snapshot.NewSnapshotServiceImplWithStore(cfg, memStore, daemon.natsConn) daemon.tagsService = handlers_ec2_tags.NewTagsServiceImplWithStore(cfg, memStore) + wireTestEBSProvider(daemon, memStore) initAccountServiceForTest(t, daemon) // Wire RunInstances deps now that image/key services exist. vpcService and @@ -79,6 +81,25 @@ func createFullTestDaemonWithStore(t *testing.T, natsURL string) (*Daemon, *obje return daemon, memStore } +// wireTestEBSProvider mirrors configureEBSProvider for the test daemon: one +// in-memory provider behind every EBS-adjacent service. +func wireTestEBSProvider(daemon *Daemon, store objectstore.ObjectStore) { + provider := ebsfake.New(store, daemon.config.Predastore.Bucket) + daemon.ebsProvider = provider + daemon.instanceService.SetEBSProvider(provider) + daemon.imageService.SetEBSProvider(provider) + daemon.snapshotService.SetEBSProvider(provider) + daemon.volumeService.SetEBSProvider(provider) +} + +// seedVolumeDocument registers a volume the only way the control plane knows +// one: as an ebsmetadata document. Tests that need a volume to exist seed it +// here rather than writing storage state the control plane never reads. +func seedVolumeDocument(t *testing.T, store objectstore.ObjectStore, volume ebsmetadata.Volume) { + t.Helper() + require.NoError(t, ebsmetadata.NewStore(store, "test-bucket").PutVolume(t.Context(), volume)) +} + // createFullTestDaemon creates a test daemon with ALL services initialized (including // key, image, snapshot, tags, eigw, account) using in-memory object stores. func createFullTestDaemon(t *testing.T, natsURL string) *Daemon { @@ -1470,24 +1491,12 @@ func TestAttachVolume_ZoneMismatch(t *testing.T) { daemon.vmMgr.Insert(instance) // Create a volume in a different AZ - wrapper := struct { - VolumeConfig viperblock.VolumeConfig `json:"VolumeConfig"` - }{ - VolumeConfig: viperblock.VolumeConfig{ - VolumeMetadata: viperblock.VolumeMetadata{ - VolumeID: volumeID, - SizeGiB: 10, - State: "available", - AvailabilityZone: "us-west-2a", - TenantID: testAccountID, - }, - }, - } - data, _ := json.Marshal(wrapper) - store.PutObject(t.Context(), &awss3.PutObjectInput{ - Bucket: aws.String("test-bucket"), - Key: aws.String(volumeID + "/config.json"), - Body: strings.NewReader(string(data)), + seedVolumeDocument(t, store, ebsmetadata.Volume{ + VolumeID: volumeID, + CapacityGiB: 10, + State: "available", + AvailabilityZone: "us-west-2a", + TenantID: testAccountID, }) // Subscribe handler @@ -2671,23 +2680,11 @@ func TestHandleEC2ModifyVolume_Success(t *testing.T) { // Seed a volume in the store volumeID := "vol-modify-success" - wrapper := struct { - VolumeConfig viperblock.VolumeConfig `json:"VolumeConfig"` - }{ - VolumeConfig: viperblock.VolumeConfig{ - VolumeMetadata: viperblock.VolumeMetadata{ - VolumeID: volumeID, - SizeGiB: 10, - State: "available", - VolumeType: "gp3", - }, - }, - } - data, _ := json.Marshal(wrapper) - store.PutObject(t.Context(), &awss3.PutObjectInput{ - Bucket: aws.String("test-bucket"), - Key: aws.String(volumeID + "/config.json"), - Body: strings.NewReader(string(data)), + seedVolumeConfig(t, daemon, store, ebsmetadata.Volume{ + VolumeID: volumeID, + CapacityGiB: 10, + State: "available", + VolumeType: "gp3", }) // Subscribe a dummy ebs.sync handler so the NATS Request doesn't time out @@ -2810,22 +2807,10 @@ func TestHandleEC2CreateImage_RunningInstanceReachesService(t *testing.T) { sourceImageID := "ami-source-001" // Seed a root volume config - wrapper := struct { - VolumeConfig viperblock.VolumeConfig `json:"VolumeConfig"` - }{ - VolumeConfig: viperblock.VolumeConfig{ - VolumeMetadata: viperblock.VolumeMetadata{ - VolumeID: rootVolumeID, - SizeGiB: 8, - State: "in-use", - }, - }, - } - volData, _ := json.Marshal(wrapper) - store.PutObject(t.Context(), &awss3.PutObjectInput{ - Bucket: aws.String("test-bucket"), - Key: aws.String(rootVolumeID + "/config.json"), - Body: strings.NewReader(string(volData)), + seedVolumeDocument(t, store, ebsmetadata.Volume{ + VolumeID: rootVolumeID, + CapacityGiB: 8, + State: "in-use", }) daemon.vmMgr.Insert(&vm.VM{ @@ -3011,23 +2996,11 @@ func TestAttachVolume_VolumeInUse(t *testing.T) { daemon.vmMgr.Insert(instance) // Seed a volume that is already in-use - wrapper := struct { - VolumeConfig viperblock.VolumeConfig `json:"VolumeConfig"` - }{ - VolumeConfig: viperblock.VolumeConfig{ - VolumeMetadata: viperblock.VolumeMetadata{ - VolumeID: volumeID, - SizeGiB: 10, - State: "in-use", - TenantID: testAccountID, - }, - }, - } - data, _ := json.Marshal(wrapper) - store.PutObject(t.Context(), &awss3.PutObjectInput{ - Bucket: aws.String("test-bucket"), - Key: aws.String(volumeID + "/config.json"), - Body: strings.NewReader(string(data)), + seedVolumeDocument(t, store, ebsmetadata.Volume{ + VolumeID: volumeID, + CapacityGiB: 10, + State: "in-use", + TenantID: testAccountID, }) sub, err := daemon.natsConn.Subscribe( diff --git a/spinifex/daemon/daemon_handlers_volume.go b/spinifex/daemon/daemon_handlers_volume.go index 1800d9fba..4cda6dd8a 100644 --- a/spinifex/daemon/daemon_handlers_volume.go +++ b/spinifex/daemon/daemon_handlers_volume.go @@ -42,31 +42,31 @@ func (d *Daemon) handleAttachVolume(ctx context.Context, msg *nats.Msg, command return } - volCfg, err := d.volumeService.GetVolumeConfig(volumeID) + volMeta, err := d.volumeService.GetVolumeMetadata(volumeID) if err != nil { - slog.ErrorContext(ctx, "AttachVolume: failed to get volume config", "volumeId", volumeID, "err", err) + slog.ErrorContext(ctx, "AttachVolume: failed to get volume metadata", "volumeId", volumeID, "err", err) respondWithError(msg, awserrors.ErrorInvalidVolumeNotFound) return } callerAccountID := utils.AccountIDFromMsg(msg) - if !volumeVisibleTo(volCfg.VolumeMetadata.TenantID, callerAccountID) { + if !volumeVisibleTo(volMeta.TenantID, callerAccountID) { slog.WarnContext(ctx, "AttachVolume: account does not own volume", "volumeId", volumeID, "callerAccount", callerAccountID, - "ownerAccount", volCfg.VolumeMetadata.TenantID) + "ownerAccount", volMeta.TenantID) respondWithError(msg, awserrors.ErrorInvalidVolumeNotFound) return } - if volCfg.VolumeMetadata.State != "available" { - if volCfg.VolumeMetadata.AttachedInstance == command.ID { + if volMeta.State != "available" { + if volMeta.AttachedInstance == command.ID { if command.AttachVolumeData.Device != "" && - command.AttachVolumeData.Device != volCfg.VolumeMetadata.DeviceName { + command.AttachVolumeData.Device != volMeta.DeviceName { slog.ErrorContext(ctx, "AttachVolume: requested device conflicts with existing attachment", "volumeId", volumeID, "instanceId", command.ID, "requestedDevice", command.AttachVolumeData.Device, - "attachedDevice", volCfg.VolumeMetadata.DeviceName) + "attachedDevice", volMeta.DeviceName) // AWS returns VolumeInUse (not InvalidParameterValue) when a // re-attach targets a device other than the one already in use. respondWithError(msg, awserrors.ErrorVolumeInUse) @@ -77,22 +77,22 @@ func (d *Daemon) handleAttachVolume(ctx context.Context, msg *nats.Msg, command // ControllerPublishVolume retry after a slow first attach). // Treat as an idempotent success instead of VolumeInUse. slog.InfoContext(ctx, "AttachVolume: volume already attached to requesting instance, returning idempotent success", - "volumeId", volumeID, "instanceId", command.ID, "device", volCfg.VolumeMetadata.DeviceName) - d.respondWithVolumeAttachment(msg, volumeID, command.ID, volCfg.VolumeMetadata.DeviceName, "attached") + "volumeId", volumeID, "instanceId", command.ID, "device", volMeta.DeviceName) + d.respondWithVolumeAttachment(msg, volumeID, command.ID, volMeta.DeviceName, "attached") return } slog.ErrorContext(ctx, "AttachVolume: volume not available", - "volumeId", volumeID, "state", volCfg.VolumeMetadata.State) + "volumeId", volumeID, "state", volMeta.State) respondWithError(msg, awserrors.ErrorVolumeInUse) return } - if volCfg.VolumeMetadata.AvailabilityZone != "" && d.config.AZ != "" && - volCfg.VolumeMetadata.AvailabilityZone != d.config.AZ { + if volMeta.AvailabilityZone != "" && d.config.AZ != "" && + volMeta.AvailabilityZone != d.config.AZ { slog.ErrorContext(ctx, "AttachVolume: volume and instance are in different AZs", "volumeId", volumeID, - "volumeAZ", volCfg.VolumeMetadata.AvailabilityZone, + "volumeAZ", volMeta.AvailabilityZone, "instanceAZ", d.config.AZ) respondWithError(msg, awserrors.ErrorInvalidVolumeZoneMismatch) return diff --git a/spinifex/daemon/daemon_handlers_volume_test.go b/spinifex/daemon/daemon_handlers_volume_test.go index 0488c0a9f..9512c204f 100644 --- a/spinifex/daemon/daemon_handlers_volume_test.go +++ b/spinifex/daemon/daemon_handlers_volume_test.go @@ -8,18 +8,17 @@ import ( "testing" "time" - "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" - awss3 "github.com/aws/aws-sdk-go/service/s3" "github.com/mulgadc/spinifex/spinifex/awserrors" + "github.com/mulgadc/spinifex/spinifex/ebsmetadata" + "github.com/mulgadc/spinifex/spinifex/ebsprovider" "github.com/mulgadc/spinifex/spinifex/objectstore" "github.com/mulgadc/spinifex/spinifex/qmp" "github.com/mulgadc/spinifex/spinifex/testutil" "github.com/mulgadc/spinifex/spinifex/types" "github.com/mulgadc/spinifex/spinifex/utils" "github.com/mulgadc/spinifex/spinifex/vm" - "github.com/mulgadc/viperblock/viperblock" "github.com/nats-io/nats.go" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -103,24 +102,12 @@ func TestAttachDetachErrorCode(t *testing.T) { } } -// seedVolumeConfig writes a minimal VolumeConfig to config.json, matching -// the seeding pattern used by the handleAttachVolume tests in -// daemon_handlers_test.go. -func seedVolumeConfig(t *testing.T, store *objectstore.MemoryObjectStore, volumeID string, meta viperblock.VolumeMetadata) { +// seedVolumeConfig makes a volume exist on both sides at once: the provider +// holds the blocks, the ebsmetadata document is what the control plane reads. +func seedVolumeConfig(t *testing.T, daemon *Daemon, store *objectstore.MemoryObjectStore, volume ebsmetadata.Volume) { t.Helper() - wrapper := struct { - VolumeConfig viperblock.VolumeConfig `json:"VolumeConfig"` - }{ - VolumeConfig: viperblock.VolumeConfig{VolumeMetadata: meta}, - } - data, err := json.Marshal(wrapper) - require.NoError(t, err) - _, err = store.PutObject(t.Context(), &awss3.PutObjectInput{ - Bucket: aws.String("test-bucket"), - Key: aws.String(volumeID + "/config.json"), - Body: strings.NewReader(string(data)), - }) - require.NoError(t, err) + seedProviderVolume(t, daemon, volume.VolumeID, utils.SafeUint64ToInt64(volume.CapacityGiB)) + seedVolumeDocument(t, store, volume) } // TestAttachVolume_IdempotentSameInstance verifies that re-attaching a @@ -156,9 +143,9 @@ func TestAttachVolume_IdempotentSameInstance(t *testing.T) { } daemon.vmMgr.Insert(instance) - seedVolumeConfig(t, store, volumeID, viperblock.VolumeMetadata{ + seedVolumeConfig(t, daemon, store, ebsmetadata.Volume{ VolumeID: volumeID, - SizeGiB: 10, + CapacityGiB: 10, State: "in-use", TenantID: testAccountID, AttachedInstance: instanceID, @@ -224,9 +211,9 @@ func TestAttachVolume_InUseDifferentInstance(t *testing.T) { } daemon.vmMgr.Insert(instance) - seedVolumeConfig(t, store, volumeID, viperblock.VolumeMetadata{ + seedVolumeConfig(t, daemon, store, ebsmetadata.Volume{ VolumeID: volumeID, - SizeGiB: 10, + CapacityGiB: 10, State: "in-use", TenantID: testAccountID, AttachedInstance: otherInstanceID, @@ -281,9 +268,9 @@ func TestAttachVolume_IdempotentSameInstance_DeviceMismatch(t *testing.T) { } daemon.vmMgr.Insert(instance) - seedVolumeConfig(t, store, volumeID, viperblock.VolumeMetadata{ + seedVolumeConfig(t, daemon, store, ebsmetadata.Volume{ VolumeID: volumeID, - SizeGiB: 10, + CapacityGiB: 10, State: "in-use", TenantID: testAccountID, AttachedInstance: instanceID, @@ -569,3 +556,17 @@ func TestDrainVolume_DispatchedGoroutineDoesNotLeak(t *testing.T) { goleak.VerifyNone(t, ignoreExisting) } + +// seedProviderVolume allocates volumeID in the daemon's EBS provider so +// operations that reach the provider (expand, delete) find it there. +func seedProviderVolume(t *testing.T, daemon *Daemon, volumeID string, sizeGiB int64) { + t.Helper() + if daemon == nil || daemon.ebsProvider == nil || volumeID == "" || sizeGiB <= 0 { + return + } + _, err := daemon.ebsProvider.CreateVolume(t.Context(), ebsprovider.CreateVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, + CapacityRange: ebsprovider.CapacityRange{RequiredBytes: sizeGiB * 1024 * 1024 * 1024}, + }) + require.NoError(t, err) +} diff --git a/spinifex/daemon/daemon_test.go b/spinifex/daemon/daemon_test.go index a59cc0622..3af83a27b 100644 --- a/spinifex/daemon/daemon_test.go +++ b/spinifex/daemon/daemon_test.go @@ -28,9 +28,9 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" - awss3 "github.com/aws/aws-sdk-go/service/s3" "github.com/mulgadc/spinifex/spinifex/awserrors" "github.com/mulgadc/spinifex/spinifex/config" + "github.com/mulgadc/spinifex/spinifex/ebsmetadata" "github.com/mulgadc/spinifex/spinifex/gpu" handlers_ec2_account "github.com/mulgadc/spinifex/spinifex/handlers/ec2/account" handlers_ec2_eigw "github.com/mulgadc/spinifex/spinifex/handlers/ec2/eigw" @@ -52,7 +52,6 @@ import ( "github.com/mulgadc/spinifex/spinifex/types" "github.com/mulgadc/spinifex/spinifex/utils" "github.com/mulgadc/spinifex/spinifex/vm" - "github.com/mulgadc/viperblock/viperblock" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/assert" @@ -171,18 +170,15 @@ func getTestInstanceType(t *testing.T) string { return "t3.micro" // Default fallback } -// seedTestAMI creates a minimal AMI config in the memory store so that +// seedTestAMI creates a minimal AMI document in the memory store so that // handleEC2RunInstances AMI validation passes. func seedTestAMI(t *testing.T, store *objectstore.MemoryObjectStore, bucket, imageID string) { t.Helper() - amiConfig := `{"VolumeConfig":{"AMIMetadata":{"ImageID":"` + imageID + `","Name":"test"}}}` - _, err := store.PutObject(t.Context(), &awss3.PutObjectInput{ - Bucket: aws.String(bucket), - Key: aws.String(imageID + "/config.json"), - Body: strings.NewReader(amiConfig), - ContentType: aws.String("application/json"), - }) - require.NoError(t, err) + require.NoError(t, ebsmetadata.NewStore(store, bucket).PutAMI(t.Context(), ebsmetadata.AMI{ + ImageID: imageID, + Name: "test", + State: "available", + })) } // TestResourceManager tests resource manager functionality. @@ -1573,12 +1569,13 @@ func TestInstanceCleanerAdapter_DeleteVolumes_BootVolumeDeletedAfterAttachmentCl }) require.NoError(t, err) daemon.volumeService = handlers_ec2_volume.NewVolumeServiceImplWithStore(daemon.config, store, daemon.natsConn, snapKV) + daemon.volumeService.SetEBSProvider(daemon.ebsProvider) volumeID := "vol-root-attached" - seedVolumeConfig(t, store, volumeID, viperblock.VolumeMetadata{ + seedVolumeConfig(t, daemon, store, ebsmetadata.Volume{ VolumeID: volumeID, TenantID: testAccountID, - SizeGiB: 10, + CapacityGiB: 10, State: "available", AttachedInstance: "i-test-boot-delete", // stale: never cleared by Stop/shutdownAndUnmount's Boot carve-out }) @@ -1597,7 +1594,7 @@ func TestInstanceCleanerAdapter_DeleteVolumes_BootVolumeDeletedAfterAttachmentCl err = cleaner.DeleteVolumes(instance) require.NoError(t, err, "the still-attached boot volume must be deleted, not rejected as VolumeInUse") - _, err = daemon.volumeService.GetVolumeConfig(volumeID) + _, err = daemon.volumeService.GetVolumeMetadata(volumeID) require.Error(t, err, "the volume must actually be deleted") assert.Contains(t, err.Error(), awserrors.ErrorInvalidVolumeNotFound) } @@ -1613,10 +1610,10 @@ func TestInstanceCleanerAdapter_DeleteVolumes_NonDoTBootVolumeDetachedNotDeleted daemon, store := createFullTestDaemonWithStore(t, sharedNATSURL) volumeID := "vol-root-nondot-attached" - seedVolumeConfig(t, store, volumeID, viperblock.VolumeMetadata{ + seedVolumeConfig(t, daemon, store, ebsmetadata.Volume{ VolumeID: volumeID, TenantID: testAccountID, - SizeGiB: 10, + CapacityGiB: 10, State: "available", AttachedInstance: "i-test-boot-detach", // stale: never cleared by Unmount's Boot carve-out }) @@ -1635,10 +1632,10 @@ func TestInstanceCleanerAdapter_DeleteVolumes_NonDoTBootVolumeDetachedNotDeleted err := cleaner.DeleteVolumes(instance) require.NoError(t, err) - cfg, err := daemon.volumeService.GetVolumeConfig(volumeID) + meta, err := daemon.volumeService.GetVolumeMetadata(volumeID) require.NoError(t, err, "a DeleteOnTermination=false volume must survive terminate") - assert.Equal(t, "available", cfg.VolumeMetadata.State) - assert.Empty(t, cfg.VolumeMetadata.AttachedInstance, "terminate must still detach it") + assert.Equal(t, "available", meta.State) + assert.Empty(t, meta.AttachedInstance, "terminate must still detach it") } // TestInstanceCleanerAdapter_DeleteVolumes_NonBootNonDoTVolumeDetachedNotDeleted @@ -1653,10 +1650,10 @@ func TestInstanceCleanerAdapter_DeleteVolumes_NonBootNonDoTVolumeDetachedNotDele daemon, store := createFullTestDaemonWithStore(t, sharedNATSURL) volumeID := "vol-data-nondot-attached" - seedVolumeConfig(t, store, volumeID, viperblock.VolumeMetadata{ + seedVolumeConfig(t, daemon, store, ebsmetadata.Volume{ VolumeID: volumeID, TenantID: testAccountID, - SizeGiB: 10, + CapacityGiB: 10, State: "in-use", AttachedInstance: "i-test-data-detach", // stale: Unmount's seal failed or never ran }) @@ -1675,10 +1672,10 @@ func TestInstanceCleanerAdapter_DeleteVolumes_NonBootNonDoTVolumeDetachedNotDele err := cleaner.DeleteVolumes(instance) require.NoError(t, err) - cfg, err := daemon.volumeService.GetVolumeConfig(volumeID) + meta, err := daemon.volumeService.GetVolumeMetadata(volumeID) require.NoError(t, err, "a DeleteOnTermination=false volume must survive terminate") - assert.Equal(t, "available", cfg.VolumeMetadata.State) - assert.Empty(t, cfg.VolumeMetadata.AttachedInstance, "terminate must still detach it") + assert.Equal(t, "available", meta.State) + assert.Empty(t, meta.AttachedInstance, "terminate must still detach it") } // TestTerminatedTeardownReaper_SelfHealsFailedVolumeTeardown proves the @@ -1707,13 +1704,14 @@ func TestTerminatedTeardownReaper_SelfHealsFailedVolumeTeardown(t *testing.T) { }) require.NoError(t, err) daemon.volumeService = handlers_ec2_volume.NewVolumeServiceImplWithStore(daemon.config, store, daemon.natsConn, snapKV) + daemon.volumeService.SetEBSProvider(daemon.ebsProvider) volumeID := "vol-root-self-heal" instanceID := "i-self-heal" - seedVolumeConfig(t, store, volumeID, viperblock.VolumeMetadata{ + seedVolumeConfig(t, daemon, store, ebsmetadata.Volume{ VolumeID: volumeID, TenantID: testAccountID, - SizeGiB: 10, + CapacityGiB: 10, State: "available", AttachedInstance: instanceID, // stale, left by the earlier failed terminate }) @@ -1744,7 +1742,7 @@ func TestTerminatedTeardownReaper_SelfHealsFailedVolumeTeardown(t *testing.T) { _, err = reaper.Sweep(context.Background()) require.NoError(t, err) - _, err = daemon.volumeService.GetVolumeConfig(volumeID) + _, err = daemon.volumeService.GetVolumeMetadata(volumeID) require.Error(t, err, "the retried delete must actually remove the volume") assert.Contains(t, err.Error(), awserrors.ErrorInvalidVolumeNotFound) @@ -1771,10 +1769,10 @@ func TestTerminatedTeardownReaper_SelfHealsFailedVolumeDetach(t *testing.T) { volumeID := "vol-data-self-heal" instanceID := "i-self-heal-detach" - seedVolumeConfig(t, store, volumeID, viperblock.VolumeMetadata{ + seedVolumeConfig(t, daemon, store, ebsmetadata.Volume{ VolumeID: volumeID, TenantID: testAccountID, - SizeGiB: 10, + CapacityGiB: 10, State: "in-use", AttachedInstance: instanceID, // stale, left by the earlier failed terminate }) @@ -1805,10 +1803,10 @@ func TestTerminatedTeardownReaper_SelfHealsFailedVolumeDetach(t *testing.T) { _, err := reaper.Sweep(context.Background()) require.NoError(t, err) - cfg, err := daemon.volumeService.GetVolumeConfig(volumeID) + meta, err := daemon.volumeService.GetVolumeMetadata(volumeID) require.NoError(t, err, "the volume must survive: DeleteOnTermination=false") - assert.Equal(t, "available", cfg.VolumeMetadata.State) - assert.Empty(t, cfg.VolumeMetadata.AttachedInstance, "the retry must clear the stale attachment") + assert.Equal(t, "available", meta.State) + assert.Empty(t, meta.AttachedInstance, "the retry must clear the stale attachment") remaining, err := fakeStore.ListTerminatedInstances() require.NoError(t, err) @@ -1832,10 +1830,10 @@ func TestStuckTerminateReaper_DetachesNonDoTVolumeWithoutUnmount(t *testing.T) { const instanceID = "i-wedged-terminate" volumeID := "vol-data-wedged" - seedVolumeConfig(t, store, volumeID, viperblock.VolumeMetadata{ + seedVolumeConfig(t, daemon, store, ebsmetadata.Volume{ VolumeID: volumeID, TenantID: testAccountID, - SizeGiB: 10, + CapacityGiB: 10, State: "in-use", AttachedInstance: instanceID, // never cleared: Unmount never ran on this path }) @@ -1863,11 +1861,11 @@ func TestStuckTerminateReaper_DetachesNonDoTVolumeWithoutUnmount(t *testing.T) { require.NoError(t, err) assert.Equal(t, 1, reaped, "the wedged terminate must be force-completed") - cfg, err := daemon.volumeService.GetVolumeConfig(volumeID) + meta, err := daemon.volumeService.GetVolumeMetadata(volumeID) require.NoError(t, err) - assert.Equal(t, "available", cfg.VolumeMetadata.State, + assert.Equal(t, "available", meta.State, "the force-complete path must still detach a non-Boot DeleteOnTermination=false volume") - assert.Empty(t, cfg.VolumeMetadata.AttachedInstance) + assert.Empty(t, meta.AttachedInstance) } // TestHandleEC2Events_AttachVolume tests the attach-volume handler in handleEC2Events. @@ -4501,3 +4499,61 @@ func TestAssertNoClusterServicesInitialised_PerField(t *testing.T) { }) } } + +// --- configureEBSProvider --- + +// newEBSProviderTestDaemon builds a daemon with real (constructor-built, not +// zero-valued) EBS services, so configureEBSProvider has something to wire the +// provider into rather than a set of zero values that would accept anything. +func newEBSProviderTestDaemon(t *testing.T, provider string) *Daemon { + t.Helper() + nc, err := nats.Connect(sharedJSNATSURL) + require.NoError(t, err) + t.Cleanup(nc.Close) + + jsManager, err := NewJetStreamManager(nc, 1) + require.NoError(t, err) + + ctx := context.Background() + cfg := &config.Config{ + EBS: config.EBSConfig{Provider: provider}, + Predastore: config.PredastoreConfig{Bucket: "test-bucket"}, + } + store := objectstore.NewMemoryObjectStore() + + return &Daemon{ + ctx: ctx, + config: cfg, + natsConn: nc, + jsManager: jsManager, + instanceService: &handlers_ec2_instance.InstanceServiceImpl{}, + imageService: handlers_ec2_image.NewImageServiceImplWithStore(store, cfg.Predastore.Bucket), + snapshotService: handlers_ec2_snapshot.NewSnapshotServiceImplWithStore(cfg, store, nc), + volumeService: handlers_ec2_volume.NewVolumeServiceImplWithStore(cfg, store, nc), + } +} + +// An unset provider must still wire viperblockd everywhere: there is no +// second provider to fall back to, so leaving it nil would silently strand +// every EBS call on a node whose spinifex.toml predates the key. +func TestConfigureEBSProvider_UnsetInjectsProviderEverywhere(t *testing.T) { + d := newEBSProviderTestDaemon(t, "") + require.NoError(t, d.configureEBSProvider()) + + require.NotNil(t, d.ebsProvider) + assert.Same(t, d.ebsProvider, d.instanceService.EBSProvider()) + assert.Same(t, d.ebsProvider, d.imageService.EBSProvider()) + assert.Same(t, d.ebsProvider, d.snapshotService.EBSProvider()) + assert.Same(t, d.ebsProvider, d.volumeService.EBSProvider()) +} + +func TestConfigureEBSProvider_ViperblockdInjectsSameInstanceEverywhere(t *testing.T) { + d := newEBSProviderTestDaemon(t, config.EBSProviderViperblockd) + require.NoError(t, d.configureEBSProvider()) + + require.NotNil(t, d.ebsProvider, "daemon must retain the provider it constructed") + assert.Same(t, d.ebsProvider, d.instanceService.EBSProvider()) + assert.Same(t, d.ebsProvider, d.imageService.EBSProvider()) + assert.Same(t, d.ebsProvider, d.snapshotService.EBSProvider()) + assert.Same(t, d.ebsProvider, d.volumeService.EBSProvider()) +} diff --git a/spinifex/daemon/eni_orphan_reaper.go b/spinifex/daemon/eni_orphan_reaper.go new file mode 100644 index 000000000..2b6643100 --- /dev/null +++ b/spinifex/daemon/eni_orphan_reaper.go @@ -0,0 +1,80 @@ +package daemon + +import ( + "context" + "log/slog" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mulgadc/spinifex/spinifex/awserrors" + handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc" + "github.com/mulgadc/spinifex/spinifex/vm" +) + +// eniOrphanMinAge is how long an unattached instance ENI must sit before the +// sweep will delete it. The launch RPC gives a node five minutes to answer, so +// this leaves a wide margin past the point where a launch could still attach. +const eniOrphanMinAge = 15 * time.Minute + +// eniOrphanReaper deletes ENI records left behind by a launch that created the +// interface and then died before attaching it. The record outlives the launch, +// and because the security group dependency check counts an ENI that lists the +// group, the SG can never be deleted afterwards. +// +// Cluster-wide rather than node-local: the record has no instance, so no node +// owns it, and nothing local would ever look at it again. +type eniOrphanReaper struct { + vpc *handlers_ec2_vpc.VPCServiceImpl + minAge time.Duration +} + +var _ vm.Reaper = (*eniOrphanReaper)(nil) + +// newENIOrphanReaper builds the sweep over the daemon's VPC service, or nil +// when there is no control plane on this node to sweep with. +func (d *Daemon) newENIOrphanReaper() *eniOrphanReaper { + if d.vpcService == nil { + return nil + } + return &eniOrphanReaper{vpc: d.vpcService, minAge: eniOrphanMinAge} +} + +func (r *eniOrphanReaper) Class() string { return "eni-orphan" } +func (r *eniOrphanReaper) Scope() vm.ReaperScope { return vm.ScopeClusterWide } + +// Sweep deletes every abandoned instance ENI. A per-record failure is logged +// and skipped so one bad record cannot stall the rest of the sweep. +func (r *eniOrphanReaper) Sweep(ctx context.Context) (int, error) { + orphans, err := r.vpc.ListAbandonedInstanceENIs(ctx, r.minAge) + if err != nil { + return 0, err + } + + reaped := 0 + for _, orphan := range orphans { + if ctx.Err() != nil { + return reaped, ctx.Err() + } + eniID := orphan.Record.NetworkInterfaceId + slog.InfoContext(ctx, "eni-orphan: deleting ENI left by an abandoned launch", + "eniId", eniID, + "accountId", orphan.AccountID, + "description", orphan.Record.Description, + "securityGroups", orphan.Record.SecurityGroupIds, + "createdAt", orphan.Record.CreatedAt) + + _, err := r.vpc.DeleteNetworkInterface(ctx, &ec2.DeleteNetworkInterfaceInput{ + NetworkInterfaceId: aws.String(eniID), + }, orphan.AccountID) + // Another sweep may have deleted it between the list and here, which + // is the outcome this asked for either way. + if err != nil && err.Error() != awserrors.ErrorInvalidNetworkInterfaceIDNotFound { + slog.WarnContext(ctx, "eni-orphan: delete failed", + "eniId", eniID, "accountId", orphan.AccountID, "err", err) + continue + } + reaped++ + } + return reaped, nil +} diff --git a/spinifex/daemon/eni_orphan_reaper_test.go b/spinifex/daemon/eni_orphan_reaper_test.go new file mode 100644 index 000000000..50fa3fc89 --- /dev/null +++ b/spinifex/daemon/eni_orphan_reaper_test.go @@ -0,0 +1,68 @@ +package daemon + +import ( + "context" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc" + "github.com/mulgadc/spinifex/spinifex/testutil" + "github.com/mulgadc/spinifex/spinifex/vm" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const reaperTestAccountID = "123456789012" + +// TestENIOrphanReaperScopeIsClusterWide pins the scope. The record it reaps has +// no instance, so no node owns it; running the sweep node-locally would have +// every node racing to delete the same records. +func TestENIOrphanReaperScopeIsClusterWide(t *testing.T) { + r := &eniOrphanReaper{} + assert.Equal(t, vm.ScopeClusterWide, r.Scope()) + assert.Equal(t, "eni-orphan", r.Class()) +} + +func TestENIOrphanReaperSweep(t *testing.T) { + _, nc, _ := testutil.StartTestJetStream(t) + vpcSvc, err := handlers_ec2_vpc.NewVPCServiceImplWithNATS(t.Context(), nil, nc) + require.NoError(t, err) + testutil.StubVpcdSGResponder(t, nc) + + ctx := context.Background() + vpcOut, err := vpcSvc.CreateVpc(ctx, &ec2.CreateVpcInput{ + CidrBlock: aws.String("10.0.0.0/16"), + }, reaperTestAccountID) + require.NoError(t, err) + subnetOut, err := vpcSvc.CreateSubnet(ctx, &ec2.CreateSubnetInput{ + VpcId: vpcOut.Vpc.VpcId, + CidrBlock: aws.String("10.0.1.0/24"), + }, reaperTestAccountID) + require.NoError(t, err) + + eniOut, err := vpcSvc.CreateNetworkInterface(ctx, &ec2.CreateNetworkInterfaceInput{ + SubnetId: subnetOut.Subnet.SubnetId, + Description: aws.String(handlers_ec2_vpc.AutoENIDescriptionPrefix + "i-abandoned"), + }, reaperTestAccountID) + require.NoError(t, err) + eniID := *eniOut.NetworkInterface.NetworkInterfaceId + require.NoError(t, vpcSvc.UpdateENI(reaperTestAccountID, eniID, func(rec *handlers_ec2_vpc.ENIRecord) { + rec.CreatedAt = time.Now().Add(-time.Hour) + })) + + r := &eniOrphanReaper{vpc: vpcSvc, minAge: 15 * time.Minute} + + reaped, err := r.Sweep(ctx) + require.NoError(t, err) + assert.Equal(t, 1, reaped) + + _, err = vpcSvc.GetENIRecord(reaperTestAccountID, eniID) + require.Error(t, err, "the record must be gone, or it keeps blocking its security group") + + // A second pass has nothing left and must not report work or fail. + reaped, err = r.Sweep(ctx) + require.NoError(t, err) + assert.Zero(t, reaped) +} diff --git a/spinifex/ebsmetadata/metadata.go b/spinifex/ebsmetadata/metadata.go new file mode 100644 index 000000000..60faf23dc --- /dev/null +++ b/spinifex/ebsmetadata/metadata.go @@ -0,0 +1,125 @@ +// Package ebsmetadata owns Spinifex's control-plane view of EBS resources. +// These documents deliberately do not mirror viperblock's VBState schema. +package ebsmetadata + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + "time" +) + +const SchemaVersion uint16 = 1 + +// ErrCorruptDocument wraps decode and schema-version failures so callers can +// tell a document that exists but cannot be read from one that is absent. The +// two deserve different answers: salvage versus not-found. +var ErrCorruptDocument = errors.New("corrupt EBS metadata document") + +// Volume is the control-plane record used for EC2 Describe/Modify/attachment +// operations. ProviderHandle is opaque and is never decoded by the API layer. +type Volume struct { + SchemaVersion uint16 `json:"schema_version"` + VolumeID string `json:"volume_id"` + VolumeName string `json:"volume_name,omitempty"` + TenantID string `json:"tenant_id"` + CapacityGiB uint64 `json:"capacity_gib"` + State string `json:"state"` + CreatedAt time.Time `json:"created_at"` + AttachedAt time.Time `json:"attached_at,omitzero"` + AvailabilityZone string `json:"availability_zone"` + AttachedInstance string `json:"attached_instance,omitempty"` + DeviceName string `json:"device_name,omitempty"` + VolumeType string `json:"volume_type"` + IOPS int `json:"iops"` + Throughput int `json:"throughput,omitempty"` + Tags map[string]string `json:"tags,omitempty"` + SnapshotID string `json:"snapshot_id,omitempty"` + DeleteOnTermination bool `json:"delete_on_termination,omitempty"` + Encrypted bool `json:"encrypted,omitempty"` + ProviderHandle string `json:"provider_handle,omitempty"` + Modification *VolumeModification `json:"modification,omitempty"` +} + +// VolumeModification is the control-plane record of a completed or in-flight +// ModifyVolume operation, read back by DescribeVolumesModifications. It owns +// its own fields rather than reusing viperblock.VolumeModification: this +// package must stay free of viperblock types. VolumeID is deliberately not +// duplicated here — callers have it from the owning Volume. +type VolumeModification struct { + ModificationState string `json:"modification_state"` + Progress int64 `json:"progress"` + StatusMessage string `json:"status_message,omitempty"` + OriginalSize int64 `json:"original_size"` + OriginalIOPS int64 `json:"original_iops"` + OriginalVolumeType string `json:"original_volume_type"` + TargetSize int64 `json:"target_size"` + TargetIOPS int64 `json:"target_iops"` + TargetVolumeType string `json:"target_volume_type"` + StartTime time.Time `json:"start_time"` + EndTime time.Time `json:"end_time,omitzero"` +} + +// AMI is the control-plane record used for EC2 image operations. +type AMI struct { + SchemaVersion uint16 `json:"schema_version"` + ImageID string `json:"image_id"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + Architecture string `json:"architecture"` + PlatformDetails string `json:"platform_details"` + CreationDate time.Time `json:"creation_date"` + RootDeviceType string `json:"root_device_type"` + Virtualization string `json:"virtualization"` + ImageOwnerAlias string `json:"image_owner_alias"` + VolumeSizeGiB uint64 `json:"volume_size_gib"` + SnapshotID string `json:"snapshot_id,omitempty"` + BootMode string `json:"boot_mode,omitempty"` + Distro string `json:"distro,omitempty"` + DistroFamily string `json:"distro_family,omitempty"` + Tags map[string]string `json:"tags,omitempty"` + State string `json:"state,omitempty"` +} + +func VolumeKey(volumeID string) (string, error) { return key("volumes", volumeID) } +func AMIKey(imageID string) (string, error) { return key("amis", imageID) } + +func key(kind, id string) (string, error) { + if id == "" || id == "." || id == ".." || strings.ContainsAny(id, "/\\") { + return "", fmt.Errorf("invalid EBS metadata ID %q", id) + } + return "spinifex/ebsmetadata/v1/" + kind + "/" + id + ".json", nil +} + +func MarshalVolume(volume Volume) ([]byte, error) { + volume.SchemaVersion = SchemaVersion + return json.Marshal(volume) +} + +func UnmarshalVolume(data []byte) (Volume, error) { + var volume Volume + if err := json.Unmarshal(data, &volume); err != nil { + return Volume{}, fmt.Errorf("%w: %w", ErrCorruptDocument, err) + } + if volume.SchemaVersion != SchemaVersion { + return Volume{}, fmt.Errorf("%w: unsupported volume metadata schema version %d", ErrCorruptDocument, volume.SchemaVersion) + } + return volume, nil +} + +func MarshalAMI(ami AMI) ([]byte, error) { + ami.SchemaVersion = SchemaVersion + return json.Marshal(ami) +} + +func UnmarshalAMI(data []byte) (AMI, error) { + var ami AMI + if err := json.Unmarshal(data, &ami); err != nil { + return AMI{}, fmt.Errorf("%w: %w", ErrCorruptDocument, err) + } + if ami.SchemaVersion != SchemaVersion { + return AMI{}, fmt.Errorf("%w: unsupported AMI metadata schema version %d", ErrCorruptDocument, ami.SchemaVersion) + } + return ami, nil +} diff --git a/spinifex/ebsmetadata/metadata_test.go b/spinifex/ebsmetadata/metadata_test.go new file mode 100644 index 000000000..d2191ccb1 --- /dev/null +++ b/spinifex/ebsmetadata/metadata_test.go @@ -0,0 +1,66 @@ +package ebsmetadata + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestVolumeMetadataRoundTripOwnsItsSchema(t *testing.T) { + want := Volume{ + VolumeID: "vol-1", TenantID: "acct-1", CapacityGiB: 20, + State: "available", CreatedAt: time.Date(2026, 8, 5, 0, 0, 0, 0, time.UTC), + ProviderHandle: "opaque-provider-handle", Tags: map[string]string{"env": "test"}, + } + data, err := MarshalVolume(want) + require.NoError(t, err) + got, err := UnmarshalVolume(data) + require.NoError(t, err) + assert.Equal(t, SchemaVersion, got.SchemaVersion) + assert.Equal(t, want.VolumeID, got.VolumeID) + assert.Equal(t, want.ProviderHandle, got.ProviderHandle) + assert.Equal(t, want.Tags, got.Tags) +} + +// TestVolumeRoundTripsEncryptedAndModification covers the two fields added +// to close the schema gaps behind the provider-branch consumer bugs: a +// volume's encryption bit and its persisted modification record must both +// survive a marshal/unmarshal round-trip. +func TestVolumeRoundTripsEncryptedAndModification(t *testing.T) { + want := Volume{ + VolumeID: "vol-1", TenantID: "acct-1", CapacityGiB: 20, + State: "available", Encrypted: true, + Modification: &VolumeModification{ + ModificationState: "completed", Progress: 100, + OriginalSize: 8, OriginalIOPS: 3000, OriginalVolumeType: "gp3", + TargetSize: 16, TargetIOPS: 3000, TargetVolumeType: "gp3", + StartTime: time.Date(2026, 8, 5, 0, 0, 0, 0, time.UTC), + EndTime: time.Date(2026, 8, 5, 0, 0, 1, 0, time.UTC), + }, + } + data, err := MarshalVolume(want) + require.NoError(t, err) + got, err := UnmarshalVolume(data) + require.NoError(t, err) + assert.True(t, got.Encrypted) + require.NotNil(t, got.Modification) + assert.Equal(t, *want.Modification, *got.Modification) +} + +func TestMetadataRejectsUnknownSchemaVersion(t *testing.T) { + _, err := UnmarshalAMI([]byte(`{"schema_version":99,"image_id":"ami-1"}`)) + require.Error(t, err) + assert.Contains(t, err.Error(), "unsupported") +} + +func TestMetadataKeysRejectPathTraversal(t *testing.T) { + for _, id := range []string{"", "..", "../escape", "a/b", "a\\b"} { + _, err := VolumeKey(id) + require.Error(t, err, "ID %q must not escape metadata prefix", id) + } + key, err := AMIKey("ami-1") + require.NoError(t, err) + assert.Equal(t, "spinifex/ebsmetadata/v1/amis/ami-1.json", key) +} diff --git a/spinifex/ebsmetadata/store.go b/spinifex/ebsmetadata/store.go new file mode 100644 index 000000000..2821382e1 --- /dev/null +++ b/spinifex/ebsmetadata/store.go @@ -0,0 +1,200 @@ +package ebsmetadata + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "log/slog" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/mulgadc/spinifex/spinifex/objectstore" +) + +// Store persists Spinifex-owned metadata documents in the existing object +// store. It intentionally knows nothing about provider state or VBState. +type Store struct { + objects objectstore.ObjectStore + bucket string +} + +func NewStore(objects objectstore.ObjectStore, bucket string) *Store { + return &Store{objects: objects, bucket: bucket} +} + +func (s *Store) PutVolume(ctx context.Context, volume Volume) error { + key, err := VolumeKey(volume.VolumeID) + if err != nil { + return err + } + data, err := MarshalVolume(volume) + if err != nil { + return fmt.Errorf("marshal volume metadata: %w", err) + } + return s.put(ctx, key, data) +} + +// GetVolume returns the volume's ebsmetadata document. A volume with no +// document does not exist as far as the control plane is concerned. +func (s *Store) GetVolume(ctx context.Context, volumeID string) (Volume, error) { + key, err := VolumeKey(volumeID) + if err != nil { + return Volume{}, err + } + data, err := s.get(ctx, key) + if err != nil { + return Volume{}, err + } + return UnmarshalVolume(data) +} + +func (s *Store) DeleteVolume(ctx context.Context, volumeID string) error { + key, err := VolumeKey(volumeID) + if err != nil { + return err + } + _, err = s.objects.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(key)}) + return err +} + +// ListVolumes returns every ebsmetadata volume document. +func (s *Store) ListVolumes(ctx context.Context) ([]Volume, error) { + if s == nil || s.objects == nil { + return nil, errors.New("metadata store is not configured") + } + return s.listVolumeDocuments(ctx) +} + +func (s *Store) listVolumeDocuments(ctx context.Context) ([]Volume, error) { + result, err := s.objects.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ + Bucket: aws.String(s.bucket), Prefix: aws.String("spinifex/ebsmetadata/v1/volumes/"), + }) + if err != nil { + return nil, err + } + volumes := make([]Volume, 0, len(result.Contents)) + for _, object := range result.Contents { + if object.Key == nil { + continue + } + data, err := s.get(ctx, *object.Key) + if err != nil { + return nil, err + } + volume, err := UnmarshalVolume(data) + if err != nil { + return nil, err + } + volumes = append(volumes, volume) + } + return volumes, nil +} + +// ListAMIs returns every AMI document, skipping any it cannot decode. One +// corrupt document must not make every image in the cluster invisible, so the +// tolerance is deliberate — the skip is logged rather than silent. +func (s *Store) ListAMIs(ctx context.Context) ([]AMI, error) { + if s == nil || s.objects == nil { + return nil, errors.New("metadata store is not configured") + } + return s.listAMIDocuments(ctx, true) +} + +// ListAMIsStrict is ListAMIs without the tolerance. It is for callers whose +// answer would be wrong rather than merely partial: a name-uniqueness check +// cannot rule out a name held by an AMI it failed to read. +func (s *Store) ListAMIsStrict(ctx context.Context) ([]AMI, error) { + if s == nil || s.objects == nil { + return nil, errors.New("metadata store is not configured") + } + return s.listAMIDocuments(ctx, false) +} + +func (s *Store) listAMIDocuments(ctx context.Context, skipCorrupt bool) ([]AMI, error) { + result, err := s.objects.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ + Bucket: aws.String(s.bucket), Prefix: aws.String("spinifex/ebsmetadata/v1/amis/"), + }) + if err != nil { + return nil, err + } + amis := make([]AMI, 0, len(result.Contents)) + for _, object := range result.Contents { + if object.Key == nil { + continue + } + data, err := s.get(ctx, *object.Key) + if err != nil { + return nil, err + } + ami, err := UnmarshalAMI(data) + if err != nil { + if skipCorrupt && errors.Is(err, ErrCorruptDocument) { + slog.WarnContext(ctx, "skipping undecodable AMI document", "key", *object.Key, "err", err) + continue + } + return nil, err + } + amis = append(amis, ami) + } + return amis, nil +} + +func (s *Store) PutAMI(ctx context.Context, ami AMI) error { + key, err := AMIKey(ami.ImageID) + if err != nil { + return err + } + data, err := MarshalAMI(ami) + if err != nil { + return fmt.Errorf("marshal AMI metadata: %w", err) + } + return s.put(ctx, key, data) +} + +// GetAMI returns the AMI's ebsmetadata document. An AMI with no document does +// not exist as far as the control plane is concerned. +func (s *Store) GetAMI(ctx context.Context, imageID string) (AMI, error) { + key, err := AMIKey(imageID) + if err != nil { + return AMI{}, err + } + data, err := s.get(ctx, key) + if err != nil { + return AMI{}, err + } + return UnmarshalAMI(data) +} + +func (s *Store) DeleteAMI(ctx context.Context, imageID string) error { + key, err := AMIKey(imageID) + if err != nil { + return err + } + _, err = s.objects.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(key)}) + return err +} + +func (s *Store) put(ctx context.Context, key string, data []byte) error { + if s == nil || s.objects == nil { + return errors.New("metadata store is not configured") + } + _, err := s.objects.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(s.bucket), Key: aws.String(key), Body: bytes.NewReader(data), + ContentType: aws.String("application/json"), + }) + return err +} + +func (s *Store) get(ctx context.Context, key string) ([]byte, error) { + if s == nil || s.objects == nil { + return nil, errors.New("metadata store is not configured") + } + result, err := s.objects.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(key)}) + if err != nil { + return nil, err + } + defer result.Body.Close() + return io.ReadAll(result.Body) +} diff --git a/spinifex/ebsmetadata/store_test.go b/spinifex/ebsmetadata/store_test.go new file mode 100644 index 000000000..47578d761 --- /dev/null +++ b/spinifex/ebsmetadata/store_test.go @@ -0,0 +1,225 @@ +package ebsmetadata + +import ( + "bytes" + "context" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/mulgadc/spinifex/spinifex/objectstore" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestStoreRoundTripsVolumeAndAMI(t *testing.T) { + objects := objectstore.NewMemoryObjectStore() + store := NewStore(objects, "control-plane") + + volume := Volume{VolumeID: "vol-1", TenantID: "acct-1", CapacityGiB: 10, State: "available", ProviderHandle: "opaque"} + require.NoError(t, store.PutVolume(context.Background(), volume)) + gotVolume, err := store.GetVolume(context.Background(), volume.VolumeID) + require.NoError(t, err) + assert.Equal(t, volume.VolumeID, gotVolume.VolumeID) + assert.Equal(t, volume.ProviderHandle, gotVolume.ProviderHandle) + assert.Equal(t, SchemaVersion, gotVolume.SchemaVersion) + + ami := AMI{ImageID: "ami-1", Name: "test", SnapshotID: "snap-1"} + require.NoError(t, store.PutAMI(context.Background(), ami)) + gotAMI, err := store.GetAMI(context.Background(), ami.ImageID) + require.NoError(t, err) + assert.Equal(t, ami.ImageID, gotAMI.ImageID) + assert.Equal(t, ami.SnapshotID, gotAMI.SnapshotID) + assert.Equal(t, SchemaVersion, gotAMI.SchemaVersion) + + require.NoError(t, store.DeleteVolume(context.Background(), volume.VolumeID)) + require.NoError(t, store.DeleteAMI(context.Background(), ami.ImageID)) +} + +func TestListVolumes_Empty(t *testing.T) { + store := NewStore(objectstore.NewMemoryObjectStore(), "control-plane") + volumes, err := store.ListVolumes(context.Background()) + require.NoError(t, err) + assert.Empty(t, volumes) +} + +func TestListVolumes_ReturnsAllStoredVolumes(t *testing.T) { + store := NewStore(objectstore.NewMemoryObjectStore(), "control-plane") + ctx := context.Background() + + want := []Volume{ + {VolumeID: "vol-1", TenantID: "acct-1", CapacityGiB: 8, State: "available"}, + {VolumeID: "vol-2", TenantID: "acct-2", CapacityGiB: 16, State: "in-use"}, + {VolumeID: "vol-3", TenantID: "acct-1", CapacityGiB: 32, State: "available"}, + } + for _, v := range want { + require.NoError(t, store.PutVolume(ctx, v)) + } + + got, err := store.ListVolumes(ctx) + require.NoError(t, err) + require.Len(t, got, len(want)) + + gotIDs := make(map[string]uint64, len(got)) + for _, v := range got { + gotIDs[v.VolumeID] = v.CapacityGiB + } + for _, v := range want { + assert.Equal(t, v.CapacityGiB, gotIDs[v.VolumeID]) + } +} + +// TestListVolumes_CorruptObjectReturnsError covers the UnmarshalVolume error +// path: an object under the volumes prefix that isn't a valid Volume record +// must fail the whole listing rather than being silently skipped. +func TestListVolumes_CorruptObjectReturnsError(t *testing.T) { + objects := objectstore.NewMemoryObjectStore() + store := NewStore(objects, "control-plane") + ctx := context.Background() + + require.NoError(t, store.PutVolume(ctx, Volume{VolumeID: "vol-good", CapacityGiB: 1})) + + key, err := VolumeKey("vol-corrupt") + require.NoError(t, err) + _, err = objects.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String("control-plane"), Key: aws.String(key), Body: bytes.NewReader([]byte("not json")), + }) + require.NoError(t, err) + + _, err = store.ListVolumes(ctx) + require.Error(t, err) +} + +// TestListVolumes_NotConfigured covers the nil-store guard shared by every +// Store method: a zero-value *Store (no ObjectStore wired up) must error +// instead of panicking on a nil dereference. +func TestListVolumes_NotConfigured(t *testing.T) { + var store *Store + _, err := store.ListVolumes(context.Background()) + require.Error(t, err) + + empty := &Store{} + _, err = empty.ListVolumes(context.Background()) + require.Error(t, err) +} + +// TestListAMIs_Empty mirrors TestListVolumes_Empty: an AMI-less store lists +// as an empty slice, not an error. +func TestListAMIs_Empty(t *testing.T) { + store := NewStore(objectstore.NewMemoryObjectStore(), "control-plane") + amis, err := store.ListAMIs(context.Background()) + require.NoError(t, err) + assert.Empty(t, amis) +} + +// TestListAMIs_ReturnsAllStoredAMIs mirrors TestListVolumes_ReturnsAllStoredVolumes. +func TestListAMIs_ReturnsAllStoredAMIs(t *testing.T) { + store := NewStore(objectstore.NewMemoryObjectStore(), "control-plane") + ctx := context.Background() + + want := []AMI{ + {ImageID: "ami-1", Name: "one", VolumeSizeGiB: 8}, + {ImageID: "ami-2", Name: "two", VolumeSizeGiB: 16}, + } + for _, a := range want { + require.NoError(t, store.PutAMI(ctx, a)) + } + + got, err := store.ListAMIs(ctx) + require.NoError(t, err) + require.Len(t, got, len(want)) + + gotSizes := make(map[string]uint64, len(got)) + for _, a := range got { + gotSizes[a.ImageID] = a.VolumeSizeGiB + } + for _, a := range want { + assert.Equal(t, a.VolumeSizeGiB, gotSizes[a.ImageID]) + } +} + +// TestListAMIs_NotConfigured mirrors TestListVolumes_NotConfigured. +func TestListAMIs_NotConfigured(t *testing.T) { + var store *Store + _, err := store.ListAMIs(context.Background()) + require.Error(t, err) + + empty := &Store{} + _, err = empty.ListAMIs(context.Background()) + require.Error(t, err) +} + +// --- Legacy fallback tests --- +// TestGetVolume_MissingDocumentIsNotFound locks the only answer a volume with +// no document gets: it does not exist as far as the control plane is +// concerned, reported as the object store's not-found rather than a zero value. +func TestGetVolume_MissingDocumentIsNotFound(t *testing.T) { + store := NewStore(objectstore.NewMemoryObjectStore(), "control-plane") + _, err := store.GetVolume(context.Background(), "vol-missing") + require.Error(t, err) + assert.True(t, objectstore.IsNoSuchKeyError(err)) +} + +// TestGetAMI_MissingDocumentIsNotFound mirrors TestGetVolume_MissingDocumentIsNotFound. +func TestGetAMI_MissingDocumentIsNotFound(t *testing.T) { + store := NewStore(objectstore.NewMemoryObjectStore(), "control-plane") + _, err := store.GetAMI(context.Background(), "ami-missing") + require.Error(t, err) + assert.True(t, objectstore.IsNoSuchKeyError(err)) +} + +// TestGet_CorruptDocumentIsDistinguishable is what lets the admin tooling tell +// salvage from not-found: an undecodable document must not read as absent, or +// a --force removal would refuse the one case it exists for. +func TestGet_CorruptDocumentIsDistinguishable(t *testing.T) { + objects := objectstore.NewMemoryObjectStore() + store := NewStore(objects, "control-plane") + ctx := context.Background() + + volKey, err := VolumeKey("vol-corrupt") + require.NoError(t, err) + writeRaw(t, objects, volKey, []byte("{not json")) + amiKey, err := AMIKey("ami-corrupt") + require.NoError(t, err) + writeRaw(t, objects, amiKey, []byte("{not json")) + + _, err = store.GetVolume(ctx, "vol-corrupt") + require.ErrorIs(t, err, ErrCorruptDocument) + assert.False(t, objectstore.IsNoSuchKeyError(err), "corrupt must not read as absent") + + _, err = store.GetAMI(ctx, "ami-corrupt") + require.ErrorIs(t, err, ErrCorruptDocument) + assert.False(t, objectstore.IsNoSuchKeyError(err), "corrupt must not read as absent") +} + +// writeRaw puts bytes at key without going through Marshal, so a test can +// store a document the store cannot decode. +func writeRaw(t *testing.T, objects objectstore.ObjectStore, key string, data []byte) { + t.Helper() + _, err := objects.PutObject(context.Background(), &s3.PutObjectInput{ + Bucket: aws.String("control-plane"), Key: aws.String(key), Body: bytes.NewReader(data), + }) + require.NoError(t, err) +} + +// TestListAMIs_SkipsCorruptButStrictDoesNot pins the blast radius of one bad +// document: DescribeImages keeps working and loses only the unreadable image, +// while a caller that cannot answer partially still gets the error. +func TestListAMIs_SkipsCorruptButStrictDoesNot(t *testing.T) { + objects := objectstore.NewMemoryObjectStore() + store := NewStore(objects, "control-plane") + ctx := context.Background() + + require.NoError(t, store.PutAMI(ctx, AMI{ImageID: "ami-good", Name: "readable"})) + badKey, err := AMIKey("ami-bad") + require.NoError(t, err) + writeRaw(t, objects, badKey, []byte("{not json")) + + got, err := store.ListAMIs(ctx) + require.NoError(t, err) + require.Len(t, got, 1, "the readable AMI must survive its neighbour being corrupt") + assert.Equal(t, "ami-good", got[0].ImageID) + + _, err = store.ListAMIsStrict(ctx) + require.ErrorIs(t, err, ErrCorruptDocument) +} diff --git a/spinifex/ebsprovider/conformance/bench.go b/spinifex/ebsprovider/conformance/bench.go new file mode 100644 index 000000000..ad9b7e513 --- /dev/null +++ b/spinifex/ebsprovider/conformance/bench.go @@ -0,0 +1,185 @@ +package conformance + +import ( + "context" + "fmt" + "testing" + + "github.com/mulgadc/spinifex/spinifex/ebsprovider" +) + +// defaultBenchVolumeBytes is the size a benchmarked volume is created at when +// BenchConfig does not say. A provider that allocates lazily costs the same at +// any size, and one that does not should be measured at a size its operator +// recognises rather than at whatever this file picked. +const defaultBenchVolumeBytes int64 = 1 << 30 + +// BenchConfig tunes a benchmark run for the provider it is pointed at. +type BenchConfig struct { + SuiteConfig + + // VolumeBytes sizes every volume the suite creates. Against a live store + // this wants to be small enough that a run finishes and large enough that + // the provider does real work. + VolumeBytes int64 +} + +func (c BenchConfig) volumeBytes() int64 { + if c.VolumeBytes <= 0 { + return defaultBenchVolumeBytes + } + return c.VolumeBytes +} + +// benchProvider is what a benchmark measures. Unlike the conformance suite it +// is built once for the whole run: constructing a provider per iteration would +// price the constructor, which is not a verb. +type benchProvider struct { + provider ebsprovider.EBSProvider + cfg BenchConfig +} + +func (p benchProvider) createVolume(b *testing.B, volumeID string) { + b.Helper() + _, err := p.provider.CreateVolume(context.Background(), ebsprovider.CreateVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), + VolumeID: volumeID, + CapacityRange: ebsprovider.CapacityRange{RequiredBytes: p.cfg.volumeBytes()}, + }) + if err != nil { + b.Fatalf("create %s: %v", volumeID, err) + } +} + +func (p benchProvider) deleteVolume(b *testing.B, volumeID string) { + b.Helper() + if err := p.provider.DeleteVolume(context.Background(), ebsprovider.DeleteVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), + VolumeID: volumeID, + }); err != nil { + b.Fatalf("delete %s: %v", volumeID, err) + } +} + +// RunBenchSuite prices one round trip per verb. Every implementation is driven +// through the same calls in the same order, so two providers' numbers are +// comparable and the only difference is what answers. +// +// It measures control operations. Guest I/O never crosses these verbs, so no +// number here says anything about data-path throughput. +func RunBenchSuite(b *testing.B, provider ebsprovider.EBSProvider, cfg BenchConfig) { + p := benchProvider{provider: provider, cfg: cfg} + capabilities := benchCapabilities(b, provider) + + b.Run("GetCapabilities", func(b *testing.B) { + for b.Loop() { + if _, err := provider.GetCapabilities(context.Background(), ebsprovider.GetCapabilitiesRequest{ + Versioned: ebsprovider.NewVersioned(), + }); err != nil { + b.Fatal(err) + } + } + }) + + // Create and delete are measured as a pair rather than separately: a + // benchmark that only created would leave b.N volumes behind on a live + // provider, and one that only deleted would have nothing to delete. + b.Run("CreateDeleteVolume", func(b *testing.B) { + i := 0 + for b.Loop() { + volumeID := cfg.id(fmt.Sprintf("vol-bench%08d", i)) + p.createVolume(b, volumeID) + p.deleteVolume(b, volumeID) + i++ + } + }) + + b.Run("GetVolume", func(b *testing.B) { + volumeID := cfg.id("vol-benchdescribe") + p.createVolume(b, volumeID) + b.Cleanup(func() { p.deleteVolume(b, volumeID) }) + b.ResetTimer() + for b.Loop() { + if _, err := provider.GetVolume(context.Background(), ebsprovider.GetVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), + VolumeID: volumeID, + }); err != nil { + b.Fatal(err) + } + } + }) + + if capabilities.VolumeEnumeration { + b.Run("ListVolumes", func(b *testing.B) { + for b.Loop() { + if _, err := provider.ListVolumes(context.Background(), ebsprovider.ListVolumesRequest{ + Versioned: ebsprovider.NewVersioned(), + }); err != nil { + b.Fatal(err) + } + } + }) + } + + // Publish and unpublish are the attach path, and the pair is what an + // instance launch actually costs. Measuring publish alone would leave the + // volume attached and the next iteration would fail on it. + b.Run("PublishUnpublishVolume", func(b *testing.B) { + volumeID := cfg.id("vol-benchpublish0") + p.createVolume(b, volumeID) + b.Cleanup(func() { p.deleteVolume(b, volumeID) }) + b.ResetTimer() + for b.Loop() { + if _, err := provider.PublishVolume(context.Background(), ebsprovider.PublishVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), + VolumeID: volumeID, + NodeID: cfg.node(), + }); err != nil { + b.Fatal(err) + } + if err := provider.UnpublishVolume(context.Background(), ebsprovider.UnpublishVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), + VolumeID: volumeID, + NodeID: cfg.node(), + }); err != nil { + b.Fatal(err) + } + } + }) + + b.Run("CreateDeleteSnapshot", func(b *testing.B) { + volumeID := cfg.id("vol-benchsnapshot") + p.createVolume(b, volumeID) + b.Cleanup(func() { p.deleteVolume(b, volumeID) }) + b.ResetTimer() + i := 0 + for b.Loop() { + snapshotID := cfg.id(fmt.Sprintf("snap-bench%08d", i)) + if _, err := provider.CreateSnapshot(context.Background(), ebsprovider.CreateSnapshotRequest{ + Versioned: ebsprovider.NewVersioned(), + VolumeID: volumeID, + SnapshotID: snapshotID, + }); err != nil { + b.Fatal(err) + } + if err := provider.DeleteSnapshot(context.Background(), ebsprovider.DeleteSnapshotRequest{ + Versioned: ebsprovider.NewVersioned(), + SnapshotID: snapshotID, + }); err != nil { + b.Fatal(err) + } + i++ + } + }) +} + +func benchCapabilities(b *testing.B, provider ebsprovider.EBSProvider) ebsprovider.Capabilities { + b.Helper() + resp, err := provider.GetCapabilities(context.Background(), ebsprovider.GetCapabilitiesRequest{ + Versioned: ebsprovider.NewVersioned(), + }) + if err != nil { + b.Fatalf("get capabilities: %v", err) + } + return resp.Capabilities +} diff --git a/spinifex/ebsprovider/conformance/bench_test.go b/spinifex/ebsprovider/conformance/bench_test.go new file mode 100644 index 000000000..f6e64356b --- /dev/null +++ b/spinifex/ebsprovider/conformance/bench_test.go @@ -0,0 +1,29 @@ +package conformance_test + +import ( + "testing" + + "github.com/mulgadc/spinifex/spinifex/ebsprovider/conformance" + "github.com/mulgadc/spinifex/spinifex/ebsprovider/natsserve" + "github.com/mulgadc/spinifex/spinifex/ebsprovider/nullprovider" + "github.com/mulgadc/spinifex/spinifex/testutil" +) + +// BenchmarkNullProvider_InProcess prices the contract with no transport and no +// storage. Anything a real provider costs above this is its own. +func BenchmarkNullProvider_InProcess(b *testing.B) { + conformance.RunBenchSuite(b, nullprovider.New(), conformance.BenchConfig{}) +} + +// BenchmarkNullProvider_OverNATS adds the transport and nothing else, so the +// gap between it and the in-process run is what the seam costs: encode, +// publish, queue dispatch, decode. +func BenchmarkNullProvider_OverNATS(b *testing.B) { + _, conn := testutil.StartTestNATS(b) + client, stop, err := nullprovider.Serve(b.Context(), conn, natsserve.Options{NoQueueGroup: true}) + if err != nil { + b.Fatal(err) + } + b.Cleanup(stop) + conformance.RunBenchSuite(b, client, conformance.BenchConfig{}) +} diff --git a/spinifex/ebsprovider/conformance/conformance.go b/spinifex/ebsprovider/conformance/conformance.go new file mode 100644 index 000000000..cda1ae8c9 --- /dev/null +++ b/spinifex/ebsprovider/conformance/conformance.go @@ -0,0 +1,933 @@ +// Package conformance exercises the ebsprovider.EBSProvider contract +// itself, not any one implementation's internals, so it can run unmodified +// against every implementation of the interface; no assertion here has changed from the original suite it was moved out of. +package conformance + +import ( + "bytes" + "context" + "strings" + "sync" + "testing" + + "github.com/mulgadc/spinifex/spinifex/ebsprovider" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// ReferenceCapabilities is what a full-featured provider advertises. It is +// only a convenience for callers constructing a MemoryProvider; the suite +// itself never assumes any implementation reports this set. +var ReferenceCapabilities = ebsprovider.Capabilities{ + OnlineExpansion: false, + SparseExtentReporting: true, + CrashConsistentSnapshot: true, + VolumeSeeding: true, + ReadOnlyPublish: true, + VolumeEnumeration: true, + SnapshotEnumeration: true, + // MemoryProvider keeps publication in one process's map, so node is the + // honest answer. A reference set that claimed cluster would make the + // suite's concurrency subtests pass against something that cannot hold. + Exclusion: ebsprovider.ExclusionSemantics{Scope: ebsprovider.ExclusionScopeNode}, +} + +// capabilitiesOf reads what a provider advertises, so the suite can branch on +// it the way the contract tells callers to. +func capabilitiesOf(t *testing.T, provider ebsprovider.EBSProvider) ebsprovider.Capabilities { + t.Helper() + resp, err := provider.GetCapabilities(context.Background(), ebsprovider.GetCapabilitiesRequest{Versioned: ebsprovider.NewVersioned()}) + require.NoError(t, err) + require.NotNil(t, resp) + return resp.Capabilities +} + +// maxListPages bounds a drain so a provider whose token fails to advance +// fails the test instead of hanging it. +const maxListPages = 100 + +// drainVolumeIDs walks every page of ListVolumes and returns the IDs in the +// order they arrived, duplicates included: detecting a token that replays or +// overshoots a boundary is the point, so this must not deduplicate. +func drainVolumeIDs(t *testing.T, provider ebsprovider.EBSProvider, maxResults int32) []string { + t.Helper() + var ids []string + var token string + for pages := 0; ; pages++ { + require.Lessf(t, pages, maxListPages, "ListVolumes did not terminate within %d pages; the next token is not advancing", maxListPages) + resp, err := provider.ListVolumes(context.Background(), ebsprovider.ListVolumesRequest{ + Versioned: ebsprovider.NewVersioned(), + MaxResults: maxResults, + StartingToken: token, + }) + require.NoError(t, err) + require.NotNil(t, resp) + for _, ref := range resp.Volumes { + ids = append(ids, ref.ID) + } + if resp.NextToken == "" { + return ids + } + token = resp.NextToken + } +} + +// drainSnapshotIDs is drainVolumeIDs for snapshots, and keeps duplicates for +// the same reason: a token that replays a boundary must be visible. +func drainSnapshotIDs(t *testing.T, provider ebsprovider.EBSProvider, maxResults int32) []string { + t.Helper() + var ids []string + var token string + for pages := 0; ; pages++ { + require.Lessf(t, pages, maxListPages, "ListSnapshots did not terminate within %d pages; the next token is not advancing", maxListPages) + resp, err := provider.ListSnapshots(context.Background(), ebsprovider.ListSnapshotsRequest{ + Versioned: ebsprovider.NewVersioned(), + MaxResults: maxResults, + StartingToken: token, + }) + require.NoError(t, err) + require.NotNil(t, resp) + for _, ref := range resp.Snapshots { + ids = append(ids, ref.ID) + } + if resp.NextToken == "" { + return ids + } + token = resp.NextToken + } +} + +// SuiteConfig tunes a run for the provider it is pointed at. +type SuiteConfig struct { + // NamePrefix is inserted into every volume and snapshot ID the suite + // creates, after the vol-/snap- token. A provider that keeps state across + // runs needs it: without it a second run collides with the first run's + // volumes and fails on already_exists. + NamePrefix string + + // NodeID is the node PublishVolume targets. A live provider answers only + // for the node it runs on, so the in-process default is wrong there. + NodeID string + + // OtherNodeID is a second node, used to prove a published volume cannot + // also be published elsewhere. Empty skips that subtest, for a caller with + // only one node to offer. + OtherNodeID string +} + +func (c SuiteConfig) node() string { + if c.NodeID == "" { + return "node-1" + } + return c.NodeID +} + +func (c SuiteConfig) otherNode() string { + if c.NodeID == "" && c.OtherNodeID == "" { + return "node-2" + } + return c.OtherNodeID +} + +// id builds a suite-owned identifier, keeping the vol-/snap- prefix the rest of +// the system reads as a type token. +func (c SuiteConfig) id(base string) string { + if c.NamePrefix == "" { + return base + } + if kind, rest, ok := strings.Cut(base, "-"); ok { + return kind + "-" + c.NamePrefix + rest + } + return c.NamePrefix + base +} + +// RunSuite runs the full EBSProvider conformance suite against a provider +// newProvider constructs fresh for each subtest. +func RunSuite(t *testing.T, newProvider func(t *testing.T) ebsprovider.EBSProvider) { + RunSuiteWithConfig(t, newProvider, SuiteConfig{}) +} + +// RunSuiteWithConfig is RunSuite with control over the identifiers it uses, so +// the same suite can run against a live provider whose state outlives the test. +// Optional behaviour is gated on what the provider advertises, never on a set +// the suite picked. +func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovider.EBSProvider, cfg SuiteConfig) { + // Everything the suite creates is tracked and torn down, so a live + // provider is left as the run found it. + newProvider := func(t *testing.T) ebsprovider.EBSProvider { + return newTrackingProvider(t, newRawProvider(t)) + } + + capabilities := capabilitiesOf(t, newProvider(t)) + + t.Run("GetCapabilities", func(t *testing.T) { + provider := newProvider(t) + resp, err := provider.GetCapabilities(context.Background(), ebsprovider.GetCapabilitiesRequest{Versioned: ebsprovider.NewVersioned()}) + require.NoError(t, err) + require.NotNil(t, resp) + assert.Equal(t, ebsprovider.SchemaVersion, resp.SchemaVersion, "response must carry the negotiated schema version") + + // Capabilities describe the implementation, so they must not vary + // between calls: a caller branches on them once and caches. + again, err := provider.GetCapabilities(context.Background(), ebsprovider.GetCapabilitiesRequest{Versioned: ebsprovider.NewVersioned()}) + require.NoError(t, err) + assert.Equal(t, resp.Capabilities, again.Capabilities, "GetCapabilities must be stable across calls") + }) + + t.Run("CreateVolume", func(t *testing.T) { + t.Run("success", func(t *testing.T) { + provider := newProvider(t) + vol, err := provider.CreateVolume(context.Background(), ebsprovider.CreateVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-create-ok"), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}, + }) + require.NoError(t, err) + require.NotNil(t, vol) + assert.Equal(t, cfg.id("vol-create-ok"), vol.ID) + assert.Equal(t, int64(1<<30), vol.CapacityBytes) + assert.Equal(t, ebsprovider.VolumeStateAvailable, vol.State) + assert.NotEmpty(t, vol.Handle) + }) + + t.Run("already_exists on conflicting recreate", func(t *testing.T) { + provider := newProvider(t) + ctx := context.Background() + _, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-conflict"), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}}) + require.NoError(t, err) + _, err = provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-conflict"), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 2 << 30}}) + require.ErrorIs(t, err, ebsprovider.ErrAlreadyExists) + }) + + t.Run("invalid_argument on empty volume id", func(t *testing.T) { + provider := newProvider(t) + _, err := provider.CreateVolume(context.Background(), ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}}) + require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument) + }) + + t.Run("not_found on absent source snapshot", func(t *testing.T) { + provider := newProvider(t) + _, err := provider.CreateVolume(context.Background(), ebsprovider.CreateVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-from-missing-snap"), + CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}, + SourceSnapshotID: cfg.id("snap-missing"), SourceSnapshotVolumeID: cfg.id("vol-missing-origin"), + }) + require.ErrorIs(t, err, ebsprovider.ErrNotFound) + }) + + // SourceSnapshotVolumeID names the volume the snapshot came from, which + // a provider may need to resolve the snapshot's blocks. Omitting it is a + // malformed request, not a request for behaviour the provider lacks. + t.Run("invalid_argument when a source snapshot has no source volume", func(t *testing.T) { + provider := newProvider(t) + _, err := provider.CreateVolume(context.Background(), ebsprovider.CreateVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-snapsrc-noorigin"), + CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}, SourceSnapshotID: cfg.id("snap-any"), + }) + require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument) + }) + + t.Run("volume is created from a snapshot", func(t *testing.T) { + provider := newProvider(t) + ctx := context.Background() + _, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-snapsrc-origin"), + CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}, + }) + require.NoError(t, err) + snap, err := provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{ + Versioned: ebsprovider.NewVersioned(), SnapshotID: cfg.id("snap-snapsrc"), VolumeID: cfg.id("vol-snapsrc-origin"), + }) + require.NoError(t, err) + require.Equal(t, cfg.id("vol-snapsrc-origin"), snap.SourceVolumeID, "a snapshot must report the volume it was taken from") + + restored, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-snapsrc-restored"), + CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}, + SourceSnapshotID: snap.ID, SourceSnapshotVolumeID: snap.SourceVolumeID, + }) + require.NoError(t, err) + assert.Equal(t, ebsprovider.VolumeStateAvailable, restored.State) + }) + + // The snapshot records its own origin, so a request naming a different + // one is checkable and wrong. Accepting it resolves the snapshot's + // blocks against a volume they were never written to. + t.Run("invalid_argument on a source volume the snapshot did not come from", func(t *testing.T) { + provider := newProvider(t) + ctx := context.Background() + _, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-wrongorigin-src"), + CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}, + }) + require.NoError(t, err) + snap, err := provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{ + Versioned: ebsprovider.NewVersioned(), SnapshotID: cfg.id("snap-wrongorigin"), VolumeID: cfg.id("vol-wrongorigin-src"), + }) + require.NoError(t, err) + + _, err = provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-wrongorigin-restored"), + CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}, + SourceSnapshotID: snap.ID, SourceSnapshotVolumeID: cfg.id("vol-wrongorigin-other"), + }) + require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument) + }) + + t.Run("unsupported_version", func(t *testing.T) { + provider := newProvider(t) + _, err := provider.CreateVolume(context.Background(), ebsprovider.CreateVolumeRequest{ + VolumeID: cfg.id("vol-noversion"), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}, + }) + require.ErrorIs(t, err, ebsprovider.ErrUnsupportedVersion) + }) + + t.Run("seeded volume is created", func(t *testing.T) { + if !capabilities.VolumeSeeding { + t.Skip("provider does not advertise VolumeSeeding") + } + provider := newProvider(t) + vol, err := provider.CreateVolume(context.Background(), ebsprovider.CreateVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-seeded"), + CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 4096}, + SeedData: bytes.Repeat([]byte{0xAB}, 4096), + }) + require.NoError(t, err) + require.NotNil(t, vol) + assert.Equal(t, int64(4096), vol.CapacityBytes) + }) + + // A provider that cannot seed must say so, not accept the seed and + // drop it: the caller has no other way to learn the volume is blank. + t.Run("unsupported_capability on seed when seeding is not advertised", func(t *testing.T) { + if capabilities.VolumeSeeding { + t.Skip("provider advertises VolumeSeeding") + } + provider := newProvider(t) + _, err := provider.CreateVolume(context.Background(), ebsprovider.CreateVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-seed-unsupported"), + CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 4096}, + SeedData: bytes.Repeat([]byte{0xAB}, 4096), + }) + require.ErrorIs(t, err, ebsprovider.ErrUnsupportedCapability) + }) + + // A seed above MaxSeedBytes must fail as invalid_argument rather than + // reaching the transport, where NATS would refuse the oversized publish + // with an error that says nothing about firmware. + t.Run("invalid_argument on oversized seed", func(t *testing.T) { + if !capabilities.VolumeSeeding { + t.Skip("provider does not advertise VolumeSeeding") + } + provider := newProvider(t) + _, err := provider.CreateVolume(context.Background(), ebsprovider.CreateVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-seed-toobig"), + CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}, + SeedData: make([]byte, ebsprovider.MaxSeedBytes+1), + }) + require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument) + }) + + t.Run("invalid_argument when seed exceeds capacity", func(t *testing.T) { + if !capabilities.VolumeSeeding { + t.Skip("provider does not advertise VolumeSeeding") + } + provider := newProvider(t) + _, err := provider.CreateVolume(context.Background(), ebsprovider.CreateVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-seed-overcap"), + CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 512}, + SeedData: make([]byte, 4096), + }) + require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument) + }) + }) + + t.Run("GetVolume", func(t *testing.T) { + t.Run("success", func(t *testing.T) { + provider := newProvider(t) + ctx := context.Background() + created, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-get-ok"), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}}) + require.NoError(t, err) + got, err := provider.GetVolume(ctx, ebsprovider.GetVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-get-ok")}) + require.NoError(t, err) + assert.Equal(t, created.Handle, got.Handle) + assert.Equal(t, created.CapacityBytes, got.CapacityBytes) + }) + + t.Run("not_found on absent volume", func(t *testing.T) { + provider := newProvider(t) + _, err := provider.GetVolume(context.Background(), ebsprovider.GetVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-never-existed")}) + require.ErrorIs(t, err, ebsprovider.ErrNotFound) + }) + + t.Run("invalid_argument on empty volume id", func(t *testing.T) { + provider := newProvider(t) + _, err := provider.GetVolume(context.Background(), ebsprovider.GetVolumeRequest{Versioned: ebsprovider.NewVersioned()}) + require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument) + }) + }) + + t.Run("ListVolumes", func(t *testing.T) { + capabilities := capabilitiesOf(t, newProvider(t)) + + t.Run("unsupported_capability when enumeration is not advertised", func(t *testing.T) { + if capabilities.VolumeEnumeration { + t.Skip("provider advertises volume enumeration") + } + _, err := newProvider(t).ListVolumes(context.Background(), ebsprovider.ListVolumesRequest{Versioned: ebsprovider.NewVersioned()}) + require.ErrorIs(t, err, ebsprovider.ErrUnsupportedCapability) + }) + + if !capabilities.VolumeEnumeration { + return + } + + t.Run("a created volume is enumerated and a deleted one is not", func(t *testing.T) { + provider := newProvider(t) + ctx := context.Background() + volumeID := cfg.id("vol-list-visible") + _, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}}) + require.NoError(t, err) + assert.Contains(t, drainVolumeIDs(t, provider, 0), volumeID, + "enumeration must report a volume the provider holds; it is the only way to find one whose control-plane document is lost") + + require.NoError(t, provider.DeleteVolume(ctx, ebsprovider.DeleteVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID})) + assert.NotContains(t, drainVolumeIDs(t, provider, 0), volumeID, + "a deleted volume must stop being enumerated, or the report it feeds accuses storage of holding blocks it released") + }) + + t.Run("pagination returns every volume exactly once", func(t *testing.T) { + provider := newProvider(t) + ctx := context.Background() + want := []string{cfg.id("vol-list-page-a"), cfg.id("vol-list-page-b"), cfg.id("vol-list-page-c")} + for _, volumeID := range want { + _, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}}) + require.NoError(t, err) + } + + // One volume per page forces every token boundary to be exercised. + paged := drainVolumeIDs(t, provider, 1) + seen := make(map[string]int, len(paged)) + for _, id := range paged { + seen[id]++ + } + for id, count := range seen { + assert.Equalf(t, 1, count, "volume %s appeared %d times across pages; a duplicate means the token replays a boundary", id, count) + } + for _, volumeID := range want { + assert.Containsf(t, paged, volumeID, "volume %s was skipped across pages; a gap means the token overshoots", volumeID) + } + }) + + t.Run("max results above the cap is clamped, not refused", func(t *testing.T) { + provider := newProvider(t) + resp, err := provider.ListVolumes(context.Background(), ebsprovider.ListVolumesRequest{ + Versioned: ebsprovider.NewVersioned(), + MaxResults: ebsprovider.MaxListResults * 10, + }) + require.NoError(t, err) + require.NotNil(t, resp) + assert.LessOrEqual(t, len(resp.Volumes), int(ebsprovider.MaxListResults), + "a page must never exceed MaxListResults; the reply has to fit one NATS message") + }) + + t.Run("unsupported_version", func(t *testing.T) { + _, err := newProvider(t).ListVolumes(context.Background(), ebsprovider.ListVolumesRequest{Versioned: ebsprovider.Versioned{SchemaVersion: ebsprovider.SchemaVersion + 1}}) + require.ErrorIs(t, err, ebsprovider.ErrUnsupportedVersion) + }) + }) + + t.Run("ExpandVolume", func(t *testing.T) { + t.Run("success", func(t *testing.T) { + provider := newProvider(t) + ctx := context.Background() + _, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-expand-ok"), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}}) + require.NoError(t, err) + expanded, err := provider.ExpandVolume(ctx, ebsprovider.ExpandVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-expand-ok"), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 2 << 30}}) + require.NoError(t, err) + assert.Equal(t, int64(2<<30), expanded.CapacityBytes) + }) + + t.Run("not_found on absent volume", func(t *testing.T) { + provider := newProvider(t) + _, err := provider.ExpandVolume(context.Background(), ebsprovider.ExpandVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-never-existed"), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}}) + require.ErrorIs(t, err, ebsprovider.ErrNotFound) + }) + + t.Run("invalid_argument on shrink", func(t *testing.T) { + provider := newProvider(t) + ctx := context.Background() + _, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-shrink"), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 2 << 30}}) + require.NoError(t, err) + _, err = provider.ExpandVolume(ctx, ebsprovider.ExpandVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-shrink"), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}}) + require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument) + }) + + // Expanding a published volume is the one operation OnlineExpansion + // describes, so both answers are contractual: succeed, or refuse with + // volume_in_use. Silently doing nothing is neither. + t.Run("expanding a published volume matches OnlineExpansion", func(t *testing.T) { + provider := newProvider(t) + ctx := context.Background() + _, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-expand-inuse"), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}}) + require.NoError(t, err) + _, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-expand-inuse"), NodeID: cfg.node()}) + require.NoError(t, err) + + expanded, err := provider.ExpandVolume(ctx, ebsprovider.ExpandVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-expand-inuse"), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 2 << 30}}) + if !capabilities.OnlineExpansion { + require.ErrorIs(t, err, ebsprovider.ErrVolumeInUse) + return + } + require.NoError(t, err, "provider advertises OnlineExpansion, so expanding a published volume must succeed") + require.NotNil(t, expanded) + assert.Equal(t, int64(2<<30), expanded.CapacityBytes) + }) + }) + + t.Run("DeleteVolume", func(t *testing.T) { + t.Run("success", func(t *testing.T) { + provider := newProvider(t) + ctx := context.Background() + _, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-delete-ok"), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}}) + require.NoError(t, err) + require.NoError(t, provider.DeleteVolume(ctx, ebsprovider.DeleteVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-delete-ok")})) + _, err = provider.GetVolume(ctx, ebsprovider.GetVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-delete-ok")}) + require.ErrorIs(t, err, ebsprovider.ErrNotFound) + }) + + // CSI's controller service treats DeleteVolume on an absent volume as + // success (idempotent-when-absent), not not_found. + t.Run("absent target is idempotent", func(t *testing.T) { + provider := newProvider(t) + require.NoError(t, provider.DeleteVolume(context.Background(), ebsprovider.DeleteVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-never-existed")})) + }) + + t.Run("volume_in_use when published", func(t *testing.T) { + provider := newProvider(t) + ctx := context.Background() + _, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-delete-inuse"), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}}) + require.NoError(t, err) + _, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-delete-inuse"), NodeID: cfg.node()}) + require.NoError(t, err) + err = provider.DeleteVolume(ctx, ebsprovider.DeleteVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-delete-inuse")}) + require.ErrorIs(t, err, ebsprovider.ErrVolumeInUse) + }) + }) + + t.Run("CreateSnapshot", func(t *testing.T) { + t.Run("success", func(t *testing.T) { + provider := newProvider(t) + ctx := context.Background() + _, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-snap-src"), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}}) + require.NoError(t, err) + snap, err := provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: cfg.id("snap-ok"), VolumeID: cfg.id("vol-snap-src")}) + require.NoError(t, err) + require.NotNil(t, snap) + assert.Equal(t, cfg.id("snap-ok"), snap.ID) + assert.Equal(t, cfg.id("vol-snap-src"), snap.SourceVolumeID) + assert.Equal(t, ebsprovider.SnapshotStateCompleted, snap.State) + }) + + t.Run("not_found on absent source volume", func(t *testing.T) { + provider := newProvider(t) + _, err := provider.CreateSnapshot(context.Background(), ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: cfg.id("snap-orphan"), VolumeID: cfg.id("vol-never-existed")}) + require.ErrorIs(t, err, ebsprovider.ErrNotFound) + }) + + t.Run("already_exists on conflicting source volume", func(t *testing.T) { + provider := newProvider(t) + ctx := context.Background() + _, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-snap-a"), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}}) + require.NoError(t, err) + _, err = provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-snap-b"), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}}) + require.NoError(t, err) + _, err = provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: cfg.id("snap-conflict"), VolumeID: cfg.id("vol-snap-a")}) + require.NoError(t, err) + _, err = provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: cfg.id("snap-conflict"), VolumeID: cfg.id("vol-snap-b")}) + require.ErrorIs(t, err, ebsprovider.ErrAlreadyExists) + }) + + t.Run("invalid_argument on missing ids", func(t *testing.T) { + provider := newProvider(t) + _, err := provider.CreateSnapshot(context.Background(), ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.NewVersioned()}) + require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument) + }) + }) + + t.Run("DeleteSnapshot", func(t *testing.T) { + t.Run("success", func(t *testing.T) { + provider := newProvider(t) + ctx := context.Background() + _, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-delsnap-src"), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}}) + require.NoError(t, err) + _, err = provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: cfg.id("snap-delete-ok"), VolumeID: cfg.id("vol-delsnap-src")}) + require.NoError(t, err) + require.NoError(t, provider.DeleteSnapshot(ctx, ebsprovider.DeleteSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: cfg.id("snap-delete-ok")})) + }) + + // memory.go implements delete-of-absent-snapshot as a no-op success, + // matching CSI's idempotency rule the same way DeleteVolume does. + t.Run("absent target is idempotent", func(t *testing.T) { + provider := newProvider(t) + require.NoError(t, provider.DeleteSnapshot(context.Background(), ebsprovider.DeleteSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: cfg.id("snap-never-existed")})) + }) + }) + + t.Run("CopySnapshot", func(t *testing.T) { + t.Run("success", func(t *testing.T) { + provider := newProvider(t) + ctx := context.Background() + _, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-copysnap-src"), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}}) + require.NoError(t, err) + _, err = provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: cfg.id("snap-copysnap-src"), VolumeID: cfg.id("vol-copysnap-src")}) + require.NoError(t, err) + + copied, err := provider.CopySnapshot(ctx, ebsprovider.CopySnapshotRequest{ + Versioned: ebsprovider.NewVersioned(), SourceSnapshotID: cfg.id("snap-copysnap-src"), DestinationSnapshotID: cfg.id("snap-copysnap-dst"), VolumeID: cfg.id("vol-copysnap-src"), + }) + require.NoError(t, err) + require.NotNil(t, copied) + assert.Equal(t, cfg.id("snap-copysnap-dst"), copied.ID) + assert.Equal(t, cfg.id("vol-copysnap-src"), copied.SourceVolumeID) + assert.Equal(t, ebsprovider.SnapshotStateCompleted, copied.State) + assert.NotEmpty(t, copied.Handle) + + // The destination is a real, independently addressable snapshot. + require.NoError(t, provider.DeleteSnapshot(ctx, ebsprovider.DeleteSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: cfg.id("snap-copysnap-src")})) + require.NoError(t, provider.DeleteSnapshot(ctx, ebsprovider.DeleteSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: cfg.id("snap-copysnap-dst")})) + }) + + t.Run("not_found on absent source snapshot", func(t *testing.T) { + provider := newProvider(t) + ctx := context.Background() + _, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-copysnap-nosrc"), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}}) + require.NoError(t, err) + _, err = provider.CopySnapshot(ctx, ebsprovider.CopySnapshotRequest{ + Versioned: ebsprovider.NewVersioned(), SourceSnapshotID: cfg.id("snap-copysnap-missing"), DestinationSnapshotID: cfg.id("snap-copysnap-missing-dst"), VolumeID: cfg.id("vol-copysnap-nosrc"), + }) + require.ErrorIs(t, err, ebsprovider.ErrNotFound) + }) + + t.Run("already_exists on conflicting destination", func(t *testing.T) { + provider := newProvider(t) + ctx := context.Background() + _, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-copysnap-conflict"), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}}) + require.NoError(t, err) + _, err = provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: cfg.id("snap-copysnap-a"), VolumeID: cfg.id("vol-copysnap-conflict")}) + require.NoError(t, err) + _, err = provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: cfg.id("snap-copysnap-b"), VolumeID: cfg.id("vol-copysnap-conflict")}) + require.NoError(t, err) + _, err = provider.CopySnapshot(ctx, ebsprovider.CopySnapshotRequest{ + Versioned: ebsprovider.NewVersioned(), SourceSnapshotID: cfg.id("snap-copysnap-a"), DestinationSnapshotID: cfg.id("snap-copysnap-b"), VolumeID: cfg.id("vol-copysnap-conflict"), + }) + require.ErrorIs(t, err, ebsprovider.ErrAlreadyExists) + }) + + t.Run("invalid_argument on missing ids", func(t *testing.T) { + provider := newProvider(t) + _, err := provider.CopySnapshot(context.Background(), ebsprovider.CopySnapshotRequest{Versioned: ebsprovider.NewVersioned()}) + require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument) + }) + + t.Run("invalid_argument when source and destination match", func(t *testing.T) { + provider := newProvider(t) + ctx := context.Background() + _, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-copysnap-same"), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}}) + require.NoError(t, err) + _, err = provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: cfg.id("snap-copysnap-same"), VolumeID: cfg.id("vol-copysnap-same")}) + require.NoError(t, err) + _, err = provider.CopySnapshot(ctx, ebsprovider.CopySnapshotRequest{ + Versioned: ebsprovider.NewVersioned(), SourceSnapshotID: cfg.id("snap-copysnap-same"), DestinationSnapshotID: cfg.id("snap-copysnap-same"), VolumeID: cfg.id("vol-copysnap-same"), + }) + require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument) + }) + + t.Run("invalid_argument when volume id does not own the source snapshot", func(t *testing.T) { + provider := newProvider(t) + ctx := context.Background() + _, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-copysnap-owner"), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}}) + require.NoError(t, err) + _, err = provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-copysnap-foreign"), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}}) + require.NoError(t, err) + _, err = provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: cfg.id("snap-copysnap-owned"), VolumeID: cfg.id("vol-copysnap-owner")}) + require.NoError(t, err) + _, err = provider.CopySnapshot(ctx, ebsprovider.CopySnapshotRequest{ + Versioned: ebsprovider.NewVersioned(), SourceSnapshotID: cfg.id("snap-copysnap-owned"), DestinationSnapshotID: cfg.id("snap-copysnap-owned-dst"), VolumeID: cfg.id("vol-copysnap-foreign"), + }) + require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument) + }) + + t.Run("unsupported_version", func(t *testing.T) { + provider := newProvider(t) + _, err := provider.CopySnapshot(context.Background(), ebsprovider.CopySnapshotRequest{ + SourceSnapshotID: cfg.id("snap-a"), DestinationSnapshotID: cfg.id("snap-b"), VolumeID: cfg.id("vol-a"), + }) + require.ErrorIs(t, err, ebsprovider.ErrUnsupportedVersion) + }) + }) + + t.Run("ListSnapshots", func(t *testing.T) { + capabilities := capabilitiesOf(t, newProvider(t)) + + t.Run("unsupported_capability when enumeration is not advertised", func(t *testing.T) { + if capabilities.SnapshotEnumeration { + t.Skip("provider advertises snapshot enumeration") + } + _, err := newProvider(t).ListSnapshots(context.Background(), ebsprovider.ListSnapshotsRequest{Versioned: ebsprovider.NewVersioned()}) + require.ErrorIs(t, err, ebsprovider.ErrUnsupportedCapability) + }) + + if !capabilities.SnapshotEnumeration { + return + } + + t.Run("a created snapshot is enumerated and a deleted one is not", func(t *testing.T) { + provider := newProvider(t) + ctx := context.Background() + volumeID := cfg.id("vol-snaplist-src") + snapshotID := cfg.id("snap-list-visible") + _, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}}) + require.NoError(t, err) + _, err = provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, VolumeID: volumeID}) + require.NoError(t, err) + assert.Contains(t, drainSnapshotIDs(t, provider, 0), snapshotID, + "enumeration must report a snapshot the provider holds; it is the only way to find one whose control-plane document is lost") + + require.NoError(t, provider.DeleteSnapshot(ctx, ebsprovider.DeleteSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID})) + assert.NotContains(t, drainSnapshotIDs(t, provider, 0), snapshotID, + "a deleted snapshot must stop being enumerated, or the report it feeds accuses storage of holding data it released") + }) + + t.Run("pagination returns every snapshot exactly once", func(t *testing.T) { + provider := newProvider(t) + ctx := context.Background() + volumeID := cfg.id("vol-snaplist-page-src") + _, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}}) + require.NoError(t, err) + want := []string{cfg.id("snap-list-page-a"), cfg.id("snap-list-page-b"), cfg.id("snap-list-page-c")} + for _, snapshotID := range want { + _, err := provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, VolumeID: volumeID}) + require.NoError(t, err) + } + + // One snapshot per page forces every token boundary to be exercised. + paged := drainSnapshotIDs(t, provider, 1) + seen := make(map[string]int, len(paged)) + for _, id := range paged { + seen[id]++ + } + for id, count := range seen { + assert.Equalf(t, 1, count, "snapshot %s appeared %d times across pages; a duplicate means the token replays a boundary", id, count) + } + for _, snapshotID := range want { + assert.Containsf(t, paged, snapshotID, "snapshot %s was skipped across pages; a gap means the token overshoots", snapshotID) + } + }) + + t.Run("max results above the cap is clamped, not refused", func(t *testing.T) { + provider := newProvider(t) + resp, err := provider.ListSnapshots(context.Background(), ebsprovider.ListSnapshotsRequest{ + Versioned: ebsprovider.NewVersioned(), + MaxResults: ebsprovider.MaxListResults * 10, + }) + require.NoError(t, err) + require.NotNil(t, resp) + assert.LessOrEqual(t, len(resp.Snapshots), int(ebsprovider.MaxListResults), + "a page must never exceed MaxListResults; the reply has to fit one NATS message") + }) + + t.Run("unsupported_version", func(t *testing.T) { + _, err := newProvider(t).ListSnapshots(context.Background(), ebsprovider.ListSnapshotsRequest{Versioned: ebsprovider.Versioned{SchemaVersion: ebsprovider.SchemaVersion + 1}}) + require.ErrorIs(t, err, ebsprovider.ErrUnsupportedVersion) + }) + }) + + t.Run("PublishVolume", func(t *testing.T) { + t.Run("success", func(t *testing.T) { + provider := newProvider(t) + ctx := context.Background() + _, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-pub-ok"), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}}) + require.NoError(t, err) + pub, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-pub-ok"), NodeID: cfg.node()}) + require.NoError(t, err) + require.NotNil(t, pub) + assert.Equal(t, cfg.id("vol-pub-ok"), pub.VolumeID) + assert.Equal(t, cfg.node(), pub.NodeID) + assertNBDURI(t, pub.NBDURI) + }) + + t.Run("idempotent republish to the same node", func(t *testing.T) { + provider := newProvider(t) + ctx := context.Background() + _, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-pub-idem"), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}}) + require.NoError(t, err) + first, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-pub-idem"), NodeID: cfg.node()}) + require.NoError(t, err) + second, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-pub-idem"), NodeID: cfg.node()}) + require.NoError(t, err) + assert.Equal(t, first, second) + }) + + t.Run("volume_in_use when published to a different node", func(t *testing.T) { + if cfg.otherNode() == "" { + t.Skip("no second node configured") + } + provider := newProvider(t) + ctx := context.Background() + _, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-pub-conflict"), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}}) + require.NoError(t, err) + _, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-pub-conflict"), NodeID: cfg.node()}) + require.NoError(t, err) + _, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-pub-conflict"), NodeID: cfg.otherNode()}) + require.ErrorIs(t, err, ebsprovider.ErrVolumeInUse) + }) + + t.Run("not_found on absent volume", func(t *testing.T) { + provider := newProvider(t) + _, err := provider.PublishVolume(context.Background(), ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-never-existed"), NodeID: cfg.node()}) + require.ErrorIs(t, err, ebsprovider.ErrNotFound) + }) + + t.Run("invalid_argument on missing ids", func(t *testing.T) { + provider := newProvider(t) + _, err := provider.PublishVolume(context.Background(), ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned()}) + require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument) + }) + }) + + t.Run("UnpublishVolume", func(t *testing.T) { + t.Run("success", func(t *testing.T) { + provider := newProvider(t) + ctx := context.Background() + _, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-unpub-ok"), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}}) + require.NoError(t, err) + _, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-unpub-ok"), NodeID: cfg.node()}) + require.NoError(t, err) + require.NoError(t, provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-unpub-ok"), NodeID: cfg.node()})) + }) + + t.Run("absent target is idempotent", func(t *testing.T) { + provider := newProvider(t) + require.NoError(t, provider.UnpublishVolume(context.Background(), ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-never-existed"), NodeID: cfg.node()})) + }) + + t.Run("invalid_argument on missing ids", func(t *testing.T) { + provider := newProvider(t) + err := provider.UnpublishVolume(context.Background(), ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned()}) + require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument) + }) + }) + + t.Run("Exclusion", func(t *testing.T) { + exclusion := capabilities.Exclusion + + // An advertised guarantee that contradicts itself is worse than a + // missing one: a caller reads it and plans around something the + // provider never claimed to do. + t.Run("advertised semantics are self-consistent", func(t *testing.T) { + switch exclusion.Scope { + case ebsprovider.ExclusionScopeNone, ebsprovider.ExclusionScopeNode, ebsprovider.ExclusionScopeCluster: + default: + t.Fatalf("unknown exclusion scope %q; a caller cannot branch on a scope it does not recognise", exclusion.Scope) + } + if exclusion.Scope == ebsprovider.ExclusionScopeNone { + assert.Zero(t, exclusion.ClaimTTLSeconds, "a provider that excludes nothing holds no claim, so a TTL on it means nothing") + assert.False(t, exclusion.FencesLostClaim, "there is no claim to lose, so nothing can be fenced on losing it") + } + assert.GreaterOrEqual(t, exclusion.ClaimTTLSeconds, 0, "a negative TTL is not a duration") + if exclusion.FencesLostClaim { + assert.NotEqual(t, ebsprovider.ExclusionScopeNone, exclusion.Scope, + "fencing a lost claim requires there to be a claim") + } + assert.Equal(t, exclusion.Scope == ebsprovider.ExclusionScopeCluster, exclusion.SingleWriter(), + "SingleWriter must agree with Scope, or callers branching on it get a different answer from callers reading the scope") + }) + + if exclusion.Scope == ebsprovider.ExclusionScopeNone { + return + } + + t.Run("a second publish elsewhere is refused while the first holds", func(t *testing.T) { + if cfg.otherNode() == "" { + t.Skip("no second node configured") + } + provider := newProvider(t) + ctx := context.Background() + volumeID := cfg.id("vol-excl-second") + _, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}}) + require.NoError(t, err) + _, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()}) + require.NoError(t, err) + + _, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.otherNode()}) + require.ErrorIsf(t, err, ebsprovider.ErrVolumeInUse, + "scope %q promises a second writer is refused; letting this through is the two-writer case the guarantee exists to prevent", exclusion.Scope) + + // Releasing the claim must make the volume publishable again, or + // the guarantee is indistinguishable from a permanent lock. + require.NoError(t, provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()})) + _, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.otherNode()}) + require.NoError(t, err, "a released claim must be reclaimable, or an unpublish strands the volume") + }) + + // Concurrent publishes settle to one winner. Both succeeding is the + // corruption case; both failing means the claim was lost by neither + // and the volume is stranded. + t.Run("racing publishes from two nodes produce exactly one winner", func(t *testing.T) { + if cfg.otherNode() == "" { + t.Skip("no second node configured") + } + provider := newProvider(t) + ctx := context.Background() + volumeID := cfg.id("vol-excl-race") + _, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}}) + require.NoError(t, err) + + nodes := []string{cfg.node(), cfg.otherNode()} + errs := make([]error, len(nodes)) + start := make(chan struct{}) + var wg sync.WaitGroup + for i, node := range nodes { + wg.Go(func() { + <-start + _, errs[i] = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: node, + }) + }) + } + close(start) + wg.Wait() + + var winners int + for i, err := range errs { + if err == nil { + winners++ + continue + } + assert.ErrorIsf(t, err, ebsprovider.ErrVolumeInUse, + "the loser of a publish race must be told the volume is in use, not %v; node %s cannot retry sensibly otherwise", err, nodes[i]) + } + assert.Equalf(t, 1, winners, "%d of %d racing publishes succeeded; two winners is the two-writer case, zero strands the volume", winners, len(nodes)) + }) + + // A delete arriving while a volume is published must lose to the + // publication, not race it. Deleting under a live writer destroys + // blocks the writer still believes it owns. + t.Run("delete racing a live publication is refused", func(t *testing.T) { + provider := newProvider(t) + ctx := context.Background() + volumeID := cfg.id("vol-excl-delrace") + _, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}}) + require.NoError(t, err) + _, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()}) + require.NoError(t, err) + + err = provider.DeleteVolume(ctx, ebsprovider.DeleteVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID}) + require.ErrorIs(t, err, ebsprovider.ErrVolumeInUse, + "deleting a published volume must be refused; the export is still serving reads and writes against it") + + require.NoError(t, provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()})) + require.NoError(t, provider.DeleteVolume(ctx, ebsprovider.DeleteVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID}), + "once unpublished the delete must go through, or a volume can never be removed after being used") + }) + }) +} diff --git a/spinifex/ebsprovider/conformance/conformance_test.go b/spinifex/ebsprovider/conformance/conformance_test.go new file mode 100644 index 000000000..9346e85c7 --- /dev/null +++ b/spinifex/ebsprovider/conformance/conformance_test.go @@ -0,0 +1,90 @@ +package conformance_test + +import ( + "testing" + "time" + + "github.com/mulgadc/spinifex/spinifex/ebsprovider" + "github.com/mulgadc/spinifex/spinifex/ebsprovider/conformance" + "github.com/mulgadc/spinifex/spinifex/ebsprovider/natsserve" + "github.com/mulgadc/spinifex/spinifex/testutil" + "github.com/stretchr/testify/require" +) + +func TestMemoryProviderConformance(t *testing.T) { + conformance.RunSuite(t, func(t *testing.T) ebsprovider.EBSProvider { + t.Helper() + return ebsprovider.NewMemoryProvider(conformance.ReferenceCapabilities) + }) +} + +// TestMemoryProviderMinimalCapabilitiesConformance is what makes the suite +// capability-driven rather than capability-asserting: the same suite, a +// provider advertising none of the optional behaviour, and it still passes. +func TestMemoryProviderMinimalCapabilitiesConformance(t *testing.T) { + conformance.RunSuite(t, func(t *testing.T) ebsprovider.EBSProvider { + t.Helper() + return ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{}) + }) +} + +// TestMemoryProviderOnlineExpansionConformance covers the branch no provider +// we ship exercises: expanding a published volume must succeed when the +// provider advertises that it can. +func TestMemoryProviderOnlineExpansionConformance(t *testing.T) { + conformance.RunSuite(t, func(t *testing.T) ebsprovider.EBSProvider { + t.Helper() + capabilities := conformance.ReferenceCapabilities + capabilities.OnlineExpansion = true + return ebsprovider.NewMemoryProvider(capabilities) + }) +} + +// TestMemoryProviderSharedInstanceConformance runs the suite twice against one +// provider instance, standing in for a live daemon whose volumes outlive the +// test binary. It passes only if the suite removes everything it creates: with +// no cleanup the second run collides with the first on already_exists. +func TestMemoryProviderSharedInstanceConformance(t *testing.T) { + shared := ebsprovider.NewMemoryProvider(conformance.ReferenceCapabilities) + newProvider := func(t *testing.T) ebsprovider.EBSProvider { + t.Helper() + return shared + } + + for _, run := range []string{"first run", "second run"} { + t.Run(run, func(t *testing.T) { + conformance.RunSuiteWithConfig(t, newProvider, conformance.SuiteConfig{}) + }) + } +} + +// TestMemoryProviderNamePrefixConformance covers the other half of running +// against a live provider: distinct prefixes let concurrent runs share one +// daemon without naming the same volumes. +func TestMemoryProviderNamePrefixConformance(t *testing.T) { + shared := ebsprovider.NewMemoryProvider(conformance.ReferenceCapabilities) + newProvider := func(t *testing.T) ebsprovider.EBSProvider { + t.Helper() + return shared + } + + for _, prefix := range []string{"alpha-", "beta-"} { + t.Run(prefix, func(t *testing.T) { + conformance.RunSuiteWithConfig(t, newProvider, conformance.SuiteConfig{NamePrefix: prefix}) + }) + } +} + +// TestNATSProviderConformance runs the suite against a NATSProvider backed +// by natsserve.Serve, the production-shaped neutral server, rather than a +// test-only twin: this exercises the transport this package's implementors actually run in production, not a hand-written stand-in for it. +func TestNATSProviderConformance(t *testing.T) { + conformance.RunSuite(t, func(t *testing.T) ebsprovider.EBSProvider { + t.Helper() + _, conn := testutil.StartTestNATS(t) + stop, err := natsserve.Serve(t.Context(), conn, ebsprovider.NewMemoryProvider(conformance.ReferenceCapabilities), natsserve.Options{}) + require.NoError(t, err) + t.Cleanup(stop) + return ebsprovider.NewNATSProvider(conn, 5*time.Second) + }) +} diff --git a/spinifex/ebsprovider/conformance/datapath.go b/spinifex/ebsprovider/conformance/datapath.go new file mode 100644 index 000000000..8c9e78e44 --- /dev/null +++ b/spinifex/ebsprovider/conformance/datapath.go @@ -0,0 +1,200 @@ +package conformance + +// The rest of this package prices the control verbs: create, publish, +// describe. Guest I/O never crosses those, so none of those numbers say +// anything about what a guest actually experiences. This file drives the +// published export with qemu-img bench, which is the same block layer a +// guest's virtio-blk sits on. + +import ( + "context" + "fmt" + "os/exec" + "regexp" + "strconv" + "testing" + "time" + + "github.com/mulgadc/spinifex/spinifex/ebsprovider" + "github.com/stretchr/testify/require" +) + +// dataPathVolumeBytes is large enough that a strided profile can skip far +// enough to defeat readahead, and small enough to stay quick over a real +// store. +const dataPathVolumeBytes int64 = 256 << 20 + +// DataPathProfile is one qemu-img bench run: a request size, a queue depth +// and how far apart consecutive requests land. +type DataPathProfile struct { + Name string + + // BufferSize is the size of each request. + BufferSize int64 + + // StepSize is the offset increment between requests. Zero means + // sequential, i.e. one buffer. + StepSize int64 + + Count int + Depth int + Write bool + + // FlushInterval, when set, flushes every N write requests. Without it a + // write profile measures how fast the WAL accepts data, not how fast the + // store takes it: chunk upload is asynchronous. + FlushInterval int +} + +func (p DataPathProfile) step() int64 { + if p.StepSize == 0 { + return p.BufferSize + } + return p.StepSize +} + +// bytes is how much payload the profile moves, which is not the span it +// covers when it strides. +func (p DataPathProfile) bytes() int64 { + return p.BufferSize * int64(p.Count) +} + +// DataPathResult is one profile's timing as qemu-img reported it. The +// harness's own wall clock is deliberately not used: it would fold in +// process startup and the NBD handshake, which a running guest pays once +// rather than per request. +type DataPathResult struct { + Profile DataPathProfile + Elapsed time.Duration +} + +// ThroughputMiBs is the payload rate. +func (r DataPathResult) ThroughputMiBs() float64 { + if r.Elapsed <= 0 { + return 0 + } + return float64(r.Profile.bytes()) / (1 << 20) / r.Elapsed.Seconds() +} + +// PerRequest is elapsed divided by request count. At depth > 1 this is +// throughput expressed per request, not the latency of one request. +func (r DataPathResult) PerRequest() time.Duration { + if r.Profile.Count == 0 { + return 0 + } + return r.Elapsed / time.Duration(r.Profile.Count) +} + +// writtenSpan is how much of the volume the leading write profile covers. +// Every read profile stays inside it: a read past the written region is +// answered as a hole, which measures nothing. +const writtenSpan int64 = 64 << 20 + +// DefaultDataPathProfiles writes before it reads, so the reads have +// something to find. A read profile that ran first would be measuring how +// fast the export can answer with zeroes. +func DefaultDataPathProfiles() []DataPathProfile { + return []DataPathProfile{ + {Name: "seq-write-64k", BufferSize: 64 << 10, Count: 1024, Depth: 16, Write: true}, + {Name: "seq-write-4k", BufferSize: 4 << 10, Count: 4096, Depth: 16, Write: true}, + {Name: "seq-write-64k-flush", BufferSize: 64 << 10, Count: 256, Depth: 16, Write: true, FlushInterval: 16}, + {Name: "seq-read-64k", BufferSize: 64 << 10, Count: 1024, Depth: 16}, + {Name: "seq-read-4k", BufferSize: 4 << 10, Count: 4096, Depth: 16}, + // 256 requests one 256K step apart covers exactly the written span. + {Name: "strided-read-4k", BufferSize: 4 << 10, StepSize: writtenSpan / 256, Count: 256, Depth: 16}, + {Name: "seq-read-64k-qd1", BufferSize: 64 << 10, Count: 256, Depth: 1}, + } +} + +// RequireDataPathTools skips when qemu-img is absent rather than reporting a +// data path nothing measured. +func RequireDataPathTools(t *testing.T) { + t.Helper() + if _, err := exec.LookPath("qemu-img"); err != nil { + t.Skip("qemu-img not installed (qemu-utils)") + } +} + +// benchElapsed pulls the timing out of qemu-img bench's own report. +var benchElapsed = regexp.MustCompile(`Run completed in ([0-9.]+) seconds`) + +// runDataPathProfile executes one profile against uri. +func runDataPathProfile(t *testing.T, uri string, p DataPathProfile) DataPathResult { + t.Helper() + + args := []string{ + "bench", "-f", "raw", + "-c", strconv.Itoa(p.Count), + "-s", strconv.FormatInt(p.BufferSize, 10), + "-S", strconv.FormatInt(p.step(), 10), + "-d", strconv.Itoa(p.Depth), + } + if p.Write { + args = append(args, "-w") + if p.FlushInterval > 0 { + args = append(args, "--flush-interval", strconv.Itoa(p.FlushInterval)) + } + } + args = append(args, uri) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + + out, err := exec.CommandContext(ctx, "qemu-img", args...).CombinedOutput() + require.NoErrorf(t, err, "qemu-img %v: %s", args, out) + + match := benchElapsed.FindSubmatch(out) + require.NotNilf(t, match, "qemu-img bench reported no elapsed time: %s", out) + seconds, err := strconv.ParseFloat(string(match[1]), 64) + require.NoError(t, err) + + return DataPathResult{Profile: p, Elapsed: time.Duration(seconds * float64(time.Second))} +} + +// RunDataPathSuite publishes one volume and drives every profile against it, +// logging a table. It asserts only that each run completes and moves a +// non-zero amount of data: what a given store is capable of is not a +// contract, so pinning a number here would only produce a flaky test. +func RunDataPathSuite(t *testing.T, newProvider func(t *testing.T) ebsprovider.EBSProvider, cfg NBDClientConfig) { + RequireNBDTools(t) + RequireDataPathTools(t) + + cfg = cfg.withDefaults() + if cfg.VolumeBytes < dataPathVolumeBytes { + cfg.VolumeBytes = dataPathVolumeBytes + } + + provider := newProvider(t) + volumeID := cfg.VolumePrefix + "datapath" + + pub := publishForNBD(t, provider, cfg, volumeID, false) + assertNBDURI(t, pub.NBDURI) + uri := dialURI(t, pub.NBDURI) + + export := nbdInfo(t, uri) + require.Equalf(t, cfg.VolumeBytes, export.Size, "export size does not match the volume that was created") + + results := make([]DataPathResult, 0, len(DefaultDataPathProfiles())) + // Profiles run back to back deliberately. Each is its own NBD connection, + // so a run that gets this far has also shown that an arriving connection + // waits out the previous one's close rather than racing it. + for _, profile := range DefaultDataPathProfiles() { + result := runDataPathProfile(t, uri, profile) + require.Positivef(t, result.Elapsed, "%s completed in no measurable time", profile.Name) + results = append(results, result) + } + + t.Log(formatDataPathResults(results)) +} + +// formatDataPathResults renders the table that is the point of the run. +func formatDataPathResults(results []DataPathResult) string { + report := fmt.Sprintf("\n%-18s %8s %6s %6s %10s %12s %12s\n", + "profile", "req", "size", "depth", "elapsed", "MiB/s", "per-req") + for _, r := range results { + report += fmt.Sprintf("%-18s %8d %5dK %6d %10s %12.1f %12s\n", + r.Profile.Name, r.Profile.Count, r.Profile.BufferSize>>10, r.Profile.Depth, + r.Elapsed.Round(time.Millisecond), r.ThroughputMiBs(), r.PerRequest().Round(time.Microsecond)) + } + return report +} diff --git a/spinifex/ebsprovider/conformance/nbdclient.go b/spinifex/ebsprovider/conformance/nbdclient.go new file mode 100644 index 000000000..488a8283a --- /dev/null +++ b/spinifex/ebsprovider/conformance/nbdclient.go @@ -0,0 +1,380 @@ +package conformance + +// Everything else in this package asserts the contract through our own Go +// client. This file asserts the published export through libnbd's tools, +// which were written with no knowledge of this codebase. + +import ( + "bytes" + "context" + "encoding/json" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/mulgadc/spinifex/spinifex/ebsprovider" + "github.com/mulgadc/spinifex/spinifex/utils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// defaultNBDVolumeBytes is small enough for a full nbdcopy round trip to be +// quick and large enough to span many allocation units. +const defaultNBDVolumeBytes int64 = 64 << 20 + +const defaultNBDNodeID = "node-a" + +const defaultNBDVolumePrefix = "vol-nbd" + +// reconnectTimeout bounds how long an export may refuse a reconnecting +// client while it finishes tearing the previous connection down. +const reconnectTimeout = 2 * time.Minute + +// NBDClientConfig adapts the suite to where it runs. An in-process provider +// needs none of it; a live cluster needs its own node name, and unique +// volume IDs so a rerun does not collide with a previous run's leftovers. +type NBDClientConfig struct { + NodeID string + VolumePrefix string + VolumeBytes int64 +} + +func (c NBDClientConfig) withDefaults() NBDClientConfig { + if c.NodeID == "" { + c.NodeID = defaultNBDNodeID + } + if c.VolumePrefix == "" { + c.VolumePrefix = defaultNBDVolumePrefix + } + if c.VolumeBytes == 0 { + c.VolumeBytes = defaultNBDVolumeBytes + } + return c +} + +// nbdExport is the subset of `nbdinfo --json` this suite asserts on. +type nbdExport struct { + Name string `json:"export-name"` + Size int64 `json:"export-size"` + ReadOnly bool `json:"is_read_only"` + BlockSizeMinimum int64 `json:"block_size_minimum"` + BlockSizePreferred int64 `json:"block_size_preferred"` + CanFlush bool `json:"can_flush"` + Contexts []string `json:"contexts"` +} + +type nbdInfoOutput struct { + Exports []nbdExport `json:"exports"` +} + +// nbdExtent is one entry of `nbdinfo --map --json`. Description is libnbd's +// rendering of the allocation flags, e.g. "data" or "hole,zero". +type nbdExtent struct { + Offset int64 `json:"offset"` + Length int64 `json:"length"` + Description string `json:"description"` +} + +// assertNBDURI requires a published URI to be one an NBD client can dial: +// nbd+unix:///?socket=/path or nbd://host:port. QEMU's legacy nbd:unix: +// filename syntax is not a URI and no NBD client but QEMU accepts it. +func assertNBDURI(t *testing.T, nbdURI string) { + t.Helper() + require.NotEmpty(t, nbdURI) + assert.Truef(t, strings.HasPrefix(nbdURI, "nbd+unix://") || strings.HasPrefix(nbdURI, "nbd://"), + "published NBD URI %q is not an NBD URI", nbdURI) +} + +// dialURI normalises whatever form a provider published into one libnbd will +// dial, so a provider emitting the legacy syntax still gets its export +// tested rather than every subtest failing on the same string. +func dialURI(t *testing.T, nbdURI string) string { + t.Helper() + serverType, socketPath, host, port, err := utils.ParseNBDURI(nbdURI) + require.NoErrorf(t, err, "parse published NBD URI %q", nbdURI) + + if serverType == "unix" { + return utils.FormatNBDSocketURI(socketPath) + } + return utils.FormatNBDTCPURI(host, port) +} + +// RequireNBDTools skips when libnbd's client tools are absent, so a host +// that cannot run an external NBD client does not report a false pass. +func RequireNBDTools(t *testing.T) { + t.Helper() + for _, bin := range []string{"nbdinfo", "nbdcopy"} { + if _, err := exec.LookPath(bin); err != nil { + // CI sets this so a runner image that loses libnbd fails rather + // than dropping the only external witness to the export. + if os.Getenv("SPINIFEX_REQUIRE_CONFORMANCE_TOOLS") != "" { + t.Fatalf("%s not installed (libnbd-bin), but SPINIFEX_REQUIRE_CONFORMANCE_TOOLS is set", bin) + } + t.Skipf("%s not installed (libnbd-bin)", bin) + } + } +} + +// nbdInfo runs nbdinfo against uri and returns its single export. +func nbdInfo(t *testing.T, uri string) nbdExport { + t.Helper() + out, err := exec.CommandContext(context.Background(), "nbdinfo", "--json", uri).CombinedOutput() + require.NoErrorf(t, err, "nbdinfo %s: %s", uri, out) + t.Logf("nbdinfo %s:\n%s", uri, out) + + var parsed nbdInfoOutput + require.NoError(t, json.Unmarshal(out, &parsed)) + require.Lenf(t, parsed.Exports, 1, "expected exactly one export from %s", uri) + return parsed.Exports[0] +} + +// nbdMap runs nbdinfo --map against uri and returns the allocation map the +// export reports. +func nbdMap(t *testing.T, uri string) []nbdExtent { + t.Helper() + out, err := exec.CommandContext(context.Background(), "nbdinfo", "--map", "--json", uri).CombinedOutput() + require.NoErrorf(t, err, "nbdinfo --map %s: %s", uri, out) + t.Logf("nbdinfo --map %s:\n%s", uri, out) + + var extents []nbdExtent + require.NoError(t, json.Unmarshal(out, &extents)) + return extents +} + +// nbdInfoRetry retries nbdinfo until the export accepts it, reporting how +// many attempts and how long it took. Each attempt is a real connection, so +// polling with a cheaper probe would only re-trigger whatever refused it. +func nbdInfoRetry(t *testing.T, uri string) (nbdExport, int, time.Duration) { + t.Helper() + start := time.Now() + deadline := start.Add(reconnectTimeout) + + var lastOut []byte + for attempt := 1; ; attempt++ { + out, err := exec.CommandContext(context.Background(), "nbdinfo", "--json", uri).CombinedOutput() + if err == nil { + var parsed nbdInfoOutput + require.NoError(t, json.Unmarshal(out, &parsed)) + require.Lenf(t, parsed.Exports, 1, "expected exactly one export from %s", uri) + return parsed.Exports[0], attempt, time.Since(start) + } + lastOut = out + if time.Now().After(deadline) { + require.FailNowf(t, "export never accepted a reconnection", + "after %s and %d attempts: %s", reconnectTimeout, attempt, lastOut) + } + time.Sleep(200 * time.Millisecond) + } +} + +// nbdReachable reports whether nbdinfo can still connect, for asserting that +// an export has gone away. +func nbdReachable(uri string) bool { + return exec.CommandContext(context.Background(), "nbdinfo", "--size", uri).Run() == nil +} + +// publishForNBD creates and publishes a volume, registering cleanup so a run +// against a real cluster does not leave volumes behind. +func publishForNBD(t *testing.T, provider ebsprovider.EBSProvider, cfg NBDClientConfig, volumeID string, readOnly bool) *ebsprovider.PublishedVolume { + t.Helper() + ctx := context.Background() + _, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), + VolumeID: volumeID, + CapacityRange: ebsprovider.CapacityRange{RequiredBytes: cfg.VolumeBytes}, + }) + require.NoError(t, err) + t.Cleanup(func() { cleanupVolume(t, provider, cfg, volumeID) }) + + pub, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), + VolumeID: volumeID, + NodeID: cfg.NodeID, + ReadOnly: readOnly, + }) + require.NoError(t, err) + require.NotNil(t, pub) + return pub +} + +// cleanupVolume unpublishes and deletes best-effort: cleanup runs after the +// assertions and must never turn a passing test red. +func cleanupVolume(t *testing.T, provider ebsprovider.EBSProvider, cfg NBDClientConfig, volumeID string) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + if err := provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.NodeID, + }); err != nil { + t.Logf("cleanup: unpublish %s: %v", volumeID, err) + } + if err := provider.DeleteVolume(ctx, ebsprovider.DeleteVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, + }); err != nil { + t.Logf("cleanup: delete %s: %v", volumeID, err) + } +} + +// RunNBDClientSuite drives a provider's published export with libnbd's +// client tools. newProvider must return a provider that serves real NBD; +// the in-memory reference implementation does not qualify. +func RunNBDClientSuite(t *testing.T, newProvider func(t *testing.T) ebsprovider.EBSProvider) { + RunNBDClientSuiteWithConfig(t, newProvider, NBDClientConfig{}) +} + +// RunNBDClientSuiteWithConfig is RunNBDClientSuite against a named node with +// caller-chosen volume IDs, for driving a real daemon over a live cluster. +func RunNBDClientSuiteWithConfig(t *testing.T, newProvider func(t *testing.T) ebsprovider.EBSProvider, cfg NBDClientConfig) { + RequireNBDTools(t) + cfg = cfg.withDefaults() + + t.Run("published URI is a standard NBD URI", func(t *testing.T) { + provider := newProvider(t) + pub := publishForNBD(t, provider, cfg, cfg.VolumePrefix+"-uri", false) + assertNBDURI(t, pub.NBDURI) + }) + + t.Run("export is connectable by a standard NBD client", func(t *testing.T) { + provider := newProvider(t) + pub := publishForNBD(t, provider, cfg, cfg.VolumePrefix+"-connect", false) + export := nbdInfo(t, dialURI(t, pub.NBDURI)) + assert.Equal(t, cfg.VolumeBytes, export.Size) + }) + + t.Run("export advertises usable block size constraints", func(t *testing.T) { + provider := newProvider(t) + pub := publishForNBD(t, provider, cfg, cfg.VolumePrefix+"-blocksize", false) + export := nbdInfo(t, dialURI(t, pub.NBDURI)) + + require.Positivef(t, export.BlockSizeMinimum, + "export advertises no minimum block size, so a client cannot know how to align its requests") + assert.GreaterOrEqual(t, export.BlockSizePreferred, export.BlockSizeMinimum) + assert.Zero(t, cfg.VolumeBytes%export.BlockSizeMinimum, + "export size must be a whole number of minimum blocks") + }) + + // A guest that reboots, or any client that drops, reconnects to the same + // export. Publication is what holds the volume, not any one connection. + t.Run("export accepts a client reconnecting after the first disconnects", func(t *testing.T) { + provider := newProvider(t) + pub := publishForNBD(t, provider, cfg, cfg.VolumePrefix+"-reconnect", false) + uri := dialURI(t, pub.NBDURI) + + first := nbdInfo(t, uri) + + second, attempts, elapsed := nbdInfoRetry(t, uri) + t.Logf("reconnect accepted on attempt %d after %s", attempts, elapsed.Round(10*time.Millisecond)) + assert.Equalf(t, 1, attempts, + "reconnect was refused for %s after the first client disconnected; a client that does not retry sees a hard failure", elapsed.Round(10*time.Millisecond)) + assert.Equal(t, first.Size, second.Size) + }) + + t.Run("data written by an external client reads back byte for byte", func(t *testing.T) { + provider := newProvider(t) + pub := publishForNBD(t, provider, cfg, cfg.VolumePrefix+"-roundtrip", false) + + dir := t.TempDir() + in := filepath.Join(dir, "in.bin") + out := filepath.Join(dir, "out.bin") + want := patternBytes(int(cfg.VolumeBytes)) + require.NoError(t, os.WriteFile(in, want, 0o600)) + + uri := dialURI(t, pub.NBDURI) + runNBDCopy(t, in, uri) + runNBDCopy(t, uri, out) + + got, err := os.ReadFile(out) + require.NoError(t, err) + require.Len(t, got, len(want)) + assert.True(t, bytes.Equal(got, want), "data read back over NBD differs from what was written") + }) + + // SparseExtentReporting is only observable at the export. base:allocation + // being offered proves nothing on its own — nbdkit offers it whether or not + // the plugin can compute extents, answering "all allocated" when it cannot. + // What separates the two is the answer: an untouched volume is entirely + // holes to a server that really tracks allocation. + t.Run("sparse extent reporting matches the advertised capability", func(t *testing.T) { + provider := newProvider(t) + capabilities := capabilitiesOf(t, provider) + pub := publishForNBD(t, provider, cfg, cfg.VolumePrefix+"-extents", false) + + holes := false + for _, extent := range nbdMap(t, dialURI(t, pub.NBDURI)) { + if strings.Contains(extent.Description, "hole") { + holes = true + break + } + } + assert.Equalf(t, capabilities.SparseExtentReporting, holes, + "SparseExtentReporting=%v but a volume with nothing written to it reports holes=%v", + capabilities.SparseExtentReporting, holes) + }) + + t.Run("read-only publish matches the advertised capability", func(t *testing.T) { + provider := newProvider(t) + capabilities := capabilitiesOf(t, provider) + + if !capabilities.ReadOnlyPublish { + // Refusing is the contract for a provider that cannot honour it; + // handing back a writable export would be the real failure. + _, err := provider.CreateVolume(context.Background(), ebsprovider.CreateVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), + VolumeID: cfg.VolumePrefix + "-readonly", + CapacityRange: ebsprovider.CapacityRange{RequiredBytes: cfg.VolumeBytes}, + }) + require.NoError(t, err) + t.Cleanup(func() { cleanupVolume(t, provider, cfg, cfg.VolumePrefix+"-readonly") }) + + _, err = provider.PublishVolume(context.Background(), ebsprovider.PublishVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.VolumePrefix + "-readonly", + NodeID: cfg.NodeID, ReadOnly: true, + }) + require.ErrorIs(t, err, ebsprovider.ErrUnsupportedCapability) + return + } + + pub := publishForNBD(t, provider, cfg, cfg.VolumePrefix+"-readonly", true) + export := nbdInfo(t, dialURI(t, pub.NBDURI)) + assert.True(t, export.ReadOnly, "a volume published ReadOnly must set the NBD read-only transmission flag") + }) + + t.Run("unpublish tears the export down", func(t *testing.T) { + provider := newProvider(t) + volumeID := cfg.VolumePrefix + "-unpublish" + pub := publishForNBD(t, provider, cfg, volumeID, false) + uri := dialURI(t, pub.NBDURI) + require.True(t, nbdReachable(uri), "export should be reachable before unpublish") + + require.NoError(t, provider.UnpublishVolume(context.Background(), ebsprovider.UnpublishVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), + VolumeID: volumeID, + NodeID: cfg.NodeID, + })) + + assert.Eventually(t, func() bool { return !nbdReachable(uri) }, 30*time.Second, 250*time.Millisecond, + "export still answers after UnpublishVolume") + }) +} + +// runNBDCopy copies between a local file and an NBD URI in either direction. +func runNBDCopy(t *testing.T, src, dst string) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + out, err := exec.CommandContext(ctx, "nbdcopy", src, dst).CombinedOutput() + require.NoErrorf(t, err, "nbdcopy %s %s: %s", src, dst, out) +} + +// patternBytes builds a position-dependent pattern, so a copy that lands at +// the wrong offset fails as loudly as one that loses data. +func patternBytes(n int) []byte { + buf := make([]byte, n) + for i := range buf { + buf[i] = byte(i*31 + i/4096) + } + return buf +} diff --git a/spinifex/ebsprovider/conformance/tracking.go b/spinifex/ebsprovider/conformance/tracking.go new file mode 100644 index 000000000..101d13243 --- /dev/null +++ b/spinifex/ebsprovider/conformance/tracking.go @@ -0,0 +1,125 @@ +package conformance + +import ( + "context" + "testing" + + "github.com/mulgadc/spinifex/spinifex/ebsprovider" +) + +// trackingProvider delegates every call to the provider under test and records +// what it created, so the suite can remove it afterwards. An in-process +// provider is discarded with the test, but a live one keeps its volumes in S3 +// and would fail the next run on already_exists. +type trackingProvider struct { + ebsprovider.EBSProvider + + t *testing.T + + volumes map[string]string + snapshots map[string]string + published map[string]string +} + +var _ ebsprovider.EBSProvider = (*trackingProvider)(nil) + +func newTrackingProvider(t *testing.T, inner ebsprovider.EBSProvider) *trackingProvider { + t.Helper() + p := &trackingProvider{ + EBSProvider: inner, + t: t, + volumes: make(map[string]string), + snapshots: make(map[string]string), + published: make(map[string]string), + } + t.Cleanup(p.cleanup) + return p +} + +func (p *trackingProvider) CreateVolume(ctx context.Context, req ebsprovider.CreateVolumeRequest) (*ebsprovider.Volume, error) { + vol, err := p.EBSProvider.CreateVolume(ctx, req) + if err == nil && vol != nil { + p.volumes[vol.ID] = vol.Handle + } + return vol, err +} + +func (p *trackingProvider) DeleteVolume(ctx context.Context, req ebsprovider.DeleteVolumeRequest) error { + err := p.EBSProvider.DeleteVolume(ctx, req) + if err == nil { + delete(p.volumes, req.VolumeID) + } + return err +} + +func (p *trackingProvider) CreateSnapshot(ctx context.Context, req ebsprovider.CreateSnapshotRequest) (*ebsprovider.Snapshot, error) { + snap, err := p.EBSProvider.CreateSnapshot(ctx, req) + if err == nil && snap != nil { + p.snapshots[snap.ID] = snap.Handle + } + return snap, err +} + +func (p *trackingProvider) CopySnapshot(ctx context.Context, req ebsprovider.CopySnapshotRequest) (*ebsprovider.Snapshot, error) { + snap, err := p.EBSProvider.CopySnapshot(ctx, req) + if err == nil && snap != nil { + p.snapshots[snap.ID] = snap.Handle + } + return snap, err +} + +func (p *trackingProvider) DeleteSnapshot(ctx context.Context, req ebsprovider.DeleteSnapshotRequest) error { + err := p.EBSProvider.DeleteSnapshot(ctx, req) + if err == nil { + delete(p.snapshots, req.SnapshotID) + } + return err +} + +func (p *trackingProvider) PublishVolume(ctx context.Context, req ebsprovider.PublishVolumeRequest) (*ebsprovider.PublishedVolume, error) { + pub, err := p.EBSProvider.PublishVolume(ctx, req) + if err == nil { + p.published[req.VolumeID] = req.NodeID + } + return pub, err +} + +func (p *trackingProvider) UnpublishVolume(ctx context.Context, req ebsprovider.UnpublishVolumeRequest) error { + err := p.EBSProvider.UnpublishVolume(ctx, req) + if err == nil { + delete(p.published, req.VolumeID) + } + return err +} + +// cleanup removes what the subtest left behind, in dependency order: a +// published volume cannot be deleted, and a provider may hold a volume for its +// snapshots. Failures are logged rather than failed on, so a teardown problem +// never masks the assertion the subtest was actually making. +func (p *trackingProvider) cleanup() { + ctx := context.Background() + + for volumeID, nodeID := range p.published { + if err := p.EBSProvider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, Handle: p.volumes[volumeID], NodeID: nodeID, + }); err != nil { + p.t.Logf("conformance cleanup: unpublish %s: %v", volumeID, err) + } + } + + for snapshotID, handle := range p.snapshots { + if err := p.EBSProvider.DeleteSnapshot(ctx, ebsprovider.DeleteSnapshotRequest{ + Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, Handle: handle, + }); err != nil { + p.t.Logf("conformance cleanup: delete snapshot %s: %v", snapshotID, err) + } + } + + for volumeID, handle := range p.volumes { + if err := p.EBSProvider.DeleteVolume(ctx, ebsprovider.DeleteVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, Handle: handle, + }); err != nil { + p.t.Logf("conformance cleanup: delete volume %s: %v", volumeID, err) + } + } +} diff --git a/spinifex/ebsprovider/errors.go b/spinifex/ebsprovider/errors.go new file mode 100644 index 000000000..925cd7757 --- /dev/null +++ b/spinifex/ebsprovider/errors.go @@ -0,0 +1,93 @@ +package ebsprovider + +import ( + "errors" + "fmt" +) + +var ( + ErrAlreadyExists = errors.New("resource already exists with different parameters") + ErrInvalidArgument = errors.New("invalid argument") + ErrNotFound = errors.New("resource not found") + ErrUnsupportedVersion = errors.New("unsupported EBS provider schema version") + ErrVolumeInUse = errors.New("volume is in use") + + // ErrUnsupportedCapability is returned when a request asks for optional + // behaviour this provider does not advertise in Capabilities. It is + // distinct from ErrInvalidArgument: the request is well formed. + ErrUnsupportedCapability = errors.New("provider does not support the requested capability") + + // ErrUnavailable is returned when the provider could not answer now and + // the same request may succeed later: a peer it had to consult was + // unreachable, or state it had to establish could not be established. + // + // It is the only code that says anything about retrying. Every other one + // classifies what went wrong, which leaves a caller unable to tell a + // permanent refusal from a transient one; returning ErrInternal for a + // transient condition tells a caller to give up when it should retry. + ErrUnavailable = errors.New("provider is temporarily unable to answer") +) + +type ErrorCode string + +const ( + ErrorCodeAlreadyExists ErrorCode = "already_exists" + ErrorCodeInvalidArgument ErrorCode = "invalid_argument" + ErrorCodeNotFound ErrorCode = "not_found" + ErrorCodeUnsupportedVersion ErrorCode = "unsupported_version" + ErrorCodeVolumeInUse ErrorCode = "volume_in_use" + ErrorCodeUnsupportedCap ErrorCode = "unsupported_capability" + ErrorCodeUnavailable ErrorCode = "unavailable" + ErrorCodeInternal ErrorCode = "internal" +) + +// Retryable reports whether repeating the identical request could succeed. +// Callers branch on this rather than on the code, so adding a transient code +// later does not need every call site to be found again. +func (c ErrorCode) Retryable() bool { + return c == ErrorCodeUnavailable +} + +// ProviderError is the stable error representation carried over NATS. +type ProviderError struct { + Code ErrorCode `json:"code"` + Message string `json:"message"` +} + +func (e *ProviderError) Error() string { + if e == nil { + return "" + } + return e.Message +} + +func (e *ProviderError) Unwrap() error { + if e == nil { + return nil + } + switch e.Code { + case ErrorCodeAlreadyExists: + return ErrAlreadyExists + case ErrorCodeInvalidArgument: + return ErrInvalidArgument + case ErrorCodeNotFound: + return ErrNotFound + case ErrorCodeUnsupportedVersion: + return ErrUnsupportedVersion + case ErrorCodeVolumeInUse: + return ErrVolumeInUse + case ErrorCodeUnsupportedCap: + return ErrUnsupportedCapability + case ErrorCodeUnavailable: + return ErrUnavailable + default: + return nil + } +} + +func checkVersion(version uint16) error { + if version != SchemaVersion { + return fmt.Errorf("%w: got %d, want %d", ErrUnsupportedVersion, version, SchemaVersion) + } + return nil +} diff --git a/spinifex/ebsprovider/errors_test.go b/spinifex/ebsprovider/errors_test.go new file mode 100644 index 000000000..43d6c2767 --- /dev/null +++ b/spinifex/ebsprovider/errors_test.go @@ -0,0 +1,78 @@ +package ebsprovider + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// TestProviderErrorUnwrapTable covers every ErrorCode's mapping back to its +// sentinel error, plus the two codes with no sentinel (internal, unknown) +// and the nil-receiver methods a caller can hit via a typed-nil *ProviderError. +func TestProviderErrorUnwrapTable(t *testing.T) { + tests := []struct { + code ErrorCode + want error + }{ + {ErrorCodeAlreadyExists, ErrAlreadyExists}, + {ErrorCodeInvalidArgument, ErrInvalidArgument}, + {ErrorCodeNotFound, ErrNotFound}, + {ErrorCodeUnsupportedVersion, ErrUnsupportedVersion}, + {ErrorCodeVolumeInUse, ErrVolumeInUse}, + {ErrorCodeUnsupportedCap, ErrUnsupportedCapability}, + {ErrorCodeUnavailable, ErrUnavailable}, + } + for _, tt := range tests { + t.Run(string(tt.code), func(t *testing.T) { + err := &ProviderError{Code: tt.code, Message: "msg: " + string(tt.code)} + assert.ErrorIs(t, err, tt.want) + assert.Equal(t, "msg: "+string(tt.code), err.Error()) + }) + } + + t.Run("internal code has no sentinel", func(t *testing.T) { + err := &ProviderError{Code: ErrorCodeInternal, Message: "boom"} + assert.NoError(t, err.Unwrap()) + }) + + t.Run("unknown code has no sentinel", func(t *testing.T) { + err := &ProviderError{Code: ErrorCode("bogus"), Message: "boom"} + assert.NoError(t, err.Unwrap()) + }) + + t.Run("nil receiver", func(t *testing.T) { + var err *ProviderError + assert.NoError(t, err.Unwrap()) + assert.Empty(t, err.Error()) + }) +} + +// TestErrorCodeRetryable pins which refusals a caller may repeat. Every code +// here is named, so adding one without deciding its retry meaning fails rather +// than defaulting to permanent and stranding whatever the caller was doing. +func TestErrorCodeRetryable(t *testing.T) { + retryable := []ErrorCode{ErrorCodeUnavailable} + permanent := []ErrorCode{ + ErrorCodeAlreadyExists, + ErrorCodeInvalidArgument, + ErrorCodeNotFound, + ErrorCodeUnsupportedVersion, + ErrorCodeVolumeInUse, + ErrorCodeUnsupportedCap, + ErrorCodeInternal, + } + + for _, code := range retryable { + assert.Truef(t, code.Retryable(), "%s must read as retryable, or a caller gives up on a request that would have succeeded", code) + } + for _, code := range permanent { + assert.Falsef(t, code.Retryable(), "%s must read as permanent, or a caller retries a request that can never succeed", code) + } + assert.False(t, ErrorCode("bogus").Retryable(), "an unrecognised code must not invite a retry loop") +} + +func TestCheckVersion(t *testing.T) { + assert.NoError(t, checkVersion(SchemaVersion)) + assert.ErrorIs(t, checkVersion(SchemaVersion+1), ErrUnsupportedVersion) + assert.ErrorIs(t, checkVersion(0), ErrUnsupportedVersion) +} diff --git a/spinifex/ebsprovider/hostile/hostile.go b/spinifex/ebsprovider/hostile/hostile.go new file mode 100644 index 000000000..d9fc35d5c --- /dev/null +++ b/spinifex/ebsprovider/hostile/hostile.go @@ -0,0 +1,263 @@ +// Package hostile wraps any EBSProvider in one that misbehaves on purpose. +// It exists because the control plane's handling of a provider that fails, +// stalls, or answers wrongly is the part nothing tests, and the part that +// bites in production. +// +// Faults are derived from a seed, not drawn from a global source, so a +// failure found at hour six of a soak run replays exactly in a unit test. A +// fault injector whose output cannot be reproduced only produces bug reports +// nobody can act on. +package hostile + +import ( + "context" + + "fmt" + "hash/fnv" + "sync" + "time" + + "github.com/mulgadc/spinifex/spinifex/ebsprovider" + "github.com/nats-io/nats.go" +) + +// Fault is what was injected into one call. +type Fault string + +const ( + // FaultNone is the call passing through untouched. + FaultNone Fault = "none" + + // FaultError fails the call without reaching the inner provider. The + // caller is told no, and nothing happened. + FaultError Fault = "error" + + // FaultErrorAfterWork fails the call after the inner provider has already + // done it. This is the interesting half of the pair: the control plane is + // told the volume was not created, and the blocks exist anyway. It is how + // orphans are made, so it is the fault an orphan scan is the oracle for. + FaultErrorAfterWork Fault = "error_after_work" + + // FaultLatency delays the call. Configure MaxLatency past the caller's + // request timeout to reach the case where the caller gives up and the + // provider is still working. + FaultLatency Fault = "latency" + + // FaultUnavailable answers as though nothing is listening, which is the + // condition owner-first routing treats as safe to retry elsewhere. + FaultUnavailable Fault = "unavailable" + + // FaultLie succeeds and returns a wrong answer: a capacity, state or + // handle that does not match what the provider holds. Error paths are + // written deliberately; a plausible wrong answer is what corrupts + // control-plane state quietly. + FaultLie Fault = "lie" +) + +// Config sets how often each fault fires. Rates are independent probabilities +// applied in the order the fields are declared, and a call takes the first +// fault that fires, so the sum need not be 1. All-zero is a pass-through +// provider, which is the right default: a caller must ask to be hurt. +type Config struct { + // Seed fixes the fault sequence. Two runs with the same seed over the + // same calls inject the same faults. + Seed uint64 + + ErrorRate float64 + ErrorAfterWorkRate float64 + LatencyRate float64 + UnavailableRate float64 + LieRate float64 + + // MaxLatency bounds FaultLatency. Zero disables the delay while leaving + // the fault recorded, which is useful for a fast test of the wiring. + MaxLatency time.Duration +} + +// Injection records one fault, with everything needed to reproduce it: the +// seed is in the provider, and Verb, Target and Sequence are the rest of the +// key the fault was derived from. +type Injection struct { + Verb string + Target string + Sequence uint64 + Fault Fault + Detail string +} + +func (i Injection) String() string { + return fmt.Sprintf("%s %s #%d: %s (%s)", i.Verb, i.Target, i.Sequence, i.Fault, i.Detail) +} + +// injectedErrors is the closed set FaultError and FaultErrorAfterWork draw +// from. Every sentinel the contract defines appears, so a soak exercises each +// branch the control plane has for them. +var injectedErrors = []error{ + ebsprovider.ErrAlreadyExists, + ebsprovider.ErrInvalidArgument, + ebsprovider.ErrNotFound, + ebsprovider.ErrVolumeInUse, + ebsprovider.ErrUnsupportedCapability, +} + +// Provider decorates an EBSProvider with faults. +type Provider struct { + inner ebsprovider.EBSProvider + config Config + + mu sync.Mutex + sequences map[string]uint64 + injections []Injection +} + +var _ ebsprovider.EBSProvider = (*Provider)(nil) + +// New wraps inner. A zero Config injects nothing. +func New(inner ebsprovider.EBSProvider, config Config) *Provider { + return &Provider{inner: inner, config: config, sequences: make(map[string]uint64)} +} + +// Injections returns what has been injected so far, in order. +func (p *Provider) Injections() []Injection { + p.mu.Lock() + defer p.mu.Unlock() + return append([]Injection(nil), p.injections...) +} + +// draw decides this call's fault. The key is (verb, target, per-key sequence) +// rather than a global counter, so concurrent calls on different volumes +// cannot reorder each other's faults; only repeated calls on the same target +// advance, and those are ordered by definition. +func (p *Provider) draw(verb, target string) Injection { + return p.drawFault(verb, target, true) +} + +// drawTruthful is draw for a verb that must never return a wrong answer. A +// drawn lie degrades to no fault, and the log says so, so the injection +// record never claims a lie that was not told. +func (p *Provider) drawTruthful(verb, target string) Injection { + return p.drawFault(verb, target, false) +} + +func (p *Provider) drawFault(verb, target string, allowLie bool) Injection { + p.mu.Lock() + key := verb + "\x00" + target + sequence := p.sequences[key] + p.sequences[key] = sequence + 1 + p.mu.Unlock() + + injection := Injection{Verb: verb, Target: target, Sequence: sequence, Fault: FaultNone} + value, selector := p.hash(key, sequence) + + switch { + case value < p.config.ErrorRate: + injection.Fault = FaultError + case value < p.config.ErrorRate+p.config.ErrorAfterWorkRate: + injection.Fault = FaultErrorAfterWork + case value < p.config.ErrorRate+p.config.ErrorAfterWorkRate+p.config.LatencyRate: + injection.Fault = FaultLatency + case value < p.config.ErrorRate+p.config.ErrorAfterWorkRate+p.config.LatencyRate+p.config.UnavailableRate: + injection.Fault = FaultUnavailable + case value < p.config.ErrorRate+p.config.ErrorAfterWorkRate+p.config.LatencyRate+p.config.UnavailableRate+p.config.LieRate: + if !allowLie { + return injection + } + injection.Fault = FaultLie + default: + return injection + } + + injection.Detail = p.detail(injection.Fault, selector) + p.mu.Lock() + p.injections = append(p.injections, injection) + p.mu.Unlock() + return injection +} + +// mix is the SplitMix64 finalizer. FNV alone is not good enough here: its +// output over keys that share long prefixes clusters badly enough to leave +// whole fault bands undrawable, which silently disables a fault class. +func mix(x uint64) uint64 { + x += 0x9e3779b97f4a7c15 + x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9 + x = (x ^ (x >> 27)) * 0x94d049bb133111eb + return x ^ (x >> 31) +} + +// hash turns the seed and key into a uniform draw plus an independent +// selector, so which fault fires and which flavour it takes do not correlate. +func (p *Provider) hash(key string, sequence uint64) (value float64, selector uint64) { + digest := fnv.New64a() + _, _ = digest.Write([]byte(key)) + sum := mix(digest.Sum64() ^ mix(p.config.Seed+sequence)) + return float64(sum>>11) / float64(1<<53), mix(sum) +} + +func (p *Provider) detail(fault Fault, selector uint64) string { + switch fault { + case FaultError, FaultErrorAfterWork: + return injectedErrors[selector%uint64(len(injectedErrors))].Error() + case FaultLatency: + return p.latency(selector).String() + case FaultUnavailable: + return nats.ErrNoResponders.Error() + case FaultLie: + return lieFlavours[selector%uint64(len(lieFlavours))] + default: + return "" + } +} + +func (p *Provider) latency(selector uint64) time.Duration { + if p.config.MaxLatency <= 0 { + return 0 + } + // Scaling a fraction of MaxLatency keeps the bound obvious and avoids + // reducing an unsigned selector into a signed Duration. + fraction := float64(selector>>11) / float64(1<<53) + return time.Duration(fraction * float64(p.config.MaxLatency)) +} + +func (p *Provider) injectedError(selector uint64) error { + return fmt.Errorf("hostile: %w", injectedErrors[selector%uint64(len(injectedErrors))]) +} + +// apply runs the part of a fault that happens before the inner provider. It +// reports whether the caller should stop, and with what. +func (p *Provider) apply(ctx context.Context, injection Injection) (stop bool, err error) { + _, selector := p.hash(injection.Verb+"\x00"+injection.Target, injection.Sequence) + switch injection.Fault { + case FaultError: + return true, p.injectedError(selector) + case FaultUnavailable: + return true, fmt.Errorf("hostile: %w", nats.ErrNoResponders) + case FaultLatency: + return false, sleep(ctx, p.latency(selector)) + default: + return false, nil + } +} + +// after runs the part of a fault that happens once the inner provider has +// already done the work. +func (p *Provider) after(injection Injection, err error) error { + if err != nil || injection.Fault != FaultErrorAfterWork { + return err + } + _, selector := p.hash(injection.Verb+"\x00"+injection.Target, injection.Sequence) + return p.injectedError(selector) +} + +func sleep(ctx context.Context, d time.Duration) error { + if d <= 0 { + return nil + } + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-timer.C: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} diff --git a/spinifex/ebsprovider/hostile/hostile_test.go b/spinifex/ebsprovider/hostile/hostile_test.go new file mode 100644 index 000000000..0195d1b0c --- /dev/null +++ b/spinifex/ebsprovider/hostile/hostile_test.go @@ -0,0 +1,250 @@ +package hostile_test + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/mulgadc/spinifex/spinifex/ebsprovider" + "github.com/mulgadc/spinifex/spinifex/ebsprovider/conformance" + "github.com/mulgadc/spinifex/spinifex/ebsprovider/hostile" + "github.com/mulgadc/spinifex/spinifex/ebsprovider/nullprovider" + "github.com/nats-io/nats.go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const volumeBytes = 1 << 30 + +// countingProvider records how many times the inner provider was actually +// reached, which is the only way to tell FaultError (nothing happened) from +// FaultErrorAfterWork (it happened and was reported as failed). +type countingProvider struct { + ebsprovider.EBSProvider + + creates int + deletes int +} + +var _ ebsprovider.EBSProvider = (*countingProvider)(nil) + +func (c *countingProvider) CreateVolume(ctx context.Context, req ebsprovider.CreateVolumeRequest) (*ebsprovider.Volume, error) { + c.creates++ + return c.EBSProvider.CreateVolume(ctx, req) +} + +func (c *countingProvider) DeleteVolume(ctx context.Context, req ebsprovider.DeleteVolumeRequest) error { + c.deletes++ + return c.EBSProvider.DeleteVolume(ctx, req) +} + +func createVolume(ctx context.Context, provider ebsprovider.EBSProvider, volumeID string) (*ebsprovider.Volume, error) { + return provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), + VolumeID: volumeID, + CapacityRange: ebsprovider.CapacityRange{RequiredBytes: volumeBytes}, + }) +} + +// drive runs a fixed sequence of calls, so two runs differ only in the seed. +func drive(t *testing.T, provider ebsprovider.EBSProvider, count int) { + t.Helper() + for i := range count { + volumeID := fmt.Sprintf("vol-drive%011d", i) + _, _ = createVolume(t.Context(), provider, volumeID) + _, _ = provider.GetVolume(t.Context(), ebsprovider.GetVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, + }) + _ = provider.DeleteVolume(t.Context(), ebsprovider.DeleteVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, + }) + } +} + +func hostileConfig(seed uint64) hostile.Config { + return hostile.Config{ + Seed: seed, + ErrorRate: 0.15, + ErrorAfterWorkRate: 0.15, + LatencyRate: 0.10, + UnavailableRate: 0.10, + LieRate: 0.15, + } +} + +// TestZeroConfigInjectsNothing is the default that matters: wrapping a +// provider must not hurt it until a caller asks. +func TestZeroConfigInjectsNothing(t *testing.T) { + provider := hostile.New(nullprovider.New(), hostile.Config{}) + drive(t, provider, 50) + assert.Empty(t, provider.Injections()) + + volume, err := createVolume(t.Context(), provider, "vol-zeroconfig00001") + require.NoError(t, err) + assert.Equal(t, int64(volumeBytes), volume.CapacityBytes) +} + +// TestPassThroughIsConformant holds the decorator to the same contract it +// decorates. Every difference a run shows must come from a fault that was +// asked for, never from the wrapping. +func TestPassThroughIsConformant(t *testing.T) { + conformance.RunSuite(t, func(t *testing.T) ebsprovider.EBSProvider { + return hostile.New(ebsprovider.NewMemoryProvider(conformance.ReferenceCapabilities), hostile.Config{}) + }) +} + +// TestSameSeedReplaysTheSameFaults is the whole reason this is seeded. A soak +// failure at hour six has to be reproducible in a test, and it can only be if +// the fault sequence is a function of the seed and the calls. +func TestSameSeedReplaysTheSameFaults(t *testing.T) { + first := hostile.New(nullprovider.New(), hostileConfig(42)) + second := hostile.New(nullprovider.New(), hostileConfig(42)) + drive(t, first, 40) + drive(t, second, 40) + + require.NotEmpty(t, first.Injections(), "the run injected nothing, so it proves nothing") + assert.Equal(t, first.Injections(), second.Injections()) +} + +func TestDifferentSeedsDiverge(t *testing.T) { + first := hostile.New(nullprovider.New(), hostileConfig(42)) + second := hostile.New(nullprovider.New(), hostileConfig(43)) + drive(t, first, 40) + drive(t, second, 40) + + assert.NotEqual(t, first.Injections(), second.Injections()) +} + +// TestEveryFaultClassIsReachable guards against a threshold arithmetic slip +// that would silently disable a class: a fault nobody can draw is a fault +// nobody tests against. +func TestEveryFaultClassIsReachable(t *testing.T) { + provider := hostile.New(nullprovider.New(), hostileConfig(7)) + drive(t, provider, 400) + + seen := map[hostile.Fault]bool{} + for _, injection := range provider.Injections() { + seen[injection.Fault] = true + assert.NotEmpty(t, injection.Detail, "%s has no detail to reproduce it from", injection) + } + for _, fault := range []hostile.Fault{ + hostile.FaultError, + hostile.FaultErrorAfterWork, + hostile.FaultLatency, + hostile.FaultUnavailable, + hostile.FaultLie, + } { + assert.True(t, seen[fault], "%s was never drawn", fault) + } +} + +// TestFaultErrorNeverReachesTheProvider and its ErrorAfterWork counterpart pin +// the distinction the two faults exist for. +func TestFaultErrorNeverReachesTheProvider(t *testing.T) { + counting := &countingProvider{EBSProvider: nullprovider.New()} + provider := hostile.New(counting, hostile.Config{Seed: 1, ErrorRate: 1}) + + _, err := createVolume(t.Context(), provider, "vol-neverreaches001") + require.Error(t, err) + assert.Zero(t, counting.creates, "the provider was reached for a call that was supposed to fail first") +} + +// TestFaultErrorAfterWorkLeavesTheVolumeBehind is the orphan-making case, and +// the reason an enumeration-based scan is the right oracle for a soak: the +// caller was told the create failed, and the volume exists. +func TestFaultErrorAfterWorkLeavesTheVolumeBehind(t *testing.T) { + inner := nullprovider.New() + counting := &countingProvider{EBSProvider: inner} + provider := hostile.New(counting, hostile.Config{Seed: 1, ErrorAfterWorkRate: 1}) + + const volumeID = "vol-leftbehind00001" + _, err := createVolume(t.Context(), provider, volumeID) + require.Error(t, err, "the caller must be told this failed") + assert.Equal(t, 1, counting.creates, "the work must actually have happened") + + // The oracle reads the provider directly. Asking through the injector + // would be asking the thing under test whether it lied. + resp, err := inner.ListVolumes(t.Context(), ebsprovider.ListVolumesRequest{Versioned: ebsprovider.NewVersioned()}) + require.NoError(t, err) + var ids []string + for _, ref := range resp.Volumes { + ids = append(ids, ref.ID) + } + assert.Contains(t, ids, volumeID, "the provider holds a volume the caller believes was never created") +} + +// TestUnavailableIsRetryable matters because owner-first routing treats +// no-responders as safe to retry elsewhere and a timeout as not. +func TestUnavailableIsRetryable(t *testing.T) { + provider := hostile.New(nullprovider.New(), hostile.Config{Seed: 1, UnavailableRate: 1}) + _, err := createVolume(t.Context(), provider, "vol-unavailable0001") + require.ErrorIs(t, err, nats.ErrNoResponders) +} + +// aliasingProvider hands back a pointer into its own state, the way a +// provider that forgot to copy would. MemoryProvider returns copies, so a +// lie told through it cannot corrupt anything and proves nothing about +// whether the injector copies. +type aliasingProvider struct { + ebsprovider.EBSProvider + + volume *ebsprovider.Volume +} + +var _ ebsprovider.EBSProvider = (*aliasingProvider)(nil) + +func (a *aliasingProvider) CreateVolume(ctx context.Context, req ebsprovider.CreateVolumeRequest) (*ebsprovider.Volume, error) { + volume, err := a.EBSProvider.CreateVolume(ctx, req) + if err != nil { + return nil, err + } + a.volume = volume + return a.volume, nil +} + +// TestLiesDoNotCorruptTheProvider keeps the fault a fault. If a wrong answer +// also changed what the provider held, the oracle would be measuring the +// injector's bugs instead of the control plane's. +func TestLiesDoNotCorruptTheProvider(t *testing.T) { + aliasing := &aliasingProvider{EBSProvider: nullprovider.New()} + provider := hostile.New(aliasing, hostile.Config{Seed: 3, LieRate: 1}) + + const volumeID = "vol-lieaboutit00001" + lied, err := createVolume(t.Context(), provider, volumeID) + require.NoError(t, err) + + require.NotNil(t, aliasing.volume) + assert.Equal(t, int64(volumeBytes), aliasing.volume.CapacityBytes, "the lie was written into the provider's own object") + assert.NotEqual(t, *aliasing.volume, *lied, "a lie that matches the truth is not a lie") +} + +// TestListVolumesNeverLies protects the oracle. A soak checks its invariants +// with enumeration, so an enumeration that can be made to agree with a broken +// control plane cannot falsify anything. +func TestListVolumesNeverLies(t *testing.T) { + provider := hostile.New(nullprovider.New(), hostile.Config{Seed: 5, LieRate: 1}) + drive(t, provider, 30) + for range 30 { + _, err := provider.ListVolumes(t.Context(), ebsprovider.ListVolumesRequest{Versioned: ebsprovider.NewVersioned()}) + require.NoError(t, err) + } + for _, injection := range provider.Injections() { + assert.NotEqual(t, "volume.list", injection.Verb, "enumeration was faulted: %s", injection) + } +} + +// TestLatencyRespectsCancellation stops a long delay from outliving the caller +// that gave up on it, which is what would make a soak run leak goroutines. +func TestLatencyRespectsCancellation(t *testing.T) { + provider := hostile.New(nullprovider.New(), hostile.Config{ + Seed: 1, LatencyRate: 1, MaxLatency: time.Hour, + }) + ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond) + defer cancel() + + started := time.Now() + _, err := createVolume(ctx, provider, "vol-cancelme0000001") + require.ErrorIs(t, err, context.DeadlineExceeded) + assert.Less(t, time.Since(started), 5*time.Second) +} diff --git a/spinifex/ebsprovider/hostile/soak_test.go b/spinifex/ebsprovider/hostile/soak_test.go new file mode 100644 index 000000000..248322ac9 --- /dev/null +++ b/spinifex/ebsprovider/hostile/soak_test.go @@ -0,0 +1,229 @@ +package hostile_test + +import ( + "context" + "fmt" + "os" + "sort" + "sync" + "testing" + "time" + + "github.com/mulgadc/spinifex/spinifex/ebsprovider" + "github.com/mulgadc/spinifex/spinifex/ebsprovider/hostile" + "github.com/mulgadc/spinifex/spinifex/ebsprovider/nullprovider" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// soakDuration keeps the default short enough for CI while letting a real +// soak run for hours from the same code. A fault injector only earns its +// keep over a long run, but a long run must not be the only way to use it. +func soakDuration(t *testing.T) time.Duration { + t.Helper() + raw := os.Getenv("EBS_SOAK_DURATION") + if raw == "" { + return 2 * time.Second + } + parsed, err := time.ParseDuration(raw) + require.NoError(t, err, "EBS_SOAK_DURATION") + return parsed +} + +// belief is what the caller was told about a volume, which is all a control +// plane ever has. The provider's own answer is the truth it is checked +// against. +type belief struct { + createdOK bool + deletedOK bool +} + +// soakResult is what one run produced, so two runs can be compared. +type soakResult struct { + beliefs map[string]belief + injections []hostile.Injection + held []string +} + +func runSoak(t *testing.T, seed uint64, workers int, duration time.Duration) soakResult { + t.Helper() + inner := nullprovider.New() + provider := hostile.New(inner, hostile.Config{ + Seed: seed, + ErrorRate: 0.10, + ErrorAfterWorkRate: 0.10, + LatencyRate: 0.05, + UnavailableRate: 0.05, + LieRate: 0.10, + MaxLatency: 2 * time.Millisecond, + }) + + var mu sync.Mutex + beliefs := map[string]belief{} + deadline := time.Now().Add(duration) + + var wg sync.WaitGroup + for worker := range workers { + wg.Go(func() { + for i := 0; time.Now().Before(deadline); i++ { + // Each worker owns its own ID space, so faults drawn per + // volume never depend on how the workers interleave. + volumeID := fmt.Sprintf("vol-soak%04d%07d", worker, i) + state := soakOneVolume(t.Context(), provider, volumeID) + mu.Lock() + beliefs[volumeID] = state + mu.Unlock() + } + }) + } + wg.Wait() + + return soakResult{ + beliefs: beliefs, + injections: provider.Injections(), + held: drainHeldVolumes(t, inner), + } +} + +// soakOneVolume drives one volume through its whole life without retrying. +// Retries would hide leaks, and what is being looked for here is what the +// control plane loses track of when a provider misbehaves once. +func soakOneVolume(ctx context.Context, provider ebsprovider.EBSProvider, volumeID string) belief { + var state belief + if _, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), + VolumeID: volumeID, + CapacityRange: ebsprovider.CapacityRange{RequiredBytes: volumeBytes}, + }); err != nil { + return state + } + state.createdOK = true + + _, _ = provider.GetVolume(ctx, ebsprovider.GetVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, + }) + if _, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: "node-1", + }); err == nil { + _ = provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: "node-1", + }) + } + + if err := provider.DeleteVolume(ctx, ebsprovider.DeleteVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, + }); err == nil { + state.deletedOK = true + } + return state +} + +func drainHeldVolumes(t *testing.T, provider ebsprovider.EBSProvider) []string { + t.Helper() + var ids []string + var token string + for range 1000 { + resp, err := provider.ListVolumes(context.Background(), ebsprovider.ListVolumesRequest{ + Versioned: ebsprovider.NewVersioned(), StartingToken: token, + }) + require.NoError(t, err) + for _, ref := range resp.Volumes { + ids = append(ids, ref.ID) + } + if resp.NextToken == "" { + sort.Strings(ids) + return ids + } + token = resp.NextToken + } + t.Fatal("ListVolumes did not terminate") + return nil +} + +// explainedBy reports whether an injection recorded for volumeID accounts for +// the provider still holding it: work that was done and then reported failed. +func explainedBy(injections []hostile.Injection, volumeID string) bool { + for _, injection := range injections { + if injection.Target != volumeID { + continue + } + switch injection.Fault { + case hostile.FaultErrorAfterWork, hostile.FaultLatency, hostile.FaultLie: + return true + } + } + return false +} + +// TestSoak_EveryLeakIsAttributable is the oracle. Injecting faults proves +// nothing on its own; what proves something is that every volume the provider +// still holds can be traced to a fault that was injected. A leftover volume +// with no injection behind it is the control plane losing track of storage on +// its own, which is the bug worth finding. +func TestSoak_EveryLeakIsAttributable(t *testing.T) { + result := runSoak(t, 20260811, 8, soakDuration(t)) + require.NotEmpty(t, result.injections, "the run injected nothing, so it proves nothing") + require.NotEmpty(t, result.beliefs) + + held := map[string]bool{} + for _, id := range result.held { + held[id] = true + } + + var unexplained []string + for volumeID, state := range result.beliefs { + switch { + case state.deletedOK && held[volumeID]: + unexplained = append(unexplained, volumeID+" (delete reported success, still held)") + case !state.createdOK && held[volumeID] && !explainedBy(result.injections, volumeID): + unexplained = append(unexplained, volumeID+" (create reported failure, held anyway)") + case state.createdOK && !state.deletedOK && !held[volumeID] && !explainedBy(result.injections, volumeID): + unexplained = append(unexplained, volumeID+" (created and not deleted, gone anyway)") + } + } + sort.Strings(unexplained) + assert.Empty(t, unexplained, "volumes the injection log cannot account for") +} + +// TestSoak_ConcurrentRunsWithTheSameSeedMatch is what makes a long run worth +// running: reproducing hour six needs the faults to be a function of the seed +// and the calls, not of how the workers happened to interleave. +func TestSoak_ConcurrentRunsWithTheSameSeedMatch(t *testing.T) { + const seed = 99 + const duration = 500 * time.Millisecond + + first := runSoak(t, seed, 4, duration) + second := runSoak(t, seed, 4, duration) + + // The runs cover different volumes, since each stops on a deadline. Only + // the volumes both reached can be compared, and for those the faults must + // be identical despite the interleaving being different. + shared := 0 + firstByVolume := indexInjections(first.injections) + secondByVolume := indexInjections(second.injections) + for volumeID, injections := range firstByVolume { + other, ok := secondByVolume[volumeID] + if !ok { + continue + } + shared++ + assert.Equal(t, injections, other, "volume %s drew different faults across runs with the same seed", volumeID) + } + require.Positive(t, shared, "the runs shared no volumes, so nothing was compared") +} + +func indexInjections(injections []hostile.Injection) map[string][]hostile.Injection { + byVolume := map[string][]hostile.Injection{} + for _, injection := range injections { + byVolume[injection.Target] = append(byVolume[injection.Target], injection) + } + for _, list := range byVolume { + sort.Slice(list, func(i, j int) bool { + if list[i].Verb != list[j].Verb { + return list[i].Verb < list[j].Verb + } + return list[i].Sequence < list[j].Sequence + }) + } + return byVolume +} diff --git a/spinifex/ebsprovider/hostile/verbs.go b/spinifex/ebsprovider/hostile/verbs.go new file mode 100644 index 000000000..aabad533e --- /dev/null +++ b/spinifex/ebsprovider/hostile/verbs.go @@ -0,0 +1,238 @@ +package hostile + +import ( + "context" + + "github.com/mulgadc/spinifex/spinifex/ebsprovider" +) + +// Verb names match the span names the contract emits, so an injection log and +// a trace can be read side by side. +const ( + verbCapabilities = "capabilities" + verbVolumeCreate = "volume.create" + verbVolumeDescribe = "volume.describe" + verbVolumeList = "volume.list" + verbVolumeExpand = "volume.expand" + verbVolumeDelete = "volume.delete" + verbVolumePublish = "volume.publish" + verbVolumeUnpublish = "volume.unpublish" + verbSnapshotCreate = "snapshot.create" + verbSnapshotDelete = "snapshot.delete" + verbSnapshotCopy = "snapshot.copy" + verbSnapshotList = "snapshot.list" +) + +// lieFlavours are the ways a successful answer can be wrong. Each one is a +// field the control plane reads and acts on. +var lieFlavours = []string{"capacity", "handle", "state"} + +// corruptVolume returns a copy of volume with one field wrong. It copies +// rather than mutating: the inner provider's own object must stay correct, or +// the fault becomes a real bug in the fake and nothing downstream means +// anything. +func corruptVolume(volume *ebsprovider.Volume, flavour string) *ebsprovider.Volume { + if volume == nil { + return nil + } + corrupted := *volume + switch flavour { + case "capacity": + corrupted.CapacityBytes *= 2 + case "handle": + corrupted.Handle = "hostile://wrong-handle" + case "state": + if corrupted.State == ebsprovider.VolumeStateAvailable { + corrupted.State = ebsprovider.VolumeStateInUse + } else { + corrupted.State = ebsprovider.VolumeStateAvailable + } + } + return &corrupted +} + +func corruptSnapshot(snapshot *ebsprovider.Snapshot, flavour string) *ebsprovider.Snapshot { + if snapshot == nil { + return nil + } + corrupted := *snapshot + switch flavour { + case "capacity": + corrupted.SizeBytes *= 2 + case "handle": + corrupted.SourceVolumeID = "vol-hostilewrongsrc" + case "state": + corrupted.State = ebsprovider.SnapshotStatePending + } + return &corrupted +} + +func corruptPublished(published *ebsprovider.PublishedVolume, flavour string) *ebsprovider.PublishedVolume { + if published == nil { + return nil + } + corrupted := *published + switch flavour { + case "handle", "capacity": + corrupted.NBDURI = "nbd+unix:///?socket=/hostile/wrong.sock" + case "state": + corrupted.NodeID = "hostile-wrong-node" + } + return &corrupted +} + +func (p *Provider) GetCapabilities(ctx context.Context, req ebsprovider.GetCapabilitiesRequest) (*ebsprovider.GetCapabilitiesResponse, error) { + injection := p.draw(verbCapabilities, "") + if stop, err := p.apply(ctx, injection); stop || err != nil { + return nil, err + } + resp, err := p.inner.GetCapabilities(ctx, req) + if err := p.after(injection, err); err != nil { + return nil, err + } + return resp, nil +} + +func (p *Provider) CreateVolume(ctx context.Context, req ebsprovider.CreateVolumeRequest) (*ebsprovider.Volume, error) { + injection := p.draw(verbVolumeCreate, req.VolumeID) + if stop, err := p.apply(ctx, injection); stop || err != nil { + return nil, err + } + volume, err := p.inner.CreateVolume(ctx, req) + if err := p.after(injection, err); err != nil { + return nil, err + } + if injection.Fault == FaultLie { + return corruptVolume(volume, injection.Detail), nil + } + return volume, nil +} + +func (p *Provider) GetVolume(ctx context.Context, req ebsprovider.GetVolumeRequest) (*ebsprovider.Volume, error) { + injection := p.draw(verbVolumeDescribe, req.VolumeID) + if stop, err := p.apply(ctx, injection); stop || err != nil { + return nil, err + } + volume, err := p.inner.GetVolume(ctx, req) + if err := p.after(injection, err); err != nil { + return nil, err + } + if injection.Fault == FaultLie { + return corruptVolume(volume, injection.Detail), nil + } + return volume, nil +} + +// ListVolumes is never made to lie. Enumeration is the oracle a soak run +// checks its invariants with, and an instrument that reports what it was told +// to report cannot falsify anything. +func (p *Provider) ListVolumes(ctx context.Context, req ebsprovider.ListVolumesRequest) (*ebsprovider.ListVolumesResponse, error) { + injection := p.drawTruthful(verbVolumeList, "") + if stop, err := p.apply(ctx, injection); stop || err != nil { + return nil, err + } + resp, err := p.inner.ListVolumes(ctx, req) + if err := p.after(injection, err); err != nil { + return nil, err + } + return resp, nil +} + +func (p *Provider) ExpandVolume(ctx context.Context, req ebsprovider.ExpandVolumeRequest) (*ebsprovider.Volume, error) { + injection := p.draw(verbVolumeExpand, req.VolumeID) + if stop, err := p.apply(ctx, injection); stop || err != nil { + return nil, err + } + volume, err := p.inner.ExpandVolume(ctx, req) + if err := p.after(injection, err); err != nil { + return nil, err + } + if injection.Fault == FaultLie { + return corruptVolume(volume, injection.Detail), nil + } + return volume, nil +} + +func (p *Provider) DeleteVolume(ctx context.Context, req ebsprovider.DeleteVolumeRequest) error { + injection := p.draw(verbVolumeDelete, req.VolumeID) + if stop, err := p.apply(ctx, injection); stop || err != nil { + return err + } + return p.after(injection, p.inner.DeleteVolume(ctx, req)) +} + +func (p *Provider) CreateSnapshot(ctx context.Context, req ebsprovider.CreateSnapshotRequest) (*ebsprovider.Snapshot, error) { + injection := p.draw(verbSnapshotCreate, req.SnapshotID) + if stop, err := p.apply(ctx, injection); stop || err != nil { + return nil, err + } + snapshot, err := p.inner.CreateSnapshot(ctx, req) + if err := p.after(injection, err); err != nil { + return nil, err + } + if injection.Fault == FaultLie { + return corruptSnapshot(snapshot, injection.Detail), nil + } + return snapshot, nil +} + +func (p *Provider) DeleteSnapshot(ctx context.Context, req ebsprovider.DeleteSnapshotRequest) error { + injection := p.draw(verbSnapshotDelete, req.SnapshotID) + if stop, err := p.apply(ctx, injection); stop || err != nil { + return err + } + return p.after(injection, p.inner.DeleteSnapshot(ctx, req)) +} + +func (p *Provider) CopySnapshot(ctx context.Context, req ebsprovider.CopySnapshotRequest) (*ebsprovider.Snapshot, error) { + injection := p.draw(verbSnapshotCopy, req.DestinationSnapshotID) + if stop, err := p.apply(ctx, injection); stop || err != nil { + return nil, err + } + snapshot, err := p.inner.CopySnapshot(ctx, req) + if err := p.after(injection, err); err != nil { + return nil, err + } + if injection.Fault == FaultLie { + return corruptSnapshot(snapshot, injection.Detail), nil + } + return snapshot, nil +} + +// ListSnapshots is never made to lie, for the same reason ListVolumes is not: +// it is an oracle, and an instrument that reports what it was told to report +// cannot falsify anything. +func (p *Provider) ListSnapshots(ctx context.Context, req ebsprovider.ListSnapshotsRequest) (*ebsprovider.ListSnapshotsResponse, error) { + injection := p.drawTruthful(verbSnapshotList, "") + if stop, err := p.apply(ctx, injection); stop || err != nil { + return nil, err + } + resp, err := p.inner.ListSnapshots(ctx, req) + if err := p.after(injection, err); err != nil { + return nil, err + } + return resp, nil +} + +func (p *Provider) PublishVolume(ctx context.Context, req ebsprovider.PublishVolumeRequest) (*ebsprovider.PublishedVolume, error) { + injection := p.draw(verbVolumePublish, req.VolumeID) + if stop, err := p.apply(ctx, injection); stop || err != nil { + return nil, err + } + published, err := p.inner.PublishVolume(ctx, req) + if err := p.after(injection, err); err != nil { + return nil, err + } + if injection.Fault == FaultLie { + return corruptPublished(published, injection.Detail), nil + } + return published, nil +} + +func (p *Provider) UnpublishVolume(ctx context.Context, req ebsprovider.UnpublishVolumeRequest) error { + injection := p.draw(verbVolumeUnpublish, req.VolumeID) + if stop, err := p.apply(ctx, injection); stop || err != nil { + return err + } + return p.after(injection, p.inner.UnpublishVolume(ctx, req)) +} diff --git a/spinifex/ebsprovider/memory.go b/spinifex/ebsprovider/memory.go new file mode 100644 index 000000000..68f6c19a3 --- /dev/null +++ b/spinifex/ebsprovider/memory.go @@ -0,0 +1,443 @@ +package ebsprovider + +import ( + "bytes" + "context" + "fmt" + "sort" + "sync" + "time" +) + +// MemoryProvider is a deterministic, concurrency-safe Provider used by +// control-plane tests. It implements the same idempotency rules expected of a +// real provider rather than acting as a programmable mock. +type MemoryProvider struct { + mu sync.RWMutex + capabilities Capabilities + volumes map[string]*memoryVolume + snapshots map[string]*Snapshot + now func() time.Time +} + +type memoryVolume struct { + volume Volume + sourceSnapshotID string + parameters []byte + seed []byte + published *PublishedVolume +} + +var _ EBSProvider = (*MemoryProvider)(nil) + +func NewMemoryProvider(capabilities Capabilities) *MemoryProvider { + // Publication lives in this process's map, so node is what this provider + // does whatever the caller passed. An unset scope is filled rather than + // left empty: the contract has no valid empty scope, and a caller reading + // one would have to guess. + if capabilities.Exclusion.Scope == "" { + capabilities.Exclusion.Scope = ExclusionScopeNode + } + return &MemoryProvider{ + capabilities: capabilities, + volumes: make(map[string]*memoryVolume), + snapshots: make(map[string]*Snapshot), + now: time.Now, + } +} + +func (m *MemoryProvider) GetCapabilities(_ context.Context, req GetCapabilitiesRequest) (*GetCapabilitiesResponse, error) { + if err := checkVersion(req.SchemaVersion); err != nil { + return nil, err + } + return &GetCapabilitiesResponse{Versioned: NewVersioned(), Capabilities: m.capabilities}, nil +} + +func (m *MemoryProvider) CreateVolume(_ context.Context, req CreateVolumeRequest) (*Volume, error) { + if err := checkVersion(req.SchemaVersion); err != nil { + return nil, err + } + if req.VolumeID == "" || req.CapacityRange.RequiredBytes <= 0 || + (req.CapacityRange.LimitBytes > 0 && req.CapacityRange.RequiredBytes > req.CapacityRange.LimitBytes) { + return nil, fmt.Errorf("%w: invalid volume ID or capacity range", ErrInvalidArgument) + } + if req.SourceSnapshotID != "" && req.SourceSnapshotVolumeID == "" { + return nil, fmt.Errorf("%w: source snapshot volume ID is required with a source snapshot", ErrInvalidArgument) + } + if err := ValidateSeedData(req.SeedData); err != nil { + return nil, err + } + if len(req.SeedData) > 0 && !m.capabilities.VolumeSeeding { + return nil, fmt.Errorf("%w: volume seeding", ErrUnsupportedCapability) + } + if int64(len(req.SeedData)) > req.CapacityRange.RequiredBytes { + return nil, fmt.Errorf("%w: seed data is larger than the requested capacity", ErrInvalidArgument) + } + + m.mu.Lock() + defer m.mu.Unlock() + // SeedData is deliberately absent from the mismatch check below: a seed + // applies only to a volume this call creates, so a repeated create returns + // the existing volume rather than overwriting bytes the guest has since set. + if existing := m.volumes[req.VolumeID]; existing != nil { + if existing.volume.CapacityBytes != req.CapacityRange.RequiredBytes || + existing.volume.AvailabilityZone != req.AvailabilityZone || + existing.sourceSnapshotID != req.SourceSnapshotID || + !bytes.Equal(existing.parameters, req.Parameters) { + return nil, fmt.Errorf("%w: volume %s", ErrAlreadyExists, req.VolumeID) + } + return cloneVolume(&existing.volume), nil + } + if req.SourceSnapshotID != "" { + snapshot := m.snapshots[req.SourceSnapshotID] + if snapshot == nil { + return nil, fmt.Errorf("%w: snapshot %s", ErrNotFound, req.SourceSnapshotID) + } + if req.CapacityRange.RequiredBytes < snapshot.SizeBytes { + return nil, fmt.Errorf("%w: requested capacity is smaller than snapshot %s", ErrInvalidArgument, req.SourceSnapshotID) + } + // The caller names the snapshot's origin volume, so a wrong name is a + // caller bug the reference provider refuses rather than resolving + // blocks against a volume the snapshot never came from. + if req.SourceSnapshotVolumeID != snapshot.SourceVolumeID { + return nil, fmt.Errorf("%w: snapshot %s was taken from volume %s, not %s", + ErrInvalidArgument, req.SourceSnapshotID, snapshot.SourceVolumeID, req.SourceSnapshotVolumeID) + } + } + + volume := Volume{ + ID: req.VolumeID, + CapacityBytes: req.CapacityRange.RequiredBytes, + State: VolumeStateAvailable, + Handle: "memory://volume/" + req.VolumeID, + AvailabilityZone: req.AvailabilityZone, + } + m.volumes[req.VolumeID] = &memoryVolume{ + volume: volume, + sourceSnapshotID: req.SourceSnapshotID, + parameters: bytes.Clone(req.Parameters), + seed: bytes.Clone(req.SeedData), + } + return cloneVolume(&volume), nil +} + +// SeedData returns the bytes a CreateVolume call seeded the volume with, so a +// control-plane test can assert what it asked the provider to write. +func (m *MemoryProvider) SeedData(volumeID string) ([]byte, bool) { + m.mu.RLock() + defer m.mu.RUnlock() + volume := m.volumes[volumeID] + if volume == nil { + return nil, false + } + return bytes.Clone(volume.seed), true +} + +func (m *MemoryProvider) GetVolume(_ context.Context, req GetVolumeRequest) (*Volume, error) { + if err := checkVersion(req.SchemaVersion); err != nil { + return nil, err + } + if req.VolumeID == "" { + return nil, fmt.Errorf("%w: volume ID is required", ErrInvalidArgument) + } + m.mu.RLock() + defer m.mu.RUnlock() + volume := m.volumes[req.VolumeID] + if volume == nil || (req.Handle != "" && req.Handle != volume.volume.Handle) { + return nil, fmt.Errorf("%w: volume %s", ErrNotFound, req.VolumeID) + } + return cloneVolume(&volume.volume), nil +} + +// ListVolumes pages through the volumes this provider holds, ordered by ID so +// a token stays meaningful across calls. The token is the ID to resume after, +// which keeps paging correct when volumes are created or deleted mid-walk. +func (m *MemoryProvider) ListVolumes(_ context.Context, req ListVolumesRequest) (*ListVolumesResponse, error) { + if err := checkVersion(req.SchemaVersion); err != nil { + return nil, err + } + if !m.capabilities.VolumeEnumeration { + return nil, fmt.Errorf("%w: volume enumeration", ErrUnsupportedCapability) + } + + m.mu.RLock() + defer m.mu.RUnlock() + ids := make([]string, 0, len(m.volumes)) + for id := range m.volumes { + if id > req.StartingToken { + ids = append(ids, id) + } + } + sort.Strings(ids) + + pageSize := int(req.PageSize()) + response := &ListVolumesResponse{Versioned: NewVersioned()} + if len(ids) > pageSize { + response.NextToken = ids[pageSize-1] + ids = ids[:pageSize] + } + for _, id := range ids { + response.Volumes = append(response.Volumes, VolumeRef{ID: id, Handle: m.volumes[id].volume.Handle}) + } + return response, nil +} + +func (m *MemoryProvider) ExpandVolume(_ context.Context, req ExpandVolumeRequest) (*Volume, error) { + if err := checkVersion(req.SchemaVersion); err != nil { + return nil, err + } + if req.VolumeID == "" || req.CapacityRange.RequiredBytes <= 0 || + (req.CapacityRange.LimitBytes > 0 && req.CapacityRange.RequiredBytes > req.CapacityRange.LimitBytes) { + return nil, fmt.Errorf("%w: invalid volume ID or capacity range", ErrInvalidArgument) + } + m.mu.Lock() + defer m.mu.Unlock() + volume := m.volumes[req.VolumeID] + if volume == nil || (req.Handle != "" && req.Handle != volume.volume.Handle) { + return nil, fmt.Errorf("%w: volume %s", ErrNotFound, req.VolumeID) + } + if req.CapacityRange.RequiredBytes < volume.volume.CapacityBytes { + return nil, fmt.Errorf("%w: volume expansion is grow-only", ErrInvalidArgument) + } + if volume.published != nil && !m.capabilities.OnlineExpansion && req.CapacityRange.RequiredBytes > volume.volume.CapacityBytes { + return nil, fmt.Errorf("%w: provider does not support online expansion", ErrVolumeInUse) + } + volume.volume.CapacityBytes = req.CapacityRange.RequiredBytes + return cloneVolume(&volume.volume), nil +} + +func (m *MemoryProvider) DeleteVolume(_ context.Context, req DeleteVolumeRequest) error { + if err := checkVersion(req.SchemaVersion); err != nil { + return err + } + if req.VolumeID == "" { + return fmt.Errorf("%w: volume ID is required", ErrInvalidArgument) + } + m.mu.Lock() + defer m.mu.Unlock() + volume := m.volumes[req.VolumeID] + if volume == nil { + return nil + } + if req.Handle != "" && req.Handle != volume.volume.Handle { + return nil + } + if volume.published != nil { + return fmt.Errorf("%w: volume %s", ErrVolumeInUse, req.VolumeID) + } + delete(m.volumes, req.VolumeID) + return nil +} + +func (m *MemoryProvider) CreateSnapshot(_ context.Context, req CreateSnapshotRequest) (*Snapshot, error) { + if err := checkVersion(req.SchemaVersion); err != nil { + return nil, err + } + if req.SnapshotID == "" || req.VolumeID == "" { + return nil, fmt.Errorf("%w: snapshot and volume IDs are required", ErrInvalidArgument) + } + m.mu.Lock() + defer m.mu.Unlock() + if existing := m.snapshots[req.SnapshotID]; existing != nil { + if existing.SourceVolumeID != req.VolumeID { + return nil, fmt.Errorf("%w: snapshot %s", ErrAlreadyExists, req.SnapshotID) + } + return cloneSnapshot(existing), nil + } + volume := m.volumes[req.VolumeID] + if volume == nil || (req.VolumeHandle != "" && req.VolumeHandle != volume.volume.Handle) { + return nil, fmt.Errorf("%w: volume %s", ErrNotFound, req.VolumeID) + } + snapshot := &Snapshot{ + ID: req.SnapshotID, + SourceVolumeID: req.VolumeID, + SizeBytes: volume.volume.CapacityBytes, + CreatedAt: m.now().UTC(), + State: SnapshotStateCompleted, + Handle: "memory://snapshot/" + req.SnapshotID, + } + m.snapshots[req.SnapshotID] = snapshot + return cloneSnapshot(snapshot), nil +} + +// Snapshot reports what the provider holds under snapshotID, so a +// control-plane test can assert a snapshot really reached the provider rather +// than existing only as a control-plane document. +func (m *MemoryProvider) Snapshot(snapshotID string) (*Snapshot, bool) { + m.mu.RLock() + defer m.mu.RUnlock() + snapshot := m.snapshots[snapshotID] + if snapshot == nil { + return nil, false + } + return cloneSnapshot(snapshot), true +} + +func (m *MemoryProvider) DeleteSnapshot(_ context.Context, req DeleteSnapshotRequest) error { + if err := checkVersion(req.SchemaVersion); err != nil { + return err + } + if req.SnapshotID == "" { + return fmt.Errorf("%w: snapshot ID is required", ErrInvalidArgument) + } + m.mu.Lock() + defer m.mu.Unlock() + snapshot := m.snapshots[req.SnapshotID] + if snapshot == nil || (req.Handle != "" && req.Handle != snapshot.Handle) { + return nil + } + delete(m.snapshots, req.SnapshotID) + return nil +} + +// CopySnapshot duplicates SourceSnapshotID under DestinationSnapshotID. +// Mirrors viperblock.CopySnapshotMeta's contract: it refuses an existing +// destination, an empty or equal ID pair, and a snapshot that does not +// belong to VolumeID (the caller's claimed source volume). +func (m *MemoryProvider) CopySnapshot(_ context.Context, req CopySnapshotRequest) (*Snapshot, error) { + if err := checkVersion(req.SchemaVersion); err != nil { + return nil, err + } + if req.SourceSnapshotID == "" || req.DestinationSnapshotID == "" || req.VolumeID == "" { + return nil, fmt.Errorf("%w: source snapshot, destination snapshot, and volume IDs are required", ErrInvalidArgument) + } + if req.SourceSnapshotID == req.DestinationSnapshotID { + return nil, fmt.Errorf("%w: destination snapshot must differ from the source", ErrInvalidArgument) + } + + m.mu.Lock() + defer m.mu.Unlock() + src := m.snapshots[req.SourceSnapshotID] + if src == nil { + return nil, fmt.Errorf("%w: snapshot %s", ErrNotFound, req.SourceSnapshotID) + } + if src.SourceVolumeID != req.VolumeID { + return nil, fmt.Errorf("%w: snapshot %s belongs to volume %s, not %s", ErrInvalidArgument, req.SourceSnapshotID, src.SourceVolumeID, req.VolumeID) + } + if existing := m.snapshots[req.DestinationSnapshotID]; existing != nil { + return nil, fmt.Errorf("%w: snapshot %s", ErrAlreadyExists, req.DestinationSnapshotID) + } + + dst := &Snapshot{ + ID: req.DestinationSnapshotID, + SourceVolumeID: src.SourceVolumeID, + SizeBytes: src.SizeBytes, + CreatedAt: m.now().UTC(), + State: SnapshotStateCompleted, + Handle: "memory://snapshot/" + req.DestinationSnapshotID, + } + m.snapshots[req.DestinationSnapshotID] = dst + return cloneSnapshot(dst), nil +} + +// ListSnapshots pages through the snapshots this provider holds, ordered by ID +// so a token stays meaningful across calls. The token is the ID to resume +// after, which keeps paging correct when snapshots come and go mid-walk. +func (m *MemoryProvider) ListSnapshots(_ context.Context, req ListSnapshotsRequest) (*ListSnapshotsResponse, error) { + if err := checkVersion(req.SchemaVersion); err != nil { + return nil, err + } + if !m.capabilities.SnapshotEnumeration { + return nil, fmt.Errorf("%w: snapshot enumeration", ErrUnsupportedCapability) + } + + m.mu.RLock() + defer m.mu.RUnlock() + ids := make([]string, 0, len(m.snapshots)) + for id := range m.snapshots { + if id > req.StartingToken { + ids = append(ids, id) + } + } + sort.Strings(ids) + + pageSize := int(req.PageSize()) + response := &ListSnapshotsResponse{Versioned: NewVersioned()} + if len(ids) > pageSize { + response.NextToken = ids[pageSize-1] + ids = ids[:pageSize] + } + for _, id := range ids { + snapshot := m.snapshots[id] + response.Snapshots = append(response.Snapshots, SnapshotRef{ + ID: id, + SourceVolumeID: snapshot.SourceVolumeID, + Handle: snapshot.Handle, + }) + } + return response, nil +} + +func (m *MemoryProvider) PublishVolume(_ context.Context, req PublishVolumeRequest) (*PublishedVolume, error) { + if err := checkVersion(req.SchemaVersion); err != nil { + return nil, err + } + if req.VolumeID == "" || req.NodeID == "" { + return nil, fmt.Errorf("%w: volume and node IDs are required", ErrInvalidArgument) + } + if req.ReadOnly && !m.capabilities.ReadOnlyPublish { + return nil, fmt.Errorf("%w: read-only publication", ErrUnsupportedCapability) + } + m.mu.Lock() + defer m.mu.Unlock() + volume := m.volumes[req.VolumeID] + if volume == nil || (req.Handle != "" && req.Handle != volume.volume.Handle) { + return nil, fmt.Errorf("%w: volume %s", ErrNotFound, req.VolumeID) + } + if volume.published != nil { + if volume.published.NodeID != req.NodeID { + return nil, fmt.Errorf("%w: volume %s is published to %s", ErrVolumeInUse, req.VolumeID, volume.published.NodeID) + } + return clonePublished(volume.published), nil + } + volume.published = &PublishedVolume{ + VolumeID: req.VolumeID, + NodeID: req.NodeID, + NBDURI: "nbd+unix:///?socket=/memory/" + req.VolumeID + ".sock", + } + volume.volume.State = VolumeStateInUse + return clonePublished(volume.published), nil +} + +func (m *MemoryProvider) UnpublishVolume(_ context.Context, req UnpublishVolumeRequest) error { + if err := checkVersion(req.SchemaVersion); err != nil { + return err + } + if req.VolumeID == "" || req.NodeID == "" { + return fmt.Errorf("%w: volume and node IDs are required", ErrInvalidArgument) + } + m.mu.Lock() + defer m.mu.Unlock() + volume := m.volumes[req.VolumeID] + if volume == nil || volume.published == nil || volume.published.NodeID != req.NodeID { + return nil + } + volume.published = nil + volume.volume.State = VolumeStateAvailable + return nil +} + +func cloneVolume(volume *Volume) *Volume { + if volume == nil { + return nil + } + clone := *volume + clone.ProviderData = bytes.Clone(volume.ProviderData) + return &clone +} + +func cloneSnapshot(snapshot *Snapshot) *Snapshot { + if snapshot == nil { + return nil + } + clone := *snapshot + return &clone +} + +func clonePublished(published *PublishedVolume) *PublishedVolume { + if published == nil { + return nil + } + clone := *published + return &clone +} diff --git a/spinifex/ebsprovider/memory_test.go b/spinifex/ebsprovider/memory_test.go new file mode 100644 index 000000000..053ccac68 --- /dev/null +++ b/spinifex/ebsprovider/memory_test.go @@ -0,0 +1,274 @@ +package ebsprovider + +import ( + "bytes" + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMemoryProviderLifecycleAndIdempotency(t *testing.T) { + provider := NewMemoryProvider(Capabilities{CrashConsistentSnapshot: true}) + createdAt := time.Date(2026, 8, 5, 6, 0, 0, 0, time.UTC) + provider.now = func() time.Time { return createdAt } + ctx := context.Background() + + create := CreateVolumeRequest{ + Versioned: NewVersioned(), + VolumeID: "vol-1", + CapacityRange: CapacityRange{RequiredBytes: 8 << 30}, + AvailabilityZone: "ap-southeast-2a", + } + volume, err := provider.CreateVolume(ctx, create) + require.NoError(t, err) + assert.Equal(t, "memory://volume/vol-1", volume.Handle) + assert.Equal(t, VolumeStateAvailable, volume.State) + + repeated, err := provider.CreateVolume(ctx, create) + require.NoError(t, err) + assert.Equal(t, volume, repeated, "same-name/same-parameters create must be idempotent") + + changed := create + changed.CapacityRange.RequiredBytes++ + _, err = provider.CreateVolume(ctx, changed) + require.ErrorIs(t, err, ErrAlreadyExists) + + published, err := provider.PublishVolume(ctx, PublishVolumeRequest{ + Versioned: NewVersioned(), VolumeID: volume.ID, Handle: volume.Handle, NodeID: "node-1", + }) + require.NoError(t, err) + assert.Equal(t, "nbd+unix:///?socket=/memory/vol-1.sock", published.NBDURI) + + repeatedPublish, err := provider.PublishVolume(ctx, PublishVolumeRequest{ + Versioned: NewVersioned(), VolumeID: volume.ID, Handle: volume.Handle, NodeID: "node-1", + }) + require.NoError(t, err) + assert.Equal(t, published, repeatedPublish) + + _, err = provider.PublishVolume(ctx, PublishVolumeRequest{ + Versioned: NewVersioned(), VolumeID: volume.ID, Handle: volume.Handle, NodeID: "node-2", + }) + require.ErrorIs(t, err, ErrVolumeInUse) + require.ErrorIs(t, provider.DeleteVolume(ctx, DeleteVolumeRequest{ + Versioned: NewVersioned(), VolumeID: volume.ID, Handle: volume.Handle, + }), ErrVolumeInUse) + + require.NoError(t, provider.UnpublishVolume(ctx, UnpublishVolumeRequest{ + Versioned: NewVersioned(), VolumeID: volume.ID, Handle: volume.Handle, NodeID: "node-1", + })) + + expanded, err := provider.ExpandVolume(ctx, ExpandVolumeRequest{ + Versioned: NewVersioned(), VolumeID: volume.ID, Handle: volume.Handle, + CapacityRange: CapacityRange{RequiredBytes: 16 << 30}, + }) + require.NoError(t, err) + assert.Equal(t, int64(16<<30), expanded.CapacityBytes) + + snapshot, err := provider.CreateSnapshot(ctx, CreateSnapshotRequest{ + Versioned: NewVersioned(), SnapshotID: "snap-1", VolumeID: volume.ID, VolumeHandle: volume.Handle, + }) + require.NoError(t, err) + assert.Equal(t, createdAt, snapshot.CreatedAt) + assert.Equal(t, SnapshotStateCompleted, snapshot.State) + assert.Equal(t, expanded.CapacityBytes, snapshot.SizeBytes) + + repeatedSnapshot, err := provider.CreateSnapshot(ctx, CreateSnapshotRequest{ + Versioned: NewVersioned(), SnapshotID: "snap-1", VolumeID: volume.ID, VolumeHandle: volume.Handle, + }) + require.NoError(t, err) + assert.Equal(t, snapshot, repeatedSnapshot) + + require.NoError(t, provider.DeleteVolume(ctx, DeleteVolumeRequest{ + Versioned: NewVersioned(), VolumeID: volume.ID, Handle: volume.Handle, + })) + require.NoError(t, provider.DeleteVolume(ctx, DeleteVolumeRequest{ + Versioned: NewVersioned(), VolumeID: volume.ID, Handle: volume.Handle, + }), "delete of an absent volume must be idempotent") + + require.NoError(t, provider.DeleteSnapshot(ctx, DeleteSnapshotRequest{ + Versioned: NewVersioned(), SnapshotID: snapshot.ID, Handle: snapshot.Handle, + })) + require.NoError(t, provider.DeleteSnapshot(ctx, DeleteSnapshotRequest{ + Versioned: NewVersioned(), SnapshotID: snapshot.ID, Handle: snapshot.Handle, + }), "delete of an absent snapshot must be idempotent") +} + +func TestMemoryProviderRequiresExplicitSchemaVersion(t *testing.T) { + provider := NewMemoryProvider(Capabilities{}) + _, err := provider.CreateVolume(context.Background(), CreateVolumeRequest{ + VolumeID: "vol-unversioned", CapacityRange: CapacityRange{RequiredBytes: 1}, + }) + require.ErrorIs(t, err, ErrUnsupportedVersion) +} + +func TestProviderErrorUnwrap(t *testing.T) { + err := &ProviderError{Code: ErrorCodeNotFound, Message: "volume disappeared"} + assert.ErrorIs(t, err, ErrNotFound) + assert.Equal(t, "volume disappeared", err.Error()) +} + +// TestMemoryProviderChecksVersionOnEveryMethod covers the checkVersion guard +// on every one of the nine EBSProvider methods directly on MemoryProvider, +// independent of NATSProvider's own client-side pre-check. +func TestMemoryProviderChecksVersionOnEveryMethod(t *testing.T) { + provider := NewMemoryProvider(Capabilities{}) + ctx := context.Background() + + _, err := provider.GetCapabilities(ctx, GetCapabilitiesRequest{}) + require.ErrorIs(t, err, ErrUnsupportedVersion) + + _, err = provider.CreateVolume(ctx, CreateVolumeRequest{VolumeID: "vol-1"}) + require.ErrorIs(t, err, ErrUnsupportedVersion) + + _, err = provider.GetVolume(ctx, GetVolumeRequest{VolumeID: "vol-1"}) + require.ErrorIs(t, err, ErrUnsupportedVersion) + + _, err = provider.ExpandVolume(ctx, ExpandVolumeRequest{VolumeID: "vol-1"}) + require.ErrorIs(t, err, ErrUnsupportedVersion) + + err = provider.DeleteVolume(ctx, DeleteVolumeRequest{VolumeID: "vol-1"}) + require.ErrorIs(t, err, ErrUnsupportedVersion) + + _, err = provider.CreateSnapshot(ctx, CreateSnapshotRequest{SnapshotID: "snap-1", VolumeID: "vol-1"}) + require.ErrorIs(t, err, ErrUnsupportedVersion) + + err = provider.DeleteSnapshot(ctx, DeleteSnapshotRequest{SnapshotID: "snap-1"}) + require.ErrorIs(t, err, ErrUnsupportedVersion) + + _, err = provider.PublishVolume(ctx, PublishVolumeRequest{VolumeID: "vol-1", NodeID: "node-1"}) + require.ErrorIs(t, err, ErrUnsupportedVersion) + + err = provider.UnpublishVolume(ctx, UnpublishVolumeRequest{VolumeID: "vol-1", NodeID: "node-1"}) + require.ErrorIs(t, err, ErrUnsupportedVersion) +} + +// TestMemoryProviderCreateVolume_SourceSnapshotTooSmall covers CreateVolume's +// snapshot-capacity guard: a volume created from a snapshot must be at least +// as large as the snapshot it is cloned from. +func TestMemoryProviderCreateVolume_SourceSnapshotTooSmall(t *testing.T) { + provider := NewMemoryProvider(Capabilities{}) + ctx := context.Background() + + _, err := provider.CreateVolume(ctx, CreateVolumeRequest{Versioned: NewVersioned(), VolumeID: "vol-src", CapacityRange: CapacityRange{RequiredBytes: 4 << 30}}) + require.NoError(t, err) + _, err = provider.CreateSnapshot(ctx, CreateSnapshotRequest{Versioned: NewVersioned(), SnapshotID: "snap-src", VolumeID: "vol-src"}) + require.NoError(t, err) + + _, err = provider.CreateVolume(ctx, CreateVolumeRequest{ + Versioned: NewVersioned(), VolumeID: "vol-too-small", CapacityRange: CapacityRange{RequiredBytes: 1 << 30}, SourceSnapshotID: "snap-src", + }) + require.ErrorIs(t, err, ErrInvalidArgument) +} + +// TestMemoryProviderExpandVolume_InvalidArgument covers ExpandVolume's own +// argument validation (empty volume ID, non-positive capacity), separate +// from the not_found and grow-only cases covered elsewhere. +func TestMemoryProviderExpandVolume_InvalidArgument(t *testing.T) { + provider := NewMemoryProvider(Capabilities{}) + _, err := provider.ExpandVolume(context.Background(), ExpandVolumeRequest{Versioned: NewVersioned()}) + require.ErrorIs(t, err, ErrInvalidArgument) +} + +// TestMemoryProviderExpandVolume_HandleMismatchIsNotFound covers the handle +// mismatch arm of ExpandVolume's not_found check: a stale/wrong handle must +// be treated the same as a missing volume, not silently ignored. +func TestMemoryProviderExpandVolume_HandleMismatchIsNotFound(t *testing.T) { + provider := NewMemoryProvider(Capabilities{}) + ctx := context.Background() + _, err := provider.CreateVolume(ctx, CreateVolumeRequest{Versioned: NewVersioned(), VolumeID: "vol-handle", CapacityRange: CapacityRange{RequiredBytes: 1 << 30}}) + require.NoError(t, err) + + _, err = provider.ExpandVolume(ctx, ExpandVolumeRequest{ + Versioned: NewVersioned(), VolumeID: "vol-handle", Handle: "memory://volume/stale", CapacityRange: CapacityRange{RequiredBytes: 2 << 30}, + }) + require.ErrorIs(t, err, ErrNotFound) +} + +// TestMemoryProviderDeleteVolume_InvalidArgumentAndHandleMismatch covers +// DeleteVolume's empty-ID rejection and its handle-mismatch no-op: a caller +// deleting by a stale handle must not delete the current volume out from +// under a newer handle. +func TestMemoryProviderDeleteVolume_InvalidArgumentAndHandleMismatch(t *testing.T) { + provider := NewMemoryProvider(Capabilities{}) + ctx := context.Background() + + err := provider.DeleteVolume(ctx, DeleteVolumeRequest{Versioned: NewVersioned()}) + require.ErrorIs(t, err, ErrInvalidArgument) + + _, createErr := provider.CreateVolume(ctx, CreateVolumeRequest{Versioned: NewVersioned(), VolumeID: "vol-stale-handle", CapacityRange: CapacityRange{RequiredBytes: 1 << 30}}) + require.NoError(t, createErr) + require.NoError(t, provider.DeleteVolume(ctx, DeleteVolumeRequest{Versioned: NewVersioned(), VolumeID: "vol-stale-handle", Handle: "memory://volume/wrong"})) + + // The volume must still exist: the mismatched-handle delete above must + // have been a no-op, not a real deletion. + got, err := provider.GetVolume(ctx, GetVolumeRequest{Versioned: NewVersioned(), VolumeID: "vol-stale-handle"}) + require.NoError(t, err) + assert.Equal(t, "vol-stale-handle", got.ID) +} + +// TestMemoryProviderDeleteSnapshot_InvalidArgument covers DeleteSnapshot's +// empty-ID rejection, the one argument-validation arm no other test reaches. +func TestMemoryProviderDeleteSnapshot_InvalidArgument(t *testing.T) { + provider := NewMemoryProvider(Capabilities{}) + err := provider.DeleteSnapshot(context.Background(), DeleteSnapshotRequest{Versioned: NewVersioned()}) + require.ErrorIs(t, err, ErrInvalidArgument) +} + +// TestCloneHelpersHandleNil covers the nil-input arm of each clone helper: +// none of MemoryProvider's current call sites pass a nil pointer, so this +// arm is otherwise unreachable dead code from the method-level tests alone. +func TestCloneHelpersHandleNil(t *testing.T) { + assert.Nil(t, cloneVolume(nil)) + assert.Nil(t, cloneSnapshot(nil)) + assert.Nil(t, clonePublished(nil)) +} + +// A provider that does not advertise VolumeSeeding must refuse a seed outright +// rather than accepting the request and silently dropping the bytes, which +// would leave a guest booting an EFI volume full of zeroes. +func TestMemoryProviderSeedRequiresCapability(t *testing.T) { + provider := NewMemoryProvider(Capabilities{}) + _, err := provider.CreateVolume(context.Background(), CreateVolumeRequest{ + Versioned: NewVersioned(), VolumeID: "vol-seed-nocap", + CapacityRange: CapacityRange{RequiredBytes: 4096}, + SeedData: bytes.Repeat([]byte{0x01}, 16), + }) + require.ErrorIs(t, err, ErrUnsupportedCapability) +} + +func TestMemoryProviderSeedIsStoredOnce(t *testing.T) { + provider := NewMemoryProvider(Capabilities{VolumeSeeding: true}) + ctx := context.Background() + seed := bytes.Repeat([]byte{0xAB}, 512) + + create := CreateVolumeRequest{ + Versioned: NewVersioned(), VolumeID: "vol-seed-once", + CapacityRange: CapacityRange{RequiredBytes: 4096}, SeedData: seed, + } + _, err := provider.CreateVolume(ctx, create) + require.NoError(t, err) + + stored, ok := provider.SeedData("vol-seed-once") + require.True(t, ok) + assert.Equal(t, seed, stored) + + // A relaunch reissues the same create. Reseeding here would overwrite an + // EFI variable store the guest has since written its own BootOrder into. + reseed := create + reseed.SeedData = bytes.Repeat([]byte{0xCD}, 512) + _, err = provider.CreateVolume(ctx, reseed) + require.NoError(t, err, "a repeat create must stay idempotent regardless of seed") + + stored, ok = provider.SeedData("vol-seed-once") + require.True(t, ok) + assert.Equal(t, seed, stored, "an existing volume must never be reseeded") +} + +func TestValidateSeedData(t *testing.T) { + require.NoError(t, ValidateSeedData(nil)) + require.NoError(t, ValidateSeedData(make([]byte, MaxSeedBytes))) + require.ErrorIs(t, ValidateSeedData(make([]byte, MaxSeedBytes+1)), ErrInvalidArgument) +} diff --git a/spinifex/ebsprovider/nats.go b/spinifex/ebsprovider/nats.go new file mode 100644 index 000000000..6fa720315 --- /dev/null +++ b/spinifex/ebsprovider/nats.go @@ -0,0 +1,455 @@ +package ebsprovider + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/nats-io/nats.go" +) + +// subjectPrefix is common to every subject in the contract, including the +// per-node mount/unmount subjects that have no constant of their own. +const subjectPrefix = "ebs.provider.v1." + +const ( + CapabilitiesSubject = "ebs.provider.v1.capabilities" + CreateVolumeSubject = "ebs.provider.v1.volume.create" + GetVolumeSubject = "ebs.provider.v1.volume.describe" + ListVolumesSubject = "ebs.provider.v1.volume.list" + ExpandVolumeSubject = "ebs.provider.v1.volume.expand" + DeleteVolumeSubject = "ebs.provider.v1.volume.delete" + DeleteSnapshotSubject = "ebs.provider.v1.snapshot.delete" + CopySnapshotSubject = "ebs.provider.v1.snapshot.copy" + ListSnapshotsSubject = "ebs.provider.v1.snapshot.list" + + // SnapshotCreateSubjectPrefix is the wildcard prefix servers subscribe to + // (SnapshotCreateSubjectPrefix + "*") to catch every per-volume create + // subject SnapshotSubject builds. + SnapshotCreateSubjectPrefix = "ebs.provider.v1.snapshot.create." +) + +const defaultRequestTimeout = 30 * time.Second + +// SnapshotSubject addresses a create by source volume so the owning node can +// serve it. The create verb is its own token: a bare volume ID here would +// collide with DeleteSnapshotSubject and make the wildcard unsubscribable. +func SnapshotSubject(volumeID string) (string, error) { + if err := validateSubjectToken(volumeID); err != nil { + return "", err + } + return SnapshotCreateSubjectPrefix + volumeID, nil +} + +func SnapshotCompletionSubject(snapshotID string) (string, error) { + if err := validateSubjectToken(snapshotID); err != nil { + return "", err + } + return "ebs.provider.v1.snapshot.response." + snapshotID, nil +} + +func PublishSubject(nodeID string) (string, error) { + if err := validateSubjectToken(nodeID); err != nil { + return "", err + } + return subjectPrefix + nodeID + ".mount", nil +} + +func UnpublishSubject(nodeID string) (string, error) { + if err := validateSubjectToken(nodeID); err != nil { + return "", err + } + return subjectPrefix + nodeID + ".unmount", nil +} + +// ownerSubjectPrefix is common to every owner-routed subject OwnerSubject +// builds: ebs.provider.v1.owner.{volumeID}.{verb}. +const ownerSubjectPrefix = subjectPrefix + "owner." + +// Owner verbs are a closed set: only these four operations have an +// owner-routed variant, so callers use the named wrapper functions below +// instead of a string literal at the call site. +const ( + verbSnapshotCreate = "snapshot.create" + verbSnapshotCopy = "snapshot.copy" + verbVolumeExpand = "volume.expand" + verbVolumeDescribe = "volume.describe" +) + +// OwnerSubject addresses volumeID's mounting node directly, bypassing the +// spinifex-workers queue group that would otherwise deliver the request to +// a random node. verb must be one of the unexported verb constants above. +func OwnerSubject(volumeID, verb string) (string, error) { + if err := validateSubjectToken(volumeID); err != nil { + return "", err + } + return ownerSubjectPrefix + volumeID + "." + verb, nil +} + +func SnapshotCreateOwnerSubject(volumeID string) (string, error) { + return OwnerSubject(volumeID, verbSnapshotCreate) +} + +func SnapshotCopyOwnerSubject(volumeID string) (string, error) { + return OwnerSubject(volumeID, verbSnapshotCopy) +} + +func ExpandVolumeOwnerSubject(volumeID string) (string, error) { + return OwnerSubject(volumeID, verbVolumeExpand) +} + +func GetVolumeOwnerSubject(volumeID string) (string, error) { + return OwnerSubject(volumeID, verbVolumeDescribe) +} + +// ParseOwnerSubject recovers the volumeID and verb OwnerSubject encoded into +// subject. volumeID cannot itself contain '.' (validateSubjectToken enforces +// this), so the first '.' after ownerSubjectPrefix is always the +// volumeID/verb boundary, even though every verb also contains one. +func ParseOwnerSubject(subject string) (volumeID, verb string, ok bool) { + rest, found := strings.CutPrefix(subject, ownerSubjectPrefix) + if !found { + return "", "", false + } + volumeID, verb, found = strings.Cut(rest, ".") + if !found || volumeID == "" || verb == "" { + return "", "", false + } + return volumeID, verb, true +} + +func validateSubjectToken(value string) error { + if value == "" || strings.ContainsAny(value, ".*>") { + return fmt.Errorf("%w: invalid NATS subject token %q", ErrInvalidArgument, value) + } + return nil +} + +// NATSProvider drives a provider daemon through the versioned ebs.* contract. +type NATSProvider struct { + conn *nats.Conn + requestTimeout time.Duration +} + +var _ EBSProvider = (*NATSProvider)(nil) + +func NewNATSProvider(conn *nats.Conn, requestTimeout time.Duration) *NATSProvider { + if requestTimeout <= 0 { + requestTimeout = defaultRequestTimeout + } + return &NATSProvider{conn: conn, requestTimeout: requestTimeout} +} + +func (p *NATSProvider) GetCapabilities(ctx context.Context, req GetCapabilitiesRequest) (*GetCapabilitiesResponse, error) { + if err := checkVersion(req.SchemaVersion); err != nil { + return nil, err + } + var response GetCapabilitiesResponse + if err := p.request(ctx, CapabilitiesSubject, req, &response); err != nil { + return nil, err + } + if err := responseError(response.SchemaVersion, response.Error); err != nil { + return nil, err + } + return &response, nil +} + +func (p *NATSProvider) CreateVolume(ctx context.Context, req CreateVolumeRequest) (*Volume, error) { + if err := checkVersion(req.SchemaVersion); err != nil { + return nil, err + } + if err := ValidateSeedData(req.SeedData); err != nil { + return nil, err + } + var response CreateVolumeResponse + if err := p.request(ctx, CreateVolumeSubject, req, &response); err != nil { + return nil, err + } + if err := responseError(response.SchemaVersion, response.Error); err != nil { + return nil, err + } + if response.Volume == nil { + return nil, fmt.Errorf("ebs.create returned no volume") + } + return response.Volume, nil +} + +func (p *NATSProvider) GetVolume(ctx context.Context, req GetVolumeRequest) (*Volume, error) { + if err := checkVersion(req.SchemaVersion); err != nil { + return nil, err + } + ownerSubject, err := GetVolumeOwnerSubject(req.VolumeID) + if err != nil { + return nil, err + } + var response GetVolumeResponse + if err := p.requestOwnerFirst(ctx, ownerSubject, GetVolumeSubject, req, &response); err != nil { + return nil, err + } + if err := responseError(response.SchemaVersion, response.Error); err != nil { + return nil, err + } + if response.Volume == nil { + return nil, fmt.Errorf("ebs.describe returned no volume") + } + return response.Volume, nil +} + +// ListVolumes asks the provider what it holds. It is not owner-routed: the +// question is about the whole provider, not one volume, so it goes to the +// queue group rather than to any single volume's mounting node. +func (p *NATSProvider) ListVolumes(ctx context.Context, req ListVolumesRequest) (*ListVolumesResponse, error) { + if err := checkVersion(req.SchemaVersion); err != nil { + return nil, err + } + var response ListVolumesResponse + if err := p.request(ctx, ListVolumesSubject, req, &response); err != nil { + return nil, err + } + if err := responseError(response.SchemaVersion, response.Error); err != nil { + return nil, err + } + return &response, nil +} + +func (p *NATSProvider) ExpandVolume(ctx context.Context, req ExpandVolumeRequest) (*Volume, error) { + if err := checkVersion(req.SchemaVersion); err != nil { + return nil, err + } + ownerSubject, err := ExpandVolumeOwnerSubject(req.VolumeID) + if err != nil { + return nil, err + } + var response ExpandVolumeResponse + if err := p.requestOwnerFirst(ctx, ownerSubject, ExpandVolumeSubject, req, &response); err != nil { + return nil, err + } + if err := responseError(response.SchemaVersion, response.Error); err != nil { + return nil, err + } + if response.Volume == nil { + return nil, fmt.Errorf("ebs.expand returned no volume") + } + return response.Volume, nil +} + +func (p *NATSProvider) DeleteVolume(ctx context.Context, req DeleteVolumeRequest) error { + if err := checkVersion(req.SchemaVersion); err != nil { + return err + } + var response DeleteVolumeResponse + if err := p.request(ctx, DeleteVolumeSubject, req, &response); err != nil { + return err + } + return responseError(response.SchemaVersion, response.Error) +} + +func (p *NATSProvider) CreateSnapshot(ctx context.Context, req CreateSnapshotRequest) (*Snapshot, error) { + if err := checkVersion(req.SchemaVersion); err != nil { + return nil, err + } + requestSubject, err := SnapshotSubject(req.VolumeID) + if err != nil { + return nil, err + } + ownerSubject, err := SnapshotCreateOwnerSubject(req.VolumeID) + if err != nil { + return nil, err + } + completionSubject, err := SnapshotCompletionSubject(req.SnapshotID) + if err != nil { + return nil, err + } + if p.conn == nil || !p.conn.IsConnected() { + return nil, nats.ErrConnectionClosed + } + + completionSub, err := p.conn.SubscribeSync(completionSubject) + if err != nil { + return nil, fmt.Errorf("subscribe to snapshot completion: %w", err) + } + defer completionSub.Unsubscribe() + if err := p.conn.Flush(); err != nil { + return nil, fmt.Errorf("flush snapshot completion subscription: %w", err) + } + + var accepted CreateSnapshotResponse + if err := p.requestOwnerFirst(ctx, ownerSubject, requestSubject, req, &accepted); err != nil { + return nil, err + } + if err := responseError(accepted.SchemaVersion, accepted.Error); err != nil { + return nil, err + } + if accepted.Snapshot != nil && accepted.Snapshot.State != SnapshotStatePending { + return accepted.Snapshot, nil + } + if accepted.OperationID == "" { + return nil, fmt.Errorf("%s returned no operation ID", requestSubject) + } + if accepted.CompletionSubject != "" && accepted.CompletionSubject != completionSubject { + return nil, fmt.Errorf("snapshot completion subject mismatch: got %q, want %q", accepted.CompletionSubject, completionSubject) + } + + waitCtx, waitSpan := startCompletionSpan(ctx, verbSnapshotCreate, completionSubject) + defer waitSpan.End() + msg, err := completionSub.NextMsgWithContext(waitCtx) + if err != nil { + RecordSpanError(waitSpan, err) + return nil, fmt.Errorf("wait for snapshot %s completion: %w", req.SnapshotID, err) + } + RecordResponseError(waitSpan, msg.Data) + var completed CreateSnapshotResponse + if err := json.Unmarshal(msg.Data, &completed); err != nil { + return nil, fmt.Errorf("decode snapshot completion: %w", err) + } + if err := responseError(completed.SchemaVersion, completed.Error); err != nil { + return nil, err + } + if completed.OperationID != accepted.OperationID { + return nil, fmt.Errorf("snapshot operation mismatch: got %q, want %q", completed.OperationID, accepted.OperationID) + } + if completed.Snapshot == nil || completed.Snapshot.State != SnapshotStateCompleted { + return nil, fmt.Errorf("snapshot %s completion returned no completed snapshot", req.SnapshotID) + } + return completed.Snapshot, nil +} + +func (p *NATSProvider) DeleteSnapshot(ctx context.Context, req DeleteSnapshotRequest) error { + if err := checkVersion(req.SchemaVersion); err != nil { + return err + } + var response DeleteSnapshotResponse + if err := p.request(ctx, DeleteSnapshotSubject, req, &response); err != nil { + return err + } + return responseError(response.SchemaVersion, response.Error) +} + +// CopySnapshot is a plain synchronous request/reply, unlike CreateSnapshot's +// accept-then-publish pattern: duplicating a snapshot's metadata is a couple +// of small object writes with no flush/upload step slow enough to warrant a +// completion subject. +func (p *NATSProvider) CopySnapshot(ctx context.Context, req CopySnapshotRequest) (*Snapshot, error) { + if err := checkVersion(req.SchemaVersion); err != nil { + return nil, err + } + ownerSubject, err := SnapshotCopyOwnerSubject(req.VolumeID) + if err != nil { + return nil, err + } + var response CopySnapshotResponse + if err := p.requestOwnerFirst(ctx, ownerSubject, CopySnapshotSubject, req, &response); err != nil { + return nil, err + } + if err := responseError(response.SchemaVersion, response.Error); err != nil { + return nil, err + } + if response.Snapshot == nil { + return nil, fmt.Errorf("ebs.snapshot.copy returned no snapshot") + } + return response.Snapshot, nil +} + +// ListSnapshots asks the provider what it holds. Like ListVolumes it is not +// owner-routed: the question is about the whole provider, not one snapshot. +func (p *NATSProvider) ListSnapshots(ctx context.Context, req ListSnapshotsRequest) (*ListSnapshotsResponse, error) { + if err := checkVersion(req.SchemaVersion); err != nil { + return nil, err + } + var response ListSnapshotsResponse + if err := p.request(ctx, ListSnapshotsSubject, req, &response); err != nil { + return nil, err + } + if err := responseError(response.SchemaVersion, response.Error); err != nil { + return nil, err + } + return &response, nil +} + +func (p *NATSProvider) PublishVolume(ctx context.Context, req PublishVolumeRequest) (*PublishedVolume, error) { + if err := checkVersion(req.SchemaVersion); err != nil { + return nil, err + } + subject, err := PublishSubject(req.NodeID) + if err != nil { + return nil, err + } + var response PublishVolumeResponse + if err := p.request(ctx, subject, req, &response); err != nil { + return nil, err + } + if err := responseError(response.SchemaVersion, response.Error); err != nil { + return nil, err + } + if response.Published == nil { + return nil, fmt.Errorf("%s returned no published volume", subject) + } + return response.Published, nil +} + +func (p *NATSProvider) UnpublishVolume(ctx context.Context, req UnpublishVolumeRequest) error { + if err := checkVersion(req.SchemaVersion); err != nil { + return err + } + subject, err := UnpublishSubject(req.NodeID) + if err != nil { + return err + } + var response UnpublishVolumeResponse + if err := p.request(ctx, subject, req, &response); err != nil { + return err + } + return responseError(response.SchemaVersion, response.Error) +} + +func (p *NATSProvider) request(ctx context.Context, subject string, input, output any) error { + if p.conn == nil || !p.conn.IsConnected() { + return nats.ErrConnectionClosed + } + payload, err := json.Marshal(input) + if err != nil { + return fmt.Errorf("encode %s request: %w", subject, err) + } + + request := &nats.Msg{Subject: subject, Data: payload, Header: nats.Header{}} + ctx, span := startClientSpan(ctx, subject, request.Header) + defer span.End() + + requestCtx, cancel := context.WithTimeout(ctx, p.requestTimeout) + defer cancel() + msg, err := p.conn.RequestMsgWithContext(requestCtx, request) + if err != nil { + RecordSpanError(span, err) + return fmt.Errorf("request %s: %w", subject, err) + } + if err := json.Unmarshal(msg.Data, output); err != nil { + RecordSpanError(span, err) + return fmt.Errorf("decode %s response: %w", subject, err) + } + RecordResponseError(span, msg.Data) + return nil +} + +// requestOwnerFirst tries ownerSubject, falling back to queueSubject only on +// nats.ErrNoResponders, which means nothing is mounted. A timeout must never +// fall back: a still-running owner would have the operation run twice. +func (p *NATSProvider) requestOwnerFirst(ctx context.Context, ownerSubject, queueSubject string, input, output any) error { + err := p.request(ctx, ownerSubject, input, output) + if err == nil || !errors.Is(err, nats.ErrNoResponders) { + return err + } + return p.request(ctx, queueSubject, input, output) +} + +func responseError(version uint16, providerErr *ProviderError) error { + if err := checkVersion(version); err != nil { + return err + } + if providerErr != nil { + return providerErr + } + return nil +} diff --git a/spinifex/ebsprovider/nats_test.go b/spinifex/ebsprovider/nats_test.go new file mode 100644 index 000000000..6c352e44b --- /dev/null +++ b/spinifex/ebsprovider/nats_test.go @@ -0,0 +1,383 @@ +package ebsprovider + +import ( + "encoding/json" + "fmt" + "sync/atomic" + "testing" + "time" + + "github.com/mulgadc/spinifex/spinifex/testutil" + "github.com/nats-io/nats.go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNATSProviderCreateVolumeUsesVersionedContract(t *testing.T) { + _, conn := testutil.StartTestNATS(t) + sub, err := conn.Subscribe(CreateVolumeSubject, func(msg *nats.Msg) { + var request CreateVolumeRequest + require.NoError(t, json.Unmarshal(msg.Data, &request)) + assert.Equal(t, SchemaVersion, request.SchemaVersion) + assert.Equal(t, "vol-1", request.VolumeID) + response, marshalErr := json.Marshal(CreateVolumeResponse{ + Versioned: NewVersioned(), + Volume: &Volume{ID: request.VolumeID, CapacityBytes: request.CapacityRange.RequiredBytes, Handle: "vb://vol-1"}, + }) + require.NoError(t, marshalErr) + require.NoError(t, msg.Respond(response)) + }) + require.NoError(t, err) + t.Cleanup(func() { _ = sub.Unsubscribe() }) + require.NoError(t, conn.Flush()) + + provider := NewNATSProvider(conn, time.Second) + volume, err := provider.CreateVolume(t.Context(), CreateVolumeRequest{ + Versioned: NewVersioned(), VolumeID: "vol-1", CapacityRange: CapacityRange{RequiredBytes: 1 << 30}, + }) + require.NoError(t, err) + assert.Equal(t, "vb://vol-1", volume.Handle) +} + +func TestNATSProviderRejectsResponseVersionSkew(t *testing.T) { + _, conn := testutil.StartTestNATS(t) + sub, err := conn.Subscribe(GetVolumeSubject, func(msg *nats.Msg) { + payload, marshalErr := json.Marshal(GetVolumeResponse{ + Versioned: Versioned{SchemaVersion: SchemaVersion + 1}, + Volume: &Volume{ID: "vol-1"}, + }) + require.NoError(t, marshalErr) + require.NoError(t, msg.Respond(payload)) + }) + require.NoError(t, err) + t.Cleanup(func() { _ = sub.Unsubscribe() }) + require.NoError(t, conn.Flush()) + + provider := NewNATSProvider(conn, time.Second) + _, err = provider.GetVolume(t.Context(), GetVolumeRequest{Versioned: NewVersioned(), VolumeID: "vol-1"}) + require.ErrorIs(t, err, ErrUnsupportedVersion) +} + +func TestNATSProviderReturnsTypedProviderError(t *testing.T) { + _, conn := testutil.StartTestNATS(t) + sub, err := conn.Subscribe(DeleteVolumeSubject, func(msg *nats.Msg) { + payload, marshalErr := json.Marshal(DeleteVolumeResponse{ + Versioned: NewVersioned(), + Error: &ProviderError{Code: ErrorCodeVolumeInUse, Message: "volume is mounted"}, + }) + require.NoError(t, marshalErr) + require.NoError(t, msg.Respond(payload)) + }) + require.NoError(t, err) + t.Cleanup(func() { _ = sub.Unsubscribe() }) + require.NoError(t, conn.Flush()) + + provider := NewNATSProvider(conn, time.Second) + err = provider.DeleteVolume(t.Context(), DeleteVolumeRequest{Versioned: NewVersioned(), VolumeID: "vol-1"}) + require.ErrorIs(t, err, ErrVolumeInUse) + assert.Equal(t, "volume is mounted", err.Error()) +} + +func TestNATSProviderSnapshotWaitsForAsyncCompletion(t *testing.T) { + _, conn := testutil.StartTestNATS(t) + subject, err := SnapshotSubject("vol-1") + require.NoError(t, err) + completionSubject, err := SnapshotCompletionSubject("snap-1") + require.NoError(t, err) + + sub, err := conn.Subscribe(subject, func(msg *nats.Msg) { + accepted, marshalErr := json.Marshal(CreateSnapshotResponse{ + Versioned: NewVersioned(), OperationID: "op-snap-1", CompletionSubject: completionSubject, + }) + require.NoError(t, marshalErr) + require.NoError(t, msg.Respond(accepted)) + + completed, marshalErr := json.Marshal(CreateSnapshotResponse{ + Versioned: NewVersioned(), OperationID: "op-snap-1", + Snapshot: &Snapshot{ID: "snap-1", SourceVolumeID: "vol-1", State: SnapshotStateCompleted}, + }) + require.NoError(t, marshalErr) + require.NoError(t, conn.Publish(completionSubject, completed)) + }) + require.NoError(t, err) + t.Cleanup(func() { _ = sub.Unsubscribe() }) + require.NoError(t, conn.Flush()) + + provider := NewNATSProvider(conn, time.Second) + snapshot, err := provider.CreateSnapshot(t.Context(), CreateSnapshotRequest{ + Versioned: NewVersioned(), SnapshotID: "snap-1", VolumeID: "vol-1", + }) + require.NoError(t, err) + assert.Equal(t, "snap-1", snapshot.ID) +} + +func TestNATSProviderRejectsUnsafeSubjectTokens(t *testing.T) { + provider := NewNATSProvider(nil, time.Second) + _, err := provider.PublishVolume(t.Context(), PublishVolumeRequest{ + Versioned: NewVersioned(), VolumeID: "vol-1", NodeID: "node.*", + }) + require.ErrorIs(t, err, ErrInvalidArgument) + assert.NotErrorIs(t, err, nats.ErrConnectionClosed, "subject validation must happen before transport use") +} + +// TestSubjectBuildersRejectUnsafeTokens tables every NATS subject-token +// validation across the four functions that embed a caller-supplied ID in a +// subject: empty, ".", "..", and any token containing a NATS wildcard +// character must be rejected before a subject string is ever built. +func TestSubjectBuildersRejectUnsafeTokens(t *testing.T) { + badTokens := []string{"", ".", "..", "a.b", "a*b", "a>b"} + + builders := []struct { + name string + build func(string) (string, error) + }{ + {"SnapshotSubject", SnapshotSubject}, + {"SnapshotCompletionSubject", SnapshotCompletionSubject}, + {"PublishSubject", PublishSubject}, + {"UnpublishSubject", UnpublishSubject}, + } + + for _, b := range builders { + t.Run(b.name, func(t *testing.T) { + for _, token := range badTokens { + t.Run(fmt.Sprintf("%q", token), func(t *testing.T) { + _, err := b.build(token) + require.ErrorIs(t, err, ErrInvalidArgument) + }) + } + t.Run("valid token", func(t *testing.T) { + subject, err := b.build("node-1") + require.NoError(t, err) + assert.NotEmpty(t, subject) + }) + }) + } +} + +// TestNATSProviderChecksVersionBeforeTransport confirms every EBSProvider +// method on NATSProvider rejects an unsupported SchemaVersion via +// checkVersion before it ever touches the connection. A nil *nats.Conn would +// panic or return nats.ErrConnectionClosed if any method reached transport, +// so seeing ErrUnsupportedVersion here proves the ordering. +func TestNATSProviderChecksVersionBeforeTransport(t *testing.T) { + provider := NewNATSProvider(nil, time.Second) + ctx := t.Context() + + t.Run("GetCapabilities", func(t *testing.T) { + _, err := provider.GetCapabilities(ctx, GetCapabilitiesRequest{}) + require.ErrorIs(t, err, ErrUnsupportedVersion) + }) + t.Run("CreateVolume", func(t *testing.T) { + _, err := provider.CreateVolume(ctx, CreateVolumeRequest{VolumeID: "vol-1"}) + require.ErrorIs(t, err, ErrUnsupportedVersion) + }) + t.Run("GetVolume", func(t *testing.T) { + _, err := provider.GetVolume(ctx, GetVolumeRequest{VolumeID: "vol-1"}) + require.ErrorIs(t, err, ErrUnsupportedVersion) + }) + t.Run("ExpandVolume", func(t *testing.T) { + _, err := provider.ExpandVolume(ctx, ExpandVolumeRequest{VolumeID: "vol-1"}) + require.ErrorIs(t, err, ErrUnsupportedVersion) + }) + t.Run("DeleteVolume", func(t *testing.T) { + err := provider.DeleteVolume(ctx, DeleteVolumeRequest{VolumeID: "vol-1"}) + require.ErrorIs(t, err, ErrUnsupportedVersion) + }) + t.Run("CreateSnapshot", func(t *testing.T) { + _, err := provider.CreateSnapshot(ctx, CreateSnapshotRequest{SnapshotID: "snap-1", VolumeID: "vol-1"}) + require.ErrorIs(t, err, ErrUnsupportedVersion) + }) + t.Run("DeleteSnapshot", func(t *testing.T) { + err := provider.DeleteSnapshot(ctx, DeleteSnapshotRequest{SnapshotID: "snap-1"}) + require.ErrorIs(t, err, ErrUnsupportedVersion) + }) + t.Run("PublishVolume", func(t *testing.T) { + _, err := provider.PublishVolume(ctx, PublishVolumeRequest{VolumeID: "vol-1", NodeID: "node-1"}) + require.ErrorIs(t, err, ErrUnsupportedVersion) + }) + t.Run("UnpublishVolume", func(t *testing.T) { + err := provider.UnpublishVolume(ctx, UnpublishVolumeRequest{VolumeID: "vol-1", NodeID: "node-1"}) + require.ErrorIs(t, err, ErrUnsupportedVersion) + }) +} + +// TestNewNATSProviderDefaultsTimeout covers the requestTimeout<=0 branch: +// callers passing a zero or negative timeout get defaultRequestTimeout +// rather than a provider that never times out. +func TestNewNATSProviderDefaultsTimeout(t *testing.T) { + assert.Equal(t, defaultRequestTimeout, NewNATSProvider(nil, 0).requestTimeout) + assert.Equal(t, defaultRequestTimeout, NewNATSProvider(nil, -time.Second).requestTimeout) + assert.Equal(t, 7*time.Second, NewNATSProvider(nil, 7*time.Second).requestTimeout) +} + +// TestNATSProviderGetCapabilities_TransportError covers the p.request error +// branch (no responder on the subject) for a method that otherwise never +// exercises it elsewhere in this suite. +func TestNATSProviderGetCapabilities_TransportError(t *testing.T) { + _, conn := testutil.StartTestNATS(t) + provider := NewNATSProvider(conn, 200*time.Millisecond) + _, err := provider.GetCapabilities(t.Context(), GetCapabilitiesRequest{Versioned: NewVersioned()}) + require.Error(t, err) +} + +// TestNATSProviderGetCapabilities_ProviderError covers responseError's +// populated-Error branch for GetCapabilities specifically. +func TestNATSProviderGetCapabilities_ProviderError(t *testing.T) { + _, conn := testutil.StartTestNATS(t) + sub, err := conn.Subscribe(CapabilitiesSubject, func(msg *nats.Msg) { + payload, marshalErr := json.Marshal(GetCapabilitiesResponse{ + Versioned: NewVersioned(), Error: &ProviderError{Code: ErrorCodeInternal, Message: "capabilities unavailable"}, + }) + require.NoError(t, marshalErr) + require.NoError(t, msg.Respond(payload)) + }) + require.NoError(t, err) + t.Cleanup(func() { _ = sub.Unsubscribe() }) + require.NoError(t, conn.Flush()) + + provider := NewNATSProvider(conn, time.Second) + _, err = provider.GetCapabilities(t.Context(), GetCapabilitiesRequest{Versioned: NewVersioned()}) + require.EqualError(t, err, "capabilities unavailable") +} + +// TestNATSProviderRequest covers p.request's three error branches directly: +// an unmarshalable input, a subject with no responder, and a responder that +// answers with invalid JSON. +func TestNATSProviderRequest(t *testing.T) { + _, conn := testutil.StartTestNATS(t) + provider := NewNATSProvider(conn, 200*time.Millisecond) + + t.Run("marshal error", func(t *testing.T) { + var out GetVolumeResponse + err := provider.request(t.Context(), "ebs.provider.v1.test.marshal", make(chan int), &out) + require.Error(t, err) + assert.Contains(t, err.Error(), "encode") + }) + + t.Run("no responder", func(t *testing.T) { + var out GetVolumeResponse + err := provider.request(t.Context(), "ebs.provider.v1.test.noresponder", GetVolumeRequest{Versioned: NewVersioned()}, &out) + require.Error(t, err) + assert.Contains(t, err.Error(), "request") + }) + + t.Run("decode error", func(t *testing.T) { + const subject = "ebs.provider.v1.test.baddecode" + sub, err := conn.Subscribe(subject, func(msg *nats.Msg) { + require.NoError(t, msg.Respond([]byte("{not valid json"))) + }) + require.NoError(t, err) + t.Cleanup(func() { _ = sub.Unsubscribe() }) + require.NoError(t, conn.Flush()) + + var out GetVolumeResponse + err = provider.request(t.Context(), subject, GetVolumeRequest{Versioned: NewVersioned()}, &out) + require.Error(t, err) + assert.Contains(t, err.Error(), "decode") + }) +} + +// TestNATSProviderOwnerFirst_RoutesToOwnerSubject proves the owner subject +// is tried first: only a responder on GetVolumeOwnerSubject exists (no +// GetVolumeSubject responder at all), so a successful reply proves the +// request never needed the queue-group subject. +func TestNATSProviderOwnerFirst_RoutesToOwnerSubject(t *testing.T) { + _, conn := testutil.StartTestNATS(t) + + ownerSubject, err := GetVolumeOwnerSubject("vol-ownerroute001") + require.NoError(t, err) + ownerSub, err := conn.Subscribe(ownerSubject, func(msg *nats.Msg) { + payload, marshalErr := json.Marshal(GetVolumeResponse{ + Versioned: NewVersioned(), Volume: &Volume{ID: "vol-ownerroute001", Handle: "owner-handle"}, + }) + require.NoError(t, marshalErr) + require.NoError(t, msg.Respond(payload)) + }) + require.NoError(t, err) + t.Cleanup(func() { _ = ownerSub.Unsubscribe() }) + require.NoError(t, conn.Flush()) + + provider := NewNATSProvider(conn, time.Second) + volume, err := provider.GetVolume(t.Context(), GetVolumeRequest{Versioned: NewVersioned(), VolumeID: "vol-ownerroute001"}) + require.NoError(t, err) + assert.Equal(t, "owner-handle", volume.Handle, "the owner subject's own responder must have answered") +} + +// TestNATSProviderOwnerFirst_NoRespondersFallsBackToQueueSubject proves the +// one condition allowed to trigger a fallback: no responder at all on the +// owner subject. Only GetVolumeSubject (the queue-group subject) has a +// responder here, so success proves the fallback fired. +func TestNATSProviderOwnerFirst_NoRespondersFallsBackToQueueSubject(t *testing.T) { + _, conn := testutil.StartTestNATS(t) + + queueSub, err := conn.Subscribe(GetVolumeSubject, func(msg *nats.Msg) { + payload, marshalErr := json.Marshal(GetVolumeResponse{ + Versioned: NewVersioned(), Volume: &Volume{ID: "vol-ownerfallback1", Handle: "queue-handle"}, + }) + require.NoError(t, marshalErr) + require.NoError(t, msg.Respond(payload)) + }) + require.NoError(t, err) + t.Cleanup(func() { _ = queueSub.Unsubscribe() }) + require.NoError(t, conn.Flush()) + + provider := NewNATSProvider(conn, time.Second) + volume, err := provider.GetVolume(t.Context(), GetVolumeRequest{Versioned: NewVersioned(), VolumeID: "vol-ownerfallback1"}) + require.NoError(t, err) + assert.Equal(t, "queue-handle", volume.Handle, "no owner responder must fall back to the queue-group subject") +} + +// TestNATSProviderOwnerFirst_NonNoRespondersDoesNotFallBack is the dangerous +// case requestOwnerFirst exists to avoid: an owner subject that has a +// responder but never replies produces a timeout, not nats.ErrNoResponders, +// and must NOT fall back to the queue subject — a queue fallback here could +// run the same operation (e.g. a snapshot) a second time while the owner is +// still mid-flight. A responsive queue-subject responder is present so a +// wrongful fallback would succeed silently instead of failing loudly. +func TestNATSProviderOwnerFirst_NonNoRespondersDoesNotFallBack(t *testing.T) { + _, conn := testutil.StartTestNATS(t) + + ownerSubject, err := GetVolumeOwnerSubject("vol-nofallback001") + require.NoError(t, err) + ownerSub, err := conn.Subscribe(ownerSubject, func(msg *nats.Msg) { + // Deliberately never responds, forcing the client to time out. + }) + require.NoError(t, err) + t.Cleanup(func() { _ = ownerSub.Unsubscribe() }) + + var queueHits atomic.Int32 + queueSub, err := conn.Subscribe(GetVolumeSubject, func(msg *nats.Msg) { + queueHits.Add(1) + payload, marshalErr := json.Marshal(GetVolumeResponse{ + Versioned: NewVersioned(), Volume: &Volume{ID: "vol-nofallback001", Handle: "queue-handle"}, + }) + require.NoError(t, marshalErr) + require.NoError(t, msg.Respond(payload)) + }) + require.NoError(t, err) + t.Cleanup(func() { _ = queueSub.Unsubscribe() }) + require.NoError(t, conn.Flush()) + + provider := NewNATSProvider(conn, 200*time.Millisecond) + _, err = provider.GetVolume(t.Context(), GetVolumeRequest{Versioned: NewVersioned(), VolumeID: "vol-nofallback001"}) + require.Error(t, err, "the owner responder never replies, so this must time out") + require.NotErrorIs(t, err, nats.ErrNoResponders, "a timeout must never be mistaken for no responders") + assert.Equal(t, int32(0), queueHits.Load(), "a timeout must never fall back to the queue-group subject") +} + +// TestResponseErrorRejectsZeroSchemaVersion covers responseError directly: a +// reply carrying the zero SchemaVersion (e.g. a server that forgot to set +// Versioned on an error path) must be rejected before its Error field is +// ever inspected, even when that field is nil. +func TestResponseErrorRejectsZeroSchemaVersion(t *testing.T) { + err := responseError(0, nil) + require.ErrorIs(t, err, ErrUnsupportedVersion) + + err = responseError(0, &ProviderError{Code: ErrorCodeNotFound, Message: "ignored"}) + require.ErrorIs(t, err, ErrUnsupportedVersion, "the version guard must win over a populated error field") + + require.NoError(t, responseError(SchemaVersion, nil)) + + err = responseError(SchemaVersion, &ProviderError{Code: ErrorCodeVolumeInUse, Message: "in use"}) + require.ErrorIs(t, err, ErrVolumeInUse) +} diff --git a/spinifex/ebsprovider/natsserve/classify_test.go b/spinifex/ebsprovider/natsserve/classify_test.go new file mode 100644 index 000000000..0092c9f82 --- /dev/null +++ b/spinifex/ebsprovider/natsserve/classify_test.go @@ -0,0 +1,42 @@ +package natsserve + +import ( + "fmt" + "testing" + + "github.com/mulgadc/spinifex/spinifex/ebsprovider" + "github.com/stretchr/testify/assert" +) + +// TestClassifyError pins every sentinel to the code it crosses the wire as. A +// sentinel missing from classifyError falls through to internal, which tells a +// caller a retryable refusal is permanent and loses the reason with it. +func TestClassifyError(t *testing.T) { + tests := []struct { + err error + want ebsprovider.ErrorCode + }{ + {ebsprovider.ErrAlreadyExists, ebsprovider.ErrorCodeAlreadyExists}, + {ebsprovider.ErrInvalidArgument, ebsprovider.ErrorCodeInvalidArgument}, + {ebsprovider.ErrNotFound, ebsprovider.ErrorCodeNotFound}, + {ebsprovider.ErrUnsupportedVersion, ebsprovider.ErrorCodeUnsupportedVersion}, + {ebsprovider.ErrVolumeInUse, ebsprovider.ErrorCodeVolumeInUse}, + {ebsprovider.ErrUnsupportedCapability, ebsprovider.ErrorCodeUnsupportedCap}, + {ebsprovider.ErrUnavailable, ebsprovider.ErrorCodeUnavailable}, + } + + for _, tt := range tests { + t.Run(string(tt.want), func(t *testing.T) { + wrapped := fmt.Errorf("vol-1: %w", tt.err) + got := classifyError(wrapped) + assert.Equal(t, tt.want, got.Code) + assert.Equal(t, wrapped.Error(), got.Message, "the wrapping context must survive classification") + }) + } + + t.Run("unrecognised error is internal", func(t *testing.T) { + got := classifyError(fmt.Errorf("something unexpected")) + assert.Equal(t, ebsprovider.ErrorCodeInternal, got.Code) + assert.False(t, got.Code.Retryable(), "an error we cannot classify must not invite a retry loop") + }) +} diff --git a/spinifex/ebsprovider/natsserve/server.go b/spinifex/ebsprovider/natsserve/server.go new file mode 100644 index 000000000..492d5d0ce --- /dev/null +++ b/spinifex/ebsprovider/natsserve/server.go @@ -0,0 +1,525 @@ +// Package natsserve is the provider-neutral NATS server for the +// ebs.provider.v1.* wire contract defined by spinifex/ebsprovider. It +// depends on nothing but the ebsprovider.EBSProvider interface, so any implementation of that interface can be served over NATS by calling Serve. +package natsserve + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "log/slog" + "strings" + "sync" + + "github.com/mulgadc/spinifex/spinifex/ebsprovider" + "github.com/nats-io/nats.go" + "go.opentelemetry.io/otel/trace" +) + +// DefaultQueueGroup is the NATS queue group Serve subscribes the +// multi-node-shared subjects (every subject except the per-node +// publish/unpublish pair) under by default, matching viperblockd's production subscriptions. +const DefaultQueueGroup = "spinifex-workers" + +// Options configures Serve's subscription behavior. +type Options struct { + // QueueGroup names the queue group the shared subjects (Capabilities, + // CreateVolume, GetVolume, ExpandVolume, DeleteVolume, DeleteSnapshot, + // CopySnapshot, and the snapshot-create wildcard) subscribe under. Empty defaults to DefaultQueueGroup; set NoQueueGroup to opt out. + QueueGroup string + + // NoQueueGroup subscribes the shared subjects plainly instead of under + // a queue group, e.g. for a single-instance server where load balancing + // across workers does not apply. + NoQueueGroup bool + + // NodeID scopes PublishVolume/UnpublishVolume to this node's own + // subjects (ebsprovider.PublishSubject / UnpublishSubject), matching a + // production daemon that can only mount volumes on its own node. Left empty (the default), Serve instead subscribes the wildcard "ebs.provider.v1.*.mount"/"unmount" subjects, answering requests for any node. + NodeID string +} + +// msgHandler takes the per-message context so the server span opened for the +// message reaches the work the handler does, rather than only timing it. +type msgHandler func(ctx context.Context, msg *nats.Msg) + +// traced opens a server span for each message, joining the caller's trace +// from its headers, and ends it when the handler returns. +func traced(base context.Context, handler msgHandler) nats.MsgHandler { + return func(msg *nats.Msg) { + ctx, span := ebsprovider.StartServerSpan(base, msg) + defer span.End() + handler(ctx, msg) + } +} + +// Serve subscribes every subject spinifex/ebsprovider/nats.go defines, +// except owner-routed subjects (see the comment in the function body), and +// delegates each to provider. The returned stop function unsubscribes everything Serve registered; it is safe to call more than once. +func Serve(ctx context.Context, nc *nats.Conn, provider ebsprovider.EBSProvider, opts Options) (stop func(), err error) { + // Owner-subject routing (ebsprovider.OwnerSubject and its per-verb + // wrappers) is not served: it routes a request to the node holding a + // volume's live engine via a per-node mounted-volume registry the EBSProvider interface does not model, so it stays viperblockd-specific. + queueGroup := opts.QueueGroup + if queueGroup == "" { + queueGroup = DefaultQueueGroup + } + if opts.NoQueueGroup { + queueGroup = "" + } + + var subs []*nats.Subscription + cleanup := func() { + for _, sub := range subs { + _ = sub.Unsubscribe() + } + } + + subscribeShared := func(subject string, handler msgHandler) error { + var sub *nats.Subscription + var subErr error + if queueGroup != "" { + sub, subErr = nc.QueueSubscribe(subject, queueGroup, traced(ctx, handler)) + } else { + sub, subErr = nc.Subscribe(subject, traced(ctx, handler)) + } + if subErr != nil { + return fmt.Errorf("subscribe to %s: %w", subject, subErr) + } + subs = append(subs, sub) + return nil + } + + // Publish/Unpublish are never queue-grouped: each subject already + // addresses exactly one node (or, in the wildcard case, is answered by + // the one backing provider standing in for every node), so load balancing across a group has nothing to add. + subscribePlain := func(subject string, handler msgHandler) error { + sub, subErr := nc.Subscribe(subject, traced(ctx, handler)) + if subErr != nil { + return fmt.Errorf("subscribe to %s: %w", subject, subErr) + } + subs = append(subs, sub) + return nil + } + + steps := []struct { + subject string + handler msgHandler + }{ + {ebsprovider.CapabilitiesSubject, handleCapabilities(provider)}, + {ebsprovider.CreateVolumeSubject, handleCreateVolume(provider)}, + {ebsprovider.GetVolumeSubject, handleGetVolume(provider)}, + {ebsprovider.ListVolumesSubject, handleListVolumes(provider)}, + {ebsprovider.ExpandVolumeSubject, handleExpandVolume(provider)}, + {ebsprovider.DeleteVolumeSubject, handleDeleteVolume(provider)}, + {ebsprovider.DeleteSnapshotSubject, handleDeleteSnapshot(provider)}, + {ebsprovider.CopySnapshotSubject, handleCopySnapshot(provider)}, + {ebsprovider.ListSnapshotsSubject, handleListSnapshots(provider)}, + {ebsprovider.SnapshotCreateSubjectPrefix + "*", handleCreateSnapshot(nc, provider)}, + } + for _, step := range steps { + if err := subscribeShared(step.subject, step.handler); err != nil { + cleanup() + return nil, err + } + } + + if opts.NodeID == "" { + if err := subscribePlain("ebs.provider.v1.*.mount", handlePublishVolume(provider)); err != nil { + cleanup() + return nil, err + } + if err := subscribePlain("ebs.provider.v1.*.unmount", handleUnpublishVolume(provider)); err != nil { + cleanup() + return nil, err + } + } else { + publishSubject, err := ebsprovider.PublishSubject(opts.NodeID) + if err != nil { + cleanup() + return nil, fmt.Errorf("build publish subject: %w", err) + } + if err := subscribePlain(publishSubject, handlePublishVolume(provider)); err != nil { + cleanup() + return nil, err + } + unpublishSubject, err := ebsprovider.UnpublishSubject(opts.NodeID) + if err != nil { + cleanup() + return nil, fmt.Errorf("build unpublish subject: %w", err) + } + if err := subscribePlain(unpublishSubject, handleUnpublishVolume(provider)); err != nil { + cleanup() + return nil, err + } + } + + if err := nc.Flush(); err != nil { + cleanup() + return nil, fmt.Errorf("flush subscriptions: %w", err) + } + + var once sync.Once + return func() { once.Do(cleanup) }, nil +} + +// respond marks the message's span with whatever error the reply carries, so +// a verb that failed shows as an error rather than a successful round trip. +func respond(ctx context.Context, msg *nats.Msg, v any) { + data, err := json.Marshal(v) + if err != nil { + slog.Error("ebs.provider: failed to marshal response", "subject", msg.Subject, "err", err) + return + } + ebsprovider.RecordResponseError(trace.SpanFromContext(ctx), data) + if err := msg.Respond(data); err != nil { + slog.Error("ebs.provider: failed to send response", "subject", msg.Subject, "err", err) + } +} + +// badRequestError maps a JSON decode failure to the wire error shape, +// mirroring viperblockd's badRequestError: a malformed request body is the +// caller's mistake, not a server-side failure. +func badRequestError(err error) *ebsprovider.ProviderError { + return &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeInvalidArgument, Message: fmt.Sprintf("bad request: %v", err)} +} + +func versionError(got uint16) *ebsprovider.ProviderError { + return &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnsupportedVersion, Message: fmt.Sprintf("unsupported schema version %d, want %d", got, ebsprovider.SchemaVersion)} +} + +func invalidArgumentError(format string, args ...any) *ebsprovider.ProviderError { + return &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeInvalidArgument, Message: fmt.Sprintf(format, args...)} +} + +func internalError(format string, args ...any) *ebsprovider.ProviderError { + return &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeInternal, Message: fmt.Sprintf(format, args...)} +} + +// classifyError maps a provider error to the wire ProviderError shape, +// using only the sentinel errors and ErrorCode values errors.go exports: +// it has no forward (error -> ProviderError) mapper of its own to call. +func classifyError(err error) *ebsprovider.ProviderError { + switch { + case errors.Is(err, ebsprovider.ErrAlreadyExists): + return &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeAlreadyExists, Message: err.Error()} + case errors.Is(err, ebsprovider.ErrInvalidArgument): + return &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeInvalidArgument, Message: err.Error()} + case errors.Is(err, ebsprovider.ErrNotFound): + return &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeNotFound, Message: err.Error()} + case errors.Is(err, ebsprovider.ErrUnsupportedVersion): + return &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnsupportedVersion, Message: err.Error()} + case errors.Is(err, ebsprovider.ErrVolumeInUse): + return &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeVolumeInUse, Message: err.Error()} + case errors.Is(err, ebsprovider.ErrUnsupportedCapability): + return &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnsupportedCap, Message: err.Error()} + case errors.Is(err, ebsprovider.ErrUnavailable): + return &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnavailable, Message: err.Error()} + default: + return &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeInternal, Message: err.Error()} + } +} + +func handleCapabilities(provider ebsprovider.EBSProvider) msgHandler { + return func(ctx context.Context, msg *nats.Msg) { + var req ebsprovider.GetCapabilitiesRequest + if err := json.Unmarshal(msg.Data, &req); err != nil { + respond(ctx, msg, ebsprovider.GetCapabilitiesResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)}) + return + } + if req.SchemaVersion != ebsprovider.SchemaVersion { + respond(ctx, msg, ebsprovider.GetCapabilitiesResponse{Versioned: ebsprovider.NewVersioned(), Error: versionError(req.SchemaVersion)}) + return + } + resp, err := provider.GetCapabilities(ctx, req) + if err != nil { + respond(ctx, msg, ebsprovider.GetCapabilitiesResponse{Versioned: ebsprovider.NewVersioned(), Error: classifyError(err)}) + return + } + respond(ctx, msg, *resp) + } +} + +func handleCreateVolume(provider ebsprovider.EBSProvider) msgHandler { + return func(ctx context.Context, msg *nats.Msg) { + var req ebsprovider.CreateVolumeRequest + if err := json.Unmarshal(msg.Data, &req); err != nil { + respond(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)}) + return + } + if req.SchemaVersion != ebsprovider.SchemaVersion { + respond(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: versionError(req.SchemaVersion)}) + return + } + vol, err := provider.CreateVolume(ctx, req) + if err != nil { + respond(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: classifyError(err)}) + return + } + respond(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Volume: vol}) + } +} + +func handleGetVolume(provider ebsprovider.EBSProvider) msgHandler { + return func(ctx context.Context, msg *nats.Msg) { + var req ebsprovider.GetVolumeRequest + if err := json.Unmarshal(msg.Data, &req); err != nil { + respond(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)}) + return + } + if req.SchemaVersion != ebsprovider.SchemaVersion { + respond(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: versionError(req.SchemaVersion)}) + return + } + vol, err := provider.GetVolume(ctx, req) + if err != nil { + respond(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: classifyError(err)}) + return + } + respond(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: ebsprovider.NewVersioned(), Volume: vol}) + } +} + +func handleListVolumes(provider ebsprovider.EBSProvider) msgHandler { + return func(ctx context.Context, msg *nats.Msg) { + var req ebsprovider.ListVolumesRequest + if err := json.Unmarshal(msg.Data, &req); err != nil { + respond(ctx, msg, ebsprovider.ListVolumesResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)}) + return + } + if req.SchemaVersion != ebsprovider.SchemaVersion { + respond(ctx, msg, ebsprovider.ListVolumesResponse{Versioned: ebsprovider.NewVersioned(), Error: versionError(req.SchemaVersion)}) + return + } + resp, err := provider.ListVolumes(ctx, req) + if err != nil { + respond(ctx, msg, ebsprovider.ListVolumesResponse{Versioned: ebsprovider.NewVersioned(), Error: classifyError(err)}) + return + } + respond(ctx, msg, *resp) + } +} + +func handleListSnapshots(provider ebsprovider.EBSProvider) msgHandler { + return func(ctx context.Context, msg *nats.Msg) { + var req ebsprovider.ListSnapshotsRequest + if err := json.Unmarshal(msg.Data, &req); err != nil { + respond(ctx, msg, ebsprovider.ListSnapshotsResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)}) + return + } + if req.SchemaVersion != ebsprovider.SchemaVersion { + respond(ctx, msg, ebsprovider.ListSnapshotsResponse{Versioned: ebsprovider.NewVersioned(), Error: versionError(req.SchemaVersion)}) + return + } + resp, err := provider.ListSnapshots(ctx, req) + if err != nil { + respond(ctx, msg, ebsprovider.ListSnapshotsResponse{Versioned: ebsprovider.NewVersioned(), Error: classifyError(err)}) + return + } + respond(ctx, msg, *resp) + } +} + +func handleExpandVolume(provider ebsprovider.EBSProvider) msgHandler { + return func(ctx context.Context, msg *nats.Msg) { + var req ebsprovider.ExpandVolumeRequest + if err := json.Unmarshal(msg.Data, &req); err != nil { + respond(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)}) + return + } + if req.SchemaVersion != ebsprovider.SchemaVersion { + respond(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: versionError(req.SchemaVersion)}) + return + } + vol, err := provider.ExpandVolume(ctx, req) + if err != nil { + respond(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: classifyError(err)}) + return + } + respond(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Volume: vol}) + } +} + +func handleDeleteVolume(provider ebsprovider.EBSProvider) msgHandler { + return func(ctx context.Context, msg *nats.Msg) { + var req ebsprovider.DeleteVolumeRequest + if err := json.Unmarshal(msg.Data, &req); err != nil { + respond(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)}) + return + } + if req.SchemaVersion != ebsprovider.SchemaVersion { + respond(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: versionError(req.SchemaVersion)}) + return + } + if err := provider.DeleteVolume(ctx, req); err != nil { + respond(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: classifyError(err)}) + return + } + respond(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: ebsprovider.NewVersioned()}) + } +} + +func handleDeleteSnapshot(provider ebsprovider.EBSProvider) msgHandler { + return func(ctx context.Context, msg *nats.Msg) { + var req ebsprovider.DeleteSnapshotRequest + if err := json.Unmarshal(msg.Data, &req); err != nil { + respond(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)}) + return + } + if req.SchemaVersion != ebsprovider.SchemaVersion { + respond(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: versionError(req.SchemaVersion)}) + return + } + if err := provider.DeleteSnapshot(ctx, req); err != nil { + respond(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: classifyError(err)}) + return + } + respond(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: ebsprovider.NewVersioned()}) + } +} + +func handleCopySnapshot(provider ebsprovider.EBSProvider) msgHandler { + return func(ctx context.Context, msg *nats.Msg) { + var req ebsprovider.CopySnapshotRequest + if err := json.Unmarshal(msg.Data, &req); err != nil { + respond(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)}) + return + } + if req.SchemaVersion != ebsprovider.SchemaVersion { + respond(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: versionError(req.SchemaVersion)}) + return + } + snap, err := provider.CopySnapshot(ctx, req) + if err != nil { + respond(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: classifyError(err)}) + return + } + respond(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: ebsprovider.NewVersioned(), Snapshot: snap}) + } +} + +// handleCreateSnapshot serves the SnapshotCreateSubjectPrefix wildcard by +// replying immediately with a pending Snapshot plus an OperationID, then +// running the blocking EBSProvider.CreateSnapshot call in the background and publishing its result: a transport-level wrapping, so no async form is needed in the interface itself. +func handleCreateSnapshot(nc *nats.Conn, provider ebsprovider.EBSProvider) msgHandler { + return func(ctx context.Context, msg *nats.Msg) { + var req ebsprovider.CreateSnapshotRequest + if err := json.Unmarshal(msg.Data, &req); err != nil { + respond(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)}) + return + } + if req.SchemaVersion != ebsprovider.SchemaVersion { + respond(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: versionError(req.SchemaVersion)}) + return + } + + // The wildcard subject carries the source volume ID as its own + // token (SnapshotCreateSubjectPrefix + volumeID); a mismatch with + // the request body means the two disagree about which volume this create is for. + if subjectVolumeID, ok := strings.CutPrefix(msg.Subject, ebsprovider.SnapshotCreateSubjectPrefix); ok && subjectVolumeID != req.VolumeID { + respond(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("volume id %q in subject does not match request volume id %q", subjectVolumeID, req.VolumeID)}) + return + } + + completionSubject, err := ebsprovider.SnapshotCompletionSubject(req.SnapshotID) + if err != nil { + respond(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: classifyError(err)}) + return + } + operationID, err := newOperationID() + if err != nil { + respond(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("generate operation id: %v", err)}) + return + } + + respond(ctx, msg, ebsprovider.CreateSnapshotResponse{ + Versioned: ebsprovider.NewVersioned(), + OperationID: operationID, + CompletionSubject: completionSubject, + Snapshot: &ebsprovider.Snapshot{ + ID: req.SnapshotID, + SourceVolumeID: req.VolumeID, + State: ebsprovider.SnapshotStatePending, + }, + }) + + go completeCreateSnapshot(ctx, nc, provider, req, operationID, completionSubject) + } +} + +func completeCreateSnapshot(ctx context.Context, nc *nats.Conn, provider ebsprovider.EBSProvider, req ebsprovider.CreateSnapshotRequest, operationID, completionSubject string) { + completion := ebsprovider.CreateSnapshotResponse{ + Versioned: ebsprovider.NewVersioned(), + OperationID: operationID, + CompletionSubject: completionSubject, + } + + snapshot, err := provider.CreateSnapshot(ctx, req) + if err != nil { + completion.Error = classifyError(err) + } else { + completion.Snapshot = snapshot + } + + data, err := json.Marshal(completion) + if err != nil { + slog.Error("ebs.provider.snapshot.create: failed to marshal completion", "volume", req.VolumeID, "snapshot", req.SnapshotID, "err", err) + return + } + if err := nc.Publish(completionSubject, data); err != nil { + slog.Error("ebs.provider.snapshot.create: failed to publish completion", "subject", completionSubject, "err", err) + } +} + +// newOperationID mints a random operation ID without depending on anything +// beyond the standard library, keeping natsserve's only non-stdlib +// dependencies ebsprovider and nats.go. +func newOperationID() (string, error) { + b := make([]byte, 9) + if _, err := rand.Read(b); err != nil { + return "", fmt.Errorf("read random bytes: %w", err) + } + return "op-" + hex.EncodeToString(b), nil +} + +func handlePublishVolume(provider ebsprovider.EBSProvider) msgHandler { + return func(ctx context.Context, msg *nats.Msg) { + var req ebsprovider.PublishVolumeRequest + if err := json.Unmarshal(msg.Data, &req); err != nil { + respond(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)}) + return + } + if req.SchemaVersion != ebsprovider.SchemaVersion { + respond(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: versionError(req.SchemaVersion)}) + return + } + pub, err := provider.PublishVolume(ctx, req) + if err != nil { + respond(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: classifyError(err)}) + return + } + respond(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Published: pub}) + } +} + +func handleUnpublishVolume(provider ebsprovider.EBSProvider) msgHandler { + return func(ctx context.Context, msg *nats.Msg) { + var req ebsprovider.UnpublishVolumeRequest + if err := json.Unmarshal(msg.Data, &req); err != nil { + respond(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)}) + return + } + if req.SchemaVersion != ebsprovider.SchemaVersion { + respond(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: versionError(req.SchemaVersion)}) + return + } + if err := provider.UnpublishVolume(ctx, req); err != nil { + respond(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: classifyError(err)}) + return + } + respond(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: ebsprovider.NewVersioned()}) + } +} diff --git a/spinifex/ebsprovider/natsserve/server_test.go b/spinifex/ebsprovider/natsserve/server_test.go new file mode 100644 index 000000000..f90ccec18 --- /dev/null +++ b/spinifex/ebsprovider/natsserve/server_test.go @@ -0,0 +1,84 @@ +package natsserve_test + +import ( + "testing" + "time" + + "github.com/mulgadc/spinifex/spinifex/ebsprovider" + "github.com/mulgadc/spinifex/spinifex/ebsprovider/natsserve" + "github.com/mulgadc/spinifex/spinifex/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/propagation" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" +) + +func spansByKind(t *testing.T, recorder *tracetest.SpanRecorder) (client, server sdktrace.ReadOnlySpan) { + t.Helper() + client, server = testutil.SpansByKind(t, recorder) + require.NotNil(t, client, "no client span recorded") + require.NotNil(t, server, "no server span recorded") + return client, server +} + +func serveMemoryProvider(t *testing.T) *ebsprovider.NATSProvider { + t.Helper() + _, conn := testutil.StartTestNATS(t) + provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{VolumeEnumeration: true}) + stop, err := natsserve.Serve(t.Context(), conn, provider, natsserve.Options{NoQueueGroup: true}) + require.NoError(t, err) + t.Cleanup(stop) + return ebsprovider.NewNATSProvider(conn, 5*time.Second) +} + +// TestServerSpan_JoinsTheCallersTrace is the property that makes provider +// latency attributable: without it the two halves of one operation are two +// unrelated traces, and time spent in the provider cannot be told from time +// spent on the wire. +func TestServerSpan_JoinsTheCallersTrace(t *testing.T) { + recorder := testutil.RecordSpans(t) + client := serveMemoryProvider(t) + + _, err := client.ListVolumes(t.Context(), ebsprovider.ListVolumesRequest{Versioned: ebsprovider.NewVersioned()}) + require.NoError(t, err) + + clientSpan, serverSpan := spansByKind(t, recorder) + assert.Equal(t, "ebs.volume.list", clientSpan.Name()) + assert.Equal(t, "ebs.volume.list", serverSpan.Name()) + assert.Equal(t, clientSpan.SpanContext().TraceID(), serverSpan.SpanContext().TraceID()) + assert.Equal(t, clientSpan.SpanContext().SpanID(), serverSpan.Parent().SpanID()) +} + +// TestServerSpan_RecordsTheErrorItReturns keeps a failed verb from looking +// like a successful round trip on the serving side. +func TestServerSpan_RecordsTheErrorItReturns(t *testing.T) { + recorder := testutil.RecordSpans(t) + client := serveMemoryProvider(t) + + _, err := client.GetVolume(t.Context(), ebsprovider.GetVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-absent", + }) + require.ErrorIs(t, err, ebsprovider.ErrNotFound) + + _, serverSpan := spansByKind(t, recorder) + assert.Equal(t, codes.Error, serverSpan.Status().Code) +} + +// TestServerSpan_SurvivesAnUntracedCaller covers a caller that never installed +// a propagator: the provider must still open its own span rather than drop the +// message or inherit a broken parent. +func TestServerSpan_SurvivesAnUntracedCaller(t *testing.T) { + recorder := testutil.RecordSpans(t) + client := serveMemoryProvider(t) + otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator()) + + _, err := client.ListVolumes(t.Context(), ebsprovider.ListVolumesRequest{Versioned: ebsprovider.NewVersioned()}) + require.NoError(t, err) + + _, serverSpan := spansByKind(t, recorder) + assert.Equal(t, "ebs.volume.list", serverSpan.Name()) + assert.False(t, serverSpan.Parent().IsValid(), "an untraced caller must not leave a dangling parent") +} diff --git a/spinifex/ebsprovider/nullprovider/nullprovider.go b/spinifex/ebsprovider/nullprovider/nullprovider.go new file mode 100644 index 000000000..dca4db537 --- /dev/null +++ b/spinifex/ebsprovider/nullprovider/nullprovider.go @@ -0,0 +1,56 @@ +// Package nullprovider serves the EBSProvider contract without any storage +// behind it. It exists to establish the floor: a measurement taken against it +// is the cost of the contract, the transport and the control plane's own +// bookkeeping, with no engine, no object store and no disk in the number. +// +// It is not a type of its own. ebsprovider.MemoryProvider already answers +// every verb from a map, and natsserve already serves any EBSProvider, so a +// null provider is those two composed; this package only names that +// composition and states what it does and does not measure. +package nullprovider + +import ( + "context" + "fmt" + + "github.com/mulgadc/spinifex/spinifex/ebsprovider" + "github.com/mulgadc/spinifex/spinifex/ebsprovider/natsserve" + "github.com/nats-io/nats.go" +) + +// Capabilities is what a null provider advertises. Everything the contract +// makes optional is on: the point is to price the verbs, and a capability +// turned off is a verb that returns early instead of being measured. +var Capabilities = ebsprovider.Capabilities{ + OnlineExpansion: true, + SparseExtentReporting: true, + CrashConsistentSnapshot: true, + VolumeSeeding: true, + ReadOnlyPublish: true, + VolumeEnumeration: true, + SnapshotEnumeration: true, + Exclusion: ebsprovider.ExclusionSemantics{Scope: ebsprovider.ExclusionScopeNode}, +} + +// New returns an in-process null provider. Calls against it skip NATS +// entirely, so it prices the contract's own validation and bookkeeping and +// nothing else. +func New() *ebsprovider.MemoryProvider { + return ebsprovider.NewMemoryProvider(Capabilities) +} + +// Serve stands a null provider up on nc and returns a client that reaches it +// the way the control plane reaches any provider: encode, publish, queue +// dispatch, decode. The difference between this and New is the transport, and +// the difference between this and a real provider is the storage. +// +// PublishVolume answers with a URI nothing serves, so this bounds control +// operations only. It cannot stand in for a data-path comparison; qemunbdd is +// the control for that. +func Serve(ctx context.Context, nc *nats.Conn, opts natsserve.Options) (client *ebsprovider.NATSProvider, stop func(), err error) { + stop, err = natsserve.Serve(ctx, nc, New(), opts) + if err != nil { + return nil, nil, fmt.Errorf("serve null provider: %w", err) + } + return ebsprovider.NewNATSProvider(nc, 0), stop, nil +} diff --git a/spinifex/ebsprovider/provider.go b/spinifex/ebsprovider/provider.go new file mode 100644 index 000000000..cb855cf78 --- /dev/null +++ b/spinifex/ebsprovider/provider.go @@ -0,0 +1,437 @@ +// Package ebsprovider defines Spinifex's provider-neutral block-storage +// contract. Provider implementations must not expose their engine's Go types +// through this package; provider-specific settings and handles stay opaque. +package ebsprovider + +import ( + "context" + "encoding/json" + "fmt" + "time" +) + +// SchemaVersion is the only wire-contract version understood by this build. +// Every request and response carries it so version skew fails at the boundary +// instead of silently decoding a changed payload. +const SchemaVersion uint16 = 1 + +// Versioned is embedded in every request and response sent over NATS. +type Versioned struct { + SchemaVersion uint16 `json:"schema_version"` +} + +func NewVersioned() Versioned { return Versioned{SchemaVersion: SchemaVersion} } + +// EBSProvider is the block-storage controller contract consumed by Spinifex. +// Its operation and idempotency shape follows the CSI controller service, but +// the transport remains NATS request/reply rather than CSI's gRPC transport. +type EBSProvider interface { + GetCapabilities(context.Context, GetCapabilitiesRequest) (*GetCapabilitiesResponse, error) + CreateVolume(context.Context, CreateVolumeRequest) (*Volume, error) + GetVolume(context.Context, GetVolumeRequest) (*Volume, error) + ListVolumes(context.Context, ListVolumesRequest) (*ListVolumesResponse, error) + ExpandVolume(context.Context, ExpandVolumeRequest) (*Volume, error) + DeleteVolume(context.Context, DeleteVolumeRequest) error + CreateSnapshot(context.Context, CreateSnapshotRequest) (*Snapshot, error) + DeleteSnapshot(context.Context, DeleteSnapshotRequest) error + CopySnapshot(context.Context, CopySnapshotRequest) (*Snapshot, error) + ListSnapshots(context.Context, ListSnapshotsRequest) (*ListSnapshotsResponse, error) + PublishVolume(context.Context, PublishVolumeRequest) (*PublishedVolume, error) + UnpublishVolume(context.Context, UnpublishVolumeRequest) error +} + +// Provider is retained as a concise alias for implementation code. New +// consumers should name the dependency EBSProvider at composition boundaries. +type Provider = EBSProvider + +// Capabilities advertises optional provider behavior. Callers must branch on +// these values instead of assuming all implementations behave like viperblock. +// +// Every field must name behaviour some request in this interface can ask for. +// A capability no verb reaches tells a caller a feature is available and then +// gives them no way to use it. +type Capabilities struct { + OnlineExpansion bool `json:"online_expansion"` + SparseExtentReporting bool `json:"sparse_extent_reporting"` + CrashConsistentSnapshot bool `json:"crash_consistent_snapshot"` + VolumeSeeding bool `json:"volume_seeding"` + + // ReadOnlyPublish advertises that PublishVolume honours ReadOnly by + // exporting the volume read-only. A provider leaving this false must + // refuse a ReadOnly request, never hand back a writable export instead. + ReadOnlyPublish bool `json:"read_only_publish"` + + // VolumeEnumeration advertises that ListVolumes answers with what the + // provider actually holds. Without it the control plane's only index of + // volumes is its own metadata, so a lost document strands the blocks. + VolumeEnumeration bool `json:"volume_enumeration"` + + // SnapshotEnumeration advertises that ListSnapshots answers with what the + // provider actually holds. It is the same exposure VolumeEnumeration + // closes, for snapshots, and is advertised separately because a provider + // can enumerate one without the other. + SnapshotEnumeration bool `json:"snapshot_enumeration"` + + // OwnerRouting advertises that CreateSnapshot, CopySnapshot, ExpandVolume + // and GetVolume are answered directly by a volume's mounting node over + // its OwnerSubject when one exists, instead of always fanning out to the + // spinifex-workers queue group. + OwnerRouting bool `json:"owner_routing"` + + // Exclusion states how strongly this provider prevents two writers from + // holding one volume at once. It is not optional behaviour a caller may + // ignore: a caller that assumes cluster-wide exclusion from a provider + // that only excludes within a node will corrupt the volume. + Exclusion ExclusionSemantics `json:"exclusion"` +} + +// ExclusionScope names how far a provider's single-writer guarantee reaches. +type ExclusionScope string + +const ( + // ExclusionScopeNone means nothing prevents two writers. Two opens of one + // volume both succeed and both write. + ExclusionScopeNone ExclusionScope = "none" + + // ExclusionScopeNode means a second writer is refused on the same node and + // nowhere else. A different node opening the same backing store is not + // seen, so this is a guarantee about a process, not about a volume. + ExclusionScopeNode ExclusionScope = "node" + + // ExclusionScopeCluster means a second writer is refused wherever it runs, + // because the claim is held somewhere both nodes must consult. + ExclusionScopeCluster ExclusionScope = "cluster" +) + +// ExclusionSemantics is the single-writer guarantee, stated so a caller can +// tell what it actually gets. Three questions have distinct answers and the +// contract used to conflate them: how far exclusion reaches, whether a dead +// owner's claim ever clears, and whether a writer that lost its claim is +// actually stopped. +type ExclusionSemantics struct { + Scope ExclusionScope `json:"scope"` + + // ClaimTTLSeconds is how long a claim outlives the writer holding it. + // Zero means it never expires: a node that dies holding one strands the + // volume until an operator clears it. Meaningless when Scope is none. + ClaimTTLSeconds int `json:"claim_ttl_seconds,omitempty"` + + // FencesLostClaim reports whether a writer whose claim lapsed is stopped + // from writing, rather than merely finding out. False is the dangerous + // case and the reason this is not a bool on its own: exclusion refuses + // the *second* opener, fencing stops the *first* one. Without fencing, a + // partitioned node whose claim expired keeps writing, and the claim + // expiring is precisely what lets a second writer in beside it. + FencesLostClaim bool `json:"fences_lost_claim"` +} + +// SingleWriter reports whether opening a volume twice anywhere is refused. +// Callers branch on this rather than comparing Scope, so adding a scope later +// does not need every call site found again. +func (e ExclusionSemantics) SingleWriter() bool { + return e.Scope == ExclusionScopeCluster +} + +type GetCapabilitiesRequest struct{ Versioned } + +type GetCapabilitiesResponse struct { + Versioned + + Capabilities Capabilities `json:"capabilities"` + Error *ProviderError `json:"error,omitempty"` +} + +// CapacityRange mirrors CSI's capacity-range semantics. RequiredBytes is the +// minimum acceptable capacity; LimitBytes, when non-zero, is the maximum. +type CapacityRange struct { + RequiredBytes int64 `json:"required_bytes"` + LimitBytes int64 `json:"limit_bytes,omitempty"` +} + +type VolumeState string + +const ( + VolumeStateAvailable VolumeState = "available" + VolumeStateInUse VolumeState = "in-use" +) + +// Volume contains only facts the control plane can rely on across providers. +// Handle and ProviderData are opaque and must be passed back uninterpreted. +type Volume struct { + ID string `json:"id"` + CapacityBytes int64 `json:"capacity_bytes"` + State VolumeState `json:"state"` + Handle string `json:"handle"` + AvailabilityZone string `json:"availability_zone,omitempty"` + ProviderData json.RawMessage `json:"provider_data,omitempty"` +} + +// MaxSeedBytes bounds CreateVolumeRequest.SeedData. JSON encodes a []byte as +// base64, inflating it by 4/3, so this leaves the encoded request comfortably +// inside the 1MB NATS max_payload the cluster runs with. +const MaxSeedBytes = 640 * 1024 + +type CreateVolumeRequest struct { + Versioned + + VolumeID string `json:"volume_id"` + CapacityRange CapacityRange `json:"capacity_range"` + AvailabilityZone string `json:"availability_zone,omitempty"` + SourceSnapshotID string `json:"source_snapshot_id,omitempty"` + Parameters json.RawMessage `json:"parameters,omitempty"` + + // SourceSnapshotVolumeID names the volume SourceSnapshotID was taken from, + // matching that snapshot's Snapshot.SourceVolumeID. It is required with + // SourceSnapshotID and meaningless without it: this is not a clone source. + SourceSnapshotVolumeID string `json:"source_snapshot_volume_id,omitempty"` + + // SeedData is written at offset 0 of a newly created volume. It exists so + // the caller can supply host-local bytes, such as a firmware VARS template + // whose layout must match the launching node, without shipping a path. + SeedData []byte `json:"seed_data,omitempty"` +} + +// ValidateSeedData rejects a seed the NATS transport cannot carry, so an +// oversized firmware template fails with an actionable error at the caller +// rather than as a truncated or refused publish. +func ValidateSeedData(seed []byte) error { + if len(seed) > MaxSeedBytes { + return fmt.Errorf("%w: seed data is %d bytes, limit is %d", ErrInvalidArgument, len(seed), MaxSeedBytes) + } + return nil +} + +type CreateVolumeResponse struct { + Versioned + + Volume *Volume `json:"volume,omitempty"` + Error *ProviderError `json:"error,omitempty"` +} + +type GetVolumeRequest struct { + Versioned + + VolumeID string `json:"volume_id"` + Handle string `json:"handle,omitempty"` +} + +type GetVolumeResponse struct { + Versioned + + Volume *Volume `json:"volume,omitempty"` + Error *ProviderError `json:"error,omitempty"` +} + +// VolumeRef identifies a volume the provider holds. It is deliberately not a +// Volume: enumeration answers "what exists", and a provider must not have to +// open every volume's engine to answer that. +type VolumeRef struct { + ID string `json:"id"` + Handle string `json:"handle,omitempty"` +} + +// MaxListResults bounds one page of ListVolumes or ListSnapshots. The reply +// rides a NATS message, so the page must fit the cluster's max_payload; a +// VolumeRef or SnapshotRef encodes to well under 1KB, leaving headroom here. +const MaxListResults int32 = 1000 + +// ListVolumesRequest pages through the volumes a provider holds. A MaxResults +// above MaxListResults is clamped rather than refused: a caller asking for +// more than fits should get a page and a token, not an error. +type ListVolumesRequest struct { + Versioned + + MaxResults int32 `json:"max_results,omitempty"` + StartingToken string `json:"starting_token,omitempty"` +} + +// ListVolumesResponse carries one page. An empty NextToken means the last +// page; an empty Volumes list with no error means the provider holds none. +type ListVolumesResponse struct { + Versioned + + Volumes []VolumeRef `json:"volumes,omitempty"` + NextToken string `json:"next_token,omitempty"` + Error *ProviderError `json:"error,omitempty"` +} + +// PageSize is the number of refs a provider should return for this request. +func (r ListVolumesRequest) PageSize() int32 { + if r.MaxResults <= 0 || r.MaxResults > MaxListResults { + return MaxListResults + } + return r.MaxResults +} + +type ExpandVolumeRequest struct { + Versioned + + VolumeID string `json:"volume_id"` + Handle string `json:"handle,omitempty"` + CapacityRange CapacityRange `json:"capacity_range"` +} + +type ExpandVolumeResponse struct { + Versioned + + Volume *Volume `json:"volume,omitempty"` + Error *ProviderError `json:"error,omitempty"` +} + +type DeleteVolumeRequest struct { + Versioned + + VolumeID string `json:"volume_id"` + Handle string `json:"handle,omitempty"` +} + +type DeleteVolumeResponse struct { + Versioned + + Error *ProviderError `json:"error,omitempty"` +} + +type SnapshotState string + +const ( + SnapshotStatePending SnapshotState = "pending" + SnapshotStateCompleted SnapshotState = "completed" + SnapshotStateError SnapshotState = "error" +) + +type Snapshot struct { + ID string `json:"id"` + SourceVolumeID string `json:"source_volume_id"` + SizeBytes int64 `json:"size_bytes"` + CreatedAt time.Time `json:"created_at"` + State SnapshotState `json:"state"` + Handle string `json:"handle"` +} + +type CreateSnapshotRequest struct { + Versioned + + SnapshotID string `json:"snapshot_id"` + VolumeID string `json:"volume_id"` + VolumeHandle string `json:"volume_handle,omitempty"` +} + +// CreateSnapshotResponse is both the immediate accepted response and the +// completion event. Pending responses carry a completion subject; completed +// responses additionally carry the provider-neutral Snapshot. +type CreateSnapshotResponse struct { + Versioned + + OperationID string `json:"operation_id"` + CompletionSubject string `json:"completion_subject,omitempty"` + Snapshot *Snapshot `json:"snapshot,omitempty"` + Error *ProviderError `json:"error,omitempty"` +} + +type DeleteSnapshotRequest struct { + Versioned + + SnapshotID string `json:"snapshot_id"` + Handle string `json:"handle,omitempty"` +} + +type DeleteSnapshotResponse struct { + Versioned + + Error *ProviderError `json:"error,omitempty"` +} + +// CopySnapshotRequest duplicates SourceSnapshotID under DestinationSnapshotID +// as a second, independently addressable snapshot over the same frozen data. +// VolumeID is the snapshot's own source volume, not a new source: the caller +// already knows it (it is the same VolumeID the original CreateSnapshotRequest +// carried), so the provider is not asked to resolve snapshot ownership itself. +type CopySnapshotRequest struct { + Versioned + + SourceSnapshotID string `json:"source_snapshot_id"` + DestinationSnapshotID string `json:"destination_snapshot_id"` + VolumeID string `json:"volume_id"` +} + +type CopySnapshotResponse struct { + Versioned + + Snapshot *Snapshot `json:"snapshot,omitempty"` + Error *ProviderError `json:"error,omitempty"` +} + +// SnapshotRef identifies a snapshot the provider holds. Like VolumeRef it is +// deliberately not a Snapshot: enumeration answers "what exists", and reading +// a snapshot's size or state can cost far more than listing it. +// +// SourceVolumeID is carried because reconciliation needs it: a snapshot found +// with no metadata document is only actionable once its volume is known. +type SnapshotRef struct { + ID string `json:"id"` + SourceVolumeID string `json:"source_volume_id,omitempty"` + Handle string `json:"handle,omitempty"` +} + +// ListSnapshotsRequest pages through the snapshots a provider holds. It is +// paged and clamped exactly as ListVolumesRequest is. +type ListSnapshotsRequest struct { + Versioned + + MaxResults int32 `json:"max_results,omitempty"` + StartingToken string `json:"starting_token,omitempty"` +} + +// ListSnapshotsResponse carries one page. An empty NextToken means the last +// page; an empty Snapshots list with no error means the provider holds none. +type ListSnapshotsResponse struct { + Versioned + + Snapshots []SnapshotRef `json:"snapshots,omitempty"` + NextToken string `json:"next_token,omitempty"` + Error *ProviderError `json:"error,omitempty"` +} + +// PageSize is the number of refs a provider should return for this request. +func (r ListSnapshotsRequest) PageSize() int32 { + if r.MaxResults <= 0 || r.MaxResults > MaxListResults { + return MaxListResults + } + return r.MaxResults +} + +type PublishVolumeRequest struct { + Versioned + + VolumeID string `json:"volume_id"` + Handle string `json:"handle,omitempty"` + NodeID string `json:"node_id"` + ReadOnly bool `json:"read_only,omitempty"` +} + +type PublishedVolume struct { + VolumeID string `json:"volume_id"` + NodeID string `json:"node_id"` + NBDURI string `json:"nbd_uri"` +} + +type PublishVolumeResponse struct { + Versioned + + Published *PublishedVolume `json:"published,omitempty"` + Error *ProviderError `json:"error,omitempty"` +} + +type UnpublishVolumeRequest struct { + Versioned + + VolumeID string `json:"volume_id"` + Handle string `json:"handle,omitempty"` + NodeID string `json:"node_id"` +} + +type UnpublishVolumeResponse struct { + Versioned + + Error *ProviderError `json:"error,omitempty"` +} diff --git a/spinifex/ebsprovider/telemetry.go b/spinifex/ebsprovider/telemetry.go new file mode 100644 index 000000000..25cc744f6 --- /dev/null +++ b/spinifex/ebsprovider/telemetry.go @@ -0,0 +1,218 @@ +package ebsprovider + +import ( + "context" + "encoding/json" + "strings" + + "github.com/nats-io/nats.go" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/trace" +) + +const tracerName = "github.com/mulgadc/spinifex/spinifex/ebsprovider" + +// spanNamePrefix puts provider spans in the same namespace as the ebs.mount +// and ebs.unmount spans the legacy path emits. +const spanNamePrefix = "ebs." + +const ( + attrSubject = "ebs.provider.subject" + attrVolumeID = "ebs.provider.volume_id" + attrNodeID = "ebs.provider.node_id" + attrOwnerRouted = "ebs.provider.owner_routed" + attrErrorCode = "ebs.provider.error_code" +) + +// VerbUnknown is the verb reported for a subject that is not part of the +// contract. Unrecognised subjects collapse to it rather than becoming span +// names of their own, which is what keeps the name set bounded. +const VerbUnknown = "unknown" + +const ( + verbCapabilities = "capabilities" + verbVolumeCreate = "volume.create" + verbVolumeList = "volume.list" + verbVolumeDelete = "volume.delete" + verbVolumePublish = "volume.publish" + verbVolumeUnpublish = "volume.unpublish" + verbSnapshotDelete = "snapshot.delete" + verbSnapshotList = "snapshot.list" +) + +// subjectVerbs maps every fixed subject to its verb. The dynamic subject +// families (owner-routed, snapshot create, per-node mount) are handled by +// subjectTarget, which strips the ID token before naming the verb. +var subjectVerbs = map[string]string{ + CapabilitiesSubject: verbCapabilities, + CreateVolumeSubject: verbVolumeCreate, + GetVolumeSubject: verbVolumeDescribe, + ListVolumesSubject: verbVolumeList, + ExpandVolumeSubject: verbVolumeExpand, + DeleteVolumeSubject: verbVolumeDelete, + DeleteSnapshotSubject: verbSnapshotDelete, + CopySnapshotSubject: verbSnapshotCopy, + ListSnapshotsSubject: verbSnapshotList, +} + +// ownerVerbs is the closed set OwnerSubject can build. Parsed subjects are +// checked against it so a malformed subject cannot invent a verb. +var ownerVerbs = map[string]bool{ + verbSnapshotCreate: true, + verbSnapshotCopy: true, + verbVolumeExpand: true, + verbVolumeDescribe: true, +} + +// subjectTarget describes what a subject addresses. The verb is drawn from a +// closed set and is safe to use as a span name; volumeID and nodeID are +// unbounded and belong in attributes only. +type subjectTarget struct { + verb string + volumeID string + nodeID string + ownerRouted bool +} + +// SubjectVerb reports the operation subject names, with any embedded volume +// or node ID removed. A subject outside the contract reports VerbUnknown. +func SubjectVerb(subject string) string { + return parseSubject(subject).verb +} + +func parseSubject(subject string) subjectTarget { + if verb, ok := subjectVerbs[subject]; ok { + return subjectTarget{verb: verb} + } + if volumeID, verb, ok := ParseOwnerSubject(subject); ok && ownerVerbs[verb] { + return subjectTarget{verb: verb, volumeID: volumeID, ownerRouted: true} + } + if volumeID, ok := strings.CutPrefix(subject, SnapshotCreateSubjectPrefix); ok && validateSubjectToken(volumeID) == nil { + return subjectTarget{verb: verbSnapshotCreate, volumeID: volumeID} + } + rest, ok := strings.CutPrefix(subject, subjectPrefix) + if !ok { + return subjectTarget{verb: VerbUnknown} + } + nodeID, suffix, ok := strings.Cut(rest, ".") + if !ok || validateSubjectToken(nodeID) != nil { + return subjectTarget{verb: VerbUnknown} + } + switch suffix { + case "mount": + return subjectTarget{verb: verbVolumePublish, nodeID: nodeID} + case "unmount": + return subjectTarget{verb: verbVolumeUnpublish, nodeID: nodeID} + default: + return subjectTarget{verb: VerbUnknown} + } +} + +func (t subjectTarget) attributes(subject string) []attribute.KeyValue { + attrs := []attribute.KeyValue{ + attribute.String("messaging.system", "nats"), + attribute.String(attrSubject, subject), + } + if t.volumeID != "" { + attrs = append(attrs, attribute.String(attrVolumeID, t.volumeID)) + } + if t.nodeID != "" { + attrs = append(attrs, attribute.String(attrNodeID, t.nodeID)) + } + if t.ownerRouted { + attrs = append(attrs, attribute.Bool(attrOwnerRouted, true)) + } + return attrs +} + +var _ propagation.TextMapCarrier = (*headerCarrier)(nil) + +// headerCarrier adapts nats.Header to the OTel carrier. It is a copy of the +// one in spinifex/utils rather than an import: this package is the wire +// contract and must not depend on the control plane's helper tree. +type headerCarrier nats.Header + +func (c headerCarrier) Get(key string) string { return nats.Header(c).Get(key) } +func (c headerCarrier) Set(key, value string) { nats.Header(c).Set(key, value) } +func (c headerCarrier) Keys() []string { + keys := make([]string, 0, len(c)) + for key := range c { + keys = append(keys, key) + } + return keys +} + +// startClientSpan opens a span for an outbound request and injects the trace +// context into hdr so the serving provider joins the same trace. +func startClientSpan(ctx context.Context, subject string, hdr nats.Header) (context.Context, trace.Span) { + target := parseSubject(subject) + ctx, span := otel.Tracer(tracerName).Start(ctx, spanNamePrefix+target.verb, + trace.WithSpanKind(trace.SpanKindClient), + trace.WithAttributes(target.attributes(subject)...)) + otel.GetTextMapPropagator().Inject(ctx, headerCarrier(hdr)) + return ctx, span +} + +// startCompletionSpan covers the wait for an asynchronous completion publish. +// A snapshot spends nearly all its time here, and the request span for the +// accept leg ends long before it. +func startCompletionSpan(ctx context.Context, verb, completionSubject string) (context.Context, trace.Span) { + return otel.Tracer(tracerName).Start(ctx, spanNamePrefix+verb+".wait", + trace.WithSpanKind(trace.SpanKindConsumer), + trace.WithAttributes( + attribute.String("messaging.system", "nats"), + attribute.String(attrSubject, completionSubject), + )) +} + +// StartServerSpan joins the trace carried in msg's headers and opens a span +// for the verb its subject names. Callers must End the returned span and +// should pass the returned context into the work the handler does. +func StartServerSpan(ctx context.Context, msg *nats.Msg) (context.Context, trace.Span) { + target := parseSubject(msg.Subject) + if msg.Header != nil { + ctx = otel.GetTextMapPropagator().Extract(ctx, headerCarrier(msg.Header)) + } + return otel.Tracer(tracerName).Start(ctx, spanNamePrefix+target.verb, + trace.WithSpanKind(trace.SpanKindServer), + trace.WithAttributes(target.attributes(msg.Subject)...)) +} + +// RecordSpanError marks span failed for a transport or handler error. +func RecordSpanError(span trace.Span, err error) { + if err == nil { + return + } + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) +} + +// RecordProviderError tags span with the wire error code a handler is about +// to return, so a failed operation is visible as an error in the trace and +// not merely as a successful round trip. +func RecordProviderError(span trace.Span, providerErr *ProviderError) { + if providerErr == nil { + return + } + span.SetAttributes(attribute.String(attrErrorCode, string(providerErr.Code))) + span.SetStatus(codes.Error, providerErr.Message) +} + +// RecordResponseError tags span with the error an encoded response carries. +// The response types share no accessor for it, so this re-reads the one field +// rather than making every verb implement one; it runs only when recording. +func RecordResponseError(span trace.Span, data []byte) { + if !span.IsRecording() { + return + } + var envelope struct { + Error *ProviderError `json:"error"` + } + if err := json.Unmarshal(data, &envelope); err != nil { + return + } + RecordProviderError(span, envelope.Error) +} diff --git a/spinifex/ebsprovider/telemetry_test.go b/spinifex/ebsprovider/telemetry_test.go new file mode 100644 index 000000000..2e4b583ed --- /dev/null +++ b/spinifex/ebsprovider/telemetry_test.go @@ -0,0 +1,197 @@ +package ebsprovider + +import ( + "encoding/json" + "testing" + "time" + + "github.com/mulgadc/spinifex/spinifex/testutil" + "github.com/nats-io/nats.go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/codes" +) + +func TestSubjectVerb_StripsIDsFromEveryDynamicSubject(t *testing.T) { + const volumeID = "vol-0123456789abcdef0" + const nodeID = "casuarina" + + ownerDescribe, err := GetVolumeOwnerSubject(volumeID) + require.NoError(t, err) + ownerExpand, err := ExpandVolumeOwnerSubject(volumeID) + require.NoError(t, err) + ownerSnapshotCreate, err := SnapshotCreateOwnerSubject(volumeID) + require.NoError(t, err) + ownerSnapshotCopy, err := SnapshotCopyOwnerSubject(volumeID) + require.NoError(t, err) + snapshotCreate, err := SnapshotSubject(volumeID) + require.NoError(t, err) + publish, err := PublishSubject(nodeID) + require.NoError(t, err) + unpublish, err := UnpublishSubject(nodeID) + require.NoError(t, err) + + cases := map[string]string{ + CapabilitiesSubject: verbCapabilities, + CreateVolumeSubject: verbVolumeCreate, + GetVolumeSubject: verbVolumeDescribe, + ListVolumesSubject: verbVolumeList, + ExpandVolumeSubject: verbVolumeExpand, + DeleteVolumeSubject: verbVolumeDelete, + DeleteSnapshotSubject: verbSnapshotDelete, + CopySnapshotSubject: verbSnapshotCopy, + ownerDescribe: verbVolumeDescribe, + ownerExpand: verbVolumeExpand, + ownerSnapshotCreate: verbSnapshotCreate, + ownerSnapshotCopy: verbSnapshotCopy, + snapshotCreate: verbSnapshotCreate, + publish: verbVolumePublish, + unpublish: verbVolumeUnpublish, + } + + for subject, want := range cases { + verb := SubjectVerb(subject) + assert.Equal(t, want, verb, "subject %q", subject) + assert.NotContains(t, verb, volumeID, "verb for %q leaks the volume ID", subject) + assert.NotContains(t, verb, nodeID, "verb for %q leaks the node ID", subject) + } +} + +func TestSubjectVerb_UnrecognisedSubjectCollapsesToUnknown(t *testing.T) { + for _, subject := range []string{ + "", + "ebs.mount", + "ebs.provider.v1.volume.invent", + "ebs.provider.v1.owner.vol-1.invent", + "ebs.provider.v1.node.rename", + "ebs.provider.v2.volume.create", + } { + assert.Equal(t, VerbUnknown, SubjectVerb(subject), "subject %q", subject) + } +} + +// TestClientSpan_NamesTheVerbNotTheSubject pins the property that makes these +// spans aggregatable: a per-volume subject must not become a span name of its +// own, so the volume ID belongs in an attribute. +func TestClientSpan_NamesTheVerbNotTheSubject(t *testing.T) { + recorder := testutil.RecordSpans(t) + _, conn := testutil.StartTestNATS(t) + + const volumeID = "vol-0123456789abcdef0" + ownerSubject, err := GetVolumeOwnerSubject(volumeID) + require.NoError(t, err) + sub, err := conn.Subscribe(ownerSubject, func(msg *nats.Msg) { + payload, marshalErr := json.Marshal(GetVolumeResponse{Versioned: NewVersioned(), Volume: &Volume{ID: volumeID}}) + require.NoError(t, marshalErr) + require.NoError(t, msg.Respond(payload)) + }) + require.NoError(t, err) + t.Cleanup(func() { _ = sub.Unsubscribe() }) + require.NoError(t, conn.Flush()) + + provider := NewNATSProvider(conn, time.Second) + _, err = provider.GetVolume(t.Context(), GetVolumeRequest{Versioned: NewVersioned(), VolumeID: volumeID}) + require.NoError(t, err) + + spans := recorder.Ended() + require.Len(t, spans, 1) + assert.Equal(t, "ebs.volume.describe", spans[0].Name()) + assert.NotContains(t, spans[0].Name(), volumeID) + assert.Equal(t, volumeID, testutil.SpanAttribute(t, spans[0], attrVolumeID).AsString()) + assert.Equal(t, ownerSubject, testutil.SpanAttribute(t, spans[0], attrSubject).AsString()) + assert.True(t, testutil.SpanAttribute(t, spans[0], attrOwnerRouted).AsBool()) +} + +// TestClientSpan_PutsTraceparentOnTheWire is the half of propagation this +// package owns. Without it the serving provider roots a trace of its own and +// the two halves of an operation can never be joined. +func TestClientSpan_PutsTraceparentOnTheWire(t *testing.T) { + recorder := testutil.RecordSpans(t) + _, conn := testutil.StartTestNATS(t) + + received := make(chan string, 1) + sub, err := conn.Subscribe(ListVolumesSubject, func(msg *nats.Msg) { + received <- msg.Header.Get("traceparent") + payload, marshalErr := json.Marshal(ListVolumesResponse{Versioned: NewVersioned()}) + require.NoError(t, marshalErr) + require.NoError(t, msg.Respond(payload)) + }) + require.NoError(t, err) + t.Cleanup(func() { _ = sub.Unsubscribe() }) + require.NoError(t, conn.Flush()) + + provider := NewNATSProvider(conn, time.Second) + _, err = provider.ListVolumes(t.Context(), ListVolumesRequest{Versioned: NewVersioned()}) + require.NoError(t, err) + + spans := recorder.Ended() + require.Len(t, spans, 1) + select { + case traceparent := <-received: + require.NotEmpty(t, traceparent, "no traceparent header reached the provider") + assert.Contains(t, traceparent, spans[0].SpanContext().TraceID().String()) + assert.Contains(t, traceparent, spans[0].SpanContext().SpanID().String()) + case <-time.After(time.Second): + t.Fatal("provider never received the request") + } +} + +// TestClientSpan_RecordsTheWireErrorCode covers the case a transport-only span +// would report as success: the round trip completed, but the operation failed. +func TestClientSpan_RecordsTheWireErrorCode(t *testing.T) { + recorder := testutil.RecordSpans(t) + _, conn := testutil.StartTestNATS(t) + + sub, err := conn.Subscribe(DeleteVolumeSubject, func(msg *nats.Msg) { + payload, marshalErr := json.Marshal(DeleteVolumeResponse{ + Versioned: NewVersioned(), + Error: &ProviderError{Code: ErrorCodeVolumeInUse, Message: "volume vol-1 is published"}, + }) + require.NoError(t, marshalErr) + require.NoError(t, msg.Respond(payload)) + }) + require.NoError(t, err) + t.Cleanup(func() { _ = sub.Unsubscribe() }) + require.NoError(t, conn.Flush()) + + provider := NewNATSProvider(conn, time.Second) + err = provider.DeleteVolume(t.Context(), DeleteVolumeRequest{Versioned: NewVersioned(), VolumeID: "vol-1"}) + require.ErrorIs(t, err, ErrVolumeInUse) + + spans := recorder.Ended() + require.Len(t, spans, 1) + assert.Equal(t, codes.Error, spans[0].Status().Code) + assert.Equal(t, string(ErrorCodeVolumeInUse), testutil.SpanAttribute(t, spans[0], attrErrorCode).AsString()) +} + +// TestClientSpan_OwnerFallbackRecordsBothLegs keeps the wasted owner attempt +// visible. A fallback that showed only the leg that answered would hide the +// cost of asking a node that holds nothing. +func TestClientSpan_OwnerFallbackRecordsBothLegs(t *testing.T) { + recorder := testutil.RecordSpans(t) + _, conn := testutil.StartTestNATS(t) + + const volumeID = "vol-0123456789abcdef0" + sub, err := conn.Subscribe(GetVolumeSubject, func(msg *nats.Msg) { + payload, marshalErr := json.Marshal(GetVolumeResponse{Versioned: NewVersioned(), Volume: &Volume{ID: volumeID}}) + require.NoError(t, marshalErr) + require.NoError(t, msg.Respond(payload)) + }) + require.NoError(t, err) + t.Cleanup(func() { _ = sub.Unsubscribe() }) + require.NoError(t, conn.Flush()) + + provider := NewNATSProvider(conn, time.Second) + _, err = provider.GetVolume(t.Context(), GetVolumeRequest{Versioned: NewVersioned(), VolumeID: volumeID}) + require.NoError(t, err) + + spans := recorder.Ended() + require.Len(t, spans, 2, "owner attempt and queue-group fallback should each be a span") + assert.Equal(t, "ebs.volume.describe", spans[0].Name()) + assert.True(t, testutil.SpanAttribute(t, spans[0], attrOwnerRouted).AsBool()) + assert.Equal(t, codes.Error, spans[0].Status().Code, "the unanswered owner attempt is a failure") + + assert.Equal(t, "ebs.volume.describe", spans[1].Name()) + assert.Equal(t, GetVolumeSubject, testutil.SpanAttribute(t, spans[1], attrSubject).AsString()) + assert.Equal(t, codes.Unset, spans[1].Status().Code) +} diff --git a/spinifex/handlers/ec2/image/ami_source_volume_test.go b/spinifex/handlers/ec2/image/ami_source_volume_test.go new file mode 100644 index 000000000..8ed58b951 --- /dev/null +++ b/spinifex/handlers/ec2/image/ami_source_volume_test.go @@ -0,0 +1,101 @@ +package handlers_ec2_image + +import ( + "context" + "testing" + + "github.com/mulgadc/spinifex/spinifex/awserrors" + "github.com/mulgadc/spinifex/spinifex/ebsmetadata" + handlers_ec2_snapshot "github.com/mulgadc/spinifex/spinifex/handlers/ec2/snapshot" + "github.com/mulgadc/spinifex/spinifex/objectstore" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// putSourceVolumeAMI registers an AMI document plus, when volumeID is set, the +// snapshot metadata recording which volume the snapshot was taken from. +func putSourceVolumeAMI(t *testing.T, svc *ImageServiceImpl, store *objectstore.MemoryObjectStore, ami ebsmetadata.AMI, volumeID string) { + t.Helper() + require.NoError(t, svc.MetadataStore().PutAMI(context.Background(), ami)) + if volumeID == "" { + return + } + require.NoError(t, handlers_ec2_snapshot.WriteSnapshotConfig(store, testBucket, ami.SnapshotID, + &handlers_ec2_snapshot.SnapshotConfig{SnapshotID: ami.SnapshotID, VolumeID: volumeID})) +} + +// TestGetAMISourceVolumeID_ReadsSnapshotMetadata locks the normal case: the +// source volume comes from the snapshot's own metadata.json. +func TestGetAMISourceVolumeID_ReadsSnapshotMetadata(t *testing.T) { + svc, store := setupProviderImageService(t) + putSourceVolumeAMI(t, svc, store, ebsmetadata.AMI{ + ImageID: "ami-src01", SnapshotID: "snap-src01", ImageOwnerAlias: testAccountID, + }, "vol-origin") + + got, err := svc.GetAMISourceVolumeID(context.Background(), "ami-src01") + require.NoError(t, err) + assert.Equal(t, "vol-origin", got) +} + +// TestGetAMISourceVolumeID_BundledSystemAMI locks the fallback for bundled +// system AMIs, whose snapshot is named after the AMI and carries no metadata.json. +func TestGetAMISourceVolumeID_BundledSystemAMI(t *testing.T) { + svc, store := setupProviderImageService(t) + putSourceVolumeAMI(t, svc, store, ebsmetadata.AMI{ + ImageID: "ami-sys01", SnapshotID: "snap-ami-sys01", ImageOwnerAlias: "system", + }, "") + + got, err := svc.GetAMISourceVolumeID(context.Background(), "ami-sys01") + require.NoError(t, err) + assert.Equal(t, "ami-sys01", got, "the bundled AMI's snapshot reads chunks from a volume named after the AMI") +} + +// TestGetAMISourceVolumeID_AccountAMIMissingSnapshotMetadata locks that the +// bundled fallback does not apply to account-owned AMIs. +func TestGetAMISourceVolumeID_AccountAMIMissingSnapshotMetadata(t *testing.T) { + svc, store := setupProviderImageService(t) + putSourceVolumeAMI(t, svc, store, ebsmetadata.AMI{ + ImageID: "ami-acct01", SnapshotID: "snap-acct01", ImageOwnerAlias: testAccountID, + }, "") + + _, err := svc.GetAMISourceVolumeID(context.Background(), "ami-acct01") + require.Error(t, err) + assert.Equal(t, awserrors.ErrorInvalidSnapshotNotFound, err.Error()) +} + +// TestGetAMISourceVolumeID_AMIWithoutSnapshot locks that an AMI carrying no +// snapshot reference is reported as not found rather than resolving to "". +func TestGetAMISourceVolumeID_AMIWithoutSnapshot(t *testing.T) { + svc, store := setupProviderImageService(t) + putSourceVolumeAMI(t, svc, store, ebsmetadata.AMI{ + ImageID: "ami-nosnap", ImageOwnerAlias: testAccountID, + }, "") + + _, err := svc.GetAMISourceVolumeID(context.Background(), "ami-nosnap") + require.Error(t, err) + assert.Equal(t, awserrors.ErrorInvalidAMIIDNotFound, err.Error()) +} + +// TestGetAMISourceVolumeID_UnknownAMI locks the missing-AMI mapping. +func TestGetAMISourceVolumeID_UnknownAMI(t *testing.T) { + svc, _ := setupProviderImageService(t) + + _, err := svc.GetAMISourceVolumeID(context.Background(), "ami-nothere") + require.Error(t, err) + assert.Equal(t, awserrors.ErrorInvalidAMIIDNotFound, err.Error()) +} + +// TestGetAMISourceVolumeID_EmptySnapshotSourceVolume locks that corrupt +// snapshot metadata naming no source volume fails instead of returning "". +func TestGetAMISourceVolumeID_EmptySnapshotSourceVolume(t *testing.T) { + svc, store := setupProviderImageService(t) + require.NoError(t, svc.MetadataStore().PutAMI(context.Background(), ebsmetadata.AMI{ + ImageID: "ami-empty", SnapshotID: "snap-empty", ImageOwnerAlias: testAccountID, + })) + require.NoError(t, handlers_ec2_snapshot.WriteSnapshotConfig(store, testBucket, "snap-empty", + &handlers_ec2_snapshot.SnapshotConfig{SnapshotID: "snap-empty"})) + + _, err := svc.GetAMISourceVolumeID(context.Background(), "ami-empty") + require.Error(t, err) + assert.Equal(t, awserrors.ErrorInvalidSnapshotNotFound, err.Error()) +} diff --git a/spinifex/handlers/ec2/image/deregister_legacy_config_test.go b/spinifex/handlers/ec2/image/deregister_legacy_config_test.go new file mode 100644 index 000000000..081ed4fca --- /dev/null +++ b/spinifex/handlers/ec2/image/deregister_legacy_config_test.go @@ -0,0 +1,52 @@ +package handlers_ec2_image + +import ( + "context" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + awss3 "github.com/aws/aws-sdk-go/service/s3" + "github.com/mulgadc/spinifex/spinifex/objectstore" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// strictDeleteStore answers a delete of an absent key with NoSuchKey, which is +// what Predastore does. MemoryObjectStore silently succeeds instead, so a +// handler that treats a missing key as a failure passes every test and still +// breaks in production. +type strictDeleteStore struct { + *objectstore.MemoryObjectStore +} + +func (s strictDeleteStore) DeleteObject(ctx context.Context, input *awss3.DeleteObjectInput) (*awss3.DeleteObjectOutput, error) { + if _, err := s.HeadObject(ctx, &awss3.HeadObjectInput{Bucket: input.Bucket, Key: input.Key}); err != nil { + return nil, err + } + return s.MemoryObjectStore.DeleteObject(ctx, input) +} + +// TestDeregisterImage_NoLegacyConfig covers every AMI created since the +// metadata store landed: they have no {imageID}/config.json, so deleting one +// answers NoSuchKey. Treating that as a failure returned InternalError after +// the AMI document was already gone, and a retrying caller then saw +// InvalidAMIID.NotFound for an image it had just been told it could not delete. +func TestDeregisterImage_NoLegacyConfig(t *testing.T) { + mem := objectstore.NewMemoryObjectStore() + svc := NewImageServiceImplWithStore(strictDeleteStore{MemoryObjectStore: mem}, testBucket) + + amiID := "ami-nolegacy001" + createTestAMIConfigWithOwner(t, mem, amiID, "no-legacy-config", testAccountID) + + out, err := svc.DeregisterImage(context.Background(), &ec2.DeregisterImageInput{ImageId: aws.String(amiID)}, testAccountID) + require.NoError(t, err) + assert.NotNil(t, out) + + _, getErr := mem.GetObject(context.Background(), &awss3.GetObjectInput{ + Bucket: aws.String(testBucket), + Key: aws.String(amiDocumentKey(t, amiID)), + }) + require.Error(t, getErr) + assert.True(t, objectstore.IsNoSuchKeyError(getErr), "the AMI document must still be deleted") +} diff --git a/spinifex/handlers/ec2/image/drain_test.go b/spinifex/handlers/ec2/image/drain_test.go index c59808667..bdfc9cec4 100644 --- a/spinifex/handlers/ec2/image/drain_test.go +++ b/spinifex/handlers/ec2/image/drain_test.go @@ -42,7 +42,8 @@ func setupDrainImageService(t *testing.T) (*ImageServiceImpl, *objectstore.Memor Bucket: testBucket, }, } - return NewImageServiceImplWithConfig(cfg, store, nc), store, nc + svc := NewImageServiceImplWithConfig(cfg, store, nc) + return svc, store, nc } func seedImageVolumeAttachment(t *testing.T, store *objectstore.MemoryObjectStore, volumeID, state, instanceID string) { diff --git a/spinifex/handlers/ec2/image/provider_snapshot_test.go b/spinifex/handlers/ec2/image/provider_snapshot_test.go new file mode 100644 index 000000000..07b53274f --- /dev/null +++ b/spinifex/handlers/ec2/image/provider_snapshot_test.go @@ -0,0 +1,163 @@ +package handlers_ec2_image + +import ( + "context" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mulgadc/spinifex/spinifex/awserrors" + "github.com/mulgadc/spinifex/spinifex/config" + "github.com/mulgadc/spinifex/spinifex/ebsmetadata" + "github.com/mulgadc/spinifex/spinifex/ebsprovider" + "github.com/mulgadc/spinifex/spinifex/objectstore" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const providerSnapshotAZ = "ap-southeast-2a" + +// setupProviderSnapshotImageService builds an image service whose only +// storage dependency for the snapshot paths is the provider. Predastore +// points at a closed port so any accidental fall-through to the embedded +// engine fails loudly (connection refused) instead of silently passing. +func setupProviderSnapshotImageService(t *testing.T) (*ImageServiceImpl, *objectstore.MemoryObjectStore, ebsprovider.EBSProvider) { + t.Helper() + store := objectstore.NewMemoryObjectStore() + provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{}) + cfg := &config.Config{ + WalDir: t.TempDir(), + AZ: providerSnapshotAZ, + Predastore: config.PredastoreConfig{ + Host: "127.0.0.1:1", + Bucket: testBucket, + Region: "us-east-1", + }, + } + svc := NewImageServiceImplWithConfig(cfg, store, nil) + svc.SetEBSProvider(provider) + return svc, store, provider +} + +// seedProviderVolume creates a provider-backed volume plus its ebsmetadata +// document, wiring the document's ProviderHandle to the volume's real handle +// so the fixture cannot drift from what the provider actually issued. +func seedProviderVolume(t *testing.T, svc *ImageServiceImpl, provider ebsprovider.EBSProvider, volumeID string, sizeGiB uint64) ebsmetadata.Volume { + t.Helper() + ctx := context.Background() + sizeBytes := int64(sizeGiB) * 1024 * 1024 * 1024 + created, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), + VolumeID: volumeID, + CapacityRange: ebsprovider.CapacityRange{RequiredBytes: sizeBytes}, + }) + require.NoError(t, err) + + doc := ebsmetadata.Volume{ + VolumeID: volumeID, + TenantID: testAccountID, + CapacityGiB: sizeGiB, + State: "available", + CreatedAt: time.Now(), + AvailabilityZone: providerSnapshotAZ, + VolumeType: "gp3", + ProviderHandle: created.Handle, + } + require.NoError(t, svc.MetadataStore().PutVolume(ctx, doc)) + return doc +} + +// TestGetVolumeMetadata_Provider_ResolvesProviderCreatedVolume is the +// regression guard for the whole change: a provider-created volume has no +// {volumeID}/config.json, so reading the legacy key made every CreateImage +// from such a volume fail with NoSuchKey, the same bug AttachVolume had +// before its volume-config read was fixed. +func TestGetVolumeMetadata_Provider_ResolvesProviderCreatedVolume(t *testing.T) { + svc, _, provider := setupProviderSnapshotImageService(t) + doc := seedProviderVolume(t, svc, provider, "vol-provider001", 10) + + meta, err := svc.getVolumeMetadata(context.Background(), doc.VolumeID) + require.NoError(t, err, "a provider-created volume with no config.json must resolve") + assert.Equal(t, doc.VolumeID, meta.VolumeID) + assert.Equal(t, testAccountID, meta.TenantID) + assert.Equal(t, uint64(10), meta.CapacityGiB) + assert.Equal(t, "available", meta.State) + assert.Equal(t, providerSnapshotAZ, meta.AvailabilityZone) +} + +// TestGetVolumeMetadata_Provider_UnknownVolumeIsNotFound locks that a missing +// ebsmetadata document maps to InvalidVolume.NotFound, not a generic error. +func TestGetVolumeMetadata_Provider_UnknownVolumeIsNotFound(t *testing.T) { + svc, _, _ := setupProviderSnapshotImageService(t) + + _, err := svc.getVolumeMetadata(context.Background(), "vol-does-not-exist") + require.Error(t, err) + assert.Equal(t, awserrors.ErrorInvalidVolumeNotFound, err.Error()) +} + +// TestSnapshotStoppedVolume_Provider_CreatesThroughProvider locks that a +// stopped-volume snapshot under a provider is created via provider.CreateSnapshot +// and never falls through to building an embedded engine — the Predastore host +// is a closed port, so any fallback would fail with connection refused. +func TestSnapshotStoppedVolume_Provider_CreatesThroughProvider(t *testing.T) { + svc, _, provider := setupProviderSnapshotImageService(t) + doc := seedProviderVolume(t, svc, provider, "vol-stopped001", 8) + + err := svc.snapshotStoppedVolume(doc.VolumeID, "snap-stopped001") + require.NoError(t, err, "the provider path must not fall through to the embedded engine against the closed Predastore port") + + // CreateSnapshot is idempotent on (SnapshotID, VolumeID): re-issuing it + // returns the existing snapshot rather than creating a second one, which + // confirms the snapshot genuinely exists in the provider (sourced from + // the right volume) instead of the call having silently no-opped. + snap, err := provider.CreateSnapshot(context.Background(), ebsprovider.CreateSnapshotRequest{ + Versioned: ebsprovider.NewVersioned(), SnapshotID: "snap-stopped001", VolumeID: doc.VolumeID, + }) + require.NoError(t, err) + assert.Equal(t, doc.VolumeID, snap.SourceVolumeID) +} + +// TestSnapshotRunningVolume_Provider_PassesProviderHandleAsVolumeHandle locks +// that the ebsmetadata document's ProviderHandle is threaded through as +// CreateSnapshotRequest.VolumeHandle. A wrong handle must be rejected by the +// provider (MemoryProvider treats an empty VolumeHandle as "skip the check", +// so only a mismatched non-empty value proves the real value is forwarded). +func TestSnapshotRunningVolume_Provider_PassesProviderHandleAsVolumeHandle(t *testing.T) { + svc, _, provider := setupProviderSnapshotImageService(t) + doc := seedProviderVolume(t, svc, provider, "vol-running001", 8) + + err := svc.snapshotRunningVolume(doc.VolumeID, "snap-running001", testAccountID) + require.NoError(t, err, "the correct provider handle must let CreateSnapshot succeed") + + doc.ProviderHandle = "wrong-handle" + require.NoError(t, svc.MetadataStore().PutVolume(context.Background(), doc)) + + err = svc.snapshotRunningVolume(doc.VolumeID, "snap-running002", testAccountID) + require.Error(t, err, "a mismatched provider handle must be rejected, proving the value came from the document") + assert.Equal(t, awserrors.ErrorServerInternal, err.Error()) +} + +// TestCreateImageFromInstance_Provider_UsesEbsMetadataVolumeSize is the +// end-to-end case: under a provider, CreateImageFromInstance's AMI.VolumeSizeGiB +// must come from the ebsmetadata document, not a legacy config.json (which a +// provider-created volume never has). +func TestCreateImageFromInstance_Provider_UsesEbsMetadataVolumeSize(t *testing.T) { + svc, _, provider := setupProviderSnapshotImageService(t) + doc := seedProviderVolume(t, svc, provider, "vol-e2e001", 20) + + out, err := svc.CreateImageFromInstance(CreateImageParams{ + Input: &ec2.CreateImageInput{ + InstanceId: aws.String("i-e2e001"), + Name: aws.String("provider-ami"), + }, + RootVolumeID: doc.VolumeID, + IsRunning: false, + }, testAccountID) + require.NoError(t, err) + require.NotNil(t, out.ImageId) + + ami, err := svc.MetadataStore().GetAMI(context.Background(), aws.StringValue(out.ImageId)) + require.NoError(t, err) + assert.Equal(t, uint64(20), ami.VolumeSizeGiB, "the AMI's volume size must come from the ebsmetadata document") +} diff --git a/spinifex/handlers/ec2/image/provider_test.go b/spinifex/handlers/ec2/image/provider_test.go new file mode 100644 index 000000000..1a4de4d93 --- /dev/null +++ b/spinifex/handlers/ec2/image/provider_test.go @@ -0,0 +1,105 @@ +package handlers_ec2_image + +import ( + "context" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + awss3 "github.com/aws/aws-sdk-go/service/s3" + "github.com/mulgadc/spinifex/spinifex/ebsmetadata" + "github.com/mulgadc/spinifex/spinifex/ebsprovider" + "github.com/mulgadc/spinifex/spinifex/objectstore" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// setupProviderImageService creates an image service with the EBS provider +// injected, so GetAMIConfig/putAMIConfig/DescribeImages take the provider +// branch instead of the legacy embedded path. +func setupProviderImageService(t *testing.T) (*ImageServiceImpl, *objectstore.MemoryObjectStore) { + svc, store := setupTestImageService(t) + svc.SetEBSProvider(ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})) + return svc, store +} + +// TestGetAMIConfig_Provider_ReturnsDocument locks that under the provider, +// GetAMIConfig reads the ebsmetadata document rather than legacy config.json. +func TestGetAMIConfig_Provider_ReturnsDocument(t *testing.T) { + svc, _ := setupProviderImageService(t) + + ami := ebsmetadata.AMI{ + ImageID: "ami-doc001", Name: "provider-ami", Architecture: "x86_64", + PlatformDetails: "Linux/UNIX", Virtualization: "hvm", RootDeviceType: "ebs", + VolumeSizeGiB: 12, ImageOwnerAlias: testAccountID, + } + require.NoError(t, svc.MetadataStore().PutAMI(context.Background(), ami)) + + got, err := svc.GetAMIConfig(context.Background(), "ami-doc001") + require.NoError(t, err) + assert.Equal(t, ami.Name, got.Name) + assert.Equal(t, ami.VolumeSizeGiB, got.VolumeSizeGiB) + assert.Equal(t, ami.ImageOwnerAlias, got.ImageOwnerAlias) +} + +// TestGetAMIConfig_Embedded_ReturnsConvertedLegacyValue locks that the +// embedded path still reads legacy config.json, converted into the same +// ebsmetadata.AMI shape the provider path returns. +func TestGetAMIConfig_Embedded_ReturnsConvertedLegacyValue(t *testing.T) { + svc, store := setupTestImageService(t) + createTestAMIConfigWithName(t, store, "ami-legacy001", "legacy-ami") + + got, err := svc.GetAMIConfig(context.Background(), "ami-legacy001") + require.NoError(t, err) + assert.Equal(t, "legacy-ami", got.Name) + assert.Equal(t, testAccountID, got.ImageOwnerAlias) + assert.Equal(t, uint64(8), got.VolumeSizeGiB) +} + +// TestPutAMIConfig_Provider_WritesDocumentNotConfigJSON locks the same +// single-source-of-truth contract as +// TestUpdateVolumeState_WritesStateJSONNotConfig in the volume package: under +// the provider, AMI metadata lives only in the ebsmetadata document, never in +// {imageID}/config.json. +func TestPutAMIConfig_Provider_WritesDocumentNotConfigJSON(t *testing.T) { + svc, store := setupProviderImageService(t) + + meta := ebsmetadata.AMI{ + ImageID: "ami-nocfg001", Name: "provider-write", Architecture: "x86_64", + PlatformDetails: "Linux/UNIX", Virtualization: "hvm", RootDeviceType: "ebs", + VolumeSizeGiB: 10, ImageOwnerAlias: testAccountID, + } + require.NoError(t, svc.putAMIConfig(context.Background(), meta.ImageID, meta)) + + _, err := store.GetObject(context.Background(), &awss3.GetObjectInput{ + Bucket: aws.String(testBucket), + Key: aws.String(meta.ImageID + "/config.json"), + }) + require.Error(t, err) + assert.True(t, objectstore.IsNoSuchKeyError(err), "putAMIConfig must not write config.json under the provider") + + got, err := svc.GetAMIConfig(context.Background(), meta.ImageID) + require.NoError(t, err) + assert.Equal(t, meta.Name, got.Name) +} + +// TestDescribeImages_Provider_ListsViaListAMIs_DocumentOnlyAMI locks that +// DescribeImages under the provider enumerates via Store.ListAMIs and +// surfaces an AMI that exists only as an ebsmetadata document. +func TestDescribeImages_Provider_ListsViaListAMIs_DocumentOnlyAMI(t *testing.T) { + svc, _ := setupProviderImageService(t) + + ami := ebsmetadata.AMI{ + ImageID: "ami-listonly001", Name: "list-only", Architecture: "x86_64", + PlatformDetails: "Linux/UNIX", Virtualization: "hvm", RootDeviceType: "ebs", + VolumeSizeGiB: 8, ImageOwnerAlias: testAccountID, CreationDate: time.Now(), State: "available", + } + require.NoError(t, svc.MetadataStore().PutAMI(context.Background(), ami)) + + out, err := svc.DescribeImages(context.Background(), &ec2.DescribeImagesInput{}, testAccountID) + require.NoError(t, err) + require.Len(t, out.Images, 1, "an AMI that exists only as an ebsmetadata document must be visible under the provider") + assert.Equal(t, "ami-listonly001", aws.StringValue(out.Images[0].ImageId)) + assert.Equal(t, "list-only", aws.StringValue(out.Images[0].Name)) +} diff --git a/spinifex/handlers/ec2/image/service_impl.go b/spinifex/handlers/ec2/image/service_impl.go index 0421a0282..4fd70921c 100644 --- a/spinifex/handlers/ec2/image/service_impl.go +++ b/spinifex/handlers/ec2/image/service_impl.go @@ -1,12 +1,9 @@ package handlers_ec2_image import ( - "bytes" "context" - "encoding/json" "errors" "fmt" - "io" "log/slog" "maps" "strconv" @@ -18,12 +15,12 @@ import ( "github.com/aws/aws-sdk-go/service/s3" "github.com/mulgadc/spinifex/spinifex/awserrors" "github.com/mulgadc/spinifex/spinifex/config" + "github.com/mulgadc/spinifex/spinifex/ebsmetadata" + "github.com/mulgadc/spinifex/spinifex/ebsprovider" "github.com/mulgadc/spinifex/spinifex/filterutil" handlers_ec2_snapshot "github.com/mulgadc/spinifex/spinifex/handlers/ec2/snapshot" "github.com/mulgadc/spinifex/spinifex/objectstore" "github.com/mulgadc/spinifex/spinifex/utils" - "github.com/mulgadc/viperblock/viperblock" - vbs3 "github.com/mulgadc/viperblock/viperblock/backends/s3" "github.com/nats-io/nats.go" ) @@ -45,6 +42,26 @@ type ImageServiceImpl struct { store objectstore.ObjectStore bucketName string natsConn *nats.Conn + metadata *ebsmetadata.Store + provider ebsprovider.EBSProvider +} + +// SetEBSProvider injects the provider boundary used by the control plane. +// Legacy AMI/snapshot paths remain in place until their metadata migration is complete. +func (s *ImageServiceImpl) SetEBSProvider(provider ebsprovider.EBSProvider) { + s.provider = provider +} + +// EBSProvider returns the injected provider boundary, or nil on the legacy +// embedded-engine path. Primarily for composition-root tests to observe wiring. +func (s *ImageServiceImpl) EBSProvider() ebsprovider.EBSProvider { + return s.provider +} + +// MetadataStore returns the control-plane metadata store. Primarily for +// composition-root tests to observe wiring. +func (s *ImageServiceImpl) MetadataStore() *ebsmetadata.Store { + return s.metadata } // NewImageServiceImpl creates a new daemon-side image service. natsConn is used @@ -63,6 +80,7 @@ func NewImageServiceImpl(cfg *config.Config, natsConn *nats.Conn) *ImageServiceI store: store, bucketName: cfg.Predastore.Bucket, natsConn: natsConn, + metadata: ebsmetadata.NewStore(store, cfg.Predastore.Bucket), } } @@ -71,6 +89,7 @@ func NewImageServiceImplWithStore(store objectstore.ObjectStore, bucketName stri return &ImageServiceImpl{ store: store, bucketName: bucketName, + metadata: ebsmetadata.NewStore(store, bucketName), } } @@ -83,6 +102,7 @@ func NewImageServiceImplWithConfig(cfg *config.Config, store objectstore.ObjectS store: store, bucketName: cfg.Predastore.Bucket, natsConn: natsConn, + metadata: ebsmetadata.NewStore(store, cfg.Predastore.Bucket), } } @@ -114,108 +134,26 @@ func (s *ImageServiceImpl) DescribeImages(ctx context.Context, input *ec2.Descri return nil, errors.New(awserrors.ErrorInvalidParameterValue) } - // List all prefixes in the bucket (AMIs are stored as ami-/ directories) - result, err := s.store.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ - Bucket: aws.String(s.bucketName), - Delimiter: aws.String("/"), - }) + return s.describeImages(ctx, input, accountID, parsedFilters) +} +// describeImages enumerates AMIs via Store.ListAMIs, then filters and renders +// each document into the AWS SDK image shape. +func (s *ImageServiceImpl) describeImages(ctx context.Context, input *ec2.DescribeImagesInput, accountID string, parsedFilters map[string][]string) (*ec2.DescribeImagesOutput, error) { + amis, err := s.metadata.ListAMIs(ctx) if err != nil { - slog.ErrorContext(ctx, "Failed to list S3 objects", "err", err) + slog.ErrorContext(ctx, "DescribeImages: failed to list AMIs", "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } - // Extract image-id filter values for early prefix skipping to avoid - // unnecessary S3 GetObject calls on non-matching AMIs. - var imageIDFilterValues []string - if parsedFilters != nil { - imageIDFilterValues = parsedFilters["image-id"] - } - - var images []*ec2.Image encryptedAtRest := s.clusterEncryptionEnabled() + var images []*ec2.Image - // Counting prefixes seen against those dropped for an unreadable config.json - // makes a mismatch with len(images) visible at INFO, so an empty result is - // distinguishable from one where every config failed to load. - var prefixesExamined, unreadableConfigs int - - // Iterate over CommonPrefixes to find ami-* directories - for _, prefix := range result.CommonPrefixes { - if prefix.Prefix == nil { - continue - } - - prefixStr := *prefix.Prefix - slog.InfoContext(ctx, "Processing S3 prefix", "prefix", prefixStr) - - // Only check prefixes that match pattern: ami-/ - if !strings.HasPrefix(prefixStr, "ami-") { - continue - } - prefixesExamined++ - - // Early skip: if image-id filter is set, check the prefix (ami-xxx/) - // against filter values before fetching config from S3. - if len(imageIDFilterValues) > 0 { - amiID := strings.TrimSuffix(prefixStr, "/") - if !filterutil.MatchesAny(imageIDFilterValues, amiID) { - continue - } - } - - // Construct path to config.json for this AMI directory - configKey := prefixStr + "config.json" - - // Get the config.json file - getResult, err := s.store.GetObject(ctx, &s3.GetObjectInput{ - Bucket: aws.String(s.bucketName), - Key: aws.String(configKey), - }) - if err != nil { - if objectstore.IsNoSuchKeyError(err) { - // No config.json means this is a legitimate non-AMI directory, - // not a fault, so it stays at DEBUG and isn't counted below. - slog.DebugContext(ctx, "Config file not found", "key", configKey) - } else { - unreadableConfigs++ - slog.WarnContext(ctx, "Failed to get config file", "key", configKey, "err", err) - } - continue - } - - body, err := io.ReadAll(getResult.Body) - // A close failure alone doesn't drop the prefix (read below may still - // succeed), so it's only warned about, not counted as unreadable. - if closeErr := getResult.Body.Close(); closeErr != nil { - slog.WarnContext(ctx, "Failed to close config body", "key", configKey, "err", closeErr) - } - if err != nil { - unreadableConfigs++ - slog.WarnContext(ctx, "Failed to read config body", "key", configKey, "err", err) - continue - } - - // Parse the viperblock config which contains VolumeConfig with AMIMetadata. - // StateBody unwraps the at-rest encryption envelope (the metadata ships - // authenticated-but-plaintext); it is a no-op for unencrypted volumes. - var vbConfig struct { - VolumeConfig viperblock.VolumeConfig `json:"VolumeConfig"` - } - if err := json.Unmarshal(viperblock.StateBody(body), &vbConfig); err != nil { - unreadableConfigs++ - slog.WarnContext(ctx, "Failed to unmarshal config", "key", configKey, "err", err) - continue - } - - amiMeta := vbConfig.VolumeConfig.AMIMetadata - - // Skip if this is not an AMI (ImageID is empty) + for _, amiMeta := range amis { if amiMeta.ImageID == "" { continue } - // Filter by ImageId if specified if len(input.ImageIds) > 0 { found := false for _, filterID := range input.ImageIds { @@ -229,8 +167,6 @@ func (s *ImageServiceImpl) DescribeImages(ctx context.Context, input *ec2.Descri } } - // AMIs with a non-account-ID owner (e.g. "spinifex") are system AMIs - // visible to all; account-ID owners are private. amiOwner := amiMeta.ImageOwnerAlias isSystemAMI := amiOwner != "" && !utils.IsAccountID(amiOwner) @@ -238,7 +174,6 @@ func (s *ImageServiceImpl) DescribeImages(ctx context.Context, input *ec2.Descri continue } - // Filter by Owner if specified if len(input.Owners) > 0 { found := false for _, owner := range input.Owners { @@ -247,14 +182,10 @@ func (s *ImageServiceImpl) DescribeImages(ctx context.Context, input *ec2.Descri } switch *owner { case "self": - // "self" matches only AMIs owned by the caller if amiOwner == accountID { found = true } default: - // Match by explicit account ID. System AMIs are stored - // with a non-account owner (e.g. "spinifex") but report - // GlobalAccountID in the response, so match against both. if amiOwner == *owner { found = true } else if isSystemAMI && *owner == utils.GlobalAccountID { @@ -270,15 +201,11 @@ func (s *ImageServiceImpl) DescribeImages(ctx context.Context, input *ec2.Descri } } - // Resolve the OwnerId for the response. System AMIs use global account. ownerID := amiOwner if isSystemAMI { ownerID = utils.GlobalAccountID } - // Build EC2 Image from AMIMetadata. Empty BootMode passes through as - // empty so legacy AMIs registered before this field existed don't get - // a synthesized value. image := &ec2.Image{ ImageId: aws.String(amiMeta.ImageID), Name: aws.String(amiMeta.Name), @@ -294,7 +221,7 @@ func (s *ImageServiceImpl) DescribeImages(ctx context.Context, input *ec2.Descri Public: aws.Bool(false), State: aws.String(amiImageState(amiMeta.State)), ImageType: aws.String("machine"), - Hypervisor: aws.String("xen"), // Default hypervisor + Hypervisor: aws.String("xen"), BootMode: aws.String(amiMeta.BootMode), } @@ -305,7 +232,6 @@ func (s *ImageServiceImpl) DescribeImages(ctx context.Context, input *ec2.Descri image.Tags = utils.MapToEC2Tags(amiMeta.Tags) - // Apply filters against the fully-built image if len(parsedFilters) > 0 && !imageMatchesFilters(image, parsedFilters, amiMeta.Tags) { continue } @@ -313,7 +239,6 @@ func (s *ImageServiceImpl) DescribeImages(ctx context.Context, input *ec2.Descri images = append(images, image) } - // If specific ImageIds were requested, verify all were found if len(input.ImageIds) > 0 { foundIDs := make(map[string]bool, len(images)) for _, img := range images { @@ -328,12 +253,8 @@ func (s *ImageServiceImpl) DescribeImages(ctx context.Context, input *ec2.Descri } } - slog.InfoContext(ctx, "DescribeImages completed", "count", len(images), - "prefixesExamined", prefixesExamined, "unreadableConfigs", unreadableConfigs) - - return &ec2.DescribeImagesOutput{ - Images: images, - }, nil + slog.InfoContext(ctx, "DescribeImages completed", "count", len(images)) + return &ec2.DescribeImagesOutput{Images: images}, nil } // amiImageState maps AMIMetadata.State to the ec2.Image state string. An @@ -452,15 +373,15 @@ func (s *ImageServiceImpl) CreateImageFromInstance(params CreateImageParams, acc } // Step 2: Read source volume config for size - volumeConfig, err := s.getVolumeConfig(context.Background(), params.RootVolumeID) + volMeta, err := s.getVolumeMetadata(context.Background(), params.RootVolumeID) if err != nil { - slog.Error("CreateImageFromInstance: failed to read volume config", "volumeId", params.RootVolumeID, "err", err) + slog.Error("CreateImageFromInstance: failed to read volume metadata", "volumeId", params.RootVolumeID, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } - volumeSizeGiB := volumeConfig.VolumeMetadata.SizeGiB + volumeSizeGiB := volMeta.CapacityGiB // Step 3: Read source AMI config for architecture, platform, etc. - sourceAMI := viperblock.AMIMetadata{ + sourceAMI := ebsmetadata.AMI{ Architecture: "x86_64", PlatformDetails: "Linux/UNIX", Virtualization: "hvm", @@ -481,7 +402,7 @@ func (s *ImageServiceImpl) CreateImageFromInstance(params CreateImageParams, acc } // Step 5: Build and store AMI config - meta := viperblock.AMIMetadata{ + meta := ebsmetadata.AMI{ ImageID: amiID, Name: name, Description: aws.StringValue(input.Description), @@ -522,283 +443,135 @@ func (s *ImageServiceImpl) snapshotRunningVolume(volumeID, snapshotID, accountID return errors.New(awserrors.ErrorServerInternal) } - volConfig, err := s.getVolumeConfig(context.Background(), volumeID) + volMeta, err := s.getVolumeMetadata(context.Background(), volumeID) if err != nil { - slog.Error("snapshotRunningVolume: failed to read volume config", "volumeId", volumeID, "err", err) + slog.Error("snapshotRunningVolume: failed to read volume metadata", "volumeId", volumeID, "err", err) return errors.New(awserrors.ErrorServerInternal) } - volumeSize := volConfig.VolumeMetadata.SizeGiB * 1024 * 1024 * 1024 // Flush the writes still buffered by whichever node serves the volume, so - // the live checkpoint below is current rather than an arbitrarily stale one + // the snapshot below is current rather than an arbitrarily stale one // predating e.g. a guest-triggered GPT rewrite (growpart on first boot). An // attached volume that cannot be drained fails here rather than silently // producing an image from a stale checkpoint. - if err := handlers_ec2_snapshot.DrainVolume(context.Background(), s.config, s.store, s.natsConn, volumeID, volConfig.VolumeMetadata, accountID); err != nil { + if err := handlers_ec2_snapshot.DrainVolume(context.Background(), s.config, s.store, s.natsConn, volumeID, volMeta.State, volMeta.AttachedInstance, accountID); err != nil { slog.Error("snapshotRunningVolume: drain failed", "volumeId", volumeID, "err", err) return errors.New(awserrors.ErrorServerInternal) } - cfg := vbs3.S3Config{ - VolumeName: volumeID, - VolumeSize: volumeSize, - Bucket: s.config.Predastore.Bucket, - Region: s.config.Predastore.Region, - AccessKey: s.config.Predastore.AccessKey, - SecretKey: s.config.Predastore.SecretKey, - Host: s.config.Predastore.Host, - } - - mkey, err := utils.LoadViperblockMasterKey(s.config.Viperblock.EncryptionKeyFile) - if err != nil { - slog.Error("snapshotRunningVolume: failed to load encryption key", "volumeId", volumeID, "err", err) - return errors.New(awserrors.ErrorServerInternal) - } - - vbconfig := viperblock.VB{ - VolumeName: volumeID, - VolumeSize: volumeSize, - BaseDir: s.config.WalDir, - Cache: viperblock.Cache{Config: viperblock.CacheConfig{Size: 0}}, - MasterKey: mkey, - EncryptionEnabled: mkey != nil, - } - - vb, err := viperblock.New(&vbconfig, "s3", cfg) - if err != nil { - slog.Error("snapshotRunningVolume: failed to create viperblock instance", "volumeId", volumeID, "err", err) - return errors.New(awserrors.ErrorServerInternal) - } - defer func() { - vb.StopChunkUploader() - vb.StopWALSyncer() - }() - - if err := vb.Backend.Init(); err != nil { - slog.Error("snapshotRunningVolume: failed to init backend", "volumeId", volumeID, "err", err) - return errors.New(awserrors.ErrorServerInternal) - } - - if err := vb.LoadState(); err != nil { - slog.Error("snapshotRunningVolume: failed to load state", "volumeId", volumeID, "err", err) - return errors.New(awserrors.ErrorServerInternal) - } - - if err := vb.LoadLiveCheckpoint(); err != nil { - slog.Error("snapshotRunningVolume: failed to load live checkpoint", "volumeId", volumeID, "err", err) - return errors.New(awserrors.ErrorServerInternal) - } - - if _, err := vb.CreateSnapshot(snapshotID); err != nil { - slog.Error("snapshotRunningVolume: CreateSnapshot failed", "volumeId", volumeID, "snapshotId", snapshotID, "err", err) - return errors.New(awserrors.ErrorServerInternal) - } - - slog.Info("snapshotRunningVolume: snapshot created", "volumeId", volumeID, "snapshotId", snapshotID) - return nil + return s.createSnapshot(context.Background(), volMeta, snapshotID, "snapshotRunningVolume") } -// snapshotStoppedVolume creates a snapshot offline by loading viperblock state from S3. +// snapshotStoppedVolume creates a snapshot of a detached volume. No drain: the +// volume is stopped/detached, so nothing is buffered on a serving node. +// viperblockd's handleCreateSnapshot prefers a live mounted engine over opening +// a second one, so it handles the mounted case itself. func (s *ImageServiceImpl) snapshotStoppedVolume(volumeID, snapshotID string) error { if s.config == nil { return errors.New(awserrors.ErrorServerInternal) } - // Read volume config to get the size (required by viperblock.New) - volConfig, err := s.getVolumeConfig(context.Background(), volumeID) + volMeta, err := s.getVolumeMetadata(context.Background(), volumeID) if err != nil { - slog.Error("snapshotStoppedVolume: failed to read volume config", "volumeId", volumeID, "err", err) - return errors.New(awserrors.ErrorServerInternal) - } - volumeSize := volConfig.VolumeMetadata.SizeGiB * 1024 * 1024 * 1024 - - cfg := vbs3.S3Config{ - VolumeName: volumeID, - VolumeSize: volumeSize, - Bucket: s.config.Predastore.Bucket, - Region: s.config.Predastore.Region, - AccessKey: s.config.Predastore.AccessKey, - SecretKey: s.config.Predastore.SecretKey, - Host: s.config.Predastore.Host, - } - - mkey, err := utils.LoadViperblockMasterKey(s.config.Viperblock.EncryptionKeyFile) - if err != nil { - slog.Error("snapshotStoppedVolume: failed to load encryption key", "volumeId", volumeID, "err", err) + slog.Error("snapshotStoppedVolume: failed to read volume metadata", "volumeId", volumeID, "err", err) return errors.New(awserrors.ErrorServerInternal) } - vbconfig := viperblock.VB{ - VolumeName: volumeID, - VolumeSize: volumeSize, - BaseDir: s.config.WalDir, - Cache: viperblock.Cache{Config: viperblock.CacheConfig{Size: 0}}, - MasterKey: mkey, - EncryptionEnabled: mkey != nil, - } + return s.createSnapshot(context.Background(), volMeta, snapshotID, "snapshotStoppedVolume") +} - vb, err := viperblock.New(&vbconfig, "s3", cfg) - if err != nil { - slog.Error("snapshotStoppedVolume: failed to create viperblock instance", "volumeId", volumeID, "err", err) +// createSnapshot delegates snapshot creation for volMeta to the EBS provider. +// op names the calling path so a failure is attributable in the log. +func (s *ImageServiceImpl) createSnapshot(ctx context.Context, volMeta ebsmetadata.Volume, snapshotID, op string) error { + if s.provider == nil { + slog.ErrorContext(ctx, "no EBS provider configured", "op", op) return errors.New(awserrors.ErrorServerInternal) } - defer func() { - vb.StopChunkUploader() - vb.StopWALSyncer() - }() - - if err := vb.Backend.Init(); err != nil { - slog.Error("snapshotStoppedVolume: failed to init backend", "volumeId", volumeID, "err", err) + created, err := s.provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{ + Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, VolumeID: volMeta.VolumeID, VolumeHandle: volMeta.ProviderHandle, + }) + if err != nil || created == nil { + slog.ErrorContext(ctx, "provider CreateSnapshot failed", "op", op, "volumeId", volMeta.VolumeID, "snapshotId", snapshotID, "err", err) return errors.New(awserrors.ErrorServerInternal) } - - if err := vb.LoadState(); err != nil { - slog.Error("snapshotStoppedVolume: failed to load state", "volumeId", volumeID, "err", err) - return errors.New(awserrors.ErrorServerInternal) - } - - if err := vb.LoadBlockState(); err != nil { - slog.Error("snapshotStoppedVolume: failed to load block state", "volumeId", volumeID, "err", err) - return errors.New(awserrors.ErrorServerInternal) - } - - // Clean up local WAL files regardless of snapshot success or failure - defer func() { - if err := vb.RemoveLocalFiles(); err != nil { - slog.Error("snapshotStoppedVolume: failed to remove local files", "volumeId", volumeID, "err", err) - } - }() - - if _, err := vb.CreateSnapshot(snapshotID); err != nil { - slog.Error("snapshotStoppedVolume: CreateSnapshot failed", "volumeId", volumeID, "snapshotId", snapshotID, "err", err) - return errors.New(awserrors.ErrorServerInternal) - } - - slog.Info("snapshotStoppedVolume: snapshot created", "volumeId", volumeID, "snapshotId", snapshotID) + slog.InfoContext(ctx, "snapshot created", "op", op, "volumeId", volMeta.VolumeID, "snapshotId", snapshotID) return nil } -// getVolumeConfig reads a volume's VBState config from S3. -func (s *ImageServiceImpl) getVolumeConfig(ctx context.Context, volumeID string) (*viperblock.VolumeConfig, error) { - configKey := fmt.Sprintf("%s/config.json", volumeID) - result, err := s.store.GetObject(ctx, &s3.GetObjectInput{ - Bucket: aws.String(s.bucketName), - Key: aws.String(configKey), - }) +// getVolumeMetadata reads a volume's control-plane document, remapping a +// missing document to InvalidVolume.NotFound. +func (s *ImageServiceImpl) getVolumeMetadata(ctx context.Context, volumeID string) (ebsmetadata.Volume, error) { + meta, err := s.metadata.GetVolume(ctx, volumeID) if err != nil { - return nil, err + if objectstore.IsNoSuchKeyError(err) { + return ebsmetadata.Volume{}, errors.New(awserrors.ErrorInvalidVolumeNotFound) + } + return ebsmetadata.Volume{}, fmt.Errorf("get volume metadata: %w", err) } - defer result.Body.Close() - - body, err := io.ReadAll(result.Body) - if err != nil { - return nil, err - } - var vbState viperblock.VBState - if err := json.Unmarshal(viperblock.StateBody(body), &vbState); err != nil { - return nil, err - } - return &vbState.VolumeConfig, nil + return meta, nil } -// amiNameExists checks if any existing AMI already uses the given name. -// NoSuchKey is skipped (concurrent deregister race); any other read/decode -// error is surfaced so we don't silently allow a duplicate. +// amiNameExists checks if any existing AMI already uses the given name. It +// lists strictly: an AMI this cannot read is a name it cannot rule out, and +// answering false would let a duplicate through and mask the corruption. func (s *ImageServiceImpl) amiNameExists(ctx context.Context, name string) (bool, error) { - listResult, err := s.store.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ - Bucket: aws.String(s.bucketName), - Prefix: aws.String("ami-"), - Delimiter: aws.String("/"), - }) + amis, err := s.metadata.ListAMIsStrict(ctx) if err != nil { return false, fmt.Errorf("amiNameExists: failed to list AMIs: %w", err) } - - for _, prefix := range listResult.CommonPrefixes { - if prefix.Prefix == nil { - continue - } - configKey := *prefix.Prefix + "config.json" - result, err := s.store.GetObject(ctx, &s3.GetObjectInput{ - Bucket: aws.String(s.bucketName), - Key: aws.String(configKey), - }) - if err != nil { - if objectstore.IsNoSuchKeyError(err) { - continue - } - return false, fmt.Errorf("amiNameExists: failed to read %s: %w", configKey, err) - } - - body, readErr := io.ReadAll(result.Body) - _ = result.Body.Close() - if readErr != nil { - return false, fmt.Errorf("amiNameExists: failed to read %s: %w", configKey, readErr) - } - var vbState viperblock.VBState - if decodeErr := json.Unmarshal(viperblock.StateBody(body), &vbState); decodeErr != nil { - return false, fmt.Errorf("amiNameExists: failed to decode %s: %w", configKey, decodeErr) - } - - if vbState.VolumeConfig.AMIMetadata.Name == name { + for _, ami := range amis { + if ami.Name == name { return true, nil } } - return false, nil } -// ErrCorruptAMIConfig wraps JSON decode failures on AMI config.json so callers -// can distinguish a truly-missing AMI from one whose config exists but can't -// be parsed. -var ErrCorruptAMIConfig = errors.New("corrupt AMI config") - -// GetAMIConfig returns NoSuchKeyError if the AMI is missing, or an error -// wrapping ErrCorruptAMIConfig on decode failure. -func (s *ImageServiceImpl) GetAMIConfig(ctx context.Context, imageID string) (viperblock.AMIMetadata, error) { - configKey := fmt.Sprintf("%s/config.json", imageID) - result, err := s.store.GetObject(ctx, &s3.GetObjectInput{ - Bucket: aws.String(s.bucketName), - Key: aws.String(configKey), - }) - if err != nil { - return viperblock.AMIMetadata{}, err - } - defer result.Body.Close() - - body, err := io.ReadAll(result.Body) - if err != nil { - return viperblock.AMIMetadata{}, fmt.Errorf("%w: %s: %w", ErrCorruptAMIConfig, configKey, err) - } - var vbState viperblock.VBState - if err := json.Unmarshal(viperblock.StateBody(body), &vbState); err != nil { - return viperblock.AMIMetadata{}, fmt.Errorf("%w: %s: %w", ErrCorruptAMIConfig, configKey, err) - } - return vbState.VolumeConfig.AMIMetadata, nil +// GetAMIConfig reads an AMI's control-plane document. Errors come from the +// metadata store. +func (s *ImageServiceImpl) GetAMIConfig(ctx context.Context, imageID string) (ebsmetadata.AMI, error) { + return s.metadata.GetAMI(ctx, imageID) } -// putAMIConfig writes AMI metadata to {imageID}/config.json, preserving the -// VBState wrapper used by GetAMIConfig. -func (s *ImageServiceImpl) putAMIConfig(ctx context.Context, imageID string, meta viperblock.AMIMetadata) error { - state := viperblock.VBState{ - VolumeConfig: viperblock.VolumeConfig{ - AMIMetadata: meta, - }, - } - - data, err := json.Marshal(state) +// GetAMISourceVolumeID returns the volume whose blocks imageID's snapshot +// references, read from the snapshot's metadata.json. Bundled system AMIs have +// no standalone snapshot metadata; their snapshot is named after the AMI. +func (s *ImageServiceImpl) GetAMISourceVolumeID(ctx context.Context, imageID string) (string, error) { + meta, err := s.GetAMIConfig(ctx, imageID) if err != nil { - return err + if objectstore.IsNoSuchKeyError(err) || errors.Is(err, ebsmetadata.ErrCorruptDocument) { + return "", errors.New(awserrors.ErrorInvalidAMIIDNotFound) + } + slog.ErrorContext(ctx, "GetAMISourceVolumeID: failed to read AMI config", "imageId", imageID, "err", err) + return "", errors.New(awserrors.ErrorServerInternal) + } + if meta.SnapshotID == "" { + return "", errors.New(awserrors.ErrorInvalidAMIIDNotFound) } - configKey := fmt.Sprintf("%s/config.json", imageID) - _, err = s.store.PutObject(ctx, &s3.PutObjectInput{ - Bucket: aws.String(s.bucketName), - Key: aws.String(configKey), - Body: bytes.NewReader(data), - ContentType: aws.String("application/json"), - }) - return err + snapCfg, err := handlers_ec2_snapshot.ReadSnapshotConfig(s.store, s.bucketName, meta.SnapshotID) + if err != nil { + if objectstore.IsNoSuchKeyError(err) && meta.ImageOwnerAlias != "" && !utils.IsAccountID(meta.ImageOwnerAlias) { + return imageID, nil + } + if objectstore.IsNoSuchKeyError(err) || errors.Is(err, handlers_ec2_snapshot.ErrCorruptSnapshotMetadata) { + return "", errors.New(awserrors.ErrorInvalidSnapshotNotFound) + } + slog.ErrorContext(ctx, "GetAMISourceVolumeID: failed to read snapshot metadata", + "imageId", imageID, "snapshotId", meta.SnapshotID, "err", err) + return "", errors.New(awserrors.ErrorServerInternal) + } + if snapCfg.VolumeID == "" { + slog.ErrorContext(ctx, "GetAMISourceVolumeID: snapshot metadata has no source volume", + "imageId", imageID, "snapshotId", meta.SnapshotID) + return "", errors.New(awserrors.ErrorInvalidSnapshotNotFound) + } + return snapCfg.VolumeID, nil +} + +// putAMIConfig writes AMI metadata to the ebsmetadata document store. +func (s *ImageServiceImpl) putAMIConfig(ctx context.Context, _ string, meta ebsmetadata.AMI) error { + return s.metadata.PutAMI(ctx, meta) } // ApplyRecordTags mirrors CreateTags into the owning AMI config so @@ -867,7 +640,7 @@ func (s *ImageServiceImpl) updateAMITags(ctx context.Context, imageID, accountID // checkAMIOwnership rejects cross-account and system-AMI mutations. Empty // owner is ServerInternal (corrupt config) rather than a misleading 403. -func (s *ImageServiceImpl) checkAMIOwnership(meta viperblock.AMIMetadata, accountID string) error { +func (s *ImageServiceImpl) checkAMIOwnership(meta ebsmetadata.AMI, accountID string) error { owner := meta.ImageOwnerAlias if owner == "" { slog.Error("checkAMIOwnership: AMI config has empty ImageOwnerAlias", "imageId", meta.ImageID) @@ -881,7 +654,7 @@ func (s *ImageServiceImpl) checkAMIOwnership(meta viperblock.AMIMetadata, accoun // callerCanReadAMI: empty owner is invisible (corrupt); non-account owner is // a system AMI visible to all; account owner is private to that account. -func callerCanReadAMI(meta viperblock.AMIMetadata, accountID string) bool { +func callerCanReadAMI(meta ebsmetadata.AMI, accountID string) bool { owner := meta.ImageOwnerAlias if owner == "" { return false @@ -895,22 +668,22 @@ func callerCanReadAMI(meta viperblock.AMIMetadata, accountID string) bool { // loadAMIForMutation fetches the AMI and enforces ownership. Cross-account // callers see UnauthorizedOperation, not NotFound (they already know the ID). // No CAS: concurrent writers last-write-wins on the full struct. -func (s *ImageServiceImpl) loadAMIForMutation(ctx context.Context, imageID, accountID string) (viperblock.AMIMetadata, error) { +func (s *ImageServiceImpl) loadAMIForMutation(ctx context.Context, imageID, accountID string) (ebsmetadata.AMI, error) { if !strings.HasPrefix(imageID, "ami-") { - return viperblock.AMIMetadata{}, errors.New(awserrors.ErrorInvalidAMIIDMalformed) + return ebsmetadata.AMI{}, errors.New(awserrors.ErrorInvalidAMIIDMalformed) } meta, err := s.GetAMIConfig(ctx, imageID) if err != nil { if objectstore.IsNoSuchKeyError(err) { - return viperblock.AMIMetadata{}, errors.New(awserrors.ErrorInvalidAMIIDNotFound) + return ebsmetadata.AMI{}, errors.New(awserrors.ErrorInvalidAMIIDNotFound) } slog.Error("loadAMIForMutation: failed to read AMI config", "imageId", imageID, "err", err) - return viperblock.AMIMetadata{}, errors.New(awserrors.ErrorServerInternal) + return ebsmetadata.AMI{}, errors.New(awserrors.ErrorServerInternal) } if err := s.checkAMIOwnership(meta, accountID); err != nil { - return viperblock.AMIMetadata{}, err + return ebsmetadata.AMI{}, err } return meta, nil } @@ -945,7 +718,7 @@ func (s *ImageServiceImpl) CopyImage(ctx context.Context, input *ec2.CopyImageIn if err != nil { // Corrupt source is treated as NotFound so callers can't tell which // half of the AMI/snapshot pair is broken. - if objectstore.IsNoSuchKeyError(err) || errors.Is(err, ErrCorruptAMIConfig) { + if objectstore.IsNoSuchKeyError(err) || errors.Is(err, ebsmetadata.ErrCorruptDocument) { return nil, errors.New(awserrors.ErrorInvalidAMIIDNotFound) } slog.ErrorContext(ctx, "CopyImage: failed to read source AMI config", "sourceImageId", sourceImageID, "err", err) @@ -1012,7 +785,7 @@ func (s *ImageServiceImpl) CopyImage(ctx context.Context, input *ec2.CopyImageIn tags := mergeCopyImageTags(srcMeta.Tags, input.TagSpecifications, aws.BoolValue(input.CopyImageTags)) - meta := viperblock.AMIMetadata{ + meta := ebsmetadata.AMI{ ImageID: newImageID, Name: name, Description: description, @@ -1129,7 +902,7 @@ func (s *ImageServiceImpl) clusterEncryptionEnabled() bool { // synthesizeRootBlockDeviceMapping returns /dev/sda1 with size+snapshot from AMIMetadata, // or nil for non-EBS AMIs. encrypted reflects the cluster-level encryption posture // (master key configured); AMI metadata carries no per-image encryption flag. -func synthesizeRootBlockDeviceMapping(meta viperblock.AMIMetadata, encrypted bool) []*ec2.BlockDeviceMapping { +func synthesizeRootBlockDeviceMapping(meta ebsmetadata.AMI, encrypted bool) []*ec2.BlockDeviceMapping { if meta.RootDeviceType != "ebs" { return nil } @@ -1218,7 +991,7 @@ func (s *ImageServiceImpl) RegisterImage(ctx context.Context, input *ec2.Registe tags := utils.ExtractTags(input.TagSpecifications, "image") amiID := utils.GenerateResourceID("ami") - meta := viperblock.AMIMetadata{ + meta := ebsmetadata.AMI{ ImageID: amiID, Name: name, Description: description, @@ -1268,8 +1041,10 @@ func pickRootSnapshotBDM(mappings []*ec2.BlockDeviceMapping, rootDeviceName *str return nil } -// DeregisterImage hard-deletes config.json. Backing snapshot is untouched, so -// operators run delete-snapshot separately to reclaim block storage. +// DeregisterImage hard-deletes the AMI document. The pre-provider +// {imageID}/config.json goes too, or the metadata store's legacy read fallback +// would resurrect the image. Backing snapshot is untouched, so operators run +// delete-snapshot separately to reclaim block storage. func (s *ImageServiceImpl) DeregisterImage(ctx context.Context, input *ec2.DeregisterImageInput, accountID string) (*ec2.DeregisterImageOutput, error) { if input == nil || input.ImageId == nil || *input.ImageId == "" { return nil, errors.New(awserrors.ErrorMissingParameter) @@ -1281,12 +1056,17 @@ func (s *ImageServiceImpl) DeregisterImage(ctx context.Context, input *ec2.Dereg return nil, err } + if err := s.metadata.DeleteAMI(ctx, imageID); err != nil { + slog.ErrorContext(ctx, "DeregisterImage: failed to delete AMI document", "imageId", imageID, "err", err) + return nil, errors.New(awserrors.ErrorServerInternal) + } + configKey := fmt.Sprintf("%s/config.json", imageID) if _, err := s.store.DeleteObject(ctx, &s3.DeleteObjectInput{ Bucket: aws.String(s.bucketName), Key: aws.String(configKey), - }); err != nil { - slog.ErrorContext(ctx, "DeregisterImage: failed to delete AMI config", "imageId", imageID, "err", err) + }); err != nil && !objectstore.IsNoSuchKeyError(err) { + slog.ErrorContext(ctx, "DeregisterImage: failed to delete legacy AMI config", "imageId", imageID, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } diff --git a/spinifex/handlers/ec2/image/service_impl_test.go b/spinifex/handlers/ec2/image/service_impl_test.go index 5ada15cff..c60466890 100644 --- a/spinifex/handlers/ec2/image/service_impl_test.go +++ b/spinifex/handlers/ec2/image/service_impl_test.go @@ -3,7 +3,6 @@ package handlers_ec2_image import ( "context" "encoding/json" - "fmt" "io" "log/slog" "net/http" @@ -17,9 +16,9 @@ import ( awss3 "github.com/aws/aws-sdk-go/service/s3" "github.com/mulgadc/spinifex/spinifex/awserrors" "github.com/mulgadc/spinifex/spinifex/config" + "github.com/mulgadc/spinifex/spinifex/ebsmetadata" handlers_ec2_snapshot "github.com/mulgadc/spinifex/spinifex/handlers/ec2/snapshot" "github.com/mulgadc/spinifex/spinifex/objectstore" - "github.com/mulgadc/viperblock/viperblock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -27,117 +26,96 @@ import ( const testBucket = "test-bucket" const testAccountID = "000000000001" -// setupTestImageService creates an image service with in-memory storage for testing. +// setupTestImageService creates an image service with in-memory storage for +// testing. func setupTestImageService(t *testing.T) (*ImageServiceImpl, *objectstore.MemoryObjectStore) { store := objectstore.NewMemoryObjectStore() svc := NewImageServiceImplWithStore(store, testBucket) return svc, store } -// createTestVolumeConfig creates a test volume config in the mock store. -func createTestVolumeConfig(t *testing.T, store *objectstore.MemoryObjectStore, volumeID string, sizeGiB int) { - volumeState := viperblock.VBState{ - VolumeConfig: viperblock.VolumeConfig{ - VolumeMetadata: viperblock.VolumeMetadata{ - SizeGiB: uint64(sizeGiB), - }, - }, - } - data, err := json.Marshal(volumeState) - require.NoError(t, err) +// putAMIDocument writes an AMI's control-plane document, which is what +// DescribeImages enumerates. +func putAMIDocument(t *testing.T, store *objectstore.MemoryObjectStore, ami ebsmetadata.AMI) { + t.Helper() + require.NoError(t, ebsmetadata.NewStore(store, testBucket).PutAMI(context.Background(), ami)) +} - _, err = store.PutObject(context.Background(), &awss3.PutObjectInput{ - Bucket: aws.String(testBucket), - Key: aws.String(volumeID + "/config.json"), - Body: strings.NewReader(string(data)), - ContentType: aws.String("application/json"), +// putRawAMIDocument writes AMI document bytes verbatim, for a corrupt document +// a typed seed cannot express. +func putRawAMIDocument(t *testing.T, store *objectstore.MemoryObjectStore, imageID, body string) { + t.Helper() + _, err := store.PutObject(context.Background(), &awss3.PutObjectInput{ + Bucket: aws.String(testBucket), + Key: aws.String(amiDocumentKey(t, imageID)), + Body: strings.NewReader(body), }) require.NoError(t, err) } +// amiDocumentKey is where an AMI's control-plane document lives. +func amiDocumentKey(t *testing.T, imageID string) string { + t.Helper() + key, err := ebsmetadata.AMIKey(imageID) + require.NoError(t, err) + return key +} + +// putVolumeDocument writes a volume's control-plane document. +func putVolumeDocument(t *testing.T, store *objectstore.MemoryObjectStore, volume ebsmetadata.Volume) { + t.Helper() + require.NoError(t, ebsmetadata.NewStore(store, testBucket).PutVolume(context.Background(), volume)) +} + +// createTestVolumeConfig creates a test volume config in the mock store. +func createTestVolumeConfig(t *testing.T, store *objectstore.MemoryObjectStore, volumeID string, sizeGiB int) { + putVolumeDocument(t, store, ebsmetadata.Volume{ + VolumeID: volumeID, + CapacityGiB: uint64(sizeGiB), + }) +} + // createTestAMIConfig creates a test AMI config in the mock store. func createTestAMIConfig(t *testing.T, store *objectstore.MemoryObjectStore, imageID string) { - amiState := viperblock.VBState{ - VolumeConfig: viperblock.VolumeConfig{ - AMIMetadata: viperblock.AMIMetadata{ - ImageID: imageID, - Name: "test-ami", - Architecture: "x86_64", - PlatformDetails: "Linux/UNIX", - Virtualization: "hvm", - RootDeviceType: "ebs", - VolumeSizeGiB: 8, - }, - }, - } - data, err := json.Marshal(amiState) - require.NoError(t, err) - - _, err = store.PutObject(context.Background(), &awss3.PutObjectInput{ - Bucket: aws.String(testBucket), - Key: aws.String(imageID + "/config.json"), - Body: strings.NewReader(string(data)), - ContentType: aws.String("application/json"), + putAMIDocument(t, store, ebsmetadata.AMI{ + ImageID: imageID, + Name: "test-ami", + Architecture: "x86_64", + PlatformDetails: "Linux/UNIX", + Virtualization: "hvm", + RootDeviceType: "ebs", + VolumeSizeGiB: 8, }) - require.NoError(t, err) } // createTestAMIConfigWithName creates a test AMI config with a specified name. // Owner defaults to testAccountID so the AMI is visible to the default caller — // an empty ImageOwnerAlias would be filtered out as a corrupt config. func createTestAMIConfigWithName(t *testing.T, store *objectstore.MemoryObjectStore, imageID, name string) { - amiState := viperblock.VBState{ - VolumeConfig: viperblock.VolumeConfig{ - AMIMetadata: viperblock.AMIMetadata{ - ImageID: imageID, - Name: name, - Architecture: "x86_64", - PlatformDetails: "Linux/UNIX", - Virtualization: "hvm", - RootDeviceType: "ebs", - VolumeSizeGiB: 8, - ImageOwnerAlias: testAccountID, - }, - }, - } - data, err := json.Marshal(amiState) - require.NoError(t, err) - - _, err = store.PutObject(context.Background(), &awss3.PutObjectInput{ - Bucket: aws.String(testBucket), - Key: aws.String(imageID + "/config.json"), - Body: strings.NewReader(string(data)), - ContentType: aws.String("application/json"), + putAMIDocument(t, store, ebsmetadata.AMI{ + ImageID: imageID, + Name: name, + Architecture: "x86_64", + PlatformDetails: "Linux/UNIX", + Virtualization: "hvm", + RootDeviceType: "ebs", + VolumeSizeGiB: 8, + ImageOwnerAlias: testAccountID, }) - require.NoError(t, err) } // createTestAMIConfigWithOwner creates a test AMI config with a specified name and owner. func createTestAMIConfigWithOwner(t *testing.T, store *objectstore.MemoryObjectStore, imageID, name, owner string) { - amiState := viperblock.VBState{ - VolumeConfig: viperblock.VolumeConfig{ - AMIMetadata: viperblock.AMIMetadata{ - ImageID: imageID, - Name: name, - Architecture: "x86_64", - PlatformDetails: "Linux/UNIX", - Virtualization: "hvm", - RootDeviceType: "ebs", - VolumeSizeGiB: 8, - ImageOwnerAlias: owner, - }, - }, - } - data, err := json.Marshal(amiState) - require.NoError(t, err) - - _, err = store.PutObject(context.Background(), &awss3.PutObjectInput{ - Bucket: aws.String(testBucket), - Key: aws.String(imageID + "/config.json"), - Body: strings.NewReader(string(data)), - ContentType: aws.String("application/json"), + putAMIDocument(t, store, ebsmetadata.AMI{ + ImageID: imageID, + Name: name, + Architecture: "x86_64", + PlatformDetails: "Linux/UNIX", + Virtualization: "hvm", + RootDeviceType: "ebs", + VolumeSizeGiB: 8, + ImageOwnerAlias: owner, }) - require.NoError(t, err) } func TestCreateImageFromInstance_NilInput(t *testing.T) { @@ -211,7 +189,7 @@ func TestDescribeImages_AfterCreate(t *testing.T) { func TestDescribeImages_BootModeProjection(t *testing.T) { svc, store := setupTestImageService(t) - createTestAMIConfigFull(t, store, viperblock.AMIMetadata{ + createTestAMIConfigFull(t, store, ebsmetadata.AMI{ ImageID: "ami-uefi001", Name: "uefi-image", Architecture: "x86_64", @@ -222,7 +200,7 @@ func TestDescribeImages_BootModeProjection(t *testing.T) { ImageOwnerAlias: testAccountID, BootMode: "uefi", }) - createTestAMIConfigFull(t, store, viperblock.AMIMetadata{ + createTestAMIConfigFull(t, store, ebsmetadata.AMI{ ImageID: "ami-legacy001", Name: "legacy-image", Architecture: "x86_64", @@ -269,7 +247,7 @@ func TestDescribeImages_StateProjection(t *testing.T) { t.Run(tt.name, func(t *testing.T) { svc, store := setupTestImageService(t) - createTestAMIConfigRich(t, store, viperblock.AMIMetadata{ + createTestAMIConfigRich(t, store, ebsmetadata.AMI{ ImageID: "ami-state001", Name: "state-image", Architecture: "x86_64", @@ -291,21 +269,20 @@ func TestDescribeImages_StateProjection(t *testing.T) { } } -func TestGetVolumeConfig(t *testing.T) { +func TestGetVolumeMetadata(t *testing.T) { svc, store := setupTestImageService(t) createTestVolumeConfig(t, store, "vol-abc123", 20) - cfg, err := svc.getVolumeConfig(context.Background(), "vol-abc123") + meta, err := svc.getVolumeMetadata(context.Background(), "vol-abc123") require.NoError(t, err) - require.NotNil(t, cfg) - assert.Equal(t, uint64(20), cfg.VolumeMetadata.SizeGiB) + assert.Equal(t, uint64(20), meta.CapacityGiB) } -func TestGetVolumeConfig_NotFound(t *testing.T) { +func TestGetVolumeMetadata_NotFound(t *testing.T) { svc, _ := setupTestImageService(t) - _, err := svc.getVolumeConfig(context.Background(), "vol-nonexistent") + _, err := svc.getVolumeMetadata(context.Background(), "vol-nonexistent") require.Error(t, err) } @@ -455,7 +432,7 @@ func TestCreateImageFromInstance_UniqueNameAllowed(t *testing.T) { func TestPutAMIConfig_RoundTrip(t *testing.T) { svc, _ := setupTestImageService(t) - meta := viperblock.AMIMetadata{ + meta := ebsmetadata.AMI{ ImageID: "ami-roundtrip01", Name: "round-trip", Description: "hello", @@ -499,7 +476,7 @@ func TestCheckAMIOwnership(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - err := svc.checkAMIOwnership(viperblock.AMIMetadata{ImageOwnerAlias: tt.owner}, caller) + err := svc.checkAMIOwnership(ebsmetadata.AMI{ImageOwnerAlias: tt.owner}, caller) if tt.wantErr == "" { assert.NoError(t, err) } else { @@ -520,10 +497,10 @@ func TestDeregisterImage_HappyPath(t *testing.T) { require.NoError(t, err) assert.NotNil(t, out) - // AMI config gone from S3 + // AMI document gone from S3 _, getErr := store.GetObject(context.Background(), &awss3.GetObjectInput{ Bucket: aws.String(testBucket), - Key: aws.String(amiID + "/config.json"), + Key: aws.String(amiDocumentKey(t, amiID)), }) require.Error(t, getErr) assert.True(t, objectstore.IsNoSuchKeyError(getErr)) @@ -573,7 +550,7 @@ func TestDeregisterImage_CrossAccount(t *testing.T) { // Confirm AMI still present after rejected mutation. _, getErr := store.GetObject(context.Background(), &awss3.GetObjectInput{ Bucket: aws.String(testBucket), - Key: aws.String(amiID + "/config.json"), + Key: aws.String(amiDocumentKey(t, amiID)), }) require.NoError(t, getErr) } @@ -594,33 +571,21 @@ func TestDeregisterImage_DoesNotTouchSnapshot(t *testing.T) { snapID := "snap-keep001" // AMI pointing at a snapshot - amiState := viperblock.VBState{ - VolumeConfig: viperblock.VolumeConfig{ - AMIMetadata: viperblock.AMIMetadata{ - ImageID: amiID, - Name: "with-snapshot", - SnapshotID: snapID, - ImageOwnerAlias: testAccountID, - Architecture: "x86_64", - Virtualization: "hvm", - RootDeviceType: "ebs", - VolumeSizeGiB: 8, - }, - }, - } - data, err := json.Marshal(amiState) - require.NoError(t, err) - _, err = store.PutObject(context.Background(), &awss3.PutObjectInput{ - Bucket: aws.String(testBucket), - Key: aws.String(amiID + "/config.json"), - Body: strings.NewReader(string(data)), + putAMIDocument(t, store, ebsmetadata.AMI{ + ImageID: amiID, + Name: "with-snapshot", + SnapshotID: snapID, + ImageOwnerAlias: testAccountID, + Architecture: "x86_64", + Virtualization: "hvm", + RootDeviceType: "ebs", + VolumeSizeGiB: 8, }) - require.NoError(t, err) // Backing snapshot metadata require.NoError(t, svc.putSnapshotMetadata(context.Background(), snapID, "vol-keep", 8, testAccountID)) - _, err = svc.DeregisterImage(context.Background(), &ec2.DeregisterImageInput{ImageId: aws.String(amiID)}, testAccountID) + _, err := svc.DeregisterImage(context.Background(), &ec2.DeregisterImageInput{ImageId: aws.String(amiID)}, testAccountID) require.NoError(t, err) // Snapshot metadata still present @@ -723,13 +688,8 @@ func TestDescribeImages_NonAMIPrefixIgnored(t *testing.T) { func TestDescribeImages_InvalidConfigJSON(t *testing.T) { svc, store := setupTestImageService(t) - // Store invalid JSON as an AMI config - _, err := store.PutObject(context.Background(), &awss3.PutObjectInput{ - Bucket: aws.String(testBucket), - Key: aws.String("ami-bad123/config.json"), - Body: strings.NewReader("not valid json"), - }) - require.NoError(t, err) + // Store invalid JSON as an AMI document + putRawAMIDocument(t, store, "ami-bad123", "not valid json") // Should skip the invalid AMI without error result, err := svc.DescribeImages(context.Background(), &ec2.DescribeImagesInput{}, testAccountID) @@ -737,38 +697,30 @@ func TestDescribeImages_InvalidConfigJSON(t *testing.T) { assert.Empty(t, result.Images) } -// TestDescribeImages_ConfigFaultCounting covers a config.json that -// exists but can't be fetched/read/parsed must be promoted to WARN and counted -// as unreadable so it stays visible at INFO, while a genuinely absent -// config.json (a non-AMI directory) stays silent at DEBUG and uncounted, and a -// healthy prefix is unaffected either way. +// TestDescribeImages_ConfigFaultCounting covers a config.json that exists but +// can't be fetched/read/parsed being promoted to WARN so it stays visible at +// the default level, while a genuinely absent config.json (a non-AMI +// directory) stays silent, and a healthy prefix is unaffected either way. func TestDescribeImages_ConfigFaultCounting(t *testing.T) { tests := []struct { name string setup func(t *testing.T, store *objectstore.MemoryObjectStore) wantImageCount int - wantUnreadable int wantWarnLogged bool }{ { - name: "malformed config.json is counted as unreadable and does not vanish silently", + name: "malformed document is counted as unreadable and does not vanish silently", setup: func(t *testing.T, store *objectstore.MemoryObjectStore) { - _, err := store.PutObject(context.Background(), &awss3.PutObjectInput{ - Bucket: aws.String(testBucket), - Key: aws.String("ami-malformed1/config.json"), - Body: strings.NewReader("{not valid json"), - }) - require.NoError(t, err) + putRawAMIDocument(t, store, "ami-malformed1", "{not valid json") }, wantImageCount: 0, - wantUnreadable: 1, wantWarnLogged: true, }, { - name: "missing config.json (NoSuchKey) is not counted as a fault", + name: "an object that is not a document is not counted as a fault", setup: func(t *testing.T, store *objectstore.MemoryObjectStore) { - // A placeholder object under the prefix makes ListObjectsV2 surface - // "ami-nokey1/" as a CommonPrefix with no config.json underneath. + // An object outside the document prefix is not an AMI at all, so + // it must neither appear nor register as a fault. _, err := store.PutObject(context.Background(), &awss3.PutObjectInput{ Bucket: aws.String(testBucket), Key: aws.String("ami-nokey1/placeholder.txt"), @@ -777,7 +729,6 @@ func TestDescribeImages_ConfigFaultCounting(t *testing.T) { require.NoError(t, err) }, wantImageCount: 0, - wantUnreadable: 0, wantWarnLogged: false, }, { @@ -786,7 +737,6 @@ func TestDescribeImages_ConfigFaultCounting(t *testing.T) { createTestAMIConfigWithOwner(t, store, "ami-healthy1", "healthy-ami", testAccountID) }, wantImageCount: 1, - wantUnreadable: 0, wantWarnLogged: false, }, } @@ -807,8 +757,6 @@ func TestDescribeImages_ConfigFaultCounting(t *testing.T) { assert.Len(t, out.Images, tt.wantImageCount) logs := buf.String() - assert.Contains(t, logs, "prefixesExamined=1", "the single ami-* prefix must be counted as examined") - assert.Contains(t, logs, fmt.Sprintf("unreadableConfigs=%d", tt.wantUnreadable)) if tt.wantWarnLogged { assert.Contains(t, logs, "level=WARN", @@ -824,24 +772,10 @@ func TestDescribeImages_ConfigFaultCounting(t *testing.T) { func TestDescribeImages_EmptyImageIDSkipped(t *testing.T) { svc, store := setupTestImageService(t) - // AMI config with empty ImageID - amiState := viperblock.VBState{ - VolumeConfig: viperblock.VolumeConfig{ - AMIMetadata: viperblock.AMIMetadata{ - ImageID: "", - Name: "empty-id-ami", - }, - }, - } - data, err := json.Marshal(amiState) - require.NoError(t, err) - - _, err = store.PutObject(context.Background(), &awss3.PutObjectInput{ - Bucket: aws.String(testBucket), - Key: aws.String("ami-emptyid/config.json"), - Body: strings.NewReader(string(data)), - }) - require.NoError(t, err) + // A document whose image_id is empty names no image, so listing it would + // hand out an AMI nothing can be launched from. + putRawAMIDocument(t, store, "ami-emptyid", + `{"schema_version":1,"image_id":"","name":"empty-id-ami"}`) result, err := svc.DescribeImages(context.Background(), &ec2.DescribeImagesInput{}, testAccountID) require.NoError(t, err) @@ -851,30 +785,17 @@ func TestDescribeImages_EmptyImageIDSkipped(t *testing.T) { func TestDescribeImages_WithTags(t *testing.T) { svc, store := setupTestImageService(t) - amiState := viperblock.VBState{ - VolumeConfig: viperblock.VolumeConfig{ - AMIMetadata: viperblock.AMIMetadata{ - ImageID: "ami-tagged123", - Name: "tagged-ami", - Architecture: "x86_64", - PlatformDetails: "Linux/UNIX", - Virtualization: "hvm", - RootDeviceType: "ebs", - VolumeSizeGiB: 8, - ImageOwnerAlias: testAccountID, - Tags: map[string]string{"Environment": "test", "Name": "my-ami"}, - }, - }, - } - data, err := json.Marshal(amiState) - require.NoError(t, err) - - _, err = store.PutObject(context.Background(), &awss3.PutObjectInput{ - Bucket: aws.String(testBucket), - Key: aws.String("ami-tagged123/config.json"), - Body: strings.NewReader(string(data)), + putAMIDocument(t, store, ebsmetadata.AMI{ + ImageID: "ami-tagged123", + Name: "tagged-ami", + Architecture: "x86_64", + PlatformDetails: "Linux/UNIX", + Virtualization: "hvm", + RootDeviceType: "ebs", + VolumeSizeGiB: 8, + ImageOwnerAlias: testAccountID, + Tags: map[string]string{"Environment": "test", "Name": "my-ami"}, }) - require.NoError(t, err) result, err := svc.DescribeImages(context.Background(), &ec2.DescribeImagesInput{ ImageIds: []*string{aws.String("ami-tagged123")}, @@ -904,28 +825,25 @@ func TestDescribeImages_OwnerFilterNilEntry(t *testing.T) { func TestGetAMIConfig_InvalidJSON(t *testing.T) { svc, store := setupTestImageService(t) - _, err := store.PutObject(context.Background(), &awss3.PutObjectInput{ - Bucket: aws.String(testBucket), - Key: aws.String("ami-badjson/config.json"), - Body: strings.NewReader("not json"), - }) - require.NoError(t, err) + putRawAMIDocument(t, store, "ami-badjson", "not json") - _, err = svc.GetAMIConfig(context.Background(), "ami-badjson") + _, err := svc.GetAMIConfig(context.Background(), "ami-badjson") assert.Error(t, err) } -func TestGetVolumeConfig_InvalidJSON(t *testing.T) { +func TestGetVolumeMetadata_InvalidJSON(t *testing.T) { svc, store := setupTestImageService(t) - _, err := store.PutObject(context.Background(), &awss3.PutObjectInput{ + key, err := ebsmetadata.VolumeKey("vol-badjson") + require.NoError(t, err) + _, err = store.PutObject(context.Background(), &awss3.PutObjectInput{ Bucket: aws.String(testBucket), - Key: aws.String("vol-badjson/config.json"), + Key: aws.String(key), Body: strings.NewReader("{invalid"), }) require.NoError(t, err) - _, err = svc.getVolumeConfig(context.Background(), "vol-badjson") + _, err = svc.getVolumeMetadata(context.Background(), "vol-badjson") assert.Error(t, err) } @@ -960,17 +878,12 @@ func TestAmiNameExists_NotFound(t *testing.T) { func TestAmiNameExists_InvalidJSON(t *testing.T) { svc, store := setupTestImageService(t) - _, err := store.PutObject(context.Background(), &awss3.PutObjectInput{ - Bucket: aws.String(testBucket), - Key: aws.String("ami-bad/config.json"), - Body: strings.NewReader("not json"), - }) - require.NoError(t, err) + putRawAMIDocument(t, store, "ami-bad", "not json") - // A corrupt AMI config is a real store-side problem. Surface it rather + // A corrupt AMI document is a real store-side problem. Surface it rather // than silently under-counting names (which would let a caller write a // duplicate and mask the corruption). - _, err = svc.amiNameExists(context.Background(), "any-name") + _, err := svc.amiNameExists(context.Background(), "any-name") require.Error(t, err) } @@ -979,25 +892,12 @@ func TestAmiNameExists_InvalidJSON(t *testing.T) { // createTestAMIConfigFull creates an AMI with all metadata fields for filter testing. // Owner defaults to testAccountID when the caller leaves ImageOwnerAlias empty — // an empty owner would be filtered out as corrupt. -func createTestAMIConfigFull(t *testing.T, store *objectstore.MemoryObjectStore, meta viperblock.AMIMetadata) { +func createTestAMIConfigFull(t *testing.T, store *objectstore.MemoryObjectStore, meta ebsmetadata.AMI) { t.Helper() if meta.ImageOwnerAlias == "" { meta.ImageOwnerAlias = testAccountID } - amiState := viperblock.VBState{ - VolumeConfig: viperblock.VolumeConfig{ - AMIMetadata: meta, - }, - } - data, err := json.Marshal(amiState) - require.NoError(t, err) - _, err = store.PutObject(context.Background(), &awss3.PutObjectInput{ - Bucket: aws.String(testBucket), - Key: aws.String(meta.ImageID + "/config.json"), - Body: strings.NewReader(string(data)), - ContentType: aws.String("application/json"), - }) - require.NoError(t, err) + putAMIDocument(t, store, meta) } // TestDescribeImages_FilterBy verifies single-attribute filters; each subtest builds @@ -1049,11 +949,11 @@ func TestDescribeImages_FilterBy(t *testing.T) { { name: "architecture", setup: func(t *testing.T, store *objectstore.MemoryObjectStore) { - createTestAMIConfigFull(t, store, viperblock.AMIMetadata{ + createTestAMIConfigFull(t, store, ebsmetadata.AMI{ ImageID: "ami-x86", Name: "x86-img", Architecture: "x86_64", RootDeviceType: "ebs", VolumeSizeGiB: 8, }) - createTestAMIConfigFull(t, store, viperblock.AMIMetadata{ + createTestAMIConfigFull(t, store, ebsmetadata.AMI{ ImageID: "ami-arm", Name: "arm-img", Architecture: "arm64", RootDeviceType: "ebs", VolumeSizeGiB: 8, }) @@ -1066,12 +966,12 @@ func TestDescribeImages_FilterBy(t *testing.T) { { name: "tag", setup: func(t *testing.T, store *objectstore.MemoryObjectStore) { - createTestAMIConfigFull(t, store, viperblock.AMIMetadata{ + createTestAMIConfigFull(t, store, ebsmetadata.AMI{ ImageID: "ami-tagged", Name: "tagged-img", Architecture: "x86_64", RootDeviceType: "ebs", VolumeSizeGiB: 8, Tags: map[string]string{"Environment": "prod"}, }) - createTestAMIConfigFull(t, store, viperblock.AMIMetadata{ + createTestAMIConfigFull(t, store, ebsmetadata.AMI{ ImageID: "ami-untagged", Name: "untagged-img", Architecture: "x86_64", RootDeviceType: "ebs", VolumeSizeGiB: 8, }) @@ -1104,11 +1004,11 @@ func TestDescribeImages_FilterBy(t *testing.T) { { name: "virtualization type", setup: func(t *testing.T, store *objectstore.MemoryObjectStore) { - createTestAMIConfigFull(t, store, viperblock.AMIMetadata{ + createTestAMIConfigFull(t, store, ebsmetadata.AMI{ ImageID: "ami-hvm", Name: "hvm-img", Architecture: "x86_64", Virtualization: "hvm", RootDeviceType: "ebs", VolumeSizeGiB: 8, }) - createTestAMIConfigFull(t, store, viperblock.AMIMetadata{ + createTestAMIConfigFull(t, store, ebsmetadata.AMI{ ImageID: "ami-pv", Name: "pv-img", Architecture: "x86_64", Virtualization: "paravirtual", RootDeviceType: "ebs", VolumeSizeGiB: 8, }) @@ -1121,11 +1021,11 @@ func TestDescribeImages_FilterBy(t *testing.T) { { name: "root device type", setup: func(t *testing.T, store *objectstore.MemoryObjectStore) { - createTestAMIConfigFull(t, store, viperblock.AMIMetadata{ + createTestAMIConfigFull(t, store, ebsmetadata.AMI{ ImageID: "ami-ebs", Name: "ebs-img", Architecture: "x86_64", Virtualization: "hvm", RootDeviceType: "ebs", VolumeSizeGiB: 8, }) - createTestAMIConfigFull(t, store, viperblock.AMIMetadata{ + createTestAMIConfigFull(t, store, ebsmetadata.AMI{ ImageID: "ami-is", Name: "is-img", Architecture: "x86_64", Virtualization: "hvm", RootDeviceType: "instance-store", VolumeSizeGiB: 8, }) @@ -1174,11 +1074,11 @@ func TestDescribeImages_FilterMultipleValues_OR(t *testing.T) { func TestDescribeImages_FilterMultipleNames_AND(t *testing.T) { svc, store := setupTestImageService(t) - createTestAMIConfigFull(t, store, viperblock.AMIMetadata{ + createTestAMIConfigFull(t, store, ebsmetadata.AMI{ ImageID: "ami-match", Name: "debian-13", Architecture: "x86_64", RootDeviceType: "ebs", VolumeSizeGiB: 8, }) - createTestAMIConfigFull(t, store, viperblock.AMIMetadata{ + createTestAMIConfigFull(t, store, ebsmetadata.AMI{ ImageID: "ami-nomatch", Name: "debian-13", Architecture: "arm64", RootDeviceType: "ebs", VolumeSizeGiB: 8, }) @@ -1602,13 +1502,13 @@ func TestRegisterImage_OwnerSetToCaller(t *testing.T) { // --- CopyImage tests --- -// readAMIConfigBytes returns the raw bytes of {imageID}/config.json from the -// store so tests can prove the source was not mutated by a copy/modify/etc. +// readAMIConfigBytes returns the raw bytes of an AMI's document from the store +// so tests can prove the source was not mutated by a copy/modify/etc. func readAMIConfigBytes(t *testing.T, store *objectstore.MemoryObjectStore, imageID string) []byte { t.Helper() result, err := store.GetObject(context.Background(), &awss3.GetObjectInput{ Bucket: aws.String(testBucket), - Key: aws.String(imageID + "/config.json"), + Key: aws.String(amiDocumentKey(t, imageID)), }) require.NoError(t, err) defer result.Body.Close() @@ -1634,7 +1534,7 @@ func readSnapshotConfigBytes(t *testing.T, store *objectstore.MemoryObjectStore, // putTestAMIConfigWithSnapshot seeds an AMI config that carries a real // SnapshotID — distinct from createTestAMIConfigWithOwner which sets no // snapshot and would be treated as orphaned by CopyImage. -func putTestAMIConfigWithSnapshot(t *testing.T, store *objectstore.MemoryObjectStore, imageID, name, owner, snapshotID string, meta viperblock.AMIMetadata) { +func putTestAMIConfigWithSnapshot(t *testing.T, store *objectstore.MemoryObjectStore, imageID, name, owner, snapshotID string, meta ebsmetadata.AMI) { t.Helper() meta.ImageID = imageID meta.Name = name @@ -1655,18 +1555,7 @@ func putTestAMIConfigWithSnapshot(t *testing.T, store *objectstore.MemoryObjectS if meta.VolumeSizeGiB == 0 { meta.VolumeSizeGiB = 8 } - amiState := viperblock.VBState{ - VolumeConfig: viperblock.VolumeConfig{AMIMetadata: meta}, - } - data, err := json.Marshal(amiState) - require.NoError(t, err) - _, err = store.PutObject(context.Background(), &awss3.PutObjectInput{ - Bucket: aws.String(testBucket), - Key: aws.String(imageID + "/config.json"), - Body: strings.NewReader(string(data)), - ContentType: aws.String("application/json"), - }) - require.NoError(t, err) + putAMIDocument(t, store, meta) } // seedCopyableAMI writes a matching (snapshot, AMI) pair so CopyImage can complete @@ -1691,7 +1580,7 @@ func seedCopyableAMI(t *testing.T, store *objectstore.MemoryObjectStore, imageID }) require.NoError(t, err) - putTestAMIConfigWithSnapshot(t, store, imageID, name, owner, snapshotID, viperblock.AMIMetadata{ + putTestAMIConfigWithSnapshot(t, store, imageID, name, owner, snapshotID, ebsmetadata.AMI{ VolumeSizeGiB: uint64(sizeGiB), Description: "source desc", }) @@ -1758,7 +1647,7 @@ func TestCopyImage_InheritsSourceFields(t *testing.T) { require.NoError(t, err) // Source AMI with non-default fields that must propagate. - putTestAMIConfigWithSnapshot(t, store, "ami-arm001", "arm-source", testAccountID, "snap-arm001", viperblock.AMIMetadata{ + putTestAMIConfigWithSnapshot(t, store, "ami-arm001", "arm-source", testAccountID, "snap-arm001", ebsmetadata.AMI{ Architecture: "arm64", PlatformDetails: "Linux/UNIX (arm64)", Virtualization: "hvm", @@ -1836,7 +1725,7 @@ func TestCopyImage_BundledSystemAMINoStandaloneSnap(t *testing.T) { // Seed only the AMI config, no snap-xxx/metadata.json object — matches the // on-disk layout produced by `spx admin images import`. putTestAMIConfigWithSnapshot(t, store, "ami-bundled01", "alpine-bundled", - "system", "snap-ami-bundled01", viperblock.AMIMetadata{ + "system", "snap-ami-bundled01", ebsmetadata.AMI{ VolumeSizeGiB: 8, Description: "bundled source", }) @@ -1894,7 +1783,7 @@ func TestCopyImage_OrphanedSource_MissingSnapshot(t *testing.T) { svc, store := setupTestImageService(t) // AMI config points at a snapshot that doesn't exist on S3. - putTestAMIConfigWithSnapshot(t, store, "ami-orphan", "orphan", testAccountID, "snap-ghost", viperblock.AMIMetadata{}) + putTestAMIConfigWithSnapshot(t, store, "ami-orphan", "orphan", testAccountID, "snap-ghost", ebsmetadata.AMI{}) _, err := svc.CopyImage(context.Background(), validCopyImageServiceInput("ami-orphan", "orphan-copy"), testAccountID) require.Error(t, err) @@ -2070,27 +1959,15 @@ func TestCopyImage_MissingRequiredFields(t *testing.T) { // --- Image attribute tests --- -func createTestAMIConfigRich(t *testing.T, store *objectstore.MemoryObjectStore, meta viperblock.AMIMetadata) { +func createTestAMIConfigRich(t *testing.T, store *objectstore.MemoryObjectStore, meta ebsmetadata.AMI) { t.Helper() - state := viperblock.VBState{ - VolumeConfig: viperblock.VolumeConfig{AMIMetadata: meta}, - } - data, err := json.Marshal(state) - require.NoError(t, err) - - _, err = store.PutObject(context.Background(), &awss3.PutObjectInput{ - Bucket: aws.String(testBucket), - Key: aws.String(meta.ImageID + "/config.json"), - Body: strings.NewReader(string(data)), - ContentType: aws.String("application/json"), - }) - require.NoError(t, err) + putAMIDocument(t, store, meta) } func TestDescribeImageAttribute_Description(t *testing.T) { svc, store := setupTestImageService(t) - createTestAMIConfigRich(t, store, viperblock.AMIMetadata{ + createTestAMIConfigRich(t, store, ebsmetadata.AMI{ ImageID: "ami-desc01", Name: "desc-ami", Description: "the stored description", @@ -2119,7 +1996,7 @@ func TestDescribeImageAttribute_Description(t *testing.T) { func TestDescribeImageAttribute_BlockDeviceMapping(t *testing.T) { svc, store := setupTestImageService(t) - createTestAMIConfigRich(t, store, viperblock.AMIMetadata{ + createTestAMIConfigRich(t, store, ebsmetadata.AMI{ ImageID: "ami-bdm01", Name: "bdm-ami", SnapshotID: "snap-bdm01", @@ -2183,7 +2060,7 @@ func TestDescribeImageAttribute_NotFound(t *testing.T) { func TestDescribeImageAttribute_CrossAccountHidesExistence(t *testing.T) { svc, store := setupTestImageService(t) - createTestAMIConfigRich(t, store, viperblock.AMIMetadata{ + createTestAMIConfigRich(t, store, ebsmetadata.AMI{ ImageID: "ami-cross01", Name: "cross-ami", Description: "secret", @@ -2202,7 +2079,7 @@ func TestDescribeImageAttribute_CrossAccountHidesExistence(t *testing.T) { func TestDescribeImageAttribute_SystemAMIReadable(t *testing.T) { svc, store := setupTestImageService(t) - createTestAMIConfigRich(t, store, viperblock.AMIMetadata{ + createTestAMIConfigRich(t, store, ebsmetadata.AMI{ ImageID: "ami-sys01", Name: "system-ami", Description: "baked-in", @@ -2240,7 +2117,7 @@ func TestDescribeImageAttribute_UnsupportedAttribute(t *testing.T) { func TestModifyImageAttribute_Description(t *testing.T) { svc, store := setupTestImageService(t) - createTestAMIConfigRich(t, store, viperblock.AMIMetadata{ + createTestAMIConfigRich(t, store, ebsmetadata.AMI{ ImageID: "ami-mod01", Name: "mod-ami", Description: "old", @@ -2273,7 +2150,7 @@ func TestModifyImageAttribute_Description(t *testing.T) { func TestModifyImageAttribute_DescriptionEmptyValueClears(t *testing.T) { svc, store := setupTestImageService(t) - createTestAMIConfigRich(t, store, viperblock.AMIMetadata{ + createTestAMIConfigRich(t, store, ebsmetadata.AMI{ ImageID: "ami-modclr01", Name: "modclr-ami", Description: "will-be-cleared", @@ -2296,7 +2173,7 @@ func TestModifyImageAttribute_DescriptionEmptyValueClears(t *testing.T) { func TestModifyImageAttribute_CrossAccount(t *testing.T) { svc, store := setupTestImageService(t) - createTestAMIConfigRich(t, store, viperblock.AMIMetadata{ + createTestAMIConfigRich(t, store, ebsmetadata.AMI{ ImageID: "ami-modx01", Name: "modx-ami", Description: "dont-touch", @@ -2321,7 +2198,7 @@ func TestModifyImageAttribute_CrossAccount(t *testing.T) { func TestModifyImageAttribute_SystemAMIImmutable(t *testing.T) { svc, store := setupTestImageService(t) - createTestAMIConfigRich(t, store, viperblock.AMIMetadata{ + createTestAMIConfigRich(t, store, ebsmetadata.AMI{ ImageID: "ami-modsys01", Name: "sys-ami", Description: "baked-in", @@ -2385,7 +2262,7 @@ func TestModifyImageAttribute_MissingParameters(t *testing.T) { func TestResetImageAttribute_Description(t *testing.T) { svc, store := setupTestImageService(t) - createTestAMIConfigRich(t, store, viperblock.AMIMetadata{ + createTestAMIConfigRich(t, store, ebsmetadata.AMI{ ImageID: "ami-reset01", Name: "reset-ami", Description: "will-be-cleared", @@ -2416,7 +2293,7 @@ func TestResetImageAttribute_Description(t *testing.T) { func TestResetImageAttribute_CrossAccount(t *testing.T) { svc, store := setupTestImageService(t) - createTestAMIConfigRich(t, store, viperblock.AMIMetadata{ + createTestAMIConfigRich(t, store, ebsmetadata.AMI{ ImageID: "ami-rstx01", Name: "rstx", Description: "dont-touch", diff --git a/spinifex/handlers/ec2/instance/metadata_options_test.go b/spinifex/handlers/ec2/instance/metadata_options_test.go index ea166d895..ca4713eb7 100644 --- a/spinifex/handlers/ec2/instance/metadata_options_test.go +++ b/spinifex/handlers/ec2/instance/metadata_options_test.go @@ -8,9 +8,9 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" "github.com/mulgadc/spinifex/spinifex/awserrors" "github.com/mulgadc/spinifex/spinifex/config" + "github.com/mulgadc/spinifex/spinifex/ebsmetadata" "github.com/mulgadc/spinifex/spinifex/utils" "github.com/mulgadc/spinifex/spinifex/vm" - "github.com/mulgadc/viperblock/viperblock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -380,7 +380,7 @@ func TestPrepareRunInstances_PlatformTokenDefault(t *testing.T) { svc := &InstanceServiceImpl{ config: &config.Config{}, instanceTypes: types, - amiLoader: &fakeAMILoader{byID: map[string]viperblock.AMIMetadata{ + amiLoader: &fakeAMILoader{byID: map[string]ebsmetadata.AMI{ "ami-1": {ImageOwnerAlias: "acc", PlatformDetails: tc.platformDetails}, }}, resourceMgr: &fakeResourceCapacityProvider{ diff --git a/spinifex/handlers/ec2/instance/prepare_efi_volume_test.go b/spinifex/handlers/ec2/instance/prepare_efi_volume_test.go deleted file mode 100644 index da6169496..000000000 --- a/spinifex/handlers/ec2/instance/prepare_efi_volume_test.go +++ /dev/null @@ -1,58 +0,0 @@ -package handlers_ec2_instance - -import ( - "context" - "os" - "path/filepath" - "testing" - - "github.com/mulgadc/spinifex/spinifex/config" - "github.com/mulgadc/spinifex/spinifex/vm" - "github.com/mulgadc/viperblock/viperblock" - "github.com/stretchr/testify/require" - "go.uber.org/goleak" -) - -// TestPrepareEFIVolume_BackendInitFailure_NoGoroutineLeak exercises the first -// of prepareEFIVolume's six error returns (Backend.Init failure) and asserts -// the chunk uploader and WAL syncer goroutines started by newViperblock do -// not survive the error return. -func TestPrepareEFIVolume_BackendInitFailure_NoGoroutineLeak(t *testing.T) { - defer goleak.VerifyNone(t) - - // Swap in fake firmware files so the test doesn't depend on OVMF being - // installed on the host; FirmwarePathCandidates is exported for exactly - // this purpose. - fwDir := t.TempDir() - codePath := filepath.Join(fwDir, "CODE.fd") - varsPath := filepath.Join(fwDir, "VARS.fd") - require.NoError(t, os.WriteFile(codePath, []byte("fake-code"), 0o600)) - require.NoError(t, os.WriteFile(varsPath, []byte("fake-vars-template"), 0o600)) - - origCandidates := vm.FirmwarePathCandidates - //nolint:reassign // test-only override of vm's exported var, restored via t.Cleanup; same pattern vm/firmware_test.go uses within its own package. - vm.FirmwarePathCandidates = map[string][]vm.FirmwareCandidate{ - "x86_64": {{Code: codePath, VarsTemplate: varsPath}}, - } - t.Cleanup(func() { - //nolint:reassign // restoring the original value swapped above - vm.FirmwarePathCandidates = origCandidates - }) - - svc := &InstanceServiceImpl{ - config: &config.Config{ - WalDir: t.TempDir(), - Predastore: config.PredastoreConfig{ - // Host/creds are unset on purpose: Backend.Init's ListObjectsV2 - // call fails fast on empty static credentials, no network needed. - Host: "127.0.0.1:1", - Bucket: "test-bucket", - Region: "us-east-1", - }, - }, - } - - instance := &vm.VM{} - err := svc.prepareEFIVolume(context.Background(), "test-image", viperblock.VolumeConfig{}, instance, "x86_64") - require.Error(t, err, "Backend.Init must fail against an unreachable predastore host") -} diff --git a/spinifex/handlers/ec2/instance/provider_efi_volume_test.go b/spinifex/handlers/ec2/instance/provider_efi_volume_test.go new file mode 100644 index 000000000..355932ce6 --- /dev/null +++ b/spinifex/handlers/ec2/instance/provider_efi_volume_test.go @@ -0,0 +1,126 @@ +package handlers_ec2_instance + +import ( + "bytes" + "context" + "testing" + + "github.com/mulgadc/spinifex/spinifex/awserrors" + "github.com/mulgadc/spinifex/spinifex/ebsprovider" + "github.com/mulgadc/spinifex/spinifex/objectstore" + "github.com/mulgadc/spinifex/spinifex/vm" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// varsTemplateBytes stands in for a firmware VARS template. The real one is +// 540,672 bytes, but nothing here depends on the size beyond it being exact. +func varsTemplateBytes() []byte { return bytes.Repeat([]byte{0xAB}, 4096) } + +func providerEFIService(t *testing.T, provider ebsprovider.EBSProvider) *InstanceServiceImpl { + t.Helper() + return providerRootVolumeService(t, provider, rootVolumeAMILoader(), objectstore.NewMemoryObjectStore()) +} + +// TestCreateEFIVolume_Provider_SeedsFromTemplate is the core of the EFI +// decouple: the variable store is allocated through the provider with the +// template as seed bytes, so no control-plane engine is built for it. +func TestCreateEFIVolume_Provider_SeedsFromTemplate(t *testing.T) { + provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{VolumeSeeding: true}) + svc := providerEFIService(t, provider) + template := varsTemplateBytes() + + require.NoError(t, svc.createEFIVolumeViaProvider(context.Background(), "vol-efiseed001-efi", template)) + + volume, err := provider.GetVolume(context.Background(), ebsprovider.GetVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-efiseed001-efi", + }) + require.NoError(t, err) + assert.Equal(t, int64(len(template)), volume.CapacityBytes, + "the store must be sized byte-exactly to the VARS template") + + seed, ok := provider.SeedData("vol-efiseed001-efi") + require.True(t, ok) + assert.Equal(t, template, seed, "the template must reach the provider as seed bytes") +} + +// A provider that rounds capacity to a GiB boundary produces a VARS region +// pflash refuses, which would surface as an unbootable guest rather than a +// failed launch. The mismatch has to fail the launch instead. +func TestCreateEFIVolume_Provider_CapacityMismatchFails(t *testing.T) { + provider := &roundingProvider{ + EBSProvider: ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{VolumeSeeding: true}), + } + svc := providerEFIService(t, provider) + + err := svc.createEFIVolumeViaProvider(context.Background(), "vol-efiround01-efi", varsTemplateBytes()) + require.Error(t, err) + assert.Equal(t, awserrors.ErrorServerInternal, err.Error()) +} + +// A relaunch reissues the same create. Reseeding would overwrite the BootOrder +// the guest wrote on its first boot, so the second call must be a no-op. +func TestCreateEFIVolume_Provider_RepeatDoesNotReseed(t *testing.T) { + provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{VolumeSeeding: true}) + svc := providerEFIService(t, provider) + ctx := context.Background() + template := varsTemplateBytes() + + require.NoError(t, svc.createEFIVolumeViaProvider(ctx, "vol-efirepeat1-efi", template)) + require.NoError(t, svc.createEFIVolumeViaProvider(ctx, "vol-efirepeat1-efi", template)) + + seed, ok := provider.SeedData("vol-efirepeat1-efi") + require.True(t, ok) + assert.Equal(t, template, seed) +} + +// A seed above the wire bound must fail the launch with an error naming the +// limit, not reach NATS and fail as an oversized publish. +func TestCreateEFIVolume_Provider_OversizedTemplateFails(t *testing.T) { + provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{VolumeSeeding: true}) + svc := providerEFIService(t, provider) + + err := svc.createEFIVolumeViaProvider(context.Background(), "vol-efitoobig1-efi", make([]byte, ebsprovider.MaxSeedBytes+1)) + require.Error(t, err) + assert.Equal(t, awserrors.ErrorServerInternal, err.Error()) +} + +// TestPrepareEFIVolume_Provider_RegistersEBSRequest covers the whole function +// against this host's real firmware, including the name the VM launches with. +func TestPrepareEFIVolume_Provider_RegistersEBSRequest(t *testing.T) { + if _, _, _, err := vm.FirmwarePaths("x86_64"); err != nil { + t.Skipf("no UEFI firmware on this host: %v", err) + } + + provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{VolumeSeeding: true}) + svc := providerEFIService(t, provider) + instance := &vm.VM{} + + require.NoError(t, svc.prepareEFIVolume(context.Background(), "vol-efiprepare001", instance, "x86_64")) + + instance.EBSRequests.Mu.Lock() + defer instance.EBSRequests.Mu.Unlock() + require.Len(t, instance.EBSRequests.Requests, 1) + assert.Equal(t, "vol-efiprepare001-efi", instance.EBSRequests.Requests[0].Name) + assert.True(t, instance.EBSRequests.Requests[0].EFI) + assert.False(t, instance.EBSRequests.Requests[0].Boot) + + seed, ok := provider.SeedData("vol-efiprepare001-efi") + require.True(t, ok) + assert.NotEmpty(t, seed, "the store must be seeded from this host's VARS template") +} + +// roundingProvider reports a GiB-rounded capacity the way a provider with no +// sub-GiB volume granularity would. +type roundingProvider struct { + ebsprovider.EBSProvider +} + +func (p *roundingProvider) CreateVolume(ctx context.Context, req ebsprovider.CreateVolumeRequest) (*ebsprovider.Volume, error) { + volume, err := p.EBSProvider.CreateVolume(ctx, req) + if err != nil { + return nil, err + } + volume.CapacityBytes = 1 << 30 + return volume, nil +} diff --git a/spinifex/handlers/ec2/instance/provider_root_volume_test.go b/spinifex/handlers/ec2/instance/provider_root_volume_test.go new file mode 100644 index 000000000..fc9690d85 --- /dev/null +++ b/spinifex/handlers/ec2/instance/provider_root_volume_test.go @@ -0,0 +1,307 @@ +package handlers_ec2_instance + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/mulgadc/spinifex/spinifex/awserrors" + "github.com/mulgadc/spinifex/spinifex/config" + "github.com/mulgadc/spinifex/spinifex/ebsmetadata" + "github.com/mulgadc/spinifex/spinifex/ebsprovider" + "github.com/mulgadc/spinifex/spinifex/objectstore" + spxtypes "github.com/mulgadc/spinifex/spinifex/types" + "github.com/mulgadc/spinifex/spinifex/vm" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/goleak" +) + +const ( + testRootVolumeBytes = 4 * 1024 * 1024 * 1024 + testRootBucket = "test-bucket" + testRootAZ = "ap-southeast-2a" + testRootAccount = "000000000001" +) + +// providerRootVolumeService builds an instance service whose only storage +// dependency is the provider. Predastore points at a closed port so any +// fallback to the embedded engine fails loudly instead of silently passing. +func providerRootVolumeService(t *testing.T, provider ebsprovider.EBSProvider, loader AMIMetaLoader, store objectstore.ObjectStore) *InstanceServiceImpl { + t.Helper() + return &InstanceServiceImpl{ + config: &config.Config{ + WalDir: t.TempDir(), + AZ: testRootAZ, + Predastore: config.PredastoreConfig{ + Host: "127.0.0.1:1", + Bucket: testRootBucket, + Region: "us-east-1", + }, + }, + objectStore: store, + metadata: ebsmetadata.NewStore(store, testRootBucket), + amiLoader: loader, + ebsProvider: provider, + } +} + +// seedProviderSnapshot registers a source volume and a snapshot of it, which is +// what an AMI's snapshot looks like to the provider. +func seedProviderSnapshot(t *testing.T, provider ebsprovider.EBSProvider, sourceVolumeID, snapshotID string) { + t.Helper() + ctx := context.Background() + _, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), + VolumeID: sourceVolumeID, + CapacityRange: ebsprovider.CapacityRange{RequiredBytes: testRootVolumeBytes}, + }) + require.NoError(t, err) + _, err = provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{ + Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, VolumeID: sourceVolumeID, + }) + require.NoError(t, err) +} + +// rootVolumeAMILoader returns a loader for one AMI backed by snap-source on +// vol-origin, the shape every provider root-volume test needs. +func rootVolumeAMILoader() *fakeAMILoader { + return &fakeAMILoader{ + byID: map[string]ebsmetadata.AMI{"ami-1": {ImageID: "ami-1", SnapshotID: "snap-source"}}, + sourceByID: map[string]string{"ami-1": "vol-origin"}, + } +} + +// TestPrepareRootVolume_Provider_CreatesFromAMISnapshot locks that an injected +// provider owns root-volume creation: the volume is restored from the AMI's +// snapshot rather than copied block-by-block by the control plane. +func TestPrepareRootVolume_Provider_CreatesFromAMISnapshot(t *testing.T) { + defer goleak.VerifyNone(t) + + provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{CrashConsistentSnapshot: true}) + seedProviderSnapshot(t, provider, "vol-origin", "snap-source") + svc := providerRootVolumeService(t, provider, rootVolumeAMILoader(), objectstore.NewMemoryObjectStore()) + + instance := &vm.VM{AccountID: testRootAccount} + input := &ec2.RunInstancesInput{ImageId: aws.String("ami-1")} + err := svc.prepareRootVolume(context.Background(), input, "vol-root", testRootVolumeBytes, 0, + instance, false) + require.NoError(t, err) + + created, err := provider.GetVolume(context.Background(), ebsprovider.GetVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-root", + }) + require.NoError(t, err, "the root volume must exist in the provider") + assert.Equal(t, int64(testRootVolumeBytes), created.CapacityBytes) + + require.Len(t, instance.EBSRequests.Requests, 1) + assert.Equal(t, "vol-root", instance.EBSRequests.Requests[0].Name) + assert.True(t, instance.EBSRequests.Requests[0].Boot, "the root volume must be marked bootable") + assert.False(t, instance.EBSRequests.Requests[0].DeleteOnTermination) +} + +// TestPrepareRootVolume_Provider_WritesMetadataDocument locks the control-plane +// record of the root volume. Without it the volume has neither a document nor a +// legacy config.json, so DescribeVolumes cannot see it at all. +func TestPrepareRootVolume_Provider_WritesMetadataDocument(t *testing.T) { + defer goleak.VerifyNone(t) + + provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{}) + seedProviderSnapshot(t, provider, "vol-origin", "snap-source") + store := objectstore.NewMemoryObjectStore() + svc := providerRootVolumeService(t, provider, rootVolumeAMILoader(), store) + + instance := &vm.VM{AccountID: testRootAccount} + err := svc.prepareRootVolume(context.Background(), &ec2.RunInstancesInput{ImageId: aws.String("ami-1")}, + "vol-root", testRootVolumeBytes, 0, instance, true) + require.NoError(t, err) + + doc, err := ebsmetadata.NewStore(store, testRootBucket).GetVolume(context.Background(), "vol-root") + require.NoError(t, err, "the root volume must have an ebsmetadata document") + + assert.Equal(t, "vol-root", doc.VolumeID) + assert.Equal(t, testRootAccount, doc.TenantID) + assert.Equal(t, uint64(4), doc.CapacityGiB) + assert.Equal(t, string(ebsprovider.VolumeStateAvailable), doc.State, + "attach flips the volume to in-use later; creation must not pre-empt it") + assert.False(t, doc.CreatedAt.IsZero()) + assert.Equal(t, testRootAZ, doc.AvailabilityZone) + assert.Equal(t, spxtypes.VolumeTypeGP3, doc.VolumeType) + assert.Equal(t, spxtypes.DefaultGP3IOPS, doc.IOPS) + assert.Equal(t, spxtypes.DefaultGP3Throughput, doc.Throughput) + assert.Equal(t, "snap-source", doc.SnapshotID) + assert.True(t, doc.DeleteOnTermination) + assert.False(t, doc.Encrypted, "no encryption key is configured in this service") + assert.NotEmpty(t, doc.ProviderHandle, "the opaque provider handle must be recorded") + assert.Empty(t, doc.AttachedInstance, "creation must not claim an attachment") +} + +// TestPrepareRootVolume_Provider_HonoursRequestedIOPS locks that a launch +// naming Iops in its block device mapping keeps that value. +func TestPrepareRootVolume_Provider_HonoursRequestedIOPS(t *testing.T) { + defer goleak.VerifyNone(t) + + provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{}) + seedProviderSnapshot(t, provider, "vol-origin", "snap-source") + store := objectstore.NewMemoryObjectStore() + svc := providerRootVolumeService(t, provider, rootVolumeAMILoader(), store) + + err := svc.prepareRootVolume(context.Background(), &ec2.RunInstancesInput{ImageId: aws.String("ami-1")}, + "vol-root", testRootVolumeBytes, 5000, &vm.VM{AccountID: testRootAccount}, true) + require.NoError(t, err) + + doc, err := ebsmetadata.NewStore(store, testRootBucket).GetVolume(context.Background(), "vol-root") + require.NoError(t, err) + assert.Equal(t, 5000, doc.IOPS) +} + +// failingPutObjectStore fails writes under failPrefix, so a test can break the +// metadata write without disturbing any other object the launch touches. +type failingPutObjectStore struct { + objectstore.ObjectStore + + failPrefix string +} + +func (s *failingPutObjectStore) PutObject(ctx context.Context, input *s3.PutObjectInput) (*s3.PutObjectOutput, error) { + if strings.HasPrefix(aws.StringValue(input.Key), s.failPrefix) { + return nil, errors.New("simulated metadata write failure") + } + return s.ObjectStore.PutObject(ctx, input) +} + +// TestPrepareRootVolume_Provider_RollbackOnMetadataWriteFailure locks that a +// failed document write deletes the volume the provider just allocated, so a +// failed launch cannot strand an untracked volume. +func TestPrepareRootVolume_Provider_RollbackOnMetadataWriteFailure(t *testing.T) { + defer goleak.VerifyNone(t) + + provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{}) + seedProviderSnapshot(t, provider, "vol-origin", "snap-source") + store := &failingPutObjectStore{ + ObjectStore: objectstore.NewMemoryObjectStore(), + failPrefix: "spinifex/ebsmetadata/v1/volumes/", + } + svc := providerRootVolumeService(t, provider, rootVolumeAMILoader(), store) + + instance := &vm.VM{AccountID: testRootAccount} + err := svc.prepareRootVolume(context.Background(), &ec2.RunInstancesInput{ImageId: aws.String("ami-1")}, + "vol-root", testRootVolumeBytes, 0, instance, true) + require.Error(t, err) + assert.Equal(t, awserrors.ErrorServerInternal, err.Error()) + assert.Empty(t, instance.EBSRequests.Requests, "a failed launch must not claim a boot volume") + + _, getErr := provider.GetVolume(context.Background(), ebsprovider.GetVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-root", + }) + require.ErrorIs(t, getErr, ebsprovider.ErrNotFound, "rollback must delete the just-created provider volume") +} + +// TestPrepareRootVolume_Provider_RepeatIsIdempotent locks that re-running the +// same launch does not re-clone or error: the provider returns the existing +// volume when the requested capacity matches. +func TestPrepareRootVolume_Provider_RepeatIsIdempotent(t *testing.T) { + defer goleak.VerifyNone(t) + + provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{}) + seedProviderSnapshot(t, provider, "vol-origin", "snap-source") + svc := providerRootVolumeService(t, provider, rootVolumeAMILoader(), objectstore.NewMemoryObjectStore()) + input := &ec2.RunInstancesInput{ImageId: aws.String("ami-1")} + + for range 2 { + instance := &vm.VM{AccountID: testRootAccount} + err := svc.prepareRootVolume(context.Background(), input, "vol-root", testRootVolumeBytes, 0, + instance, true) + require.NoError(t, err) + require.Len(t, instance.EBSRequests.Requests, 1) + } +} + +// TestPrepareRootVolume_Provider_AMIWithoutSnapshotFails locks that an AMI with +// no snapshot still fails, and allocates nothing. +func TestPrepareRootVolume_Provider_AMIWithoutSnapshotFails(t *testing.T) { + defer goleak.VerifyNone(t) + + provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{}) + svc := providerRootVolumeService(t, provider, &fakeAMILoader{ + byID: map[string]ebsmetadata.AMI{"ami-1": {ImageID: "ami-1"}}, + }, objectstore.NewMemoryObjectStore()) + + instance := &vm.VM{AccountID: testRootAccount} + err := svc.prepareRootVolume(context.Background(), &ec2.RunInstancesInput{ImageId: aws.String("ami-1")}, + "vol-root", testRootVolumeBytes, 0, instance, true) + require.Error(t, err) + assert.Equal(t, awserrors.ErrorServerInternal, err.Error()) + assert.Empty(t, instance.EBSRequests.Requests, "a failed launch must not claim a boot volume") + + _, getErr := provider.GetVolume(context.Background(), ebsprovider.GetVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-root", + }) + require.ErrorIs(t, getErr, ebsprovider.ErrNotFound) +} + +// TestPrepareRootVolume_Provider_UnknownAMIFails locks the AMI lookup failure +// mapping the legacy clone path already used. +func TestPrepareRootVolume_Provider_UnknownAMIFails(t *testing.T) { + defer goleak.VerifyNone(t) + + svc := providerRootVolumeService(t, ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{}), + &fakeAMILoader{err: errors.New("missing")}, objectstore.NewMemoryObjectStore()) + + instance := &vm.VM{AccountID: testRootAccount} + err := svc.prepareRootVolume(context.Background(), &ec2.RunInstancesInput{ImageId: aws.String("ami-gone")}, + "vol-root", testRootVolumeBytes, 0, instance, true) + require.Error(t, err) + assert.Equal(t, awserrors.ErrorInvalidAMIIDNotFound, err.Error()) + assert.Empty(t, instance.EBSRequests.Requests) +} + +// TestPrepareRootVolume_Provider_SourceVolumeUnresolvableFails locks that an +// unresolvable snapshot source aborts rather than sending an incomplete +// CreateVolume the provider would reject. +func TestPrepareRootVolume_Provider_SourceVolumeUnresolvableFails(t *testing.T) { + defer goleak.VerifyNone(t) + + provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{}) + seedProviderSnapshot(t, provider, "vol-origin", "snap-source") + svc := providerRootVolumeService(t, provider, &fakeAMILoader{ + byID: map[string]ebsmetadata.AMI{"ami-1": {ImageID: "ami-1", SnapshotID: "snap-source"}}, + sourceErr: errors.New("snapshot metadata missing"), + }, objectstore.NewMemoryObjectStore()) + + instance := &vm.VM{AccountID: testRootAccount} + err := svc.prepareRootVolume(context.Background(), &ec2.RunInstancesInput{ImageId: aws.String("ami-1")}, + "vol-root", testRootVolumeBytes, 0, instance, true) + require.Error(t, err) + assert.Equal(t, awserrors.ErrorServerInternal, err.Error()) + assert.Empty(t, instance.EBSRequests.Requests) + + _, getErr := provider.GetVolume(context.Background(), ebsprovider.GetVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-root", + }) + require.ErrorIs(t, getErr, ebsprovider.ErrNotFound) +} + +// TestPrepareRootVolume_Provider_CreateFailureIsFatal locks that a provider +// error aborts the launch rather than falling back to the embedded engine. +func TestPrepareRootVolume_Provider_CreateFailureIsFatal(t *testing.T) { + defer goleak.VerifyNone(t) + + // No snapshot seeded, so the provider rejects the clone request. + provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{}) + svc := providerRootVolumeService(t, provider, &fakeAMILoader{ + byID: map[string]ebsmetadata.AMI{"ami-1": {ImageID: "ami-1", SnapshotID: "snap-missing"}}, + sourceByID: map[string]string{"ami-1": "vol-origin"}, + }, objectstore.NewMemoryObjectStore()) + + instance := &vm.VM{AccountID: testRootAccount} + err := svc.prepareRootVolume(context.Background(), &ec2.RunInstancesInput{ImageId: aws.String("ami-1")}, + "vol-root", testRootVolumeBytes, 0, instance, true) + require.Error(t, err) + assert.Equal(t, awserrors.ErrorServerInternal, err.Error()) + assert.Empty(t, instance.EBSRequests.Requests) +} diff --git a/spinifex/handlers/ec2/instance/service.go b/spinifex/handlers/ec2/instance/service.go index 05ec20935..04fbb6cca 100644 --- a/spinifex/handlers/ec2/instance/service.go +++ b/spinifex/handlers/ec2/instance/service.go @@ -4,9 +4,9 @@ import ( "context" "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mulgadc/spinifex/spinifex/ebsmetadata" "github.com/mulgadc/spinifex/spinifex/gpu" "github.com/mulgadc/spinifex/spinifex/vm" - "github.com/mulgadc/viperblock/viperblock" ) // InstanceService defines the interface for EC2 instance operations business logic. @@ -137,9 +137,15 @@ type PublicIPReleaser interface { } // AMIMetaLoader resolves an AMI ID to its metadata for ownership/validation -// during RunInstances. Implemented by handlers/ec2/image.ImageServiceImpl. +// during RunInstances. ImageServiceImpl converts on its embedded path, so +// this boundary names no viperblock type whichever EBS provider is selected. type AMIMetaLoader interface { - GetAMIConfig(ctx context.Context, imageID string) (viperblock.AMIMetadata, error) + GetAMIConfig(ctx context.Context, imageID string) (ebsmetadata.AMI, error) + + // GetAMISourceVolumeID returns the volume whose blocks the AMI's snapshot + // references. An EBS provider needs it alongside the snapshot ID to resolve + // the clone's base data. + GetAMISourceVolumeID(ctx context.Context, imageID string) (string, error) } // KeyPairValidator checks that a named key pair exists for an account during diff --git a/spinifex/handlers/ec2/instance/service_impl.go b/spinifex/handlers/ec2/instance/service_impl.go index 65a419cf8..cf8428cc1 100644 --- a/spinifex/handlers/ec2/instance/service_impl.go +++ b/spinifex/handlers/ec2/instance/service_impl.go @@ -18,6 +18,8 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" "github.com/mulgadc/spinifex/spinifex/awserrors" "github.com/mulgadc/spinifex/spinifex/config" + "github.com/mulgadc/spinifex/spinifex/ebsmetadata" + "github.com/mulgadc/spinifex/spinifex/ebsprovider" "github.com/mulgadc/spinifex/spinifex/filterutil" "github.com/mulgadc/spinifex/spinifex/gpu" handlers_dns "github.com/mulgadc/spinifex/spinifex/handlers/dns" @@ -28,13 +30,14 @@ import ( spxtypes "github.com/mulgadc/spinifex/spinifex/types" "github.com/mulgadc/spinifex/spinifex/utils" "github.com/mulgadc/spinifex/spinifex/vm" - "github.com/mulgadc/viperblock/types" - "github.com/mulgadc/viperblock/viperblock" - "github.com/mulgadc/viperblock/viperblock/backends/s3" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" ) +// bytesPerGiB converts the byte sizes the launch path works in to the GiB +// units ebsmetadata records. +const bytesPerGiB = 1024 * 1024 * 1024 + // VolumeInfo holds volume information returned from GenerateVolumes // for populating BlockDeviceMappings in the EC2 API response. type VolumeInfo struct { @@ -135,6 +138,8 @@ type InstanceServiceImpl struct { tagWriter InstanceTagWriter gpuClaimer GPUClaimer amiLoader AMIMetaLoader + ebsProvider ebsprovider.EBSProvider + metadata *ebsmetadata.Store keyValidator KeyPairValidator eniCreator ENICreator ipAllocator PublicIPAllocator @@ -142,6 +147,19 @@ type InstanceServiceImpl struct { dnsInternalDomain string } +// SetEBSProvider injects the provider boundary used for instance-created +// volumes. Once set, root volumes are allocated through it instead of a +// control-plane storage engine. +func (s *InstanceServiceImpl) SetEBSProvider(provider ebsprovider.EBSProvider) { + s.ebsProvider = provider +} + +// EBSProvider returns the injected provider boundary, or nil on the legacy +// embedded-engine path. Primarily for composition-root tests to observe wiring. +func (s *InstanceServiceImpl) EBSProvider() ebsprovider.EBSProvider { + return s.ebsProvider +} + // NewInstanceServiceImpl creates a new instance service implementation for daemon use. func NewInstanceServiceImpl( cfg *config.Config, @@ -152,6 +170,12 @@ func NewInstanceServiceImpl( resourceMgr InstanceTypeAllocator, stoppedStore StoppedInstanceStore, ) *InstanceServiceImpl { + // cfg is nil-tolerated by the DNS resolvers below, so keep the metadata + // store's bucket lookup equally tolerant. + bucket := "" + if cfg != nil { + bucket = cfg.Predastore.Bucket + } return &InstanceServiceImpl{ config: cfg, instanceTypes: instanceTypes, @@ -160,6 +184,7 @@ func NewInstanceServiceImpl( vmMgr: vmMgr, resourceMgr: resourceMgr, stoppedStore: stoppedStore, + metadata: ebsmetadata.NewStore(store, bucket), dnsBaseDomain: handlers_dns.ResolveBaseDomain(cfg), dnsInternalDomain: handlers_dns.ResolveInternalDomain(cfg), } @@ -621,7 +646,7 @@ func (s *InstanceServiceImpl) PrepareRunInstances(ctx context.Context, input *ec if input.SubnetId != nil && *input.SubnetId != "" && s.eniCreator != nil { eniOut, eniErr := s.eniCreator.CreateNetworkInterface(ctx, &ec2.CreateNetworkInterfaceInput{ SubnetId: input.SubnetId, - Description: aws.String("Primary network interface for " + instance.ID), + Description: aws.String(handlers_ec2_vpc.AutoENIDescriptionPrefix + instance.ID), Groups: input.SecurityGroupIds, }, accountID) if eniErr != nil { @@ -1392,27 +1417,13 @@ func (s *InstanceServiceImpl) GenerateVolumes(ctx context.Context, input *ec2.Ru // Capture attach time for the root volume attachTime := time.Now() - volumeConfig := viperblock.VolumeConfig{ - VolumeMetadata: viperblock.VolumeMetadata{ - VolumeID: p.imageId, - SizeGiB: utils.SafeIntToUint64(p.size / 1024 / 1024 / 1024), - CreatedAt: attachTime, - DeviceName: p.deviceName, - VolumeType: p.volumeType, - IOPS: p.iops, - SnapshotID: p.snapshotId, - DeleteOnTermination: p.deleteOnTermination, - TenantID: instance.AccountID, - }, - } - size := p.size imageId := p.imageId deviceName := p.deviceName deleteOnTermination := p.deleteOnTermination // Step 1: Create or validate root volume - err := s.prepareRootVolume(ctx, input, imageId, size, volumeConfig, instance, deleteOnTermination) + err := s.prepareRootVolume(ctx, input, imageId, size, p.iops, instance, deleteOnTermination) if err != nil { return nil, err } @@ -1421,7 +1432,7 @@ func (s *InstanceServiceImpl) GenerateVolumes(ctx context.Context, input *ec2.Ru // guests must not allocate an orphan VARS volume). if instance.BootMode == "uefi" || instance.BootMode == "uefi-preferred" { arch := instanceArchitecture(s.instanceTypes[*input.InstanceType]) - err = s.prepareEFIVolume(ctx, imageId, volumeConfig, instance, arch) + err = s.prepareEFIVolume(ctx, imageId, instance, arch) if err != nil { return nil, err } @@ -1440,133 +1451,137 @@ func (s *InstanceServiceImpl) GenerateVolumes(ctx context.Context, input *ec2.Ru return volumeInfos, nil } -// newViperblock creates a viperblock instance with the service's S3/Predastore credentials. -func (s *InstanceServiceImpl) newViperblock(volumeName string, size int, volumeConfig viperblock.VolumeConfig) (*viperblock.VB, error) { - cfg := s3.S3Config{ - VolumeName: volumeName, - VolumeSize: utils.SafeIntToUint64(size), - Bucket: s.config.Predastore.Bucket, - Region: s.config.Predastore.Region, - AccessKey: s.config.Predastore.AccessKey, - SecretKey: s.config.Predastore.SecretKey, - Host: s.config.Predastore.Host, +// prepareRootVolume allocates the root volume through the EBS provider and +// records the boot volume the launcher must attach. +func (s *InstanceServiceImpl) prepareRootVolume(ctx context.Context, input *ec2.RunInstancesInput, imageId string, size, iops int, instance *vm.VM, deleteOnTermination bool) error { + if s.ebsProvider == nil { + slog.ErrorContext(ctx, "no EBS provider configured", "op", "prepareRootVolume") + return errors.New(awserrors.ErrorServerInternal) } - - mkey, err := utils.LoadViperblockMasterKey(s.config.Viperblock.EncryptionKeyFile) - if err != nil { - return nil, err + if err := s.createRootVolumeViaProvider(ctx, rootVolumeSpec{ + amiID: aws.StringValue(input.ImageId), + volumeID: imageId, + accountID: instance.AccountID, + sizeBytes: size, + iops: iops, + deleteOnTermination: deleteOnTermination, + }); err != nil { + return err } - - vbconfig := viperblock.VB{ - VolumeName: volumeName, - VolumeSize: utils.SafeIntToUint64(size), - BaseDir: s.config.WalDir, - Cache: viperblock.Cache{Config: viperblock.CacheConfig{Size: 0}}, - VolumeConfig: volumeConfig, - MasterKey: mkey, - EncryptionEnabled: mkey != nil, - } - - vb, err := viperblock.New(&vbconfig, "s3", cfg) - return vb, err + appendRootEBSRequest(instance, imageId, deleteOnTermination) + return nil } -// prepareRootVolume handles creation/cloning of the root volume. -func (s *InstanceServiceImpl) prepareRootVolume(ctx context.Context, input *ec2.RunInstancesInput, imageId string, size int, volumeConfig viperblock.VolumeConfig, instance *vm.VM, deleteOnTermination bool) error { - vb, err := s.newViperblock(imageId, size, volumeConfig) - if err != nil { - slog.ErrorContext(ctx, "Failed to connect to Viperblock store", "err", err) - return errors.New(awserrors.ErrorServerInternal) - } - defer func() { - vb.StopChunkUploader() - vb.StopWALSyncer() - }() - - // Initialize the backend - err = vb.Backend.Init() - if err != nil { - slog.ErrorContext(ctx, "Failed to initialize backend", "err", err) - return errors.New(awserrors.ErrorServerInternal) - } - - // Only os.ErrNotExist means "clone from AMI"; any other error must abort. - // Treating a tamper/mismatch as missing would overwrite live volume state. - _, err = vb.LoadStateRequest("") - if err != nil && !errors.Is(err, os.ErrNotExist) { - slog.ErrorContext(ctx, "Failed to load root volume state from backend", - "imageId", imageId, "err", err) - return errors.New(awserrors.ErrorServerInternal) - } - if err != nil { - slog.InfoContext(ctx, "Volume does not yet exist, creating from AMI ...") - if err = s.cloneAMIToVolume(ctx, input, vb); err != nil { - return err - } - } - - // Append root volume to instance +// appendRootEBSRequest records the boot volume the launcher must attach. +func appendRootEBSRequest(instance *vm.VM, volumeID string, deleteOnTermination bool) { instance.EBSRequests.Mu.Lock() instance.EBSRequests.Requests = append(instance.EBSRequests.Requests, spxtypes.EBSRequest{ - Name: imageId, + Name: volumeID, Boot: true, DeleteOnTermination: deleteOnTermination, }) instance.EBSRequests.Mu.Unlock() +} +// rootVolumeSpec describes the boot volume a launch needs, in provider-neutral +// terms taken from the launch request. +type rootVolumeSpec struct { + amiID string + volumeID string + accountID string + sizeBytes int + iops int + deleteOnTermination bool +} + +// createRootVolumeViaProvider allocates the root volume through the injected +// provider, cloning the AMI's snapshot, then records it in ebsmetadata. No +// control-plane engine is built, so a launch never becomes a second writer. +func (s *InstanceServiceImpl) createRootVolumeViaProvider(ctx context.Context, spec rootVolumeSpec) error { + amiConfig, err := s.amiLoader.GetAMIConfig(ctx, spec.amiID) + if err != nil { + slog.ErrorContext(ctx, "Could not load AMI config", "imageId", spec.amiID, "err", err) + return errors.New(awserrors.ErrorInvalidAMIIDNotFound) + } + if amiConfig.SnapshotID == "" { + slog.ErrorContext(ctx, "AMI has no snapshot ID, cannot perform zero-copy clone", "imageId", spec.amiID) + return errors.New(awserrors.ErrorServerInternal) + } + + // The provider resolves a clone's base blocks against the snapshot's source + // volume, so the wire contract requires both IDs together. + sourceVolumeID, err := s.amiLoader.GetAMISourceVolumeID(ctx, spec.amiID) + if err != nil { + slog.ErrorContext(ctx, "Could not resolve AMI snapshot source volume", + "imageId", spec.amiID, "snapshotId", amiConfig.SnapshotID, "err", err) + return errors.New(awserrors.ErrorServerInternal) + } + + // The control plane cannot see how a provider encrypts its volumes, but this + // shared config knob is the same one the legacy path derives Encrypted from. + mkey, err := utils.LoadViperblockMasterKey(s.config.Viperblock.EncryptionKeyFile) + if err != nil { + slog.ErrorContext(ctx, "Could not load encryption key for root volume", "volumeId", spec.volumeID, "err", err) + return errors.New(awserrors.ErrorServerInternal) + } + + slog.InfoContext(ctx, "Creating root volume from AMI snapshot via EBS provider", + "imageId", spec.amiID, "volumeId", spec.volumeID, "snapshotId", amiConfig.SnapshotID, "sourceVolumeId", sourceVolumeID) + + created, err := s.ebsProvider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), + VolumeID: spec.volumeID, + CapacityRange: ebsprovider.CapacityRange{RequiredBytes: int64(spec.sizeBytes)}, + AvailabilityZone: s.config.AZ, + SourceSnapshotID: amiConfig.SnapshotID, + SourceSnapshotVolumeID: sourceVolumeID, + }) + if err != nil { + slog.ErrorContext(ctx, "Provider root volume creation failed", + "volumeId", spec.volumeID, "snapshotId", amiConfig.SnapshotID, "err", err) + return errors.New(awserrors.ErrorServerInternal) + } + if created == nil { + slog.ErrorContext(ctx, "Provider returned no volume for the root volume", "volumeId", spec.volumeID) + return errors.New(awserrors.ErrorServerInternal) + } + + // DescribeVolumes enumerates ebsmetadata documents only, so a root volume + // without one is invisible and its attach-time state write fails. + if err := s.metadata.PutVolume(ctx, ebsmetadata.Volume{ + VolumeID: spec.volumeID, TenantID: spec.accountID, + CapacityGiB: utils.SafeIntToUint64(spec.sizeBytes / bytesPerGiB), + State: string(ebsprovider.VolumeStateAvailable), + CreatedAt: time.Now(), AvailabilityZone: s.config.AZ, + VolumeType: spxtypes.VolumeTypeGP3, IOPS: rootVolumeIOPS(spec.iops), + Throughput: spxtypes.DefaultGP3Throughput, SnapshotID: amiConfig.SnapshotID, + DeleteOnTermination: spec.deleteOnTermination, Encrypted: mkey != nil, + ProviderHandle: created.Handle, + }); err != nil { + slog.ErrorContext(ctx, "Failed to persist root volume metadata", "volumeId", spec.volumeID, "err", err) + if delErr := s.ebsProvider.DeleteVolume(ctx, ebsprovider.DeleteVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), VolumeID: spec.volumeID, Handle: created.Handle, + }); delErr != nil { + slog.ErrorContext(ctx, "Failed to roll back untracked root volume", "volumeId", spec.volumeID, "err", delErr) + } + return errors.New(awserrors.ErrorServerInternal) + } return nil } -// cloneAMIToVolume creates a new volume from an AMI using snapshot-based -// zero-copy cloning. The destination volume points at the AMI's frozen block -// map and reads on-demand from the AMI's chunks (copy-on-write). -func (s *InstanceServiceImpl) cloneAMIToVolume(ctx context.Context, input *ec2.RunInstancesInput, destVb *viperblock.VB) error { - amiConfig, err := s.amiLoader.GetAMIConfig(ctx, *input.ImageId) - if err != nil { - slog.ErrorContext(ctx, "Could not load AMI config", "imageId", *input.ImageId, "err", err) - return errors.New(awserrors.ErrorInvalidAMIIDNotFound) +// rootVolumeIOPS falls back to the gp3 baseline when the launch request named +// no Iops, so DescribeVolumes never reports a root volume as having zero. +func rootVolumeIOPS(requested int) int { + if requested <= 0 { + return spxtypes.DefaultGP3IOPS } - - snapshotID := amiConfig.SnapshotID - if snapshotID == "" { - slog.ErrorContext(ctx, "AMI has no snapshot ID, cannot perform zero-copy clone", "imageId", *input.ImageId) - return errors.New(awserrors.ErrorServerInternal) - } - - slog.InfoContext(ctx, "Cloning AMI via snapshot", "imageId", *input.ImageId, "snapshotID", snapshotID) - - // Set up destination volume from the snapshot (zero-copy) - err = destVb.OpenFromSnapshot(snapshotID) - if err != nil { - slog.ErrorContext(ctx, "Failed to open from snapshot", "snapshotID", snapshotID, "err", err) - return errors.New(awserrors.ErrorServerInternal) - } - - // Persist the snapshot relationship to the backend - err = destVb.SaveState() - if err != nil { - slog.ErrorContext(ctx, "Failed to save state", "err", err) - return errors.New(awserrors.ErrorServerInternal) - } - - err = destVb.SaveBlockState() - if err != nil { - slog.ErrorContext(ctx, "Failed to save block state", "err", err) - return errors.New(awserrors.ErrorServerInternal) - } - - err = destVb.RemoveLocalFiles() - if err != nil { - slog.WarnContext(ctx, "Failed to remove local files", "err", err) - } - - return nil + return requested } // prepareEFIVolume creates the per-VM EFI variable store, sized exactly to the // firmware VARS template (pflash requires byte-exact size) and seeded from it. // arch is "x86_64" | "arm64". -func (s *InstanceServiceImpl) prepareEFIVolume(ctx context.Context, imageId string, volumeConfig viperblock.VolumeConfig, instance *vm.VM, arch string) error { +func (s *InstanceServiceImpl) prepareEFIVolume(ctx context.Context, volumeID string, instance *vm.VM, arch string) error { codePath, varsTemplate, varsSize, err := vm.FirmwarePaths(arch) if err != nil { slog.ErrorContext(ctx, "UEFI firmware not installed on this host", "arch", arch, "err", err) @@ -1583,80 +1598,18 @@ func (s *InstanceServiceImpl) prepareEFIVolume(ctx context.Context, imageId stri } slog.InfoContext(ctx, "Preparing EFI variable store", "arch", arch, "firmwarePath", codePath, "varsTemplate", varsTemplate, "size", varsSize) - efiVolumeName := fmt.Sprintf("%s-efi", imageId) - efiVolumeConfig := volumeConfig - efiVolumeConfig.VolumeMetadata.VolumeID = efiVolumeName - // Zero SizeGiB prevents viperblock from rounding the EFI volume size - // up to GiB boundaries; pflash rejects any size beyond the VARS region. - efiVolumeConfig.VolumeMetadata.SizeGiB = 0 - - efiVb, err := s.newViperblock(efiVolumeName, int(varsSize), efiVolumeConfig) - if err != nil { - slog.ErrorContext(ctx, "Could not create EFI viperblock", "err", err) + efiVolumeName := fmt.Sprintf("%s-efi", volumeID) + if s.ebsProvider == nil { + slog.ErrorContext(ctx, "no EBS provider configured", "op", "prepareEFIVolume") return errors.New(awserrors.ErrorServerInternal) } - // newViperblock starts the chunk uploader and WAL syncer goroutines - // unconditionally, so every return path below must release them, not - // just the happy path Close() at the bottom of this function. - defer func() { - efiVb.StopChunkUploader() - efiVb.StopWALSyncer() - }() - - slog.DebugContext(ctx, "Initializing EFI Viperblock store backend") - if err := efiVb.Backend.Init(); err != nil { - slog.ErrorContext(ctx, "Failed to initialize EFI Viperblock store backend", "err", err) - return errors.New(awserrors.ErrorServerInternal) - } - - // Only os.ErrNotExist means "seed from template"; other errors must abort. - // Treating a transient failure as missing would clobber guest-set BootOrder. - _, loadErr := efiVb.LoadStateRequest("") - if loadErr != nil && !errors.Is(loadErr, os.ErrNotExist) { - slog.ErrorContext(ctx, "Failed to load EFI volume state from backend", "name", efiVolumeName, "err", loadErr) - return errors.New(awserrors.ErrorServerInternal) - } - if loadErr != nil { - slog.InfoContext(ctx, "EFI volume does not yet exist, seeding from firmware VARS template", "name", efiVolumeName) - - var walErr error - if efiVb.UseShardedWAL { - walErr = efiVb.OpenShardedWAL() - } else { - walErr = efiVb.OpenWAL(&efiVb.WAL, fmt.Sprintf("%s/%s", efiVb.WAL.BaseDir, types.GetFilePath(types.FileTypeWALChunk, efiVb.WAL.WallNum.Load(), efiVb.GetVolume()))) - } - if walErr != nil { - slog.ErrorContext(ctx, "Failed to load WAL", "err", walErr) - return errors.New(awserrors.ErrorServerInternal) - } - - if err := efiVb.OpenWAL(&efiVb.BlockToObjectWAL, fmt.Sprintf("%s/%s", efiVb.WAL.BaseDir, types.GetFilePath(types.FileTypeWALBlock, efiVb.BlockToObjectWAL.WallNum.Load(), efiVb.GetVolume()))); err != nil { - slog.ErrorContext(ctx, "Failed to load block WAL", "err", err) - return errors.New(awserrors.ErrorServerInternal) - } - - if err := efiVb.WriteAt(0, template); err != nil { - slog.ErrorContext(ctx, "Failed to seed EFI volume with VARS template", "err", err) - return errors.New(awserrors.ErrorServerInternal) - } - if err := efiVb.Flush(); err != nil { - slog.ErrorContext(ctx, "Failed to flush EFI volume", "err", err) - return errors.New(awserrors.ErrorServerInternal) - } - } - - // Close is the durability boundary; a partial VARS write causes pflash to refuse launch. - if err := efiVb.Close(); err != nil { - slog.ErrorContext(ctx, "Failed to close EFI Viperblock store", "err", err) - return errors.New(awserrors.ErrorServerInternal) - } - if err := efiVb.RemoveLocalFiles(); err != nil { - slog.ErrorContext(ctx, "Failed to remove local files", "err", err) + if err := s.createEFIVolumeViaProvider(ctx, efiVolumeName, template); err != nil { + return err } instance.EBSRequests.Mu.Lock() instance.EBSRequests.Requests = append(instance.EBSRequests.Requests, spxtypes.EBSRequest{ - Name: efiVb.VolumeName, + Name: efiVolumeName, Boot: false, EFI: true, }) @@ -1665,6 +1618,41 @@ func (s *InstanceServiceImpl) prepareEFIVolume(ctx context.Context, imageId stri return nil } +// createEFIVolumeViaProvider allocates the EFI variable store through the +// provider, seeding it with this node's firmware VARS template. The bytes +// travel with the request so the region matches the firmware that will run. +func (s *InstanceServiceImpl) createEFIVolumeViaProvider(ctx context.Context, efiVolumeName string, template []byte) error { + slog.InfoContext(ctx, "Creating EFI variable store via EBS provider", "name", efiVolumeName, "seedBytes", len(template)) + + // The store is internal to the launch, never attachable and filtered out of + // DescribeVolumes, so it gets no ebsmetadata document the way a root volume + // does. Its lifecycle is the root volume's DeleteVolume. + created, err := s.ebsProvider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), + VolumeID: efiVolumeName, + CapacityRange: ebsprovider.CapacityRange{RequiredBytes: int64(len(template))}, + AvailabilityZone: s.config.AZ, + SeedData: template, + }) + if err != nil { + slog.ErrorContext(ctx, "Provider EFI volume creation failed", "name", efiVolumeName, "err", err) + return errors.New(awserrors.ErrorServerInternal) + } + if created == nil { + slog.ErrorContext(ctx, "Provider returned no volume for the EFI variable store", "name", efiVolumeName) + return errors.New(awserrors.ErrorServerInternal) + } + + // pflash rejects a VARS region that is not byte-exact, so a provider that + // rounded the capacity up to a GiB boundary yields a guest that cannot boot. + if created.CapacityBytes != int64(len(template)) { + slog.ErrorContext(ctx, "Provider EFI volume capacity does not match the VARS template", + "name", efiVolumeName, "wantBytes", len(template), "gotBytes", created.CapacityBytes) + return errors.New(awserrors.ErrorServerInternal) + } + return nil +} + // instanceArchitecture returns the AWS architecture string from the instance type. // Returns "" on nil/malformed spec; the caller's firmware probe surfaces the error. func instanceArchitecture(it *ec2.InstanceTypeInfo) string { diff --git a/spinifex/handlers/ec2/instance/service_impl_test.go b/spinifex/handlers/ec2/instance/service_impl_test.go index 766dc22df..f3d834208 100644 --- a/spinifex/handlers/ec2/instance/service_impl_test.go +++ b/spinifex/handlers/ec2/instance/service_impl_test.go @@ -14,12 +14,12 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" "github.com/mulgadc/spinifex/spinifex/awserrors" "github.com/mulgadc/spinifex/spinifex/config" + "github.com/mulgadc/spinifex/spinifex/ebsmetadata" "github.com/mulgadc/spinifex/spinifex/gpu" "github.com/mulgadc/spinifex/spinifex/tags" spxtypes "github.com/mulgadc/spinifex/spinifex/types" "github.com/mulgadc/spinifex/spinifex/utils" "github.com/mulgadc/spinifex/spinifex/vm" - "github.com/mulgadc/viperblock/viperblock" "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" @@ -275,7 +275,7 @@ func TestRunInstance_UniqueIDs(t *testing.T) { } func TestFloorVolumeSizeToAMI(t *testing.T) { - loader := &fakeAMILoader{byID: map[string]viperblock.AMIMetadata{ + loader := &fakeAMILoader{byID: map[string]ebsmetadata.AMI{ "ami-rocky": {VolumeSizeGiB: 10}, "ami-debian": {VolumeSizeGiB: 3}, "ami-no-size": {VolumeSizeGiB: 0}, @@ -2492,7 +2492,7 @@ func (raceVolumeMounter) UnmountOne(spxtypes.EBSRequest) error { return nil } // TestStartStoppedInstance_ConcurrentClaimRace is the regression test for // the double-start bug this claim closes: two nodes (or a forwarded call // racing a local fallback) could both observe the same stopped instance as -// claimable and both launch it onto the same viperblock volume. It fires two +// claimable and both launch it onto the same EBS volume. It fires two // concurrent StartStoppedInstance calls at the same stopped instance id and // asserts the atomic claim lets exactly one of them proceed past it — // reaching resourceMgr.Allocate / vmMgr.Insert / vmMgr.Run — while the other @@ -2576,18 +2576,35 @@ func TestStartStoppedInstance_ConcurrentClaimRace(t *testing.T) { // --- PrepareRunInstances / ec2.cmd dispatch tests --------------------------- type fakeAMILoader struct { - byID map[string]viperblock.AMIMetadata - err error + byID map[string]ebsmetadata.AMI + err error + sourceByID map[string]string + sourceErr error } -func (f *fakeAMILoader) GetAMIConfig(_ context.Context, id string) (viperblock.AMIMetadata, error) { +func (f *fakeAMILoader) GetAMIConfig(_ context.Context, id string) (ebsmetadata.AMI, error) { if f.err != nil { - return viperblock.AMIMetadata{}, f.err + return ebsmetadata.AMI{}, f.err } if meta, ok := f.byID[id]; ok { return meta, nil } - return viperblock.AMIMetadata{}, errors.New("not found") + return ebsmetadata.AMI{}, errors.New("not found") +} + +// GetAMISourceVolumeID defaults to the bundled system AMI convention, where the +// snapshot's source volume is named after the AMI itself. +func (f *fakeAMILoader) GetAMISourceVolumeID(_ context.Context, id string) (string, error) { + if f.sourceErr != nil { + return "", f.sourceErr + } + if volumeID, ok := f.sourceByID[id]; ok { + return volumeID, nil + } + if _, ok := f.byID[id]; !ok { + return "", errors.New("not found") + } + return id, nil } type fakeKeyValidator struct { @@ -2665,7 +2682,7 @@ func TestPrepareRunInstances_AMINotOwnedByCaller(t *testing.T) { types, _ := defaultPrepareInstanceTypes() svc := &InstanceServiceImpl{ instanceTypes: types, - amiLoader: &fakeAMILoader{byID: map[string]viperblock.AMIMetadata{ + amiLoader: &fakeAMILoader{byID: map[string]ebsmetadata.AMI{ "ami-other": {ImageOwnerAlias: "999988887777"}, }}, } @@ -2681,7 +2698,7 @@ func TestPrepareRunInstances_KeyPairNotFound(t *testing.T) { types, _ := defaultPrepareInstanceTypes() svc := &InstanceServiceImpl{ instanceTypes: types, - amiLoader: &fakeAMILoader{byID: map[string]viperblock.AMIMetadata{ + amiLoader: &fakeAMILoader{byID: map[string]ebsmetadata.AMI{ "ami-1": {ImageOwnerAlias: "acc"}, }}, keyValidator: &fakeKeyValidator{err: errors.New("no key")}, @@ -2706,7 +2723,7 @@ func TestPrepareRunInstances_InsufficientCapacity(t *testing.T) { _ = it svc := &InstanceServiceImpl{ instanceTypes: types, - amiLoader: &fakeAMILoader{byID: map[string]viperblock.AMIMetadata{ + amiLoader: &fakeAMILoader{byID: map[string]ebsmetadata.AMI{ "ami-1": {ImageOwnerAlias: "acc"}, }}, resourceMgr: prov, @@ -2730,7 +2747,7 @@ func TestPrepareRunInstances_HappyPathNoENI(t *testing.T) { svc := &InstanceServiceImpl{ config: &config.Config{}, instanceTypes: types, - amiLoader: &fakeAMILoader{byID: map[string]viperblock.AMIMetadata{ + amiLoader: &fakeAMILoader{byID: map[string]ebsmetadata.AMI{ "ami-1": {ImageOwnerAlias: "acc"}, }}, resourceMgr: prov, @@ -2764,7 +2781,7 @@ func TestPrepareRunInstances_PersistsIamInstanceProfile(t *testing.T) { svc := &InstanceServiceImpl{ config: &config.Config{}, instanceTypes: types, - amiLoader: &fakeAMILoader{byID: map[string]viperblock.AMIMetadata{ + amiLoader: &fakeAMILoader{byID: map[string]ebsmetadata.AMI{ "ami-1": {ImageOwnerAlias: "acc"}, }}, resourceMgr: prov, @@ -2798,7 +2815,7 @@ func TestPrepareRunInstances_ConsumesReservation(t *testing.T) { svc := &InstanceServiceImpl{ config: &config.Config{}, instanceTypes: types, - amiLoader: &fakeAMILoader{byID: map[string]viperblock.AMIMetadata{ + amiLoader: &fakeAMILoader{byID: map[string]ebsmetadata.AMI{ "ami-1": {ImageOwnerAlias: "acc"}, }}, resourceMgr: prov, @@ -2829,7 +2846,7 @@ func TestPrepareRunInstances_ReservationCapsNoSpill(t *testing.T) { svc := &InstanceServiceImpl{ config: &config.Config{}, instanceTypes: types, - amiLoader: &fakeAMILoader{byID: map[string]viperblock.AMIMetadata{ + amiLoader: &fakeAMILoader{byID: map[string]ebsmetadata.AMI{ "ami-1": {ImageOwnerAlias: "acc"}, }}, resourceMgr: prov, @@ -2856,7 +2873,7 @@ func TestPrepareRunInstances_ReservationExceeded(t *testing.T) { svc := &InstanceServiceImpl{ config: &config.Config{}, instanceTypes: types, - amiLoader: &fakeAMILoader{byID: map[string]viperblock.AMIMetadata{ + amiLoader: &fakeAMILoader{byID: map[string]ebsmetadata.AMI{ "ami-1": {ImageOwnerAlias: "acc"}, }}, resourceMgr: prov, @@ -2886,7 +2903,7 @@ func TestPrepareRunInstances_ReservationRollbackNoGeneralPoolLeak(t *testing.T) svc := &InstanceServiceImpl{ config: &config.Config{Region: "us-east-1", AZ: "us-east-1a"}, instanceTypes: types, - amiLoader: &fakeAMILoader{byID: map[string]viperblock.AMIMetadata{ + amiLoader: &fakeAMILoader{byID: map[string]ebsmetadata.AMI{ "ami-1": {ImageOwnerAlias: "acc"}, }}, resourceMgr: prov, @@ -2918,7 +2935,7 @@ func TestPrepareRunInstances_AmiLaunchIndexContiguous(t *testing.T) { svc := &InstanceServiceImpl{ config: &config.Config{}, instanceTypes: types, - amiLoader: &fakeAMILoader{byID: map[string]viperblock.AMIMetadata{ + amiLoader: &fakeAMILoader{byID: map[string]ebsmetadata.AMI{ "ami-1": {ImageOwnerAlias: "acc"}, }}, resourceMgr: prov, @@ -2994,7 +3011,7 @@ func TestPrepareRunInstances_BootModePropagated(t *testing.T) { svc := &InstanceServiceImpl{ config: &config.Config{}, instanceTypes: types, - amiLoader: &fakeAMILoader{byID: map[string]viperblock.AMIMetadata{ + amiLoader: &fakeAMILoader{byID: map[string]ebsmetadata.AMI{ "ami-1": {ImageOwnerAlias: "acc", BootMode: tc.amiBootMode}, }}, resourceMgr: prov, @@ -3189,7 +3206,7 @@ func prepareSvcWithENI(t *testing.T, eni *fakeENICreator, ipam *fakeIPAllocator) svc := &InstanceServiceImpl{ config: &config.Config{Region: "us-east-1", AZ: "us-east-1a"}, instanceTypes: types, - amiLoader: &fakeAMILoader{byID: map[string]viperblock.AMIMetadata{ + amiLoader: &fakeAMILoader{byID: map[string]ebsmetadata.AMI{ "ami-1": {ImageOwnerAlias: "acc"}, }}, resourceMgr: prov, @@ -3571,7 +3588,7 @@ func TestPrepareRunInstances_PlacementGroup(t *testing.T) { svc := &InstanceServiceImpl{ config: &config.Config{}, instanceTypes: types, - amiLoader: &fakeAMILoader{byID: map[string]viperblock.AMIMetadata{ + amiLoader: &fakeAMILoader{byID: map[string]ebsmetadata.AMI{ "ami-1": {ImageOwnerAlias: "acc"}, }}, resourceMgr: prov, @@ -3598,7 +3615,7 @@ func TestPrepareRunInstances_AllocateFailsMidLoop(t *testing.T) { svc := &InstanceServiceImpl{ config: &config.Config{}, instanceTypes: types, - amiLoader: &fakeAMILoader{byID: map[string]viperblock.AMIMetadata{ + amiLoader: &fakeAMILoader{byID: map[string]ebsmetadata.AMI{ "ami-1": {ImageOwnerAlias: "acc"}, }}, resourceMgr: prov, diff --git a/spinifex/handlers/ec2/snapshot/drain_test.go b/spinifex/handlers/ec2/snapshot/drain_test.go index d6d98911e..927b224dd 100644 --- a/spinifex/handlers/ec2/snapshot/drain_test.go +++ b/spinifex/handlers/ec2/snapshot/drain_test.go @@ -12,12 +12,13 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" "github.com/mulgadc/spinifex/spinifex/awserrors" "github.com/mulgadc/spinifex/spinifex/config" + "github.com/mulgadc/spinifex/spinifex/ebsmetadata" + "github.com/mulgadc/spinifex/spinifex/ebsprovider" "github.com/mulgadc/spinifex/spinifex/handlers/ec2/volumestate" "github.com/mulgadc/spinifex/spinifex/objectstore" "github.com/mulgadc/spinifex/spinifex/testutil" "github.com/mulgadc/spinifex/spinifex/types" "github.com/mulgadc/spinifex/spinifex/utils" - "github.com/mulgadc/viperblock/viperblock" "github.com/nats-io/nats.go" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -124,7 +125,7 @@ func TestDrainVolume_AttachedRoutesToHostingNode(t *testing.T) { seedVolumeAttachment(t, store, "vol-attached", "in-use", drainInstanceID) got := drainResponder(t, nc, drainInstanceID, drainedAck(t, "vol-attached")) - require.NoError(t, svc.drainVolume(context.Background(), "vol-attached", viperblock.VolumeMetadata{}, testAccountID)) + require.NoError(t, svc.drainVolume(context.Background(), "vol-attached", "", "", testAccountID)) command := awaitDrainCommand(t, got) assert.True(t, command.Attributes.DrainVolume) @@ -141,7 +142,7 @@ func TestDrainVolume_AttachedToStoppedInstanceTakesStoppedPath(t *testing.T) { svc, store, _ := setupDrainService(t) seedVolumeAttachment(t, store, "vol-no-host", "in-use", drainInstanceID) - require.NoError(t, svc.drainVolume(context.Background(), "vol-no-host", viperblock.VolumeMetadata{}, testAccountID)) + require.NoError(t, svc.drainVolume(context.Background(), "vol-no-host", "", "", testAccountID)) } // A host that still holds the instance but reports it not running has nothing @@ -156,7 +157,7 @@ func TestDrainVolume_NotRunningAckTakesStoppedPath(t *testing.T) { require.NoError(t, err) got := drainResponder(t, nc, drainInstanceID, ack) - require.NoError(t, svc.drainVolume(context.Background(), "vol-not-running", viperblock.VolumeMetadata{}, testAccountID)) + require.NoError(t, svc.drainVolume(context.Background(), "vol-not-running", "", "", testAccountID)) assert.True(t, awaitDrainCommand(t, got).Attributes.DrainVolume) } @@ -168,7 +169,7 @@ func TestDrainVolume_AttachedWithFailedAckFails(t *testing.T) { drainResponder(t, nc, drainInstanceID, []byte(`{"Code":"`+awserrors.ErrorServerInternal+`","Message":"drain failed"}`)) - err := svc.drainVolume(context.Background(), "vol-drain-err", viperblock.VolumeMetadata{}, testAccountID) + err := svc.drainVolume(context.Background(), "vol-drain-err", "", "", testAccountID) require.Error(t, err) assert.Contains(t, err.Error(), "drain failed") } @@ -179,7 +180,7 @@ func TestDrainVolume_AttachedWithUnexpectedAckFails(t *testing.T) { seedVolumeAttachment(t, store, "vol-odd-ack", "in-use", drainInstanceID) drainResponder(t, nc, drainInstanceID, []byte(`{}`)) - err := svc.drainVolume(context.Background(), "vol-odd-ack", viperblock.VolumeMetadata{}, testAccountID) + err := svc.drainVolume(context.Background(), "vol-odd-ack", "", "", testAccountID) require.Error(t, err) assert.Contains(t, err.Error(), "unexpected ack") } @@ -200,7 +201,7 @@ func TestDrainVolume_AvailableDoesNotDialLocalSocket(t *testing.T) { svc.natsConn = nil requireReturnsWithin(t, 2*time.Second, func() error { - return svc.drainVolume(context.Background(), "vol-available", viperblock.VolumeMetadata{}, testAccountID) + return svc.drainVolume(context.Background(), "vol-available", "", "", testAccountID) }) } @@ -211,7 +212,7 @@ func TestDrainVolume_InUseWithoutInstanceTakesStoppedPath(t *testing.T) { seedVolumeAttachment(t, store, "vol-no-instance", "in-use", "") svc.natsConn = nil - require.NoError(t, svc.drainVolume(context.Background(), "vol-no-instance", viperblock.VolumeMetadata{}, testAccountID)) + require.NoError(t, svc.drainVolume(context.Background(), "vol-no-instance", "", "", testAccountID)) } // When the volume is served by this node the socket is local, so the drain @@ -225,7 +226,7 @@ func TestDrainVolume_LocalSocketShortCircuits(t *testing.T) { // drainInstanceID, so a routed command would fail. svc.natsConn = nil - require.NoError(t, svc.drainVolume(context.Background(), "vol-local", viperblock.VolumeMetadata{}, testAccountID)) + require.NoError(t, svc.drainVolume(context.Background(), "vol-local", "", "", testAccountID)) } // A socket that answers without acking means this node does serve the volume @@ -237,7 +238,7 @@ func TestDrainVolume_LocalSocketErrorAckFailsWithoutRouting(t *testing.T) { testutil.StartDrainSocket(t, svc.config.DataDir, "vol-local-err", "ERR\n") got := drainResponder(t, nc, drainInstanceID, drainedAck(t, "vol-local-err")) - err := svc.drainVolume(context.Background(), "vol-local-err", viperblock.VolumeMetadata{}, testAccountID) + err := svc.drainVolume(context.Background(), "vol-local-err", "", "", testAccountID) require.Error(t, err) assert.Empty(t, got, "a local drain that failed must not be re-issued to the host node") @@ -251,8 +252,9 @@ func TestDrainVolume_StateJSONOverridesConfigJSON(t *testing.T) { seedVolumeAttachment(t, store, "vol-stale-config", "in-use", drainInstanceID) got := drainResponder(t, nc, drainInstanceID, drainedAck(t, "vol-stale-config")) - stale := viperblock.VolumeMetadata{State: "available"} - require.NoError(t, svc.drainVolume(context.Background(), "vol-stale-config", stale, testAccountID)) + // "available" mirrors the stale State a live NBD plugin leaves in + // config.json; drainVolume must not trust it (see the test's own doc above). + require.NoError(t, svc.drainVolume(context.Background(), "vol-stale-config", "available", "", testAccountID)) assert.True(t, awaitDrainCommand(t, got).Attributes.DrainVolume) } @@ -263,8 +265,8 @@ func TestDrainVolume_FallsBackToConfigJSONWhenNoStateObject(t *testing.T) { svc, _, nc := setupDrainService(t) got := drainResponder(t, nc, drainInstanceID, drainedAck(t, "vol-legacy")) - legacy := viperblock.VolumeMetadata{State: "in-use", AttachedInstance: drainInstanceID} - require.NoError(t, svc.drainVolume(context.Background(), "vol-legacy", legacy, testAccountID)) + // The state/instance embedded in a legacy config.json, with no state.json to overlay it. + require.NoError(t, svc.drainVolume(context.Background(), "vol-legacy", "in-use", drainInstanceID, testAccountID)) assert.True(t, awaitDrainCommand(t, got).Attributes.DrainVolume) } @@ -277,40 +279,66 @@ func TestDrainVolume_MetadataOnlySkipsDrain(t *testing.T) { svc.config.Predastore.Host = "" svc.natsConn = nil - require.NoError(t, svc.drainVolume(context.Background(), "vol-meta-only", viperblock.VolumeMetadata{}, testAccountID)) + require.NoError(t, svc.drainVolume(context.Background(), "vol-meta-only", "", "", testAccountID)) } -// CreateSnapshot drains before it reads the checkpoint. The snapshot itself -// then fails against the rejecting Predastore stub; what is asserted here is -// that the drain was routed to the volume's host first. -func TestCreateSnapshot_DrainsAttachedVolume(t *testing.T) { - svc, store, nc := setupDrainService(t) - createTestVolume(t, store, "vol-snap-drain", 10) - seedVolumeAttachment(t, store, "vol-snap-drain", "in-use", drainInstanceID) - got := drainResponder(t, nc, drainInstanceID, drainedAck(t, "vol-snap-drain")) +// putProviderVolume seeds both the control-plane document CreateSnapshot's +// provider branch reads for attachment state, and the provider's own record, +// mirroring what CreateVolume's provider branch leaves behind. +func putProviderVolume(t *testing.T, svc *SnapshotServiceImpl, provider ebsprovider.EBSProvider, volumeID, state, instanceID string) { + t.Helper() + require.NoError(t, svc.metadata.PutVolume(context.Background(), ebsmetadata.Volume{ + VolumeID: volumeID, TenantID: testAccountID, CapacityGiB: 4, + State: state, AttachedInstance: instanceID, AvailabilityZone: "us-east-1a", + ProviderHandle: "memory://volume/" + volumeID, + })) + _, err := provider.CreateVolume(context.Background(), ebsprovider.CreateVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, + CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 4 * 1024 * 1024 * 1024}, + AvailabilityZone: "us-east-1a", + }) + require.NoError(t, err) +} + +// The provider branch used to skip the drain entirely, silently snapshotting a +// stale checkpoint of an attached, actively-written volume. It must drain the +// same way the legacy branch does. +func TestCreateSnapshot_Provider_DrainsAttachedVolume(t *testing.T) { + svc, _, nc := setupDrainService(t) + provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{}) + svc.SetEBSProvider(provider) + putProviderVolume(t, svc, provider, "vol-provider-drain", "in-use", drainInstanceID) + + got := drainResponder(t, nc, drainInstanceID, drainedAck(t, "vol-provider-drain")) _, err := svc.CreateSnapshot(context.Background(), - &ec2.CreateSnapshotInput{VolumeId: aws.String("vol-snap-drain")}, testAccountID) - require.Error(t, err) + &ec2.CreateSnapshotInput{VolumeId: aws.String("vol-provider-drain")}, testAccountID) + require.NoError(t, err) command := awaitDrainCommand(t, got) assert.True(t, command.Attributes.DrainVolume) - assert.Equal(t, "vol-snap-drain", command.DrainVolumeData.VolumeID) + assert.Equal(t, "vol-provider-drain", command.DrainVolumeData.VolumeID) } -// An attached volume whose host reports the drain failed fails the snapshot: a -// retryable error is strictly better than a successful snapshot of stale data. -func TestCreateSnapshot_UndrainableAttachedVolumeFails(t *testing.T) { - svc, store, nc := setupDrainService(t) - createTestVolume(t, store, "vol-snap-nodrain", 10) - seedVolumeAttachment(t, store, "vol-snap-nodrain", "in-use", drainInstanceID) +// A provider-branch snapshot whose drain fails must fail closed rather than +// return a snapshot of stale data: the provider must never be asked to +// snapshot, and no snapshot metadata may be left behind. +func TestCreateSnapshot_Provider_UndrainableAttachedVolumeFails(t *testing.T) { + svc, _, nc := setupDrainService(t) + provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{}) + svc.SetEBSProvider(provider) + putProviderVolume(t, svc, provider, "vol-provider-nodrain", "in-use", drainInstanceID) + got := drainResponder(t, nc, drainInstanceID, []byte(`{"Code":"`+awserrors.ErrorServerInternal+`","Message":"drain failed"}`)) _, err := svc.CreateSnapshot(context.Background(), - &ec2.CreateSnapshotInput{VolumeId: aws.String("vol-snap-nodrain")}, testAccountID) + &ec2.CreateSnapshotInput{VolumeId: aws.String("vol-provider-nodrain")}, testAccountID) require.Error(t, err) assert.Equal(t, awserrors.ErrorServerInternal, err.Error()) - assert.True(t, awaitDrainCommand(t, got).Attributes.DrainVolume) + + out, err := svc.DescribeSnapshots(context.Background(), &ec2.DescribeSnapshotsInput{}, testAccountID) + require.NoError(t, err) + assert.Empty(t, out.Snapshots, "a failed drain must not leave a snapshot behind") } diff --git a/spinifex/handlers/ec2/snapshot/service_impl.go b/spinifex/handlers/ec2/snapshot/service_impl.go index c4c83b5ae..693437ca9 100644 --- a/spinifex/handlers/ec2/snapshot/service_impl.go +++ b/spinifex/handlers/ec2/snapshot/service_impl.go @@ -6,9 +6,7 @@ import ( "encoding/json" "errors" "fmt" - "io" "log/slog" - "maps" "net" "path/filepath" "strconv" @@ -21,6 +19,8 @@ import ( "github.com/aws/aws-sdk-go/service/s3" "github.com/mulgadc/spinifex/spinifex/awserrors" "github.com/mulgadc/spinifex/spinifex/config" + "github.com/mulgadc/spinifex/spinifex/ebsmetadata" + "github.com/mulgadc/spinifex/spinifex/ebsprovider" "github.com/mulgadc/spinifex/spinifex/filterutil" "github.com/mulgadc/spinifex/spinifex/handlers/ec2/volumestate" "github.com/mulgadc/spinifex/spinifex/kvutil" @@ -28,8 +28,6 @@ import ( "github.com/mulgadc/spinifex/spinifex/objectstore" "github.com/mulgadc/spinifex/spinifex/types" "github.com/mulgadc/spinifex/spinifex/utils" - "github.com/mulgadc/viperblock/viperblock" - vbs3 "github.com/mulgadc/viperblock/viperblock/backends/s3" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" ) @@ -50,6 +48,26 @@ type SnapshotServiceImpl struct { natsConn *nats.Conn snapKV jetstream.KeyValue mutex sync.RWMutex + metadata *ebsmetadata.Store + provider ebsprovider.EBSProvider +} + +// SetEBSProvider injects the provider boundary for snapshot operations. +// Legacy snapshot behavior remains until snapshot metadata is migrated. +func (s *SnapshotServiceImpl) SetEBSProvider(provider ebsprovider.EBSProvider) { + s.provider = provider +} + +// EBSProvider returns the injected provider boundary, or nil on the legacy +// embedded-engine path. Primarily for composition-root tests to observe wiring. +func (s *SnapshotServiceImpl) EBSProvider() ebsprovider.EBSProvider { + return s.provider +} + +// MetadataStore returns the control-plane metadata store. Primarily for +// composition-root tests to observe wiring. +func (s *SnapshotServiceImpl) MetadataStore() *ebsmetadata.Store { + return s.metadata } // SnapshotConfig represents snapshot metadata stored in S3. @@ -65,6 +83,7 @@ type SnapshotConfig struct { OwnerID string `json:"owner_id"` AvailabilityZone string `json:"availability_zone"` Tags map[string]string `json:"tags"` + ProviderHandle string `json:"provider_handle,omitempty"` } // NewSnapshotServiceImplWithNATS creates a snapshot service with JetStream KV for volume-snapshot tracking. @@ -96,6 +115,7 @@ func NewSnapshotServiceImplWithNATS(ctx context.Context, cfg *config.Config, nat store: store, natsConn: natsConn, snapKV: kv, + metadata: ebsmetadata.NewStore(store, cfg.Predastore.Bucket), }, kv, nil } @@ -106,6 +126,7 @@ func NewSnapshotServiceImplWithStore(cfg *config.Config, store objectstore.Objec config: cfg, store: store, natsConn: natsConn, + metadata: ebsmetadata.NewStore(store, cfg.Predastore.Bucket), } if len(snapshotKV) > 0 { svc.snapKV = snapshotKV[0] @@ -206,79 +227,49 @@ func (s *SnapshotServiceImpl) CreateSnapshot(ctx context.Context, input *ec2.Cre slog.InfoContext(ctx, "CreateSnapshot request", "volumeId", volumeID) snapshotID := utils.GenerateResourceID("snap") + if s.provider == nil { + slog.ErrorContext(ctx, "no EBS provider configured", "op", "CreateSnapshot") + return nil, errors.New(awserrors.ErrorServerInternal) + } - volumeConfigKey := fmt.Sprintf("%s/config.json", volumeID) - volumeResult, err := s.store.GetObject(ctx, &s3.GetObjectInput{ - Bucket: aws.String(s.config.Predastore.Bucket), - Key: aws.String(volumeConfigKey), - }) - if err != nil { - slog.ErrorContext(ctx, "CreateSnapshot failed to get volume config", "volumeId", volumeID, "err", err) + // A volume predating tenancy records no owner. Refusing it would strand + // every pre-tenancy volume as unsnapshottable, so an absent TenantID stays + // open to any caller; a recorded one must match. + volume, err := s.metadata.GetVolume(ctx, volumeID) + if err != nil || (accountID != "" && volume.TenantID != "" && volume.TenantID != accountID) { return nil, errors.New(awserrors.ErrorInvalidVolumeNotFound) } - defer volumeResult.Body.Close() - - volumeBody, err := io.ReadAll(volumeResult.Body) - if err != nil { - slog.ErrorContext(ctx, "CreateSnapshot failed to read volume config", "volumeId", volumeID, "err", err) + if volume.CapacityGiB == 0 { return nil, errors.New(awserrors.ErrorServerInternal) } - // config.json may be an at-rest encryption envelope; StateBody unwraps it to - // the inner VBState. Decoding the raw envelope yields a zero state - // (SizeGiB==0), which the size guard below would reject as a 500. - var volumeState viperblock.VBState - if err := json.Unmarshal(viperblock.StateBody(volumeBody), &volumeState); err != nil { - slog.ErrorContext(ctx, "CreateSnapshot failed to decode volume config", "volumeId", volumeID, "err", err) - return nil, errors.New(awserrors.ErrorServerInternal) - } - volumeConfig := volumeState.VolumeConfig - - // Verify the caller owns the source volume - if accountID != "" && volumeConfig.VolumeMetadata.TenantID != "" && volumeConfig.VolumeMetadata.TenantID != accountID { - slog.WarnContext(ctx, "CreateSnapshot: account does not own volume", "volumeId", volumeID, "accountID", accountID, "tenantID", volumeConfig.VolumeMetadata.TenantID) - return nil, errors.New(awserrors.ErrorInvalidVolumeNotFound) - } - - if volumeConfig.VolumeMetadata.SizeGiB == 0 { - slog.ErrorContext(ctx, "CreateSnapshot: source volume has zero size in config", "volumeId", volumeID) - return nil, errors.New(awserrors.ErrorServerInternal) - } - - // Flush the writes still buffered by whichever node serves the volume, so the - // live checkpoint this snapshot is about to read is current. An attached - // volume that cannot be drained fails here rather than silently producing a - // snapshot of an older checkpoint. - if err := s.drainVolume(ctx, volumeID, volumeConfig.VolumeMetadata, accountID); err != nil { + // Flush the writes buffered by whichever node serves the volume, so the + // provider reads a current checkpoint. An attached volume that cannot be + // drained fails here rather than silently snapshotting stale data. + if err := s.drainVolume(ctx, volumeID, volume.State, volume.AttachedInstance, accountID); err != nil { slog.ErrorContext(ctx, "CreateSnapshot: drain failed", "volumeId", volumeID, "snapshotId", snapshotID, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } - // Snapshot the viperblock volume by reading the live checkpoint from S3. - // The live checkpoint is updated on every NBD Flush by the running nbdkit process. - // If the volume is not mounted (stopped), LoadLiveCheckpoint falls back to the - // numbered checkpoint written by Close. - if err := s.snapshotVolume(volumeID, snapshotID, volumeConfig.VolumeMetadata.SizeGiB*1024*1024*1024); err != nil { - slog.ErrorContext(ctx, "CreateSnapshot: viperblock snapshot failed", "volumeId", volumeID, "snapshotId", snapshotID, "err", err) + created, err := s.provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{ + Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, VolumeID: volumeID, VolumeHandle: volume.ProviderHandle, + }) + if err != nil || created == nil { return nil, errors.New(awserrors.ErrorServerInternal) } - slog.InfoContext(ctx, "CreateSnapshot: viperblock snapshot created", "volumeId", volumeID, "snapshotId", snapshotID) - now := time.Now() - snapshotCfg := &SnapshotConfig{ - SnapshotID: snapshotID, - VolumeID: volumeID, - VolumeSize: utils.SafeUint64ToInt64(volumeConfig.VolumeMetadata.SizeGiB), - State: "completed", - Progress: "100%", - StartTime: now, - Encrypted: volumeState.EncryptionEnabled, - OwnerID: accountID, - AvailabilityZone: volumeConfig.VolumeMetadata.AvailabilityZone, - Tags: utils.ExtractTags(input.TagSpecifications, "snapshot"), + SnapshotID: snapshotID, VolumeID: volumeID, + VolumeSize: utils.SafeUint64ToInt64(volume.CapacityGiB), + State: string(created.State), Progress: "100%", StartTime: now, + Encrypted: volume.Encrypted, + OwnerID: accountID, AvailabilityZone: volume.AvailabilityZone, + Tags: utils.ExtractTags(input.TagSpecifications, "snapshot"), + ProviderHandle: created.Handle, + } + if snapshotCfg.State == "" { + snapshotCfg.State = "completed" } - if input.Description != nil { snapshotCfg.Description = *input.Description } @@ -287,14 +278,13 @@ func (s *SnapshotServiceImpl) CreateSnapshot(ctx context.Context, input *ec2.Cre // This ensures we never have an untracked snapshot in S3. if err := s.addSnapshotRef(ctx, volumeID, snapshotID); err != nil { slog.ErrorContext(ctx, "CreateSnapshot failed to add snapshot ref to KV", "snapshotId", snapshotID, "volumeId", volumeID, "err", err) + _ = s.provider.DeleteSnapshot(ctx, ebsprovider.DeleteSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, Handle: created.Handle}) return nil, errors.New(awserrors.ErrorServerInternal) } - if err := s.putSnapshotConfig(snapshotID, snapshotCfg); err != nil { slog.ErrorContext(ctx, "CreateSnapshot failed to write config", "snapshotId", snapshotID, "err", err) - if cleanupErr := s.removeSnapshotRefForCleanup(ctx, volumeID, snapshotID); cleanupErr != nil { - slog.Error("CreateSnapshot failed to roll back snapshot reference", "snapshotId", snapshotID, "volumeId", volumeID, "err", cleanupErr) - } + _ = s.removeSnapshotRefForCleanup(ctx, volumeID, snapshotID) + _ = s.provider.DeleteSnapshot(ctx, ebsprovider.DeleteSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, Handle: created.Handle}) return nil, errors.New(awserrors.ErrorServerInternal) } @@ -353,8 +343,8 @@ func DrainVolumeSocket(dataDir, volumeID string) error { // drainVolume flushes the volume's in-flight writes to S3 before the snapshot // reads the live checkpoint from there. -func (s *SnapshotServiceImpl) drainVolume(ctx context.Context, volumeID string, meta viperblock.VolumeMetadata, accountID string) error { - return DrainVolume(ctx, s.config, s.store, s.natsConn, volumeID, meta, accountID) +func (s *SnapshotServiceImpl) drainVolume(ctx context.Context, volumeID string, fallbackState, fallbackInstance string, accountID string) error { + return DrainVolume(ctx, s.config, s.store, s.natsConn, volumeID, fallbackState, fallbackInstance, accountID) } // DrainVolume flushes the volume's in-flight writes to S3 before a live @@ -368,14 +358,14 @@ func (s *SnapshotServiceImpl) drainVolume(ctx context.Context, volumeID string, // volume's attachment record, never from whether the socket happens to be // local: an attached volume is being written to right now, and reading its // checkpoint without draining silently captures an older one. -func DrainVolume(ctx context.Context, cfg *config.Config, store objectstore.ObjectStore, natsConn *nats.Conn, volumeID string, meta viperblock.VolumeMetadata, accountID string) error { +func DrainVolume(ctx context.Context, cfg *config.Config, store objectstore.ObjectStore, natsConn *nats.Conn, volumeID string, fallbackState, fallbackInstance string, accountID string) error { // A metadata-only snapshot never reads the live checkpoint, so there is // nothing for a drain to make current. if cfg == nil || cfg.Predastore.Host == "" { return nil } - state, instanceID, err := volumeAttachment(ctx, store, cfg.Predastore.Bucket, volumeID, meta) + state, instanceID, err := volumeAttachment(ctx, store, cfg.Predastore.Bucket, volumeID, fallbackState, fallbackInstance) if err != nil { return err } @@ -410,10 +400,11 @@ func DrainVolume(ctx context.Context, cfg *config.Config, store objectstore.Obje } // volumeAttachment returns the volume's authoritative state and attached -// instance. state.json is control-plane-owned and authoritative; the copy in -// config.json (passed in as meta) is rewritten by the live NBD plugin from its -// stale in-memory state and is only used for volumes predating the split. -func volumeAttachment(ctx context.Context, store objectstore.ObjectStore, bucket, volumeID string, meta viperblock.VolumeMetadata) (state, instanceID string, err error) { +// instance. state.json is control-plane-owned and authoritative; the fallback +// pair (passed in from the caller's own volume record) is rewritten by the +// live NBD plugin from its stale in-memory state and is only used for volumes +// predating the split. +func volumeAttachment(ctx context.Context, store objectstore.ObjectStore, bucket, volumeID string, fallbackState, fallbackInstance string) (state, instanceID string, err error) { rec, found, err := volumestate.Read(ctx, store, bucket, volumeID) if err != nil { return "", "", fmt.Errorf("read volume state for %s: %w", volumeID, err) @@ -421,7 +412,7 @@ func volumeAttachment(ctx context.Context, store objectstore.ObjectStore, bucket if found { return rec.State, rec.AttachedInstance, nil } - return meta.State, meta.AttachedInstance, nil + return fallbackState, fallbackInstance, nil } // drainOnHostNode issues the drain on the node hosting instanceID. Only that @@ -473,66 +464,6 @@ func drainOnHostNode(ctx context.Context, natsConn *nats.Conn, volumeID, instanc } } -// snapshotVolume opens a read-only viperblock instance, reads the live checkpoint from S3 -// (written by nbdkit on every NBD Flush, and by the drain the caller has already run), -// and calls CreateSnapshot. Falls back to the numbered checkpoint from Close if no live -// checkpoint exists (stopped volume path). -// If Predastore is not configured the snapshot proceeds as metadata-only. -func (s *SnapshotServiceImpl) snapshotVolume(volumeID, snapshotID string, volumeSize uint64) error { - if s.config == nil || s.config.Predastore.Host == "" { - slog.Warn("snapshotVolume: Predastore not configured, skipping viperblock snapshot (metadata-only)", "volumeId", volumeID) - return nil - } - - cfg := vbs3.S3Config{ - VolumeName: volumeID, - VolumeSize: volumeSize, - Bucket: s.config.Predastore.Bucket, - Region: s.config.Predastore.Region, - AccessKey: s.config.Predastore.AccessKey, - SecretKey: s.config.Predastore.SecretKey, - Host: s.config.Predastore.Host, - } - - mkey, err := utils.LoadViperblockMasterKey(s.config.Viperblock.EncryptionKeyFile) - if err != nil { - return fmt.Errorf("load encryption key: %w", err) - } - - vbconfig := viperblock.VB{ - VolumeName: volumeID, - VolumeSize: volumeSize, - BaseDir: s.config.WalDir, - Cache: viperblock.Cache{Config: viperblock.CacheConfig{Size: 0}}, - MasterKey: mkey, - EncryptionEnabled: mkey != nil, - } - - vb, err := viperblock.New(&vbconfig, "s3", cfg) - if err != nil { - return fmt.Errorf("new viperblock: %w", err) - } - defer func() { - vb.StopChunkUploader() - vb.StopWALSyncer() - }() - - if err := vb.Backend.Init(); err != nil { - return fmt.Errorf("backend init: %w", err) - } - if err := vb.LoadState(); err != nil { - return fmt.Errorf("load state: %w", err) - } - - if err := vb.LoadLiveCheckpoint(); err != nil { - return fmt.Errorf("load live checkpoint: %w", err) - } - if _, err := vb.CreateSnapshot(snapshotID); err != nil { - return fmt.Errorf("create snapshot: %w", err) - } - return nil -} - // describeSnapshotsValidFilters defines the set of filter names accepted by DescribeSnapshots. var describeSnapshotsValidFilters = map[string]bool{ "snapshot-id": true, @@ -686,49 +617,19 @@ func snapshotMatchesFilters(cfg *SnapshotConfig, filters map[string][]string) bo return filterutil.MatchesTags(filters, cfg.Tags) } -// snapshotInUseByVolumes checks if any volume was created from the given snapshot. +// snapshotInUseByVolumes checks if any volume was created from the given +// snapshot. A clone records its source in its ebsmetadata document; missing one +// here would let a delete strip the chunks a live clone still reads. func (s *SnapshotServiceImpl) snapshotInUseByVolumes(ctx context.Context, snapshotID string) (bool, error) { - listResult, err := s.store.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ - Bucket: aws.String(s.config.Predastore.Bucket), - Prefix: aws.String("vol-"), - Delimiter: aws.String("/"), - }) + volumes, err := s.metadata.ListVolumes(ctx) if err != nil { return false, fmt.Errorf("snapshotInUseByVolumes: failed to list volumes: %w", err) } - - for _, prefix := range listResult.CommonPrefixes { - if prefix.Prefix == nil { - continue - } - volumeID := strings.TrimSuffix(*prefix.Prefix, "/") - configKey := fmt.Sprintf("%s/config.json", volumeID) - - result, err := s.store.GetObject(ctx, &s3.GetObjectInput{ - Bucket: aws.String(s.config.Predastore.Bucket), - Key: aws.String(configKey), - }) - if err != nil { - continue // volume may not have a config yet - } - - scanBody, readErr := io.ReadAll(result.Body) - _ = result.Body.Close() - if readErr != nil { - continue - } - // Unwrap the encryption envelope so encrypted volumes are scanned too; - // a raw decode yields a zero state and silently drops their snapshots. - var state viperblock.VBState - if decodeErr := json.Unmarshal(viperblock.StateBody(scanBody), &state); decodeErr != nil { - continue - } - - if state.VolumeConfig.VolumeMetadata.SnapshotID == snapshotID { + for _, volume := range volumes { + if volume.SnapshotID == snapshotID { return true, nil } } - return false, nil } @@ -764,6 +665,13 @@ func (s *SnapshotServiceImpl) DeleteSnapshot(ctx context.Context, input *ec2.Del slog.InfoContext(ctx, "DeleteSnapshot blocked: snapshot in use by volume", "snapshotId", snapshotID) return nil, errors.New(awserrors.ErrorInvalidSnapshotInUse) } + if s.provider == nil { + slog.ErrorContext(ctx, "no EBS provider configured", "op", "DeleteSnapshot") + return nil, errors.New(awserrors.ErrorServerInternal) + } + if err := s.provider.DeleteSnapshot(ctx, ebsprovider.DeleteSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, Handle: cfg.ProviderHandle}); err != nil { + return nil, errors.New(awserrors.ErrorServerInternal) + } listResult, err := s.store.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ Bucket: aws.String(s.config.Predastore.Bucket), @@ -798,8 +706,10 @@ func (s *SnapshotServiceImpl) DeleteSnapshot(ctx context.Context, input *ec2.Del return &ec2.DeleteSnapshotOutput{}, nil } -// CopySnapshot copies a snapshot (within same region for now). -// The copied snapshot is owned by the caller's account. +// CopySnapshot duplicates a snapshot under a new ID owned by the caller. +// Served only through the provider: the copy needs its own snapshot metadata +// on the backend, and on an encrypted volume that metadata is AEAD-sealed with +// the snapshot ID in the AAD, so only the storage engine can re-seal it. func (s *SnapshotServiceImpl) CopySnapshot(ctx context.Context, input *ec2.CopySnapshotInput, accountID string) (*ec2.CopySnapshotOutput, error) { if input == nil || input.SourceSnapshotId == nil { return nil, errors.New(awserrors.ErrorInvalidParameterValue) @@ -815,45 +725,59 @@ func (s *SnapshotServiceImpl) CopySnapshot(ctx context.Context, input *ec2.CopyS return nil, err } - // Verify the caller owns the source snapshot if accountID != "" && sourceCfg.OwnerID != "" && sourceCfg.OwnerID != accountID { slog.WarnContext(ctx, "CopySnapshot: account does not own source snapshot", "snapshotId", sourceSnapshotID, "accountID", accountID, "ownerID", sourceCfg.OwnerID) return nil, errors.New(awserrors.ErrorUnauthorizedOperation) } + // Without a provider the only reachable path writes the control-plane + // config alone. That copy described and created volumes fine, then failed + // at attach with no backend metadata behind the new ID, so refuse instead. + if s.provider == nil { + slog.WarnContext(ctx, "CopySnapshot is not supported without an EBS provider", "sourceSnapshotId", sourceSnapshotID) + return nil, errors.New(awserrors.ErrorUnsupportedOperation) + } + newSnapshotID := utils.GenerateResourceID("snap") - - newCfg := &SnapshotConfig{ - SnapshotID: newSnapshotID, - VolumeID: sourceCfg.VolumeID, - VolumeSize: sourceCfg.VolumeSize, - State: "completed", - Progress: "100%", - StartTime: time.Now(), - Description: sourceCfg.Description, - Encrypted: sourceCfg.Encrypted, - OwnerID: accountID, - AvailabilityZone: sourceCfg.AvailabilityZone, - Tags: make(map[string]string), - } - - if input.Description != nil { - newCfg.Description = *input.Description - } - - maps.Copy(newCfg.Tags, sourceCfg.Tags) - - // Track the volume→snapshot dependency in KV before persisting to S3. - if err := s.addSnapshotRef(ctx, sourceCfg.VolumeID, newSnapshotID); err != nil { - slog.ErrorContext(ctx, "CopySnapshot failed to add snapshot ref to KV", "snapshotId", newSnapshotID, "volumeId", sourceCfg.VolumeID, "err", err) + copied, err := s.provider.CopySnapshot(ctx, ebsprovider.CopySnapshotRequest{ + Versioned: ebsprovider.NewVersioned(), + SourceSnapshotID: sourceSnapshotID, + DestinationSnapshotID: newSnapshotID, + VolumeID: sourceCfg.VolumeID, + }) + if err != nil || copied == nil { + slog.ErrorContext(ctx, "CopySnapshot: provider copy failed", "sourceSnapshotId", sourceSnapshotID, "newSnapshotId", newSnapshotID, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } - if err := s.putSnapshotConfig(newSnapshotID, newCfg); err != nil { - slog.ErrorContext(ctx, "CopySnapshot failed to write config", "snapshotId", newSnapshotID, "err", err) - if cleanupErr := s.removeSnapshotRefForCleanup(ctx, sourceCfg.VolumeID, newSnapshotID); cleanupErr != nil { - slog.Error("CopySnapshot failed to roll back snapshot reference", "snapshotId", newSnapshotID, "volumeId", sourceCfg.VolumeID, "err", cleanupErr) - } + now := time.Now() + snapshotCfg := &SnapshotConfig{ + SnapshotID: newSnapshotID, VolumeID: sourceCfg.VolumeID, + VolumeSize: sourceCfg.VolumeSize, + State: string(copied.State), Progress: "100%", StartTime: now, + Encrypted: sourceCfg.Encrypted, + OwnerID: accountID, AvailabilityZone: sourceCfg.AvailabilityZone, + Tags: utils.ExtractTags(input.TagSpecifications, "snapshot"), + ProviderHandle: copied.Handle, + } + if snapshotCfg.State == "" { + snapshotCfg.State = "completed" + } + snapshotCfg.Description = sourceCfg.Description + if input.Description != nil { + snapshotCfg.Description = *input.Description + } + + // The copy pins the same chunks as its source, so it must join the volume's + // snapshot list before it is visible: a copy missing from that list would + // let the volume be deleted out from under it. + if err := s.addSnapshotRef(ctx, sourceCfg.VolumeID, newSnapshotID); err != nil { + _ = s.provider.DeleteSnapshot(ctx, ebsprovider.DeleteSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: newSnapshotID, Handle: copied.Handle}) + return nil, errors.New(awserrors.ErrorServerInternal) + } + if err := s.putSnapshotConfig(newSnapshotID, snapshotCfg); err != nil { + _ = s.removeSnapshotRefForCleanup(ctx, sourceCfg.VolumeID, newSnapshotID) + _ = s.provider.DeleteSnapshot(ctx, ebsprovider.DeleteSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: newSnapshotID, Handle: copied.Handle}) return nil, errors.New(awserrors.ErrorServerInternal) } @@ -861,6 +785,7 @@ func (s *SnapshotServiceImpl) CopySnapshot(ctx context.Context, input *ec2.CopyS return &ec2.CopySnapshotOutput{ SnapshotId: aws.String(newSnapshotID), + Tags: utils.MapToEC2Tags(snapshotCfg.Tags), }, nil } diff --git a/spinifex/handlers/ec2/snapshot/service_impl_test.go b/spinifex/handlers/ec2/snapshot/service_impl_test.go index d9b962349..347364ddf 100644 --- a/spinifex/handlers/ec2/snapshot/service_impl_test.go +++ b/spinifex/handlers/ec2/snapshot/service_impl_test.go @@ -3,9 +3,7 @@ package handlers_ec2_snapshot import ( "context" "encoding/json" - "io" - "net/http" - "net/http/httptest" + "errors" "strings" "testing" @@ -14,9 +12,10 @@ import ( "github.com/aws/aws-sdk-go/service/s3" "github.com/mulgadc/spinifex/spinifex/awserrors" "github.com/mulgadc/spinifex/spinifex/config" + "github.com/mulgadc/spinifex/spinifex/ebsmetadata" + "github.com/mulgadc/spinifex/spinifex/ebsprovider" "github.com/mulgadc/spinifex/spinifex/objectstore" "github.com/mulgadc/spinifex/spinifex/testutil" - "github.com/mulgadc/viperblock/viperblock" "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -25,7 +24,9 @@ import ( const testAccountID = "111122223333" const otherAccountID = "444455556666" -// setupTestSnapshotService creates a snapshot service with in-memory storage for testing. +// setupTestSnapshotService creates a snapshot service with in-memory storage, +// an in-memory EBS provider, and the same legacy volume read fallback the +// composition root wires, so pre-migration config.json fixtures resolve here. func setupTestSnapshotService(t *testing.T) (*SnapshotServiceImpl, *objectstore.MemoryObjectStore) { store := objectstore.NewMemoryObjectStore() cfg := &config.Config{ @@ -36,28 +37,40 @@ func setupTestSnapshotService(t *testing.T) (*SnapshotServiceImpl, *objectstore. } svc := NewSnapshotServiceImplWithStore(cfg, store, nil) + svc.SetEBSProvider(ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})) return svc, store } -// createTestVolume creates a test volume in the mock store -// The real S3 stores VBState (which wraps VolumeConfig), so we match that format. -func createTestVolume(t *testing.T, store *objectstore.MemoryObjectStore, volumeID string, sizeGiB int) { - volumeState := viperblock.VBState{ - VolumeConfig: viperblock.VolumeConfig{ - VolumeMetadata: viperblock.VolumeMetadata{ - SizeGiB: uint64(sizeGiB), - AvailabilityZone: "us-east-1a", - }, - }, - } - data, err := json.Marshal(volumeState) - require.NoError(t, err) +// createTestVolume seeds a volume on both sides: the provider holds the blocks +// a snapshot copies, the document is what the control plane resolves. +func createTestVolume(t *testing.T, svc *SnapshotServiceImpl, store *objectstore.MemoryObjectStore, volumeID string, sizeGiB int) { + t.Helper() + seedProviderVolume(t, svc, volumeID, sizeGiB) + seedVolumeDocument(t, store, ebsmetadata.Volume{ + VolumeID: volumeID, + CapacityGiB: uint64(sizeGiB), + AvailabilityZone: "us-east-1a", + }) +} - _, err = store.PutObject(context.Background(), &s3.PutObjectInput{ - Bucket: aws.String("test-bucket"), - Key: aws.String(volumeID + "/config.json"), - Body: strings.NewReader(string(data)), - ContentType: aws.String("application/json"), +// seedVolumeDocument writes the control-plane document for a volume. +func seedVolumeDocument(t *testing.T, store objectstore.ObjectStore, volume ebsmetadata.Volume) { + t.Helper() + require.NoError(t, ebsmetadata.NewStore(store, "test-bucket").PutVolume(context.Background(), volume)) +} + +// seedProviderVolume gives the service's provider a volume to snapshot. A +// control-plane document alone is not a volume: the provider owns the blocks, +// and CreateSnapshot fails against one it has never allocated. +func seedProviderVolume(t *testing.T, svc *SnapshotServiceImpl, volumeID string, sizeGiB int) { + t.Helper() + if svc == nil || svc.provider == nil { + return + } + _, err := svc.provider.CreateVolume(context.Background(), ebsprovider.CreateVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, + CapacityRange: ebsprovider.CapacityRange{RequiredBytes: int64(sizeGiB) * 1024 * 1024 * 1024}, + AvailabilityZone: "us-east-1a", }) require.NoError(t, err) } @@ -68,7 +81,7 @@ func TestCreateSnapshot(t *testing.T) { // Create a test volume first volumeID := "vol-test123" - createTestVolume(t, store, volumeID, 100) + createTestVolume(t, svc, store, volumeID, 100) // Create snapshot result, err := svc.CreateSnapshot(context.Background(), &ec2.CreateSnapshotInput{ @@ -111,71 +124,40 @@ func TestCreateSnapshot_MissingVolumeId(t *testing.T) { // TestCreateSnapshot_VolumeZeroSize tests that creating a snapshot from a volume with zero SizeGiB fails. func TestCreateSnapshot_VolumeZeroSize(t *testing.T) { - svc, store := setupTestSnapshotService(t) + svc, _ := setupTestSnapshotService(t) - // Store a volume config with SizeGiB == 0 - volumeState := viperblock.VBState{ - VolumeConfig: viperblock.VolumeConfig{ - VolumeMetadata: viperblock.VolumeMetadata{ - SizeGiB: 0, - }, - }, - } - data, err := json.Marshal(volumeState) - require.NoError(t, err) + require.NoError(t, svc.metadata.PutVolume(context.Background(), ebsmetadata.Volume{ + VolumeID: "vol-zerosize", TenantID: testAccountID, CapacityGiB: 0, + State: "available", AvailabilityZone: "us-east-1a", + })) - _, err = store.PutObject(context.Background(), &s3.PutObjectInput{ - Bucket: aws.String("test-bucket"), - Key: aws.String("vol-zerosize/config.json"), - Body: strings.NewReader(string(data)), - }) - require.NoError(t, err) - - _, err = svc.CreateSnapshot(context.Background(), &ec2.CreateSnapshotInput{ + _, err := svc.CreateSnapshot(context.Background(), &ec2.CreateSnapshotInput{ VolumeId: aws.String("vol-zerosize"), }, testAccountID) require.Error(t, err) assert.Contains(t, err.Error(), awserrors.ErrorServerInternal) } -// createEncryptedTestVolume seeds config.json as an at-rest encryption envelope -// ({payload, authtag}) wrapping a VBState, matching what an encrypted volume -// persists. The snapshot handler must unwrap it via StateBody, not decode raw. -func createEncryptedTestVolume(t *testing.T, store *objectstore.MemoryObjectStore, volumeID string, sizeGiB int) { - inner := viperblock.VBState{ - EncryptionEnabled: true, - VolumeConfig: viperblock.VolumeConfig{ - VolumeMetadata: viperblock.VolumeMetadata{ - SizeGiB: uint64(sizeGiB), - AvailabilityZone: "us-east-1a", - }, - }, - } - payload, err := json.Marshal(inner) - require.NoError(t, err) - - envelope, err := json.Marshal(map[string]any{ - "payload": json.RawMessage(payload), - "authtag": "deadbeefdeadbeefdeadbeefdeadbeef", +// createEncryptedTestVolume seeds an encrypted volume. Encryption is a fact +// the document records; the provider's own at-rest envelope is its business. +func createEncryptedTestVolume(t *testing.T, svc *SnapshotServiceImpl, store *objectstore.MemoryObjectStore, volumeID string, sizeGiB int) { + t.Helper() + seedProviderVolume(t, svc, volumeID, sizeGiB) + seedVolumeDocument(t, store, ebsmetadata.Volume{ + VolumeID: volumeID, + CapacityGiB: uint64(sizeGiB), + AvailabilityZone: "us-east-1a", + Encrypted: true, }) - require.NoError(t, err) - - _, err = store.PutObject(context.Background(), &s3.PutObjectInput{ - Bucket: aws.String("test-bucket"), - Key: aws.String(volumeID + "/config.json"), - Body: strings.NewReader(string(envelope)), - }) - require.NoError(t, err) } -// TestCreateSnapshot_EncryptedVolumeEnvelope verifies an encrypted volume (whose -// config.json is an envelope) snapshots successfully instead of 500ing on a -// zero-decoded VBState. -func TestCreateSnapshot_EncryptedVolumeEnvelope(t *testing.T) { +// TestCreateSnapshot_EncryptedVolume verifies an encrypted volume snapshots +// successfully and carries its Encrypted flag onto the snapshot. +func TestCreateSnapshot_EncryptedVolume(t *testing.T) { svc, store := setupTestSnapshotService(t) volumeID := "vol-enc-snap" - createEncryptedTestVolume(t, store, volumeID, 100) + createEncryptedTestVolume(t, svc, store, volumeID, 100) result, err := svc.CreateSnapshot(context.Background(), &ec2.CreateSnapshotInput{ VolumeId: aws.String(volumeID), @@ -186,31 +168,15 @@ func TestCreateSnapshot_EncryptedVolumeEnvelope(t *testing.T) { assert.True(t, *result.Encrypted) } -// TestSnapshotInUseByVolumes_EncryptedVolume verifies the membership scan sees an -// encrypted volume's source SnapshotID through the envelope (a raw decode would -// miss it, letting an in-use snapshot be deleted). +// TestSnapshotInUseByVolumes_EncryptedVolume verifies the membership scan sees +// an encrypted clone's source SnapshotID, without which an in-use snapshot +// could be deleted out from under the volume built on it. func TestSnapshotInUseByVolumes_EncryptedVolume(t *testing.T) { svc, store := setupTestSnapshotService(t) - inner := viperblock.VBState{ - EncryptionEnabled: true, - VolumeConfig: viperblock.VolumeConfig{ - VolumeMetadata: viperblock.VolumeMetadata{SizeGiB: 50, SnapshotID: "snap-src"}, - }, - } - payload, err := json.Marshal(inner) - require.NoError(t, err) - envelope, err := json.Marshal(map[string]any{ - "payload": json.RawMessage(payload), - "authtag": "deadbeefdeadbeefdeadbeefdeadbeef", + seedVolumeDocument(t, store, ebsmetadata.Volume{ + VolumeID: "vol-fromsnap", CapacityGiB: 50, SnapshotID: "snap-src", Encrypted: true, }) - require.NoError(t, err) - _, err = store.PutObject(context.Background(), &s3.PutObjectInput{ - Bucket: aws.String("test-bucket"), - Key: aws.String("vol-fromsnap/config.json"), - Body: strings.NewReader(string(envelope)), - }) - require.NoError(t, err) inUse, err := svc.snapshotInUseByVolumes(context.Background(), "snap-src") require.NoError(t, err) @@ -233,8 +199,8 @@ func TestDescribeSnapshots(t *testing.T) { svc, store := setupTestSnapshotService(t) // Create test volumes - createTestVolume(t, store, "vol-1", 50) - createTestVolume(t, store, "vol-2", 100) + createTestVolume(t, svc, store, "vol-1", 50) + createTestVolume(t, svc, store, "vol-2", 100) // Create multiple snapshots snap1, err := svc.CreateSnapshot(context.Background(), &ec2.CreateSnapshotInput{ @@ -290,7 +256,7 @@ func TestDescribeSnapshots_ByID(t *testing.T) { svc, store := setupTestSnapshotService(t) // Create test volume - createTestVolume(t, store, "vol-1", 50) + createTestVolume(t, svc, store, "vol-1", 50) // Create multiple snapshots snap1, err := svc.CreateSnapshot(context.Background(), &ec2.CreateSnapshotInput{ @@ -319,7 +285,7 @@ func TestDescribeSnapshots_ByID(t *testing.T) { // TestDescribeSnapshots_FilterNoResults). func TestDescribeSnapshots_NotFound(t *testing.T) { svc, store := setupTestSnapshotService(t) - createTestVolume(t, store, "vol-1", 50) + createTestVolume(t, svc, store, "vol-1", 50) snap1, err := svc.CreateSnapshot(context.Background(), &ec2.CreateSnapshotInput{ VolumeId: aws.String("vol-1"), @@ -347,7 +313,7 @@ func TestDescribeSnapshots_Empty(t *testing.T) { func TestDescribeSnapshots_AccountScoping(t *testing.T) { svc, store := setupTestSnapshotService(t) - createTestVolume(t, store, "vol-1", 50) + createTestVolume(t, svc, store, "vol-1", 50) // Account A creates a snapshot snapA, err := svc.CreateSnapshot(context.Background(), &ec2.CreateSnapshotInput{ @@ -379,7 +345,7 @@ func TestDeleteSnapshot(t *testing.T) { svc, store := setupTestSnapshotService(t) // Create test volume and snapshot - createTestVolume(t, store, "vol-1", 50) + createTestVolume(t, svc, store, "vol-1", 50) snap, err := svc.CreateSnapshot(context.Background(), &ec2.CreateSnapshotInput{ VolumeId: aws.String("vol-1"), }, testAccountID) @@ -416,7 +382,7 @@ func TestDeleteSnapshot(t *testing.T) { func TestDeleteSnapshot_WrongAccount(t *testing.T) { svc, store := setupTestSnapshotService(t) - createTestVolume(t, store, "vol-1", 50) + createTestVolume(t, svc, store, "vol-1", 50) snap, err := svc.CreateSnapshot(context.Background(), &ec2.CreateSnapshotInput{ VolumeId: aws.String("vol-1"), }, testAccountID) @@ -442,29 +408,16 @@ func TestDeleteSnapshot_InUseByVolume(t *testing.T) { svc, store := setupTestSnapshotService(t) // Create a test volume and snapshot - createTestVolume(t, store, "vol-source", 50) + createTestVolume(t, svc, store, "vol-source", 50) snap, err := svc.CreateSnapshot(context.Background(), &ec2.CreateSnapshotInput{ VolumeId: aws.String("vol-source"), }, testAccountID) require.NoError(t, err) // Create a volume that references this snapshot (simulates CreateVolume from snapshot) - volumeState := viperblock.VBState{ - VolumeConfig: viperblock.VolumeConfig{ - VolumeMetadata: viperblock.VolumeMetadata{ - SizeGiB: 50, - SnapshotID: *snap.SnapshotId, - }, - }, - } - data, err := json.Marshal(volumeState) - require.NoError(t, err) - _, err = store.PutObject(context.Background(), &s3.PutObjectInput{ - Bucket: aws.String("test-bucket"), - Key: aws.String("vol-cloned/config.json"), - Body: strings.NewReader(string(data)), + seedVolumeDocument(t, store, ebsmetadata.Volume{ + VolumeID: "vol-cloned", CapacityGiB: 50, SnapshotID: *snap.SnapshotId, }) - require.NoError(t, err) // Attempt to delete the snapshot — should fail _, err = svc.DeleteSnapshot(context.Background(), &ec2.DeleteSnapshotInput{ @@ -501,64 +454,41 @@ func TestDeleteSnapshot_MissingID(t *testing.T) { assert.Contains(t, err.Error(), awserrors.ErrorInvalidParameterValue) } -// TestCopySnapshot tests copying a snapshot. -func TestCopySnapshot(t *testing.T) { +// Without a provider a copy would need snapshot metadata under the new ID that +// the control plane cannot re-seal on an encrypted volume. Writing only the +// control-plane config produced a snapshot that described fine and then failed +// at attach, so this path refuses instead of producing one. +func TestCopySnapshot_UnsupportedWithoutProvider(t *testing.T) { svc, store := setupTestSnapshotService(t) - // Create test volume and snapshot - createTestVolume(t, store, "vol-1", 50) - snap, err := svc.CreateSnapshot(context.Background(), &ec2.CreateSnapshotInput{ - VolumeId: aws.String("vol-1"), - Description: aws.String("Original snapshot"), - }, testAccountID) - require.NoError(t, err) - - // Copy snapshot - copyResult, err := svc.CopySnapshot(context.Background(), &ec2.CopySnapshotInput{ - SourceSnapshotId: snap.SnapshotId, - Description: aws.String("Copied snapshot"), - }, testAccountID) - require.NoError(t, err) - require.NotNil(t, copyResult) - assert.True(t, strings.HasPrefix(*copyResult.SnapshotId, "snap-")) - assert.NotEqual(t, *snap.SnapshotId, *copyResult.SnapshotId) - - // Verify both snapshots exist - result, err := svc.DescribeSnapshots(context.Background(), &ec2.DescribeSnapshotsInput{}, testAccountID) - require.NoError(t, err) - assert.Len(t, result.Snapshots, 2) -} - -// TestCopySnapshot_SetsCallerAsOwner tests that copied snapshot is owned by the caller. -func TestCopySnapshot_SetsCallerAsOwner(t *testing.T) { - svc, store := setupTestSnapshotService(t) - - createTestVolume(t, store, "vol-1", 50) + createTestVolume(t, svc, store, "vol-1", 50) snap, err := svc.CreateSnapshot(context.Background(), &ec2.CreateSnapshotInput{ VolumeId: aws.String("vol-1"), }, testAccountID) require.NoError(t, err) - // Copy snapshot as the same account - copyResult, err := svc.CopySnapshot(context.Background(), &ec2.CopySnapshotInput{ - SourceSnapshotId: snap.SnapshotId, - }, testAccountID) - require.NoError(t, err) + // Drop the provider only for the copy: the source snapshot has to exist + // for the refusal to be about the copy rather than about the source. + svc.SetEBSProvider(nil) - // Verify copied snapshot owner is the caller - result, err := svc.DescribeSnapshots(context.Background(), &ec2.DescribeSnapshotsInput{ - SnapshotIds: []*string{copyResult.SnapshotId}, + _, err = svc.CopySnapshot(context.Background(), &ec2.CopySnapshotInput{ + SourceSnapshotId: snap.SnapshotId, + Description: aws.String("Copied snapshot"), }, testAccountID) + require.Error(t, err) + assert.Equal(t, awserrors.ErrorUnsupportedOperation, err.Error()) + + // The refusal must not leave a half-created snapshot behind. + result, err := svc.DescribeSnapshots(context.Background(), &ec2.DescribeSnapshotsInput{}, testAccountID) require.NoError(t, err) - require.Len(t, result.Snapshots, 1) - assert.Equal(t, testAccountID, *result.Snapshots[0].OwnerId) + assert.Len(t, result.Snapshots, 1, "only the source snapshot should exist") } // TestCopySnapshot_WrongAccount tests that account B cannot copy account A's snapshot. func TestCopySnapshot_WrongAccount(t *testing.T) { svc, store := setupTestSnapshotService(t) - createTestVolume(t, store, "vol-1", 50) + createTestVolume(t, svc, store, "vol-1", 50) _, err := svc.CreateSnapshot(context.Background(), &ec2.CreateSnapshotInput{ VolumeId: aws.String("vol-1"), }, testAccountID) @@ -597,42 +527,6 @@ func TestCopySnapshot_MissingSourceID(t *testing.T) { assert.Contains(t, err.Error(), awserrors.ErrorInvalidParameterValue) } -// TestCopySnapshot_PreservesTags tests that tags are copied. -func TestCopySnapshot_PreservesTags(t *testing.T) { - svc, store := setupTestSnapshotService(t) - - // Create test volume and snapshot with tags - createTestVolume(t, store, "vol-1", 50) - snap, err := svc.CreateSnapshot(context.Background(), &ec2.CreateSnapshotInput{ - VolumeId: aws.String("vol-1"), - TagSpecifications: []*ec2.TagSpecification{ - { - ResourceType: aws.String("snapshot"), - Tags: []*ec2.Tag{ - {Key: aws.String("Environment"), Value: aws.String("test")}, - }, - }, - }, - }, testAccountID) - require.NoError(t, err) - - // Copy snapshot - copyResult, err := svc.CopySnapshot(context.Background(), &ec2.CopySnapshotInput{ - SourceSnapshotId: snap.SnapshotId, - }, testAccountID) - require.NoError(t, err) - - // Verify copied snapshot has tags - result, err := svc.DescribeSnapshots(context.Background(), &ec2.DescribeSnapshotsInput{ - SnapshotIds: []*string{copyResult.SnapshotId}, - }, testAccountID) - require.NoError(t, err) - require.Len(t, result.Snapshots, 1) - assert.Len(t, result.Snapshots[0].Tags, 1) - assert.Equal(t, "Environment", *result.Snapshots[0].Tags[0].Key) - assert.Equal(t, "test", *result.Snapshots[0].Tags[0].Value) -} - // setupTestNATSKV creates a NATS JetStream test server and returns a KV bucket for testing. func setupTestNATSKV(t *testing.T) jetstream.KeyValue { t.Helper() @@ -747,9 +641,10 @@ func TestCreateSnapshot_WritesKVEntry(t *testing.T) { }, } svc := NewSnapshotServiceImplWithStore(cfg, store, nil, kv) + svc.SetEBSProvider(ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})) volumeID := "vol-kvtest" - createTestVolume(t, store, volumeID, 10) + createTestVolume(t, svc, store, volumeID, 10) snap, err := svc.CreateSnapshot(context.Background(), &ec2.CreateSnapshotInput{ VolumeId: aws.String(volumeID), @@ -779,9 +674,10 @@ func TestDeleteSnapshot_RemovesKVEntry(t *testing.T) { }, } svc := NewSnapshotServiceImplWithStore(cfg, store, nil, kv) + svc.SetEBSProvider(ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})) volumeID := "vol-kvdelete" - createTestVolume(t, store, volumeID, 10) + createTestVolume(t, svc, store, volumeID, 10) snap, err := svc.CreateSnapshot(context.Background(), &ec2.CreateSnapshotInput{ VolumeId: aws.String(volumeID), @@ -800,66 +696,22 @@ func TestDeleteSnapshot_RemovesKVEntry(t *testing.T) { assert.False(t, has) } -func TestCopySnapshot_AddsKVEntry(t *testing.T) { - kv := setupTestNATSKV(t) - store := objectstore.NewMemoryObjectStore() - cfg := &config.Config{ - Predastore: config.PredastoreConfig{ - Bucket: "test-bucket", - AccessKey: "test-owner-123", - }, - } - svc := NewSnapshotServiceImplWithStore(cfg, store, nil, kv) - - volumeID := "vol-kvcopy" - createTestVolume(t, store, volumeID, 10) - - snap, err := svc.CreateSnapshot(context.Background(), &ec2.CreateSnapshotInput{ - VolumeId: aws.String(volumeID), - }, testAccountID) - require.NoError(t, err) - - copyResult, err := svc.CopySnapshot(context.Background(), &ec2.CopySnapshotInput{ - SourceSnapshotId: snap.SnapshotId, - }, testAccountID) - require.NoError(t, err) - - // Both snapshot IDs should be in KV - entry, err := kv.Get(t.Context(), volumeID) - require.NoError(t, err) - var snapshots []string - require.NoError(t, json.Unmarshal(entry.Value(), &snapshots)) - assert.Contains(t, snapshots, *snap.SnapshotId) - assert.Contains(t, snapshots, *copyResult.SnapshotId) - assert.Len(t, snapshots, 2) -} - // TestCreateSnapshot_CrossAccountVolumeRejected tests that snapshotting another account's volume is rejected. func TestCreateSnapshot_CrossAccountVolumeRejected(t *testing.T) { svc, store := setupTestSnapshotService(t) // Create a volume owned by testAccountID (via TenantID) volumeID := "vol-owned-by-alpha" - volumeState := viperblock.VBState{ - VolumeConfig: viperblock.VolumeConfig{ - VolumeMetadata: viperblock.VolumeMetadata{ - SizeGiB: 100, - TenantID: testAccountID, - AvailabilityZone: "us-east-1a", - }, - }, - } - data, err := json.Marshal(volumeState) - require.NoError(t, err) - _, err = store.PutObject(context.Background(), &s3.PutObjectInput{ - Bucket: aws.String("test-bucket"), - Key: aws.String(volumeID + "/config.json"), - Body: strings.NewReader(string(data)), + seedProviderVolume(t, svc, volumeID, 100) + seedVolumeDocument(t, store, ebsmetadata.Volume{ + VolumeID: volumeID, + CapacityGiB: 100, + TenantID: testAccountID, + AvailabilityZone: "us-east-1a", }) - require.NoError(t, err) // Another account tries to snapshot the volume — should fail - _, err = svc.CreateSnapshot(context.Background(), &ec2.CreateSnapshotInput{ + _, err := svc.CreateSnapshot(context.Background(), &ec2.CreateSnapshotInput{ VolumeId: aws.String(volumeID), }, otherAccountID) require.Error(t, err) @@ -879,7 +731,7 @@ func TestCreateSnapshot_PrePhase4VolumeAllowed(t *testing.T) { // Create a volume with no TenantID (pre-phase4) volumeID := "vol-legacy" - createTestVolume(t, store, volumeID, 50) + createTestVolume(t, svc, store, volumeID, 50) // Any account can snapshot — backward compatibility result, err := svc.CreateSnapshot(context.Background(), &ec2.CreateSnapshotInput{ @@ -894,7 +746,7 @@ func TestCreateSnapshot_PrePhase4VolumeAllowed(t *testing.T) { // createTestSnapshot creates a snapshot and returns its ID. func createTestSnapshot(t *testing.T, svc *SnapshotServiceImpl, store *objectstore.MemoryObjectStore, volumeID string, sizeGiB int, tags map[string]string) string { t.Helper() - createTestVolume(t, store, volumeID, sizeGiB) + createTestVolume(t, svc, store, volumeID, sizeGiB) tagSpecs := []*ec2.TagSpecification{} if len(tags) > 0 { @@ -1106,94 +958,264 @@ func TestDescribeSnapshots_FilterNoFilters(t *testing.T) { assert.Len(t, out.Snapshots, 2) } -// TestCreateSnapshot_PredastoreInitFails exercises snapshotVolume past the nil-host guard. -// The viperblock S3 backend fails at Init() when the test server returns 403, covering the -// snapshotVolume body (S3Config build, LoadViperblockMasterKey, viperblock.New, Backend.Init) -// and the CreateSnapshot error return path. -func TestCreateSnapshot_PredastoreInitFails(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusForbidden) - })) - defer srv.Close() - +func TestCreateDeleteSnapshot_UsesInjectedProvider(t *testing.T) { store := objectstore.NewMemoryObjectStore() - cfg := &config.Config{ - Predastore: config.PredastoreConfig{ - Host: srv.URL, - Region: "us-east-1", - Bucket: "test-bucket", - }, - } + cfg := &config.Config{Predastore: config.PredastoreConfig{Bucket: "test-bucket"}} svc := NewSnapshotServiceImplWithStore(cfg, store, nil) - createTestVolume(t, store, "vol-initfail", 10) + provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{}) + svc.SetEBSProvider(provider) + require.NoError(t, svc.metadata.PutVolume(context.Background(), ebsmetadata.Volume{ + VolumeID: "vol-provider", TenantID: testAccountID, CapacityGiB: 4, State: "available", AvailabilityZone: "us-east-1a", ProviderHandle: "memory://volume/vol-provider", + })) + _, err := provider.CreateVolume(context.Background(), ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-provider", CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 4 * 1024 * 1024 * 1024}, AvailabilityZone: "us-east-1a"}) + require.NoError(t, err) - _, err := svc.CreateSnapshot(context.Background(), &ec2.CreateSnapshotInput{ - VolumeId: aws.String("vol-initfail"), + snapshot, err := svc.CreateSnapshot(context.Background(), &ec2.CreateSnapshotInput{VolumeId: aws.String("vol-provider")}, testAccountID) + require.NoError(t, err) + require.NotNil(t, snapshot) + cfgStored, err := svc.getSnapshotConfig(aws.StringValue(snapshot.SnapshotId)) + require.NoError(t, err) + assert.Equal(t, "memory://snapshot/"+aws.StringValue(snapshot.SnapshotId), cfgStored.ProviderHandle) + _, err = svc.DeleteSnapshot(context.Background(), &ec2.DeleteSnapshotInput{SnapshotId: snapshot.SnapshotId}, testAccountID) + require.NoError(t, err) +} + +// A provider-managed clone records its source snapshot only in ebsmetadata, so +// the legacy config.json scan cannot see it. Deleting the snapshot underneath +// one would strip chunks the clone still reads. +func TestDeleteSnapshot_Provider_BlockedByCloneRecordedInMetadata(t *testing.T) { + ctx := context.Background() + store := objectstore.NewMemoryObjectStore() + cfg := &config.Config{Predastore: config.PredastoreConfig{Bucket: "test-bucket"}} + svc := NewSnapshotServiceImplWithStore(cfg, store, nil) + provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{}) + svc.SetEBSProvider(provider) + + require.NoError(t, svc.metadata.PutVolume(ctx, ebsmetadata.Volume{ + VolumeID: "vol-source", TenantID: testAccountID, CapacityGiB: 4, + State: "available", AvailabilityZone: "us-east-1a", ProviderHandle: "memory://volume/vol-source", + })) + _, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-source", + CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 4 * 1024 * 1024 * 1024}, AvailabilityZone: "us-east-1a", + }) + require.NoError(t, err) + + snapshot, err := svc.CreateSnapshot(ctx, &ec2.CreateSnapshotInput{VolumeId: aws.String("vol-source")}, testAccountID) + require.NoError(t, err) + snapshotID := aws.StringValue(snapshot.SnapshotId) + + // The clone exists only as a control-plane document, as CreateVolume's + // provider branch writes it. + require.NoError(t, svc.metadata.PutVolume(ctx, ebsmetadata.Volume{ + VolumeID: "vol-clone", TenantID: testAccountID, CapacityGiB: 4, State: "available", + AvailabilityZone: "us-east-1a", SnapshotID: snapshotID, ProviderHandle: "memory://volume/vol-clone", + })) + + _, err = svc.DeleteSnapshot(ctx, &ec2.DeleteSnapshotInput{SnapshotId: aws.String(snapshotID)}, testAccountID) + assert.EqualError(t, err, awserrors.ErrorInvalidSnapshotInUse) + + // Once the clone is gone the snapshot is deletable again. + require.NoError(t, svc.metadata.DeleteVolume(ctx, "vol-clone")) + _, err = svc.DeleteSnapshot(ctx, &ec2.DeleteSnapshotInput{SnapshotId: aws.String(snapshotID)}, testAccountID) + require.NoError(t, err) +} + +// snapshotMetadataFailingObjectStore fails PutObject for any snapshot +// metadata.json write and records the last such key, so a test can recover +// the randomly generated snapshot ID CreateSnapshot attempted to persist. +type snapshotMetadataFailingObjectStore struct { + objectstore.ObjectStore + + attemptedKey string +} + +func (s *snapshotMetadataFailingObjectStore) PutObject(ctx context.Context, input *s3.PutObjectInput) (*s3.PutObjectOutput, error) { + key := aws.StringValue(input.Key) + if strings.HasSuffix(key, "/metadata.json") { + s.attemptedKey = key + return nil, errors.New("simulated metadata write failure") + } + return s.ObjectStore.PutObject(ctx, input) +} + +// TestCreateSnapshot_Provider_RollbackOnMetadataWriteFailure covers +// CreateSnapshot's rollback path: when persisting the snapshot config fails +// after the provider has already created the snapshot, CreateSnapshot must +// delete the just-created provider snapshot rather than orphan it. +func TestCreateSnapshot_Provider_RollbackOnMetadataWriteFailure(t *testing.T) { + store := &snapshotMetadataFailingObjectStore{ObjectStore: objectstore.NewMemoryObjectStore()} + cfg := &config.Config{Predastore: config.PredastoreConfig{Bucket: "test-bucket"}} + svc := NewSnapshotServiceImplWithStore(cfg, store, nil) + provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{}) + svc.SetEBSProvider(provider) + + require.NoError(t, svc.metadata.PutVolume(context.Background(), ebsmetadata.Volume{ + VolumeID: "vol-rollback", TenantID: testAccountID, CapacityGiB: 4, State: "available", + AvailabilityZone: "us-east-1a", ProviderHandle: "memory://volume/vol-rollback", + })) + _, err := provider.CreateVolume(context.Background(), ebsprovider.CreateVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-rollback", + CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 4 * 1024 * 1024 * 1024}, AvailabilityZone: "us-east-1a", + }) + require.NoError(t, err) + + _, err = svc.CreateSnapshot(context.Background(), &ec2.CreateSnapshotInput{VolumeId: aws.String("vol-rollback")}, testAccountID) + require.EqualError(t, err, awserrors.ErrorServerInternal) + require.NotEmpty(t, store.attemptedKey, "the snapshot metadata write must have been attempted") + + snapshotID := strings.TrimSuffix(store.attemptedKey, "/metadata.json") + _, err = provider.GetVolume(context.Background(), ebsprovider.GetVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-rollback"}) + require.NoError(t, err, "the source volume must be unaffected by the snapshot rollback") + + // Recreating the same snapshot ID against a different source volume only + // succeeds if the original snapshot no longer exists (MemoryProvider + // returns already_exists for a conflicting source volume on a live + // snapshot ID), proving the rollback actually deleted it rather than + // merely leaving it orphaned but reachable. + _, err = provider.CreateVolume(context.Background(), ebsprovider.CreateVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-rollback-check", CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 * 1024 * 1024 * 1024}, + }) + require.NoError(t, err) + _, err = provider.CreateSnapshot(context.Background(), ebsprovider.CreateSnapshotRequest{ + Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, VolumeID: "vol-rollback-check", + }) + require.NoError(t, err, "rollback must have deleted the just-created provider snapshot") +} + +// providerSnapshotService builds a snapshot service backed by the memory +// provider with one volume registered, ready to snapshot. +func providerSnapshotService(t *testing.T, volumeID string) (*SnapshotServiceImpl, *ebsprovider.MemoryProvider) { + t.Helper() + store := objectstore.NewMemoryObjectStore() + cfg := &config.Config{Predastore: config.PredastoreConfig{Bucket: "test-bucket"}} + svc := NewSnapshotServiceImplWithStore(cfg, store, nil) + provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{}) + svc.SetEBSProvider(provider) + require.NoError(t, svc.metadata.PutVolume(context.Background(), ebsmetadata.Volume{ + VolumeID: volumeID, TenantID: testAccountID, CapacityGiB: 4, State: "available", + AvailabilityZone: "us-east-1a", ProviderHandle: "memory://volume/" + volumeID, + })) + _, err := provider.CreateVolume(context.Background(), ebsprovider.CreateVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, + CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 4 * 1024 * 1024 * 1024}, + AvailabilityZone: "us-east-1a", + }) + require.NoError(t, err) + return svc, provider +} + +// The copy must exist in the provider under its own ID, not just as a +// control-plane document. That gap is what made the previous implementation +// describe a snapshot that failed at attach. +func TestCopySnapshot_Provider_CreatesBackendSnapshot(t *testing.T) { + ctx := context.Background() + svc, provider := providerSnapshotService(t, "vol-copysrc") + + src, err := svc.CreateSnapshot(ctx, &ec2.CreateSnapshotInput{VolumeId: aws.String("vol-copysrc")}, testAccountID) + require.NoError(t, err) + + out, err := svc.CopySnapshot(ctx, &ec2.CopySnapshotInput{ + SourceSnapshotId: src.SnapshotId, Description: aws.String("copied"), }, testAccountID) + require.NoError(t, err) + require.NotNil(t, out.SnapshotId) + newID := aws.StringValue(out.SnapshotId) + assert.NotEqual(t, aws.StringValue(src.SnapshotId), newID) + + got, ok := provider.Snapshot(newID) + require.True(t, ok, "the copy must exist in the provider, not only in the control plane") + assert.Equal(t, "vol-copysrc", got.SourceVolumeID) + + stored, err := svc.getSnapshotConfig(newID) + require.NoError(t, err) + assert.Equal(t, "vol-copysrc", stored.VolumeID) + assert.Equal(t, "copied", stored.Description) + assert.Equal(t, testAccountID, stored.OwnerID) +} + +// Both the copy and its source must describe. This service is built without +// JetStream, so it covers the control-plane documents only — the snapshot-ref +// rollback needs a KV-backed service and is not exercised here. +func TestCopySnapshot_Provider_CopyAndSourceBothDescribe(t *testing.T) { + ctx := context.Background() + svc, _ := providerSnapshotService(t, "vol-copyref") + + src, err := svc.CreateSnapshot(ctx, &ec2.CreateSnapshotInput{VolumeId: aws.String("vol-copyref")}, testAccountID) + require.NoError(t, err) + out, err := svc.CopySnapshot(ctx, &ec2.CopySnapshotInput{SourceSnapshotId: src.SnapshotId}, testAccountID) + require.NoError(t, err) + + listed, err := svc.DescribeSnapshots(ctx, &ec2.DescribeSnapshotsInput{}, testAccountID) + require.NoError(t, err) + ids := make([]string, 0, len(listed.Snapshots)) + for _, s := range listed.Snapshots { + ids = append(ids, aws.StringValue(s.SnapshotId)) + } + assert.Contains(t, ids, aws.StringValue(out.SnapshotId)) + assert.Contains(t, ids, aws.StringValue(src.SnapshotId)) +} + +// The copy inherits the source description when the caller supplies none, +// rather than landing with an empty one. +func TestCopySnapshot_Provider_InheritsSourceDescription(t *testing.T) { + ctx := context.Background() + svc, _ := providerSnapshotService(t, "vol-copydesc") + + src, err := svc.CreateSnapshot(ctx, &ec2.CreateSnapshotInput{ + VolumeId: aws.String("vol-copydesc"), Description: aws.String("original text"), + }, testAccountID) + require.NoError(t, err) + + out, err := svc.CopySnapshot(ctx, &ec2.CopySnapshotInput{SourceSnapshotId: src.SnapshotId}, testAccountID) + require.NoError(t, err) + stored, err := svc.getSnapshotConfig(aws.StringValue(out.SnapshotId)) + require.NoError(t, err) + assert.Equal(t, "original text", stored.Description) +} + +// A provider failure must leave nothing behind: no control-plane document and +// no entry in the volume's snapshot list. +func TestCopySnapshot_Provider_FailureLeavesNothingBehind(t *testing.T) { + ctx := context.Background() + svc, provider := providerSnapshotService(t, "vol-copyfail") + + src, err := svc.CreateSnapshot(ctx, &ec2.CreateSnapshotInput{VolumeId: aws.String("vol-copyfail")}, testAccountID) + require.NoError(t, err) + + svc.SetEBSProvider(©FailingProvider{EBSProvider: provider}) + _, err = svc.CopySnapshot(ctx, &ec2.CopySnapshotInput{SourceSnapshotId: src.SnapshotId}, testAccountID) require.Error(t, err) assert.Equal(t, awserrors.ErrorServerInternal, err.Error()) + + listed, err := svc.DescribeSnapshots(ctx, &ec2.DescribeSnapshotsInput{}, testAccountID) + require.NoError(t, err) + assert.Len(t, listed.Snapshots, 1, "only the source snapshot should exist") } -// s3MockServer returns a test server that passes ListObjectsV2 (Backend.Init) but -// returns 404 for every other request (GetObject, PutObject, etc.). -func s3MockServer(t *testing.T) *httptest.Server { - t.Helper() - return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if strings.Contains(r.URL.RawQuery, "list-type=2") { - w.Header().Set("Content-Type", "application/xml") - w.WriteHeader(http.StatusOK) - io.WriteString(w, ``+ - ``+ - `test-bucket0`+ - `1000false`) - return - } - w.Header().Set("Content-Type", "application/xml") - w.WriteHeader(http.StatusNotFound) - io.WriteString(w, ``+ - `NoSuchKey`+ - `The specified key does not exist.`) - })) -} +// Ownership is checked before the copy runs, so another tenant's snapshot is +// refused rather than duplicated into the caller's account. +func TestCopySnapshot_Provider_ForeignSnapshotRefused(t *testing.T) { + ctx := context.Background() + svc, provider := providerSnapshotService(t, "vol-copyown") -func TestSnapshotVolume_EncryptionKeyLoadError(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - })) - defer srv.Close() + src, err := svc.CreateSnapshot(ctx, &ec2.CreateSnapshotInput{VolumeId: aws.String("vol-copyown")}, testAccountID) + require.NoError(t, err) - cfg := &config.Config{ - Predastore: config.PredastoreConfig{ - Host: srv.URL, - Region: "us-east-1", - Bucket: "test-bucket", - }, - Viperblock: config.ViperblockConfig{ - EncryptionKeyFile: "/nonexistent-snap-test-key.bin", - }, - } - svc := NewSnapshotServiceImplWithStore(cfg, objectstore.NewMemoryObjectStore(), nil) - - err := svc.snapshotVolume("vol-enckey", "snap-enckey", 10*1024*1024*1024) + _, err = svc.CopySnapshot(ctx, &ec2.CopySnapshotInput{SourceSnapshotId: src.SnapshotId}, "999988887777") require.Error(t, err) - assert.Contains(t, err.Error(), "load encryption key") + assert.Equal(t, awserrors.ErrorUnauthorizedOperation, err.Error()) + + _, ok := provider.Snapshot(aws.StringValue(src.SnapshotId)) + require.True(t, ok, "the source must survive a refused copy") } -func TestSnapshotVolume_LoadStateFails(t *testing.T) { - srv := s3MockServer(t) - defer srv.Close() - - cfg := &config.Config{ - Predastore: config.PredastoreConfig{ - Host: srv.URL, - Region: "us-east-1", - Bucket: "test-bucket", - AccessKey: "test-key", - SecretKey: "test-secret", - }, - } - svc := NewSnapshotServiceImplWithStore(cfg, objectstore.NewMemoryObjectStore(), nil) - - err := svc.snapshotVolume("vol-lsf", "snap-lsf", 10*1024*1024*1024) - require.Error(t, err) +// copyFailingProvider fails only CopySnapshot, leaving every other operation +// intact so the rollback path is what the test exercises. +type copyFailingProvider struct { + ebsprovider.EBSProvider +} + +func (p *copyFailingProvider) CopySnapshot(context.Context, ebsprovider.CopySnapshotRequest) (*ebsprovider.Snapshot, error) { + return nil, errors.New("provider copy failed") } diff --git a/spinifex/handlers/ec2/volume/config_route_invariants_test.go b/spinifex/handlers/ec2/volume/config_route_invariants_test.go index 62f7adbd5..9da0a880a 100644 --- a/spinifex/handlers/ec2/volume/config_route_invariants_test.go +++ b/spinifex/handlers/ec2/volume/config_route_invariants_test.go @@ -1,54 +1,29 @@ package handlers_ec2_volume import ( - "context" - "encoding/json" - "strings" "testing" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/mulgadc/spinifex/spinifex/handlers/ec2/volumestate" + "github.com/mulgadc/spinifex/spinifex/ebsmetadata" "github.com/mulgadc/spinifex/spinifex/objectstore" - "github.com/mulgadc/viperblock/viperblock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -// seedUnencryptedConfig writes a plain (EncryptionEnabled=false) full VBState. -func seedUnencryptedConfig(t *testing.T, store *objectstore.MemoryObjectStore, volumeID string) { - t.Helper() - state := viperblock.VBState{ - VolumeName: volumeID, - VolumeSize: 10 * 1024 * 1024 * 1024, - BlockSize: 4096, - SeqNum: 7, - VolumeConfig: viperblock.VolumeConfig{ - VolumeMetadata: viperblock.VolumeMetadata{VolumeID: volumeID, SizeGiB: 10, State: "available"}, - }, - } - data, err := json.Marshal(state) - require.NoError(t, err) - _, err = store.PutObject(context.Background(), &s3.PutObjectInput{ - Bucket: aws.String("test-bucket"), - Key: aws.String(volumeID + "/config.json"), - Body: strings.NewReader(string(data)), - }) - require.NoError(t, err) -} - -// TestUpdateVolumeState_WritesStateJSONNotConfig locks the single-writer +// TestUpdateVolumeState_WritesDocumentNotConfig locks the single-writer // contract: the live nbdkit VB owns config.json and rewrites it from its stale // in-memory State on every SaveState, so the control plane MUST persist -// attachment state to the separate state.json object and leave config.json -// byte-for-byte untouched. Writing State into config.json out-of-band is -// silently clobbered under write load (the EKS-worker root-volume flip). -func TestUpdateVolumeState_WritesStateJSONNotConfig(t *testing.T) { +// attachment state to its own document and leave config.json byte-for-byte +// untouched. Writing State into config.json out-of-band is silently clobbered +// under write load (the EKS-worker root-volume flip). +func TestUpdateVolumeState_WritesDocumentNotConfig(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) volumeID := "vol-single-writer" - seedUnencryptedConfig(t, store, volumeID) + seedProviderConfig(t, store, volumeID) + seedVolumeDocument(t, store, ebsmetadata.Volume{ + VolumeID: volumeID, TenantID: testVolAccountID, CapacityGiB: 10, State: "available", + }) before := getStoredConfig(t, store, volumeID) require.NoError(t, svc.UpdateVolumeState(volumeID, "in-use", "i-abc", "/dev/nbd0")) @@ -57,23 +32,27 @@ func TestUpdateVolumeState_WritesStateJSONNotConfig(t *testing.T) { assert.Equal(t, string(before), string(after), "UpdateVolumeState must not write config.json; the live VB owns it") - rec := getStoredState(t, store, volumeID) - assert.Equal(t, "in-use", rec.State) - assert.Equal(t, "i-abc", rec.AttachedInstance) - assert.Equal(t, "/dev/nbd0", rec.DeviceName) + meta, err := svc.GetVolumeMetadata(volumeID) + require.NoError(t, err) + assert.Equal(t, "in-use", meta.State) + assert.Equal(t, "i-abc", meta.AttachedInstance) + assert.Equal(t, "/dev/nbd0", meta.DeviceName) } // TestUpdateVolumeState_EncryptedConfigUntouched locks the corruption-safety -// half of the contract: config.json for an encrypted volume is a sealed VBState -// whose AES-GCM nonce is derived from StateSeqNum. A second out-of-band writer -// advances the nonce and reuses it (catastrophic for AES-GCM — the decrypted -// garbage image layers). UpdateVolumeState MUST never touch the sealed object. +// half of the contract: config.json for an encrypted volume is a sealed blob +// whose AES-GCM nonce is derived from its sequence number. A second out-of-band +// writer advances the nonce and reuses it (catastrophic for AES-GCM — the +// decrypted garbage image layers). UpdateVolumeState MUST never touch it. func TestUpdateVolumeState_EncryptedConfigUntouched(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) volumeID := "vol-enc-single-writer" seedEncryptedConfig(t, store, volumeID) + seedVolumeDocument(t, store, ebsmetadata.Volume{ + VolumeID: volumeID, TenantID: testVolAccountID, CapacityGiB: 10, State: "available", + }) before := getStoredConfig(t, store, volumeID) require.NoError(t, svc.UpdateVolumeState(volumeID, "in-use", "i-enc", "/dev/nbd0")) @@ -81,54 +60,48 @@ func TestUpdateVolumeState_EncryptedConfigUntouched(t *testing.T) { after := getStoredConfig(t, store, volumeID) assert.Equal(t, string(before), string(after), "UpdateVolumeState must not rewrite the sealed config.json (AES-GCM nonce reuse)") - assert.Equal(t, "in-use", getStoredState(t, store, volumeID).State) + + meta, err := svc.GetVolumeMetadata(volumeID) + require.NoError(t, err) + assert.Equal(t, "in-use", meta.State) } -// TestGetVolumeConfig_OverlaysStateJSON locks the read side: the authoritative -// attachment state is state.json, which MUST override the (stale) State the live -// VB persisted into config.json. -func TestGetVolumeConfig_OverlaysStateJSON(t *testing.T) { +// TestGetVolumeMetadata_DocumentBeatsConfig locks the read side of the same +// contract: config.json carries a State the provider wrote and the control +// plane does not own, so a document that disagrees with it MUST win. Reading +// State back out of config.json is what let a stale SaveState resurrect a +// detached volume as attached. +func TestGetVolumeMetadata_DocumentBeatsConfig(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - volumeID := "vol-overlay" - seedUnencryptedConfig(t, store, volumeID) // config.json State == "available" - require.NoError(t, svc.UpdateVolumeState(volumeID, "in-use", "i-overlay", "/dev/nbd0")) - - cfg, err := svc.GetVolumeConfig(volumeID) - require.NoError(t, err) - assert.Equal(t, "in-use", cfg.VolumeMetadata.State, - "state.json must override the stale State in config.json") - assert.Equal(t, "i-overlay", cfg.VolumeMetadata.AttachedInstance) - assert.Equal(t, "/dev/nbd0", cfg.VolumeMetadata.DeviceName) -} - -// TestGetVolumeConfig_FallsBackWhenNoStateJSON locks the migration path: a volume -// predating the state.json split has no state.json, so the State embedded in -// config.json is used unchanged. -func TestGetVolumeConfig_FallsBackWhenNoStateJSON(t *testing.T) { - store := objectstore.NewMemoryObjectStore() - svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - - volumeID := "vol-legacy" - seedUnencryptedConfig(t, store, volumeID) // State == "available", no state.json - - cfg, err := svc.GetVolumeConfig(volumeID) - require.NoError(t, err) - assert.Equal(t, "available", cfg.VolumeMetadata.State, - "absent state.json must fall back to the State embedded in config.json") -} - -// getStoredState reads and decodes the raw state.json from the memory store. -func getStoredState(t *testing.T, store *objectstore.MemoryObjectStore, volumeID string) volumestate.Record { - t.Helper() - res, err := store.GetObject(context.Background(), &s3.GetObjectInput{ - Bucket: aws.String("test-bucket"), - Key: aws.String(volumeID + "/state.json"), + volumeID := "vol-document-wins" + seedProviderConfig(t, store, volumeID) // its embedded State says "available" + seedVolumeDocument(t, store, ebsmetadata.Volume{ + VolumeID: volumeID, TenantID: testVolAccountID, CapacityGiB: 10, + State: "in-use", AttachedInstance: "i-overlay", DeviceName: "/dev/nbd0", }) + + meta, err := svc.GetVolumeMetadata(volumeID) require.NoError(t, err) - defer res.Body.Close() - var rec volumestate.Record - require.NoError(t, json.NewDecoder(res.Body).Decode(&rec)) - return rec + assert.Equal(t, "in-use", meta.State, + "the ebsmetadata document must override any State in config.json") + assert.Equal(t, "i-overlay", meta.AttachedInstance) + assert.Equal(t, "/dev/nbd0", meta.DeviceName) +} + +// TestGetVolumeMetadata_ConfigAloneIsNotAVolume is the other half: provider +// state without a document is not a volume the control plane knows. Falling +// back to config.json would resurrect volumes whose document was deleted, so +// an absent document must read as not-found. +func TestGetVolumeMetadata_ConfigAloneIsNotAVolume(t *testing.T) { + store := objectstore.NewMemoryObjectStore() + svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) + + volumeID := "vol-config-only" + seedProviderConfig(t, store, volumeID) + + _, err := svc.GetVolumeMetadata(volumeID) + require.Error(t, err) + assert.Contains(t, err.Error(), "InvalidVolume.NotFound") } diff --git a/spinifex/handlers/ec2/volume/config_route_test.go b/spinifex/handlers/ec2/volume/config_route_test.go deleted file mode 100644 index e306af23a..000000000 --- a/spinifex/handlers/ec2/volume/config_route_test.go +++ /dev/null @@ -1,147 +0,0 @@ -package handlers_ec2_volume - -import ( - "context" - "encoding/json" - "io" - "strings" - "sync/atomic" - "testing" - "time" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/mulgadc/spinifex/spinifex/objectstore" - "github.com/mulgadc/spinifex/spinifex/types" - "github.com/mulgadc/viperblock/viperblock" - "github.com/nats-io/nats-server/v2/server" - "github.com/nats-io/nats.go" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// startTestNATS spins up an in-process NATS server and returns a connected client. -func startTestNATS(t *testing.T) *nats.Conn { - t.Helper() - opts := &server.Options{ - Host: "127.0.0.1", - Port: -1, - JetStream: true, - StoreDir: t.TempDir(), - NoLog: true, - NoSigs: true, - } - ns, err := server.NewServer(opts) - require.NoError(t, err) - go ns.Start() - require.True(t, ns.ReadyForConnections(5*time.Second)) - t.Cleanup(func() { ns.Shutdown() }) - - nc, err := nats.Connect(ns.ClientURL()) - require.NoError(t, err) - t.Cleanup(func() { nc.Close() }) - return nc -} - -// seedEncryptedConfig writes a sealed (EncryptionEnabled=true) VBState to the store. -func seedEncryptedConfig(t *testing.T, store *objectstore.MemoryObjectStore, volumeID string) { - t.Helper() - state := viperblock.VBState{ - VolumeName: volumeID, - VolumeSize: 10 * 1024 * 1024 * 1024, - BlockSize: 4096, - EncryptionEnabled: true, - VolumeConfig: viperblock.VolumeConfig{ - VolumeMetadata: viperblock.VolumeMetadata{ - VolumeID: volumeID, - TenantID: testVolAccountID, - SizeGiB: 10, - State: "available", - }, - }, - } - data, err := json.Marshal(state) - require.NoError(t, err) - _, err = store.PutObject(context.Background(), &s3.PutObjectInput{ - Bucket: aws.String("test-bucket"), - Key: aws.String(volumeID + "/config.json"), - Body: strings.NewReader(string(data)), - }) - require.NoError(t, err) -} - -// getStoredConfig reads the raw config.json bytes from the memory store. -func getStoredConfig(t *testing.T, store *objectstore.MemoryObjectStore, volumeID string) []byte { - t.Helper() - res, err := store.GetObject(context.Background(), &s3.GetObjectInput{ - Bucket: aws.String("test-bucket"), - Key: aws.String(volumeID + "/config.json"), - }) - require.NoError(t, err) - defer res.Body.Close() - data, err := io.ReadAll(res.Body) - require.NoError(t, err) - return data -} - -// encConfig builds a VolumeConfig for an encrypted-volume putVolumeConfig call. -func encConfig(volumeID string) *viperblock.VolumeConfig { - return &viperblock.VolumeConfig{ - VolumeMetadata: viperblock.VolumeMetadata{VolumeID: volumeID, SizeGiB: 10}, - } -} - -// TestPutVolumeConfig_EncryptedRoutesToQueueGroup verifies a config write for an -// encrypted volume is delegated to the ebs.config keyholder queue group rather -// than re-sealing the object out-of-band. The control plane lacks the master key, -// so a direct PutObject would strip the AES-GCM tag and brick the volume. This is -// the safe non-live writer path (markVolumeOrphaned / ModifyVolume); UpdateVolumeState -// no longer routes here at all (it writes state.json). -func TestPutVolumeConfig_EncryptedRoutesToQueueGroup(t *testing.T) { - store := objectstore.NewMemoryObjectStore() - svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - svc.natsConn = startTestNATS(t) - - volumeID := "vol-enc-detached" - seedEncryptedConfig(t, store, volumeID) - - var hits atomic.Int32 - sub, err := svc.natsConn.QueueSubscribe("ebs.config", "spinifex-workers", func(msg *nats.Msg) { - hits.Add(1) - data, _ := json.Marshal(types.EBSConfigUpdateResponse{Volume: volumeID, Success: true}) - _ = msg.Respond(data) - }) - require.NoError(t, err) - defer sub.Unsubscribe() - - require.NoError(t, svc.putVolumeConfig(context.Background(), volumeID, encConfig(volumeID))) - assert.Equal(t, int32(1), hits.Load(), "encrypted config write must hit the ebs.config keyholder queue group") - - // The sealed object must NOT have been rewritten out-of-band by the handler. - raw := getStoredConfig(t, store, volumeID) - var still viperblock.VBState - require.NoError(t, json.Unmarshal(viperblock.StateBody(raw), &still)) - assert.True(t, still.EncryptionEnabled) -} - -// TestPutVolumeConfig_EncryptedKeyholderError surfaces a keyholder failure -// instead of silently succeeding. -func TestPutVolumeConfig_EncryptedKeyholderError(t *testing.T) { - store := objectstore.NewMemoryObjectStore() - svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - svc.natsConn = startTestNATS(t) - - volumeID := "vol-enc-err" - seedEncryptedConfig(t, store, volumeID) - - sub, err := svc.natsConn.QueueSubscribe("ebs.config", "spinifex-workers", func(msg *nats.Msg) { - data, _ := json.Marshal(types.EBSConfigUpdateResponse{Volume: volumeID, Error: "reseal failed"}) - _ = msg.Respond(data) - }) - require.NoError(t, err) - defer sub.Unsubscribe() - - err = svc.putVolumeConfig(context.Background(), volumeID, encConfig(volumeID)) - require.Error(t, err) - assert.Contains(t, err.Error(), "reseal failed") -} diff --git a/spinifex/handlers/ec2/volume/describe_volumes_provider_bench_test.go b/spinifex/handlers/ec2/volume/describe_volumes_provider_bench_test.go new file mode 100644 index 000000000..d1eaddce5 --- /dev/null +++ b/spinifex/handlers/ec2/volume/describe_volumes_provider_bench_test.go @@ -0,0 +1,101 @@ +package handlers_ec2_volume + +import ( + "context" + "fmt" + "log/slog" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mulgadc/spinifex/spinifex/ebsprovider" +) + +// silenceLogs discards slog output for the duration of a benchmark's timed +// loop, so per-call INFO logging (which scales with iteration count) doesn't +// dominate the measured cost. Restores the previous default logger on return. +func silenceLogs(b *testing.B) func() { + b.Helper() + prev := slog.Default() + slog.SetDefault(slog.New(slog.DiscardHandler)) + return func() { slog.SetDefault(prev) } +} + +// seedProviderVolumes creates n volumes under accountID and returns the ID of +// the last one created, so callers can benchmark a single-ID lookup against a +// bucket holding n volume documents. +func seedProviderVolumes(b *testing.B, svc *VolumeServiceImpl, n int, accountID string) string { + b.Helper() + ctx := context.Background() + var lastID string + for i := range n { + vol, err := svc.CreateVolume(ctx, &ec2.CreateVolumeInput{ + Size: aws.Int64(8), AvailabilityZone: aws.String("ap-southeast-2a"), + }, accountID) + if err != nil { + b.Fatalf("seed volume %d: %v", i, err) + } + lastID = aws.StringValue(vol.VolumeId) + } + return lastID +} + +// BenchmarkDescribeVolumes_Provider_SingleID measures the direct-GetVolume +// fast path as the bucket grows. Before this change, a single-ID request +// went through ListVolumes and filtered in memory, so cost scaled with the +// number of volumes in the bucket (N document reads for 1 requested). The +// fast path should stay flat across bucket sizes. +func BenchmarkDescribeVolumes_Provider_SingleID(b *testing.B) { + for _, n := range []int{1, 10, 100, 500} { + b.Run(fmt.Sprintf("volumes=%d", n), func(b *testing.B) { + svc := newTestVolumeService("ap-southeast-2a") + svc.SetEBSProvider(ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})) + targetID := seedProviderVolumes(b, svc, n, "acct-bench") + ctx := context.Background() + input := &ec2.DescribeVolumesInput{VolumeIds: []*string{aws.String(targetID)}} + + defer silenceLogs(b)() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err := svc.DescribeVolumes(ctx, input, "acct-bench"); err != nil { + b.Fatalf("DescribeVolumes: %v", err) + } + } + }) + } +} + +// BenchmarkDescribeVolumes_Provider_ListAndFilter measures the pre-fix +// approach directly (ListVolumes + in-memory filter for one ID), reusing the +// same seeded buckets as BenchmarkDescribeVolumes_Provider_SingleID so the +// two can be compared at matching bucket sizes to show how listing scales +// with N while the fast path (above) does not. +func BenchmarkDescribeVolumes_Provider_ListAndFilter(b *testing.B) { + for _, n := range []int{1, 10, 100, 500} { + b.Run(fmt.Sprintf("volumes=%d", n), func(b *testing.B) { + svc := newTestVolumeService("ap-southeast-2a") + svc.SetEBSProvider(ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})) + targetID := seedProviderVolumes(b, svc, n, "acct-bench") + ctx := context.Background() + + defer silenceLogs(b)() + b.ResetTimer() + for i := 0; i < b.N; i++ { + metadata, err := svc.metadata.ListVolumes(ctx) + if err != nil { + b.Fatalf("ListVolumes: %v", err) + } + var found *ec2.Volume + for _, meta := range metadata { + if meta.TenantID != "acct-bench" || meta.VolumeID != targetID { + continue + } + found = metadataVolumeToEC2(meta) + } + if found == nil { + b.Fatalf("volume %s not found", targetID) + } + } + }) + } +} diff --git a/spinifex/handlers/ec2/volume/describe_volumes_provider_fastpath_test.go b/spinifex/handlers/ec2/volume/describe_volumes_provider_fastpath_test.go new file mode 100644 index 000000000..59cf64c54 --- /dev/null +++ b/spinifex/handlers/ec2/volume/describe_volumes_provider_fastpath_test.go @@ -0,0 +1,135 @@ +package handlers_ec2_volume + +import ( + "context" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mulgadc/spinifex/spinifex/awserrors" + "github.com/mulgadc/spinifex/spinifex/ebsprovider" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestDescribeVolumes_Provider_FastPath_SingleID is the regression for the +// +18ms single-volume DescribeVolumes latency: under the provider path, a +// single requested ID must resolve via a direct GetVolume, not by listing +// (and decoding) every volume document in the bucket. +func TestDescribeVolumes_Provider_FastPath_SingleID(t *testing.T) { + svc := newTestVolumeService("ap-southeast-2a") + svc.SetEBSProvider(ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})) + ctx := context.Background() + + vol, err := svc.CreateVolume(ctx, &ec2.CreateVolumeInput{Size: aws.Int64(8), AvailabilityZone: aws.String("ap-southeast-2a")}, "acct-1") + require.NoError(t, err) + + out, err := svc.DescribeVolumes(ctx, &ec2.DescribeVolumesInput{VolumeIds: []*string{vol.VolumeId}}, "acct-1") + require.NoError(t, err) + require.Len(t, out.Volumes, 1) + assert.Equal(t, aws.StringValue(vol.VolumeId), aws.StringValue(out.Volumes[0].VolumeId)) + assert.Equal(t, int64(8), aws.Int64Value(out.Volumes[0].Size)) +} + +// TestDescribeVolumes_Provider_FastPath_CrossTenantIsNotFound is the most +// important test in this file: a directly-fetched volume belonging to a +// different tenant must never be returned, and must report NotFound exactly +// like an unknown ID — a regression here is a cross-tenant data leak. +func TestDescribeVolumes_Provider_FastPath_CrossTenantIsNotFound(t *testing.T) { + svc := newTestVolumeService("ap-southeast-2a") + svc.SetEBSProvider(ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})) + ctx := context.Background() + + otherTenantVol, err := svc.CreateVolume(ctx, &ec2.CreateVolumeInput{Size: aws.Int64(8), AvailabilityZone: aws.String("ap-southeast-2a")}, "acct-2") + require.NoError(t, err) + + out, err := svc.DescribeVolumes(ctx, &ec2.DescribeVolumesInput{VolumeIds: []*string{otherTenantVol.VolumeId}}, "acct-1") + require.Error(t, err) + assert.EqualError(t, err, awserrors.ErrorInvalidVolumeNotFound) + assert.Nil(t, out) +} + +// TestDescribeVolumes_Provider_FastPath_UnknownIDIsNotFound covers a +// requested ID with no ebsmetadata document at all. +func TestDescribeVolumes_Provider_FastPath_UnknownIDIsNotFound(t *testing.T) { + svc := newTestVolumeService("ap-southeast-2a") + svc.SetEBSProvider(ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})) + ctx := context.Background() + + out, err := svc.DescribeVolumes(ctx, &ec2.DescribeVolumesInput{VolumeIds: []*string{aws.String("vol-doesnotexist0")}}, "acct-1") + require.Error(t, err) + assert.EqualError(t, err, awserrors.ErrorInvalidVolumeNotFound) + assert.Nil(t, out) +} + +// TestDescribeVolumes_Provider_FastPath_FiltersApply confirms parsedFilters +// still apply to directly-fetched volumes exactly as they do to enumerated +// ones: a requested ID that fails the filter is dropped, not returned. +func TestDescribeVolumes_Provider_FastPath_FiltersApply(t *testing.T) { + svc := newTestVolumeService("ap-southeast-2a") + svc.SetEBSProvider(ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})) + ctx := context.Background() + + vol, err := svc.CreateVolume(ctx, &ec2.CreateVolumeInput{Size: aws.Int64(8), AvailabilityZone: aws.String("ap-southeast-2a")}, "acct-1") + require.NoError(t, err) + + t.Run("matching filter keeps the volume", func(t *testing.T) { + out, err := svc.DescribeVolumes(ctx, &ec2.DescribeVolumesInput{ + VolumeIds: []*string{vol.VolumeId}, + Filters: []*ec2.Filter{{Name: aws.String("size"), Values: []*string{aws.String("8")}}}, + }, "acct-1") + require.NoError(t, err) + require.Len(t, out.Volumes, 1) + }) + + t.Run("non-matching filter excludes the volume", func(t *testing.T) { + out, err := svc.DescribeVolumes(ctx, &ec2.DescribeVolumesInput{ + VolumeIds: []*string{vol.VolumeId}, + Filters: []*ec2.Filter{{Name: aws.String("size"), Values: []*string{aws.String("16")}}}, + }, "acct-1") + require.NoError(t, err) + assert.Empty(t, out.Volumes) + }) +} + +// TestDescribeVolumes_Provider_FastPath_MultipleIDs covers fetching more +// than one directly-requested ID in the same call. +func TestDescribeVolumes_Provider_FastPath_MultipleIDs(t *testing.T) { + svc := newTestVolumeService("ap-southeast-2a") + svc.SetEBSProvider(ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})) + ctx := context.Background() + + volA, err := svc.CreateVolume(ctx, &ec2.CreateVolumeInput{Size: aws.Int64(8), AvailabilityZone: aws.String("ap-southeast-2a")}, "acct-1") + require.NoError(t, err) + volB, err := svc.CreateVolume(ctx, &ec2.CreateVolumeInput{Size: aws.Int64(16), AvailabilityZone: aws.String("ap-southeast-2a")}, "acct-1") + require.NoError(t, err) + + out, err := svc.DescribeVolumes(ctx, &ec2.DescribeVolumesInput{VolumeIds: []*string{volA.VolumeId, volB.VolumeId}}, "acct-1") + require.NoError(t, err) + require.Len(t, out.Volumes, 2) + + ids := map[string]bool{} + for _, v := range out.Volumes { + ids[aws.StringValue(v.VolumeId)] = true + } + assert.True(t, ids[aws.StringValue(volA.VolumeId)]) + assert.True(t, ids[aws.StringValue(volB.VolumeId)]) +} + +// TestDescribeVolumes_Provider_NoIDs_StillEnumerates locks that the no-IDs +// case keeps using ListVolumes, not the direct-fetch fast path. +func TestDescribeVolumes_Provider_NoIDs_StillEnumerates(t *testing.T) { + svc := newTestVolumeService("ap-southeast-2a") + svc.SetEBSProvider(ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})) + ctx := context.Background() + + volA, err := svc.CreateVolume(ctx, &ec2.CreateVolumeInput{Size: aws.Int64(8), AvailabilityZone: aws.String("ap-southeast-2a")}, "acct-1") + require.NoError(t, err) + _, err = svc.CreateVolume(ctx, &ec2.CreateVolumeInput{Size: aws.Int64(16), AvailabilityZone: aws.String("ap-southeast-2a")}, "acct-2") + require.NoError(t, err) + + out, err := svc.DescribeVolumes(ctx, &ec2.DescribeVolumesInput{}, "acct-1") + require.NoError(t, err) + require.Len(t, out.Volumes, 1) + assert.Equal(t, aws.StringValue(volA.VolumeId), aws.StringValue(out.Volumes[0].VolumeId)) +} diff --git a/spinifex/handlers/ec2/volume/get_volume_metadata_provider_test.go b/spinifex/handlers/ec2/volume/get_volume_metadata_provider_test.go new file mode 100644 index 000000000..92a571b81 --- /dev/null +++ b/spinifex/handlers/ec2/volume/get_volume_metadata_provider_test.go @@ -0,0 +1,78 @@ +package handlers_ec2_volume + +import ( + "context" + "testing" + "time" + + "github.com/mulgadc/spinifex/spinifex/awserrors" + "github.com/mulgadc/spinifex/spinifex/ebsmetadata" + "github.com/mulgadc/spinifex/spinifex/ebsprovider" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newProviderVolumeService(t *testing.T) *VolumeServiceImpl { + t.Helper() + svc := newTestVolumeService("ap-southeast-2a") + svc.SetEBSProvider(ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})) + return svc +} + +// TestGetVolumeMetadata_Provider_ResolvesProviderCreatedVolume is the regression +// for the attach 404: a provider-created volume has no vol-*/config.json, so +// reading the legacy key made the daemon report a healthy volume as +// InvalidVolume.NotFound while DescribeVolumes showed it available. +func TestGetVolumeMetadata_Provider_ResolvesProviderCreatedVolume(t *testing.T) { + svc := newProviderVolumeService(t) + + volumeID := "vol-provider00001" + require.NoError(t, svc.metadata.PutVolume(context.Background(), ebsmetadata.Volume{ + VolumeID: volumeID, + TenantID: "acct-provider", + CapacityGiB: 8, + State: "available", + CreatedAt: time.Now(), + AvailabilityZone: "ap-southeast-2a", + VolumeType: "gp3", + IOPS: 3000, + Encrypted: true, + })) + + meta, err := svc.GetVolumeMetadata(volumeID) + require.NoError(t, err, "a provider-created volume must resolve without a legacy config.json") + assert.Equal(t, volumeID, meta.VolumeID) + assert.Equal(t, "acct-provider", meta.TenantID) + assert.Equal(t, "available", meta.State) + assert.Equal(t, "ap-southeast-2a", meta.AvailabilityZone) + assert.Equal(t, uint64(8), meta.CapacityGiB) +} + +// The daemon's attach validation reads State/AttachedInstance/DeviceName off +// this config to decide VolumeInUse versus idempotent re-attach, so the +// document's attachment fields have to survive the projection. +func TestGetVolumeMetadata_Provider_ReflectsAttachmentState(t *testing.T) { + svc := newProviderVolumeService(t) + + volumeID := "vol-provider00002" + require.NoError(t, svc.metadata.PutVolume(context.Background(), ebsmetadata.Volume{ + VolumeID: volumeID, TenantID: "acct-provider", CapacityGiB: 1, + State: "available", AvailabilityZone: "ap-southeast-2a", VolumeType: "gp3", + })) + + require.NoError(t, svc.UpdateVolumeState(volumeID, "in-use", "i-0123456789abcdef0", "/dev/sdf")) + + meta, err := svc.GetVolumeMetadata(volumeID) + require.NoError(t, err) + assert.Equal(t, "in-use", meta.State) + assert.Equal(t, "i-0123456789abcdef0", meta.AttachedInstance) + assert.Equal(t, "/dev/sdf", meta.DeviceName) +} + +func TestGetVolumeMetadata_Provider_UnknownVolumeIsNotFound(t *testing.T) { + svc := newProviderVolumeService(t) + + _, err := svc.GetVolumeMetadata("vol-doesnotexist0") + require.Error(t, err) + assert.Equal(t, awserrors.ErrorInvalidVolumeNotFound, err.Error()) +} diff --git a/spinifex/handlers/ec2/volume/lifecycle_invariants_test.go b/spinifex/handlers/ec2/volume/lifecycle_invariants_test.go index 44fc2dace..5e612e93a 100644 --- a/spinifex/handlers/ec2/volume/lifecycle_invariants_test.go +++ b/spinifex/handlers/ec2/volume/lifecycle_invariants_test.go @@ -11,10 +11,9 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/s3" "github.com/mulgadc/spinifex/spinifex/awserrors" - "github.com/mulgadc/spinifex/spinifex/handlers/ec2/volumestate" + "github.com/mulgadc/spinifex/spinifex/ebsmetadata" "github.com/mulgadc/spinifex/spinifex/objectstore" "github.com/mulgadc/spinifex/spinifex/types" - "github.com/mulgadc/viperblock/viperblock" "github.com/nats-io/nats.go" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -42,22 +41,29 @@ func (s *recordingObjectStore) PutObject(ctx context.Context, input *s3.PutObjec return s.ObjectStore.PutObject(ctx, input) } -// seedVolume writes a volume config.json into the store so listAllVolumeIDs / -// GetVolumeConfig see it. CreateVolume cannot be used in unit tests — it needs a -// live viperblock S3 backend. +// seedVolume writes the control-plane document the service reads, and gives the +// provider the matching volume. CreateVolume cannot be used in unit tests — it +// needs the cluster's encryption key file. func seedVolume(t *testing.T, svc *VolumeServiceImpl, volID, state, attachedInstance string) { t.Helper() - cfg := &viperblock.VolumeConfig{ - VolumeMetadata: viperblock.VolumeMetadata{ - VolumeID: volID, - TenantID: testVolAccountID, - SizeGiB: 8, - State: state, - AttachedInstance: attachedInstance, - AvailabilityZone: "ap-southeast-2a", - }, - } - require.NoError(t, svc.putVolumeConfig(context.Background(), volID, cfg)) + seedTestProviderVolume(t, svc, volID, 8) + require.NoError(t, svc.metadata.PutVolume(context.Background(), ebsmetadata.Volume{ + VolumeID: volID, + TenantID: testVolAccountID, + CapacityGiB: 8, + State: state, + AttachedInstance: attachedInstance, + AvailabilityZone: "ap-southeast-2a", + VolumeType: "gp3", + })) +} + +// metadataKey is the object key the control-plane document for volumeID lives at. +func metadataKey(t *testing.T, volumeID string) string { + t.Helper() + key, err := ebsmetadata.VolumeKey(volumeID) + require.NoError(t, err) + return key } // TestRLC1_VolumeDeleteNotFoundOnAbsent enforces the Common Resource Lifecycle @@ -91,23 +97,23 @@ func TestRLC3_VolumeDeleteRequiresDetach(t *testing.T) { "ADR-0005 §2: deleting an attached volume must return VolumeInUse (detach-before-delete, rule #3)") } -// TestVolumeTagMirror_WritesOnlyControlPlaneTagsObject locks the single-writer +// TestVolumeTagMirror_WritesOnlyControlPlaneDocument locks the single-writer // invariant: tag mutations must never rewrite config.json or route an encrypted // volume through the ebs.config keyholder queue. -func TestVolumeTagMirror_WritesOnlyControlPlaneTagsObject(t *testing.T) { +func TestVolumeTagMirror_WritesOnlyControlPlaneDocument(t *testing.T) { memoryStore := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", memoryStore) svc.natsConn = startTestNATS(t) volumeID := "vol-enc-tag-writer" seedEncryptedConfig(t, memoryStore, volumeID) - require.NoError(t, svc.putVolumeState(context.Background(), volumeID, volumestate.Record{ - State: "in-use", - AttachedInstance: "i-live0000000000", - DeviceName: "/dev/nbd0", - })) + seedVolumeDocument(t, memoryStore, ebsmetadata.Volume{ + VolumeID: volumeID, TenantID: testVolAccountID, CapacityGiB: 10, State: "available", + }) + require.NoError(t, svc.UpdateVolumeState(volumeID, "in-use", "i-live0000000000", "/dev/nbd0")) before := getStoredConfig(t, memoryStore, volumeID) store := &recordingObjectStore{ObjectStore: memoryStore} svc.store = store + svc.metadata = ebsmetadata.NewStore(store, "test-bucket") var configRequests atomic.Int32 sub, err := svc.natsConn.Subscribe("ebs.config", func(msg *nats.Msg) { @@ -130,27 +136,30 @@ func TestVolumeTagMirror_WritesOnlyControlPlaneTagsObject(t *testing.T) { Tags: []*ec2.Tag{{Key: aws.String("owner"), Value: aws.String("control-plane")}}, }, testVolAccountID)) - assert.Equal(t, []string{volumeTagsKey(volumeID), volumeTagsKey(volumeID)}, store.putKeys, - "tag mutations must write only tags.json") + key := metadataKey(t, volumeID) + assert.Equal(t, []string{key, key}, store.putKeys, + "tag mutations must write only the control-plane document") assert.Equal(t, int32(0), configRequests.Load(), "tag mutations must never issue an ebs.config request") assert.Equal(t, before, getStoredConfig(t, memoryStore, volumeID), "tag mutations must leave the sealed config.json untouched") } -// TestVolumeTagMirror_ReturnsTagsObjectWriteFailure prevents tag persistence +// TestVolumeTagMirror_ReturnsDocumentWriteFailure prevents tag persistence // errors from becoming false-successful CreateTags responses. -func TestVolumeTagMirror_ReturnsTagsObjectWriteFailure(t *testing.T) { +func TestVolumeTagMirror_ReturnsDocumentWriteFailure(t *testing.T) { memoryStore := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", memoryStore) volumeID := "vol-tag-write-error" seedVolume(t, svc, volumeID, "in-use", "i-live0000000000") - svc.store = &recordingObjectStore{ + failing := &recordingObjectStore{ ObjectStore: memoryStore, putError: map[string]error{ - volumeTagsKey(volumeID): errors.New("tags store unavailable"), + metadataKey(t, volumeID): errors.New("tags store unavailable"), }, } + svc.store = failing + svc.metadata = ebsmetadata.NewStore(failing, "test-bucket") err := svc.ApplyRecordTags(&ec2.CreateTagsInput{ Resources: []*string{aws.String(volumeID)}, @@ -182,24 +191,24 @@ func TestRLC5_VolumeGCMarksLeakedVolumeNeverDeletes(t *testing.T) { require.NoError(t, err) assert.Equal(t, 1, marked, "ADR-0005 §3: the GC must surface exactly the leaked volume") - leaked, err := svc.GetVolumeConfig("vol-leaked0000000") + leaked, err := svc.GetVolumeMetadata("vol-leaked0000000") require.NoErrorf(t, err, "ADR-0005 §3: the GC must NOT delete the leaked volume — it carries data; it may only mark + alarm") - assert.NotEmpty(t, leaked.VolumeMetadata.Tags[orphanTagKey], + assert.NotEmpty(t, leaked.Tags[orphanTagKey], "ADR-0005 §3: a leaked volume must be marked orphaned") - assert.Equal(t, "preserved", leaked.VolumeMetadata.Tags["customer"], - "orphan marking must preserve tags from the authoritative tags.json") + assert.Equal(t, "preserved", leaked.Tags["customer"], + "orphan marking must preserve the volume's existing tags") // A volume attached to an instance NOT in the leaked set must be untouched — // node-local detection never false-marks another node's live-instance volume. - live, err := svc.GetVolumeConfig("vol-live000000000") + live, err := svc.GetVolumeMetadata("vol-live000000000") require.NoError(t, err) - assert.Empty(t, live.VolumeMetadata.Tags[orphanTagKey], + assert.Empty(t, live.Tags[orphanTagKey], "a volume on a live instance must never be marked orphaned") // Idempotent: a second sweep re-marks nothing and still deletes nothing. marked, err = reaper.Sweep(context.Background()) require.NoError(t, err) assert.Equal(t, 0, marked, "an already-marked orphan must not be re-marked") - _, err = svc.GetVolumeConfig("vol-leaked0000000") + _, err = svc.GetVolumeMetadata("vol-leaked0000000") require.NoError(t, err, "ADR-0005 §3: the volume must still exist after repeated sweeps") } diff --git a/spinifex/handlers/ec2/volume/provider_snapshot_source_test.go b/spinifex/handlers/ec2/volume/provider_snapshot_source_test.go new file mode 100644 index 000000000..07543f730 --- /dev/null +++ b/spinifex/handlers/ec2/volume/provider_snapshot_source_test.go @@ -0,0 +1,175 @@ +package handlers_ec2_volume + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/mulgadc/spinifex/spinifex/config" + "github.com/mulgadc/spinifex/spinifex/ebsmetadata" + "github.com/mulgadc/spinifex/spinifex/ebsprovider" + handlers_ec2_instance "github.com/mulgadc/spinifex/spinifex/handlers/ec2/instance" + "github.com/mulgadc/spinifex/spinifex/objectstore" + spxtypes "github.com/mulgadc/spinifex/spinifex/types" + "github.com/mulgadc/spinifex/spinifex/vm" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const providerTestAZ = "ap-southeast-2a" + +// snapshotSourceProvider mirrors viperblockd's handleCreateVolume validation: +// a clone resolves its base blocks against the source volume's prefix, so a +// snapshot source without a source volume is rejected outright. +type snapshotSourceProvider struct { + *ebsprovider.MemoryProvider + + lastRequest ebsprovider.CreateVolumeRequest +} + +func (p *snapshotSourceProvider) CreateVolume(ctx context.Context, req ebsprovider.CreateVolumeRequest) (*ebsprovider.Volume, error) { + p.lastRequest = req + if req.SourceSnapshotID != "" && req.SourceSnapshotVolumeID == "" { + return nil, fmt.Errorf("%w: source_snapshot_volume_id is required with source_snapshot_id", ebsprovider.ErrInvalidArgument) + } + // The snapshot exists only in the control plane's metadata for this test, + // so bypass the in-memory provider's own snapshot bookkeeping. + req.SourceSnapshotID = "" + return p.MemoryProvider.CreateVolume(ctx, req) +} + +func newSnapshotSourceProvider() *snapshotSourceProvider { + return &snapshotSourceProvider{MemoryProvider: ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})} +} + +// writeSnapshotMetadata publishes the control-plane record CreateVolume reads +// to learn which volume a snapshot was taken from. The owner must be set: +// CreateVolume fails closed on a snapshot whose owner it cannot match. +func writeSnapshotMetadata(t *testing.T, store objectstore.ObjectStore, snapshotID, sourceVolumeID, ownerID string, sizeGiB int64) { + t.Helper() + data, err := json.Marshal(snapshotMetadata{VolumeID: sourceVolumeID, VolumeSize: sizeGiB, OwnerID: ownerID}) + require.NoError(t, err) + _, err = store.PutObject(context.Background(), &s3.PutObjectInput{ + Bucket: aws.String("test-bucket"), + Key: aws.String(snapshotID + "/metadata.json"), + Body: strings.NewReader(string(data)), + }) + require.NoError(t, err) +} + +// TestCreateVolume_Provider_FromSnapshotSendsSourceVolumeID locks that a +// snapshot-backed CreateVolume names the snapshot's source volume. Without it +// the provider rejects the request and create-from-snapshot cannot work at all. +func TestCreateVolume_Provider_FromSnapshotSendsSourceVolumeID(t *testing.T) { + store := objectstore.NewMemoryObjectStore() + cfg := &config.Config{AZ: providerTestAZ, Predastore: config.PredastoreConfig{Bucket: "test-bucket"}} + svc := NewVolumeServiceImplWithStore(cfg, store, nil) + provider := newSnapshotSourceProvider() + svc.SetEBSProvider(provider) + + writeSnapshotMetadata(t, store, "snap-src", "vol-origin", "acct-1", 8) + + created, err := svc.CreateVolume(context.Background(), &ec2.CreateVolumeInput{ + Size: aws.Int64(8), AvailabilityZone: aws.String(providerTestAZ), SnapshotId: aws.String("snap-src"), + }, "acct-1") + require.NoError(t, err) + require.NotNil(t, created) + + assert.Equal(t, "snap-src", provider.lastRequest.SourceSnapshotID) + assert.Equal(t, "vol-origin", provider.lastRequest.SourceSnapshotVolumeID, + "the snapshot's source volume must travel with the snapshot ID") +} + +// TestCreateVolume_Provider_WithoutSnapshotSendsNoSource locks that an ordinary +// blank volume does not gain a spurious source volume. +func TestCreateVolume_Provider_WithoutSnapshotSendsNoSource(t *testing.T) { + store := objectstore.NewMemoryObjectStore() + cfg := &config.Config{AZ: providerTestAZ, Predastore: config.PredastoreConfig{Bucket: "test-bucket"}} + svc := NewVolumeServiceImplWithStore(cfg, store, nil) + provider := newSnapshotSourceProvider() + svc.SetEBSProvider(provider) + + _, err := svc.CreateVolume(context.Background(), &ec2.CreateVolumeInput{ + Size: aws.Int64(8), AvailabilityZone: aws.String(providerTestAZ), + }, "acct-1") + require.NoError(t, err) + + assert.Empty(t, provider.lastRequest.SourceSnapshotID) + assert.Empty(t, provider.lastRequest.SourceSnapshotVolumeID) +} + +// fixedAMILoader resolves one AMI to a snapshot on a known source volume. +type fixedAMILoader struct { + meta ebsmetadata.AMI + sourceVolumeID string +} + +func (l *fixedAMILoader) GetAMIConfig(_ context.Context, _ string) (ebsmetadata.AMI, error) { + return l.meta, nil +} + +func (l *fixedAMILoader) GetAMISourceVolumeID(_ context.Context, _ string) (string, error) { + return l.sourceVolumeID, nil +} + +// TestDescribeVolumes_Provider_SeesInstanceRootVolume is the end-to-end guard +// for the launch path: a root volume allocated through the provider carries no +// legacy config.json, so it is visible only if the instance service records an +// ebsmetadata document for it. +func TestDescribeVolumes_Provider_SeesInstanceRootVolume(t *testing.T) { + store := objectstore.NewMemoryObjectStore() + cfg := &config.Config{ + AZ: providerTestAZ, + WalDir: t.TempDir(), + Predastore: config.PredastoreConfig{Bucket: "test-bucket", Host: "127.0.0.1:1", Region: "us-east-1"}, + } + provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{}) + + // The AMI's snapshot must already exist in the provider, as it would after + // the image was registered. + const amiBytes = 4 * 1024 * 1024 * 1024 + _, err := provider.CreateVolume(context.Background(), ebsprovider.CreateVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-origin", + CapacityRange: ebsprovider.CapacityRange{RequiredBytes: amiBytes}, + }) + require.NoError(t, err) + _, err = provider.CreateSnapshot(context.Background(), ebsprovider.CreateSnapshotRequest{ + Versioned: ebsprovider.NewVersioned(), SnapshotID: "snap-src", VolumeID: "vol-origin", + }) + require.NoError(t, err) + + instanceSvc := handlers_ec2_instance.NewInstanceServiceImpl(cfg, nil, nil, store, nil, nil, nil) + instanceSvc.SetEBSProvider(provider) + instanceSvc.SetRunInstancesDeps(&fixedAMILoader{ + meta: ebsmetadata.AMI{ImageID: "ami-1", SnapshotID: "snap-src", VolumeSizeGiB: 4}, + sourceVolumeID: "vol-origin", + }, nil, nil, nil) + + instance := &vm.VM{AccountID: "acct-1"} + volumeInfos, err := instanceSvc.GenerateVolumes(context.Background(), &ec2.RunInstancesInput{ + ImageId: aws.String("ami-1"), + }, instance) + require.NoError(t, err, "the launch must allocate its root volume through the provider") + require.Len(t, volumeInfos, 1) + rootVolumeID := volumeInfos[0].VolumeId + + volumeSvc := NewVolumeServiceImplWithStore(cfg, store, nil) + volumeSvc.SetEBSProvider(provider) + + out, err := volumeSvc.DescribeVolumes(context.Background(), &ec2.DescribeVolumesInput{}, "acct-1") + require.NoError(t, err) + require.Len(t, out.Volumes, 1, "the instance root volume must be visible to DescribeVolumes") + + got := out.Volumes[0] + assert.Equal(t, rootVolumeID, aws.StringValue(got.VolumeId)) + assert.Equal(t, "available", aws.StringValue(got.State)) + assert.Equal(t, providerTestAZ, aws.StringValue(got.AvailabilityZone)) + assert.Equal(t, spxtypes.VolumeTypeGP3, aws.StringValue(got.VolumeType)) + assert.Equal(t, int64(spxtypes.DefaultGP3IOPS), aws.Int64Value(got.Iops)) + assert.Equal(t, "snap-src", aws.StringValue(got.SnapshotId)) +} diff --git a/spinifex/handlers/ec2/volume/record_tags_test.go b/spinifex/handlers/ec2/volume/record_tags_test.go index f4126e9b2..e7a66e92f 100644 --- a/spinifex/handlers/ec2/volume/record_tags_test.go +++ b/spinifex/handlers/ec2/volume/record_tags_test.go @@ -8,6 +8,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/s3" + "github.com/mulgadc/spinifex/spinifex/ebsmetadata" "github.com/mulgadc/spinifex/spinifex/objectstore" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -26,10 +27,10 @@ func TestVolumeRecordTagsMirror(t *testing.T) { }, }, testVolAccountID)) - cfg, err := svc.GetVolumeConfig("vol-tagmirror0001") + meta, err := svc.GetVolumeMetadata("vol-tagmirror0001") require.NoError(t, err) - assert.Equal(t, "yes", cfg.VolumeMetadata.Tags["keep"]) - assert.Equal(t, "v", cfg.VolumeMetadata.Tags["drop"]) + assert.Equal(t, "yes", meta.Tags["keep"]) + assert.Equal(t, "v", meta.Tags["drop"]) // Value-mismatched delete is a no-op; matched delete removes. require.NoError(t, svc.RemoveRecordTags(&ec2.DeleteTagsInput{ @@ -40,10 +41,10 @@ func TestVolumeRecordTagsMirror(t *testing.T) { }, }, testVolAccountID)) - cfg, err = svc.GetVolumeConfig("vol-tagmirror0001") + meta, err = svc.GetVolumeMetadata("vol-tagmirror0001") require.NoError(t, err) - assert.Equal(t, "yes", cfg.VolumeMetadata.Tags["keep"]) - _, ok := cfg.VolumeMetadata.Tags["drop"] + assert.Equal(t, "yes", meta.Tags["keep"]) + _, ok := meta.Tags["drop"] assert.False(t, ok) } @@ -69,11 +70,15 @@ func TestApplyRecordTags_AttachedVolumePersistsForTagFilterDiscovery(t *testing. assert.Equal(t, volumeID, aws.StringValue(out.Volumes[0].VolumeId)) } +// A tag must survive the live nbdkit VB rewriting config.json from its own +// mount-time state: the tag lives in the control-plane document, which that +// writer does not own. func TestApplyRecordTags_SurvivesStaleConfigRewrite(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) volumeID := "vol-staleconfigtag" - seedVolume(t, svc, volumeID, "in-use", "i-live0000000000") + seedVolume(t, svc, volumeID, "available", "") + seedProviderConfig(t, store, volumeID) staleConfig := getStoredConfig(t, store, volumeID) require.NoError(t, svc.ApplyRecordTags(&ec2.CreateTagsInput{ @@ -89,9 +94,9 @@ func TestApplyRecordTags_SurvivesStaleConfigRewrite(t *testing.T) { }) require.NoError(t, err) - cfg, err := svc.GetVolumeConfig(volumeID) + meta, err := svc.GetVolumeMetadata(volumeID) require.NoError(t, err) - assert.Equal(t, "retain", cfg.VolumeMetadata.Tags["sweep"]) + assert.Equal(t, "retain", meta.Tags["sweep"]) out, err := svc.DescribeVolumes(context.Background(), &ec2.DescribeVolumesInput{ Filters: []*ec2.Filter{{ @@ -104,55 +109,51 @@ func TestApplyRecordTags_SurvivesStaleConfigRewrite(t *testing.T) { assert.Equal(t, volumeID, aws.StringValue(out.Volumes[0].VolumeId)) } -func TestApplyRecordTags_FirstWriteSeedsEmbeddedTags(t *testing.T) { +// A tag write must merge into the tags already on the document rather than +// replacing the set wholesale. +func TestApplyRecordTags_MergesWithExistingTags(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) volumeID := "vol-seedembedded" - seedVolumeWithEmbeddedTags(t, svc, volumeID, map[string]string{"created-with": "volume"}) + seedVolume(t, svc, volumeID, "available", "") + seedVolumeDocument(t, store, ebsmetadata.Volume{ + VolumeID: volumeID, TenantID: testVolAccountID, CapacityGiB: 8, State: "available", + Tags: map[string]string{"created-with": "volume"}, + }) require.NoError(t, svc.ApplyRecordTags(&ec2.CreateTagsInput{ Resources: []*string{aws.String(volumeID)}, Tags: []*ec2.Tag{{Key: aws.String("added-later"), Value: aws.String("yes")}}, }, testVolAccountID)) - cfg, err := svc.GetVolumeConfig(volumeID) + meta, err := svc.GetVolumeMetadata(volumeID) require.NoError(t, err) assert.Equal(t, map[string]string{ "created-with": "volume", "added-later": "yes", - }, cfg.VolumeMetadata.Tags) + }, meta.Tags) } -func TestRemoveRecordTags_EmptyTagsJSONOverridesEmbeddedTags(t *testing.T) { +// Removing the last tag must leave an authoritative empty tag set rather than +// any other source of tags resurrecting the deleted one. +func TestRemoveRecordTags_LastTagLeavesEmptySet(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) volumeID := "vol-emptytagsjson" - seedVolumeWithEmbeddedTags(t, svc, volumeID, map[string]string{"legacy": "tag"}) + seedVolume(t, svc, volumeID, "available", "") + seedVolumeDocument(t, store, ebsmetadata.Volume{ + VolumeID: volumeID, TenantID: testVolAccountID, CapacityGiB: 8, State: "available", + Tags: map[string]string{"legacy": "tag"}, + }) require.NoError(t, svc.RemoveRecordTags(&ec2.DeleteTagsInput{ Resources: []*string{aws.String(volumeID)}, Tags: []*ec2.Tag{{Key: aws.String("legacy"), Value: aws.String("tag")}}, }, testVolAccountID)) - tags, found, err := svc.getVolumeTags(context.Background(), volumeID) + meta, err := svc.GetVolumeMetadata(volumeID) require.NoError(t, err) - assert.True(t, found, "removing the final tag must leave an authoritative tags.json") - assert.Empty(t, tags) - - cfg, err := svc.GetVolumeConfig(volumeID) - require.NoError(t, err) - assert.Empty(t, cfg.VolumeMetadata.Tags, "present empty tags.json must not fall back to embedded tags") -} - -// seedVolumeWithEmbeddedTags creates a legacy or create-time tagged volume with -// no tags.json so tests exercise the migration fallback. -func seedVolumeWithEmbeddedTags(t *testing.T, svc *VolumeServiceImpl, volumeID string, tags map[string]string) { - t.Helper() - seedVolume(t, svc, volumeID, "available", "") - cfg, err := svc.GetVolumeConfig(volumeID) - require.NoError(t, err) - cfg.VolumeMetadata.Tags = tags - require.NoError(t, svc.putVolumeConfig(context.Background(), volumeID, cfg)) + assert.Empty(t, meta.Tags, "the deleted tag must not come back from anywhere") } func TestVolumeRecordTagsMirror_CrossTenantNoMutation(t *testing.T) { @@ -166,9 +167,9 @@ func TestVolumeRecordTagsMirror_CrossTenantNoMutation(t *testing.T) { Tags: []*ec2.Tag{{Key: aws.String("evil"), Value: aws.String("1")}}, }, "999999999999")) - cfg, err := svc.GetVolumeConfig("vol-tenantguard01") + meta, err := svc.GetVolumeMetadata("vol-tenantguard01") require.NoError(t, err) - assert.Empty(t, cfg.VolumeMetadata.Tags) + assert.Empty(t, meta.Tags) } func TestVolumeRecordTagsMirror_UnownedNoError(t *testing.T) { diff --git a/spinifex/handlers/ec2/volume/service_impl.go b/spinifex/handlers/ec2/volume/service_impl.go index 8d3feede8..b9b4f1562 100644 --- a/spinifex/handlers/ec2/volume/service_impl.go +++ b/spinifex/handlers/ec2/volume/service_impl.go @@ -1,7 +1,6 @@ package handlers_ec2_volume import ( - "bytes" "context" "encoding/json" "errors" @@ -17,35 +16,20 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/s3" - "github.com/mulgadc/bluebottle/pkg/masterkey" "github.com/mulgadc/spinifex/spinifex/awserrors" "github.com/mulgadc/spinifex/spinifex/config" + "github.com/mulgadc/spinifex/spinifex/ebsmetadata" + "github.com/mulgadc/spinifex/spinifex/ebsprovider" "github.com/mulgadc/spinifex/spinifex/filterutil" handlers_ec2_instance "github.com/mulgadc/spinifex/spinifex/handlers/ec2/instance" - "github.com/mulgadc/spinifex/spinifex/handlers/ec2/volumestate" "github.com/mulgadc/spinifex/spinifex/objectstore" "github.com/mulgadc/spinifex/spinifex/types" "github.com/mulgadc/spinifex/spinifex/utils" "github.com/mulgadc/spinifex/spinifex/vm" - "github.com/mulgadc/viperblock/viperblock" - s3backend "github.com/mulgadc/viperblock/viperblock/backends/s3" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" ) -const ( - // gp3 IOPS envelope (AWS): 3000 baseline on any size, up to 500 IOPS/GiB, - // capped at 16000. - defaultGP3IOPS = 3000 - maxGP3IOPS = 16000 - gp3IOPSPerGiB = 500 - - // gp3 Throughput envelope (AWS): 125 MiB/s baseline, 1000 MiB/s ceiling, - // flat range independent of volume size. - defaultGP3Throughput = 125 - maxGP3Throughput = 1000 -) - // Ensure VolumeServiceImpl implements VolumeService. var _ VolumeService = (*VolumeServiceImpl)(nil) @@ -65,6 +49,27 @@ type VolumeServiceImpl struct { bucketName string natsConn *nats.Conn snapshotKV jetstream.KeyValue + provider ebsprovider.EBSProvider + metadata *ebsmetadata.Store +} + +// SetEBSProvider injects the provider boundary used by the control plane. It +// is set once by the composition root; a service without it can serve reads +// from ebsmetadata but cannot allocate, expand or destroy storage. +func (s *VolumeServiceImpl) SetEBSProvider(provider ebsprovider.EBSProvider) { + s.provider = provider +} + +// EBSProvider returns the injected provider boundary. Primarily for +// composition-root tests to observe wiring. +func (s *VolumeServiceImpl) EBSProvider() ebsprovider.EBSProvider { + return s.provider +} + +// MetadataStore returns the control-plane metadata store. Primarily for +// composition-root tests to observe wiring. +func (s *VolumeServiceImpl) MetadataStore() *ebsmetadata.Store { + return s.metadata } // NewVolumeServiceImpl creates a new daemon-side volume service. @@ -84,6 +89,7 @@ func NewVolumeServiceImpl(cfg *config.Config, natsConn *nats.Conn, snapshotKV je bucketName: cfg.Predastore.Bucket, natsConn: natsConn, snapshotKV: snapshotKV, + metadata: ebsmetadata.NewStore(store, cfg.Predastore.Bucket), } } @@ -98,6 +104,7 @@ func NewVolumeServiceImplWithStore(cfg *config.Config, store objectstore.ObjectS store: store, bucketName: bucketName, natsConn: natsConn, + metadata: ebsmetadata.NewStore(store, bucketName), } if len(snapshotKV) > 0 { svc.snapshotKV = snapshotKV[0] @@ -105,17 +112,18 @@ func NewVolumeServiceImplWithStore(cfg *config.Config, store objectstore.ObjectS return svc } -// CreateVolume creates a new EBS volume via viperblock and persists its config to S3. +// CreateVolume allocates a new EBS volume through the EBS provider and records +// its control-plane metadata document. func (s *VolumeServiceImpl) CreateVolume(ctx context.Context, input *ec2.CreateVolumeInput, accountID string) (*ec2.Volume, error) { if input == nil { return nil, errors.New(awserrors.ErrorInvalidParameterValue) } // Validate volume type: only gp3 supported (or empty defaults to gp3) - if input.VolumeType != nil && *input.VolumeType != "" && *input.VolumeType != "gp3" { + if input.VolumeType != nil && *input.VolumeType != "" && *input.VolumeType != types.VolumeTypeGP3 { return nil, errors.New(awserrors.ErrorUnknownVolumeType) } - volumeType := "gp3" + volumeType := types.VolumeTypeGP3 // Validate availability zone matches this node's AZ if input.AvailabilityZone == nil || *input.AvailabilityZone == "" { @@ -173,22 +181,22 @@ func (s *VolumeServiceImpl) CreateVolume(ctx context.Context, input *ec2.CreateV // Honor caller-supplied Iops for gp3, else the 3000 baseline. The ceiling is // min(16000, 500*size) but never below the free baseline, so small volumes // still get 3000. - iops := defaultGP3IOPS + iops := types.DefaultGP3IOPS if input.Iops != nil { iops = int(*input.Iops) } - maxIOPS := min(max(int(size)*gp3IOPSPerGiB, defaultGP3IOPS), maxGP3IOPS) - if iops < defaultGP3IOPS || iops > maxIOPS { + maxIOPS := min(max(int(size)*types.GP3IOPSPerGiB, types.DefaultGP3IOPS), types.MaxGP3IOPS) + if iops < types.DefaultGP3IOPS || iops > maxIOPS { return nil, errors.New(awserrors.ErrorInvalidParameterValue) } // Honor caller-supplied Throughput for gp3, else the 125 MiB/s baseline. // Range is flat (125-1000), unlike Iops it does not scale with size. - throughput := defaultGP3Throughput + throughput := types.DefaultGP3Throughput if input.Throughput != nil { throughput = int(*input.Throughput) } - if throughput < defaultGP3Throughput || throughput > maxGP3Throughput { + if throughput < types.DefaultGP3Throughput || throughput > types.MaxGP3Throughput { return nil, errors.New(awserrors.ErrorInvalidParameterValue) } @@ -197,120 +205,74 @@ func (s *VolumeServiceImpl) CreateVolume(ctx context.Context, input *ec2.CreateV tags := utils.ExtractTags(input.TagSpecifications, "volume") - // Volume size in bytes for viperblock - sizeGiB := utils.SafeInt64ToUint64(size) - volumeSizeBytes := sizeGiB * 1024 * 1024 * 1024 - - // Build VolumeConfig with metadata - volumeConfig := viperblock.VolumeConfig{ - VolumeMetadata: viperblock.VolumeMetadata{ - VolumeID: volumeID, - TenantID: accountID, - SizeGiB: sizeGiB, - State: "available", - CreatedAt: now, - AvailabilityZone: *input.AvailabilityZone, - VolumeType: volumeType, - IOPS: iops, - Throughput: throughput, - SnapshotID: snapshotID, - Tags: tags, - }, + if err := s.requireProvider(ctx, "CreateVolume"); err != nil { + return nil, err } - // Create S3 backend config - cfg := s3backend.S3Config{ - VolumeName: volumeID, - VolumeSize: volumeSizeBytes, - Bucket: s.bucketName, - Region: s.config.Predastore.Region, - AccessKey: s.config.Predastore.AccessKey, - SecretKey: s.config.Predastore.SecretKey, - Host: s.config.Predastore.Host, + // SourceVolumeName must travel with the snapshot ID: a clone resolves its + // base blocks against the source volume's prefix, and the provider rejects + // a snapshot source without it. + created, err := s.provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), + VolumeID: volumeID, + CapacityRange: ebsprovider.CapacityRange{RequiredBytes: size * 1024 * 1024 * 1024}, + AvailabilityZone: *input.AvailabilityZone, + SourceSnapshotID: snapshotID, + SourceSnapshotVolumeID: sourceVolumeName, + }) + if err != nil { + slog.ErrorContext(ctx, "CreateVolume: provider allocation failed", "volumeId", volumeID, "err", err) + return nil, errors.New(awserrors.ErrorServerInternal) + } + if created == nil { + return nil, errors.New(awserrors.ErrorServerInternal) } + // The control plane cannot see how a provider encrypts its volumes, but this + // shared config knob is the one the provider resolves the same way. mkey, err := utils.LoadViperblockMasterKey(s.config.Viperblock.EncryptionKeyFile) if err != nil { - slog.ErrorContext(ctx, "CreateVolume failed to load encryption key", "err", err) + slog.ErrorContext(ctx, "CreateVolume: failed to load encryption key", "volumeId", volumeID, "err", err) + _ = s.provider.DeleteVolume(ctx, ebsprovider.DeleteVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, Handle: created.Handle}) return nil, errors.New(awserrors.ErrorServerInternal) } - - vbconfig := s.buildVBConfig(volumeID, volumeSizeBytes, volumeConfig, mkey, snapshotID, sourceVolumeName) - - vb, err := viperblock.New(vbconfig, "s3", cfg) - if err != nil { - slog.ErrorContext(ctx, "CreateVolume failed to create viperblock instance", "err", err) - return nil, errors.New(awserrors.ErrorServerInternal) - } - defer func() { - vb.StopChunkUploader() - vb.StopWALSyncer() - }() - - vb.SetDebug(false) - - // Initialize the backend (creates bucket structure in S3) - if err := vb.Backend.Init(); err != nil { - slog.ErrorContext(ctx, "CreateVolume failed to initialize backend", "err", err) - return nil, errors.New(awserrors.ErrorServerInternal) - } - - // Persist volume state to S3 (writes config.json) - if err := vb.SaveState(); err != nil { - slog.ErrorContext(ctx, "CreateVolume failed to save state", "err", err) + if err := s.metadata.PutVolume(ctx, ebsmetadata.Volume{ + VolumeID: volumeID, TenantID: accountID, CapacityGiB: uint64(size), State: string(ebsprovider.VolumeStateAvailable), + CreatedAt: now, AvailabilityZone: *input.AvailabilityZone, VolumeType: volumeType, + IOPS: iops, Throughput: throughput, SnapshotID: snapshotID, Tags: tags, ProviderHandle: created.Handle, + Encrypted: mkey != nil, + }); err != nil { + slog.ErrorContext(ctx, "CreateVolume: failed to persist provider metadata", "volumeId", volumeID, "err", err) + _ = s.provider.DeleteVolume(ctx, ebsprovider.DeleteVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, Handle: created.Handle}) return nil, errors.New(awserrors.ErrorServerInternal) } slog.InfoContext(ctx, "CreateVolume completed", "volumeId", volumeID, "size", size, "type", volumeType) - vol := &ec2.Volume{ - VolumeId: aws.String(volumeID), - Size: aws.Int64(size), - VolumeType: aws.String(volumeType), - State: aws.String("available"), - AvailabilityZone: input.AvailabilityZone, - CreateTime: aws.Time(now), - Iops: aws.Int64(int64(iops)), - Throughput: aws.Int64(int64(throughput)), - Encrypted: aws.Bool(mkey != nil), - Tags: utils.MapToEC2Tags(tags), - } - - if snapshotID != "" { - vol.SnapshotId = aws.String(snapshotID) - } - - return vol, nil + return &ec2.Volume{VolumeId: aws.String(volumeID), Size: aws.Int64(size), VolumeType: aws.String(volumeType), + State: aws.String("available"), AvailabilityZone: input.AvailabilityZone, CreateTime: aws.Time(now), + Iops: aws.Int64(int64(iops)), Throughput: aws.Int64(int64(throughput)), Encrypted: aws.Bool(mkey != nil), + SnapshotId: snapshotIDOrNil(snapshotID), Tags: utils.MapToEC2Tags(tags)}, nil } -// buildVBConfig assembles the viperblock.VB config CreateVolume hands to -// viperblock.New. Split out from CreateVolume so the GC-enablement wiring can -// be asserted directly against the returned config, without needing a live -// viperblock/S3 backend to observe it: GCEnabled is not part of -// viperblock.VBState, so it never round-trips through a saved config.json and -// can't be checked by reading volume state back. -func (s *VolumeServiceImpl) buildVBConfig(volumeID string, volumeSizeBytes uint64, volumeConfig viperblock.VolumeConfig, mkey *masterkey.Key, snapshotID, sourceVolumeName string) *viperblock.VB { - vbconfig := &viperblock.VB{ - VolumeName: volumeID, - VolumeSize: volumeSizeBytes, - BaseDir: s.config.WalDir, - Cache: viperblock.Cache{Config: viperblock.CacheConfig{Size: 0}}, - VolumeConfig: volumeConfig, - MasterKey: mkey, - EncryptionEnabled: mkey != nil, - // GCEnabled: default false unless explicitly set to true, matching the - // nbdkit plugin and viperblockd resolution of the same config field. - GCEnabled: s.config.Viperblock.GCEnabled != nil && *s.config.Viperblock.GCEnabled, +// snapshotIDOrNil keeps SnapshotId absent for a volume that was not cloned, +// rather than serialising an empty string. +func snapshotIDOrNil(snapshotID string) *string { + if snapshotID == "" { + return nil } + return aws.String(snapshotID) +} - // If created from a snapshot, set the snapshot fields so viperblock's - // LoadState will call OpenFromSnapshot to load the base block map. - if snapshotID != "" { - vbconfig.SnapshotID = snapshotID - vbconfig.SourceVolumeName = sourceVolumeName +// requireProvider fails the call when no EBS provider is wired. Every write +// path delegates allocation to the provider, so a nil one is a wiring fault +// rather than a mode the control plane can serve. +func (s *VolumeServiceImpl) requireProvider(ctx context.Context, op string) error { + if s.provider == nil { + slog.ErrorContext(ctx, "no EBS provider configured", "op", op) + return errors.New(awserrors.ErrorServerInternal) } - - return vbconfig + return nil } // describeVolumesValidFilters defines the set of filter names accepted by DescribeVolumes. @@ -339,83 +301,43 @@ func (s *VolumeServiceImpl) DescribeVolumes(ctx context.Context, input *ec2.Desc return nil, errors.New(awserrors.ErrorInvalidParameterValue) } - var volumes []*ec2.Volume - - // Fast path: if specific volume IDs are requested, fetch them directly + // Fast path: specific volume IDs requested. Fetch each document + // directly instead of enumerating every volume in the bucket. if len(input.VolumeIds) > 0 { - // Count non-nil requested IDs - requested := 0 - for _, id := range input.VolumeIds { - if id != nil { - requested++ + results := s.fetchVolumesByIDs(ctx, input.VolumeIds, accountID) + volumes := make([]*ec2.Volume, 0, len(results)) + for _, r := range results { + if r.err != nil { + return nil, r.err + } + if r.volume == nil { + continue // nil VolumeIds entry + } + if len(parsedFilters) == 0 || volumeMatchesFilters(r.volume, parsedFilters) { + volumes = append(volumes, r.volume) } } - volumes = s.fetchVolumesByIDs(ctx, input.VolumeIds, accountID) - // AWS returns InvalidVolume.NotFound if any requested ID is missing - if len(volumes) != requested { - return nil, errors.New(awserrors.ErrorInvalidVolumeNotFound) - } - - // Apply filters to the fetched volumes - if len(parsedFilters) > 0 { - filtered := make([]*ec2.Volume, 0, len(volumes)) - for _, vol := range volumes { - if volumeMatchesFilters(vol, parsedFilters) { - filtered = append(filtered, vol) - } - } - volumes = filtered - } - slog.InfoContext(ctx, "DescribeVolumes completed", "count", len(volumes)) return &ec2.DescribeVolumesOutput{Volumes: volumes}, nil } - // Slow path: list all volumes (no specific IDs requested) - volumeIDs, err := s.listAllVolumeIDs(ctx) + // Slow path: no specific IDs requested, enumerate the bucket. + metadata, err := s.metadata.ListVolumes(ctx) if err != nil { - return nil, err + return nil, errors.New(awserrors.ErrorServerInternal) } - - // Extract volume-id filter values for early skipping to avoid - // unnecessary S3 GetObject calls on non-matching volumes. - var volumeIDFilterValues []string - if parsedFilters != nil { - volumeIDFilterValues = parsedFilters["volume-id"] - } - - for _, volumeID := range volumeIDs { - // Early skip: if volume-id filter is set, check the ID before - // fetching the full volume config from S3. - if len(volumeIDFilterValues) > 0 { - if !filterutil.MatchesAny(volumeIDFilterValues, volumeID) { - continue - } - } - - result, err := s.getVolumeByID(ctx, volumeID) - if err != nil { - slog.ErrorContext(ctx, "Failed to get volume", "volumeId", volumeID, "err", err) + volumes := make([]*ec2.Volume, 0, len(metadata)) + for _, meta := range metadata { + if meta.TenantID != accountID { continue } - - // Skip volumes not owned by the caller's account. - if result.tenantID != accountID { - continue + volume := metadataVolumeToEC2(meta) + if len(parsedFilters) == 0 || volumeMatchesFilters(volume, parsedFilters) { + volumes = append(volumes, volume) } - - if len(parsedFilters) > 0 && !volumeMatchesFilters(result.volume, parsedFilters) { - continue - } - - volumes = append(volumes, result.volume) } - slog.InfoContext(ctx, "DescribeVolumes completed", "count", len(volumes)) - - return &ec2.DescribeVolumesOutput{ - Volumes: volumes, - }, nil + return &ec2.DescribeVolumesOutput{Volumes: volumes}, nil } // volumeMatchesFilters checks whether an ec2.Volume satisfies all parsed filters. @@ -525,77 +447,7 @@ func (s *VolumeServiceImpl) DescribeVolumeStatus(ctx context.Context, input *ec2 return nil, errors.New(awserrors.ErrorInvalidParameterValue) } - var statusItems []*ec2.VolumeStatusItem - - // Fast path: if specific volume IDs are requested, fetch them directly - if len(input.VolumeIds) > 0 { - for _, vid := range input.VolumeIds { - if vid == nil { - continue - } - item, tenantID, err := s.getVolumeStatusByID(ctx, *vid) - if err != nil { - slog.ErrorContext(ctx, "DescribeVolumeStatus volume not found", "volumeId", *vid, "err", err) - return nil, errors.New(awserrors.ErrorInvalidVolumeNotFound) - } - // Skip volumes not owned by the caller's account - if tenantID != accountID { - return nil, errors.New(awserrors.ErrorInvalidVolumeNotFound) - } - if len(parsedFilters) > 0 && !volumeStatusMatchesFilters(item, parsedFilters) { - continue - } - statusItems = append(statusItems, item) - } - slog.InfoContext(ctx, "DescribeVolumeStatus completed", "count", len(statusItems)) - return &ec2.DescribeVolumeStatusOutput{VolumeStatuses: statusItems}, nil - } - - // Slow path: list all volumes (no specific IDs requested) - volumeIDs, err := s.listAllVolumeIDs(ctx) - if err != nil { - return nil, err - } - - // Extract volume-id filter values for early skipping to avoid - // unnecessary S3 GetObject calls on non-matching volumes. - var volumeStatusIDFilterValues []string - if parsedFilters != nil { - volumeStatusIDFilterValues = parsedFilters["volume-id"] - } - - for _, volumeID := range volumeIDs { - // Early skip: if volume-id filter is set, check the ID before - // fetching the full volume status from S3. - if len(volumeStatusIDFilterValues) > 0 { - if !filterutil.MatchesAny(volumeStatusIDFilterValues, volumeID) { - continue - } - } - - item, tenantID, err := s.getVolumeStatusByID(ctx, volumeID) - if err != nil { - slog.ErrorContext(ctx, "Failed to get volume status", "volumeId", volumeID, "err", err) - continue - } - - // Skip volumes not owned by the caller's account. - if tenantID != accountID { - continue - } - - if len(parsedFilters) > 0 && !volumeStatusMatchesFilters(item, parsedFilters) { - continue - } - - statusItems = append(statusItems, item) - } - - slog.InfoContext(ctx, "DescribeVolumeStatus completed", "count", len(statusItems)) - - return &ec2.DescribeVolumeStatusOutput{ - VolumeStatuses: statusItems, - }, nil + return s.describeVolumeStatus(ctx, input, accountID, parsedFilters) } // volumeStatusMatchesFilters checks whether a VolumeStatusItem satisfies all parsed filters. @@ -631,34 +483,71 @@ func volumeStatusMatchesFilters(item *ec2.VolumeStatusItem, filters map[string][ return true } -// getVolumeStatusByID builds a VolumeStatusItem by reusing getVolumeByID -// to validate the volume exists, then returning static health status. -// Returns the status item, the tenant ID for account scoping, and any error. -func (s *VolumeServiceImpl) getVolumeStatusByID(ctx context.Context, volumeID string) (*ec2.VolumeStatusItem, string, error) { - result, err := s.getVolumeByID(ctx, volumeID) +// describeVolumeStatus answers from the ebsmetadata index. Volume health is +// static: the control plane has no per-volume health signal to report, so +// every known volume reports ok. +func (s *VolumeServiceImpl) describeVolumeStatus(ctx context.Context, input *ec2.DescribeVolumeStatusInput, accountID string, parsedFilters map[string][]string) (*ec2.DescribeVolumeStatusOutput, error) { + metas, err := s.metadata.ListVolumes(ctx) if err != nil { - return nil, "", err + return nil, errors.New(awserrors.ErrorServerInternal) + } + byID := make(map[string]ebsmetadata.Volume, len(metas)) + for _, meta := range metas { + byID[meta.VolumeID] = meta } + var statusItems []*ec2.VolumeStatusItem + + if len(input.VolumeIds) > 0 { + for _, vid := range input.VolumeIds { + if vid == nil { + continue + } + meta, ok := byID[*vid] + if !ok || meta.TenantID != accountID { + return nil, errors.New(awserrors.ErrorInvalidVolumeNotFound) + } + item := metadataVolumeStatusItem(meta) + if len(parsedFilters) > 0 && !volumeStatusMatchesFilters(item, parsedFilters) { + continue + } + statusItems = append(statusItems, item) + } + slog.InfoContext(ctx, "DescribeVolumeStatus completed", "count", len(statusItems)) + return &ec2.DescribeVolumeStatusOutput{VolumeStatuses: statusItems}, nil + } + + for _, meta := range metas { + if meta.TenantID != accountID { + continue + } + item := metadataVolumeStatusItem(meta) + if len(parsedFilters) > 0 && !volumeStatusMatchesFilters(item, parsedFilters) { + continue + } + statusItems = append(statusItems, item) + } + + slog.InfoContext(ctx, "DescribeVolumeStatus completed", "count", len(statusItems)) + return &ec2.DescribeVolumeStatusOutput{VolumeStatuses: statusItems}, nil +} + +// metadataVolumeStatusItem builds a static-health VolumeStatusItem from an +// ebsmetadata document. +func metadataVolumeStatusItem(meta ebsmetadata.Volume) *ec2.VolumeStatusItem { return &ec2.VolumeStatusItem{ - VolumeId: result.volume.VolumeId, - AvailabilityZone: result.volume.AvailabilityZone, + VolumeId: aws.String(meta.VolumeID), + AvailabilityZone: aws.String(meta.AvailabilityZone), VolumeStatus: &ec2.VolumeStatusInfo{ Status: aws.String("ok"), Details: []*ec2.VolumeStatusDetails{ - { - Name: aws.String("io-enabled"), - Status: aws.String("passed"), - }, - { - Name: aws.String("io-performance"), - Status: aws.String("not-applicable"), - }, + {Name: aws.String("io-enabled"), Status: aws.String("passed")}, + {Name: aws.String("io-performance"), Status: aws.String("not-applicable")}, }, }, Actions: []*ec2.VolumeStatusAction{}, Events: []*ec2.VolumeStatusEvent{}, - }, result.tenantID, nil + } } // volumeModificationTimeFormat is the AWS-CLI compatible RFC3339-ish format @@ -680,21 +569,23 @@ var describeVolumesModificationsValidFilters = map[string]bool{ "volume-id": true, } -// vbModificationToEC2 converts a persisted viperblock.VolumeModification into -// the AWS SDK shape returned by ModifyVolume / DescribeVolumesModifications. -func vbModificationToEC2(m *viperblock.VolumeModification) *ec2.VolumeModification { +// ebsModificationToEC2 converts a persisted ebsmetadata.VolumeModification +// into the AWS SDK shape returned by ModifyVolume / +// DescribeVolumesModifications. volumeID comes from the owning +// ebsmetadata.Volume, since VolumeModification does not duplicate it. +func ebsModificationToEC2(volumeID string, m *ebsmetadata.VolumeModification) *ec2.VolumeModification { if m == nil { return nil } out := &ec2.VolumeModification{ - VolumeId: aws.String(m.VolumeID), + VolumeId: aws.String(volumeID), ModificationState: aws.String(m.ModificationState), Progress: aws.Int64(m.Progress), OriginalSize: aws.Int64(m.OriginalSize), - OriginalIops: aws.Int64(m.OriginalIops), + OriginalIops: aws.Int64(m.OriginalIOPS), OriginalVolumeType: aws.String(m.OriginalVolumeType), TargetSize: aws.Int64(m.TargetSize), - TargetIops: aws.Int64(m.TargetIops), + TargetIops: aws.Int64(m.TargetIOPS), TargetVolumeType: aws.String(m.TargetVolumeType), StartTime: aws.Time(m.StartTime), } @@ -783,10 +674,8 @@ func (s *VolumeServiceImpl) DescribeVolumesModifications(ctx context.Context, in var modifications []*ec2.VolumeModification - // Fast path: specific volume IDs requested. if len(input.VolumeIds) > 0 { results := s.fetchVolumeModificationsByIDs(ctx, input.VolumeIds, accountID) - // AWS contract: any unknown / cross-tenant ID fails the whole call. for i, vid := range input.VolumeIds { if vid == nil { continue @@ -809,10 +698,9 @@ func (s *VolumeServiceImpl) DescribeVolumesModifications(ctx context.Context, in return &ec2.DescribeVolumesModificationsOutput{VolumesModifications: modifications}, nil } - // Slow path: list all volumes. - volumeIDs, err := s.listAllVolumeIDs(ctx) + metas, err := s.metadata.ListVolumes(ctx) if err != nil { - return nil, err + return nil, errors.New(awserrors.ErrorServerInternal) } var volumeIDFilterValues []string @@ -820,26 +708,17 @@ func (s *VolumeServiceImpl) DescribeVolumesModifications(ctx context.Context, in volumeIDFilterValues = parsedFilters["volume-id"] } - for _, volumeID := range volumeIDs { - if len(volumeIDFilterValues) > 0 { - if !filterutil.MatchesAny(volumeIDFilterValues, volumeID) { - continue - } - } - - cfg, err := s.getVolumeConfig(ctx, volumeID) - if err != nil { - slog.ErrorContext(ctx, "Failed to get volume config", "volumeId", volumeID, "err", err) + for _, meta := range metas { + if meta.TenantID != accountID { continue } - if cfg.VolumeMetadata.TenantID != accountID { + if len(volumeIDFilterValues) > 0 && !filterutil.MatchesAny(volumeIDFilterValues, meta.VolumeID) { continue } - if cfg.Modification == nil { + if meta.Modification == nil { continue } - - mod := vbModificationToEC2(cfg.Modification) + mod := ebsModificationToEC2(meta.VolumeID, meta.Modification) if len(parsedFilters) > 0 && !volumeModificationMatchesFilters(mod, parsedFilters) { continue } @@ -847,9 +726,7 @@ func (s *VolumeServiceImpl) DescribeVolumesModifications(ctx context.Context, in } slog.InfoContext(ctx, "DescribeVolumesModifications completed", "count", len(modifications)) - return &ec2.DescribeVolumesModificationsOutput{ - VolumesModifications: modifications, - }, nil + return &ec2.DescribeVolumesModificationsOutput{VolumesModifications: modifications}, nil } // volumeModificationResult bundles a per-ID lookup result so the fast path @@ -859,9 +736,9 @@ type volumeModificationResult struct { err error } -// fetchVolumeModificationsByIDs reads each requested volume's config in parallel, -// returning results positionally aligned with volumeIDs. Cross-tenant volumes surface -// as InvalidVolume.NotFound. +// fetchVolumeModificationsByIDs reads each requested volume's document in +// parallel, returning results positionally aligned with volumeIDs. +// Cross-tenant volumes surface as InvalidVolume.NotFound. func (s *VolumeServiceImpl) fetchVolumeModificationsByIDs(ctx context.Context, volumeIDs []*string, accountID string) []volumeModificationResult { results := make([]volumeModificationResult, len(volumeIDs)) var wg sync.WaitGroup @@ -873,16 +750,12 @@ func (s *VolumeServiceImpl) fetchVolumeModificationsByIDs(ctx context.Context, v wg.Add(1) go func(idx int, volID string) { defer wg.Done() - cfg, err := s.GetVolumeConfig(volID) - if err != nil { + meta, err := s.metadata.GetVolume(ctx, volID) + if err != nil || meta.TenantID != accountID { results[idx] = volumeModificationResult{err: errors.New(awserrors.ErrorInvalidVolumeNotFound)} return } - if cfg.VolumeMetadata.TenantID != accountID { - results[idx] = volumeModificationResult{err: errors.New(awserrors.ErrorInvalidVolumeNotFound)} - return - } - results[idx] = volumeModificationResult{modification: vbModificationToEC2(cfg.Modification)} + results[idx] = volumeModificationResult{modification: ebsModificationToEC2(meta.VolumeID, meta.Modification)} }(i, *volumeID) } @@ -890,478 +763,137 @@ func (s *VolumeServiceImpl) fetchVolumeModificationsByIDs(ctx context.Context, v return results } -// listAllVolumeIDs lists all volume IDs from S3 by scanning bucket prefixes. -// It filters for vol-* prefixes and skips internal sub-volumes (EFI and cloud-init). +// listAllVolumeIDs lists every volume ID the control plane knows about, taking +// ebsmetadata as the index. func (s *VolumeServiceImpl) listAllVolumeIDs(ctx context.Context) ([]string, error) { - result, err := s.store.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ - Bucket: aws.String(s.bucketName), - Delimiter: aws.String("/"), - }) + volumes, err := s.metadata.ListVolumes(ctx) if err != nil { - slog.Error("Failed to list S3 objects", "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } - - var volumeIDs []string - for _, prefix := range result.CommonPrefixes { - if prefix.Prefix == nil { - continue - } - - prefixStr := *prefix.Prefix - - if !strings.HasPrefix(prefixStr, "vol-") { - continue - } - - volumeID := strings.TrimSuffix(prefixStr, "/") - - // Skip internal sub-volumes: the EFI partition and the legacy cloud-init - // seed (no longer created post boot-from-IMDS, still filtered so any - // pre-cutover volume never surfaces as a user volume). - if strings.HasSuffix(volumeID, "-efi") || strings.HasSuffix(volumeID, "-cloudinit") { - continue - } - - volumeIDs = append(volumeIDs, volumeID) + ids := make([]string, 0, len(volumes)) + for _, v := range volumes { + ids = append(ids, v.VolumeID) } - - return volumeIDs, nil + return ids, nil } -// fetchVolumesByIDs fetches multiple volumes in parallel by their IDs, -// filtering by accountID for account scoping. -func (s *VolumeServiceImpl) fetchVolumesByIDs(ctx context.Context, volumeIDs []*string, accountID string) []*ec2.Volume { - var ( - volumes []*ec2.Volume - mu sync.Mutex - wg sync.WaitGroup - ) +// volumeFetchResult bundles a single volume-ID lookup result so the fast path +// can preserve input ordering and surface per-ID errors after the parallel +// fan-out, matching fetchVolumeModificationsByIDs. +type volumeFetchResult struct { + volume *ec2.Volume + err error +} - for _, volumeID := range volumeIDs { - if volumeID == nil { +// fetchVolumesByIDs fetches each requested volume's ebsmetadata document +// directly via GetVolume, instead of enumerating every volume in the bucket +// via ListVolumes. A missing document or a cross-tenant volume both surface as +// InvalidVolume.NotFound; any other store error surfaces as ErrorServerInternal. +func (s *VolumeServiceImpl) fetchVolumesByIDs(ctx context.Context, volumeIDs []*string, accountID string) []volumeFetchResult { + results := make([]volumeFetchResult, len(volumeIDs)) + var wg sync.WaitGroup + + for i, id := range volumeIDs { + if id == nil { continue } - wg.Add(1) - go func(volID string) { + go func(idx int, volID string) { defer wg.Done() - - result, err := s.getVolumeByID(ctx, volID) + meta, err := s.metadata.GetVolume(ctx, volID) if err != nil { - slog.Debug("Volume not found", "volumeId", volID, "err", err) + if objectstore.IsNoSuchKeyError(err) { + results[idx] = volumeFetchResult{err: errors.New(awserrors.ErrorInvalidVolumeNotFound)} + } else { + results[idx] = volumeFetchResult{err: errors.New(awserrors.ErrorServerInternal)} + } return } - - // Skip volumes not owned by the caller's account. - if result.tenantID != accountID { + if meta.TenantID != accountID { + results[idx] = volumeFetchResult{err: errors.New(awserrors.ErrorInvalidVolumeNotFound)} return } - - mu.Lock() - volumes = append(volumes, result.volume) - mu.Unlock() - }(*volumeID) + results[idx] = volumeFetchResult{volume: metadataVolumeToEC2(meta)} + }(i, *id) } wg.Wait() - return volumes + return results } -// volumeResult bundles an ec2.Volume with the owning account's TenantID -// so callers can filter by account without a second config read. -type volumeResult struct { - volume *ec2.Volume - tenantID string -} - -// getVolumeByID fetches a single volume's config from S3 and builds an EC2 Volume. -// Returns the volume and the stored TenantID for account scoping. -func (s *VolumeServiceImpl) getVolumeByID(ctx context.Context, volumeID string) (*volumeResult, error) { - cfg, encryptionEnabled, err := s.getVolumeConfigAndEncryption(ctx, volumeID) - if err != nil { - return nil, err - } - - volMeta := cfg.VolumeMetadata - - if volMeta.VolumeID == "" { - slog.Debug("Volume ID is empty in config", "key", volumeID+"/config.json") - return nil, errors.New("volume ID is empty") - } - - if volMeta.SizeGiB == 0 { - slog.Error("Volume has zero size in config", "volumeId", volumeID) - return nil, fmt.Errorf("volume %s has zero size in config", volumeID) - } - +// metadataVolumeToEC2 renders an ebsmetadata document as the AWS SDK volume +// shape. Fields that are zero because they were never set are omitted rather +// than surfaced as a misleading 0 or empty string. +func metadataVolumeToEC2(meta ebsmetadata.Volume) *ec2.Volume { // An empty State is internal drift, not a valid AWS state. Derive the // effective state from ground truth (the attachment) rather than blindly // rendering "available", which would hide an empty-but-attached volume. - state := volMeta.State + state := meta.State if state == "" { - if volMeta.AttachedInstance != "" { + if meta.AttachedInstance != "" { state = "in-use" } else { state = "available" } } - volumeType := volMeta.VolumeType + volumeType := meta.VolumeType if volumeType == "" { volumeType = "gp3" } - volume := &ec2.Volume{ - VolumeId: aws.String(volMeta.VolumeID), - Size: aws.Int64(utils.SafeUint64ToInt64(volMeta.SizeGiB)), - State: aws.String(state), - AvailabilityZone: aws.String(volMeta.AvailabilityZone), - CreateTime: aws.Time(volMeta.CreatedAt), - VolumeType: aws.String(volumeType), - Encrypted: aws.Bool(encryptionEnabled), + VolumeId: aws.String(meta.VolumeID), Size: aws.Int64(utils.SafeUint64ToInt64(meta.CapacityGiB)), + State: aws.String(state), AvailabilityZone: aws.String(meta.AvailabilityZone), CreateTime: aws.Time(meta.CreatedAt), + VolumeType: aws.String(volumeType), Encrypted: aws.Bool(meta.Encrypted), Tags: utils.MapToEC2Tags(meta.Tags), } - - if volMeta.IOPS > 0 { - volume.Iops = aws.Int64(int64(volMeta.IOPS)) + if meta.IOPS > 0 { + volume.Iops = aws.Int64(int64(meta.IOPS)) } - - // Zero means unset (volume created before the Throughput field existed); - // omit rather than surface a misleading 0 MiB/s. - if volMeta.Throughput > 0 { - volume.Throughput = aws.Int64(int64(volMeta.Throughput)) + if meta.Throughput > 0 { + volume.Throughput = aws.Int64(int64(meta.Throughput)) } - - if volMeta.SnapshotID != "" { - volume.SnapshotId = aws.String(volMeta.SnapshotID) + if meta.SnapshotID != "" { + volume.SnapshotId = aws.String(meta.SnapshotID) } - - if volMeta.AttachedInstance != "" { + if meta.AttachedInstance != "" { + // A recorded attachment on a volume that is not in-use is drift, so the + // attachment reports detached rather than claiming a live attach. attachState := "attached" - if volMeta.State != "in-use" { + if meta.State != "in-use" { attachState = "detached" } - volume.Attachments = []*ec2.VolumeAttachment{ - { - VolumeId: aws.String(volMeta.VolumeID), - InstanceId: aws.String(volMeta.AttachedInstance), - Device: aws.String(volMeta.DeviceName), - State: aws.String(attachState), - DeleteOnTermination: aws.Bool(volMeta.DeleteOnTermination), - AttachTime: aws.Time(volMeta.AttachedAt), - }, - } + volume.Attachments = []*ec2.VolumeAttachment{{VolumeId: aws.String(meta.VolumeID), InstanceId: aws.String(meta.AttachedInstance), Device: aws.String(meta.DeviceName), State: aws.String(attachState), DeleteOnTermination: aws.Bool(meta.DeleteOnTermination), AttachTime: aws.Time(meta.AttachedAt)}} } - - volume.Tags = utils.MapToEC2Tags(volMeta.Tags) - - return &volumeResult{volume: volume, tenantID: volMeta.TenantID}, nil + return volume } -// volumeConfigWrapper matches the JSON structure stored in S3 config.json files. -type volumeConfigWrapper struct { - VolumeConfig viperblock.VolumeConfig `json:"VolumeConfig"` +// GetVolumeMetadata reads the control-plane document for a volume. It is the +// ownership and attachment record the daemon validates attach/detach against. +func (s *VolumeServiceImpl) GetVolumeMetadata(volumeID string) (ebsmetadata.Volume, error) { + return s.getVolumeMetadata(context.Background(), volumeID) } -// volumeTagsKey is the S3 key for a volume's control-plane tags object. -func volumeTagsKey(volumeID string) string { return volumeID + "/tags.json" } - -// putVolumeState writes the control-plane attachment state to state.json. -func (s *VolumeServiceImpl) putVolumeState(ctx context.Context, volumeID string, rec volumestate.Record) error { - return volumestate.Write(ctx, s.store, s.bucketName, volumeID, rec) -} - -// getVolumeState reads state.json. found=false with a nil error means the object -// is absent (a volume predating the state.json split), in which case the caller -// falls back to the State embedded in config.json. -func (s *VolumeServiceImpl) getVolumeState(ctx context.Context, volumeID string) (volumestate.Record, bool, error) { - return volumestate.Read(ctx, s.store, s.bucketName, volumeID) -} - -// putVolumeTags writes the control-plane-owned tag set to tags.json. -func (s *VolumeServiceImpl) putVolumeTags(ctx context.Context, volumeID string, tags map[string]string) error { - if tags == nil { - tags = map[string]string{} - } - data, err := json.Marshal(tags) - if err != nil { - return fmt.Errorf("marshal volume tags: %w", err) - } - _, err = s.store.PutObject(ctx, &s3.PutObjectInput{ - Bucket: aws.String(s.bucketName), - Key: aws.String(volumeTagsKey(volumeID)), - Body: bytes.NewReader(data), - }) - if err != nil { - return fmt.Errorf("write volume tags to S3: %w", err) - } - return nil -} - -// getVolumeTags reads tags.json. found=false means the object is absent, while -// a present empty map is an authoritative empty tag set. -func (s *VolumeServiceImpl) getVolumeTags(ctx context.Context, volumeID string) (map[string]string, bool, error) { - getResult, err := s.store.GetObject(ctx, &s3.GetObjectInput{ - Bucket: aws.String(s.bucketName), - Key: aws.String(volumeTagsKey(volumeID)), - }) +// getVolumeMetadata is GetVolumeMetadata carrying the caller's context. A +// missing document is remapped to InvalidVolume.NotFound so callers surface the +// AWS error rather than a store error. +func (s *VolumeServiceImpl) getVolumeMetadata(ctx context.Context, volumeID string) (ebsmetadata.Volume, error) { + meta, err := s.metadata.GetVolume(ctx, volumeID) if err != nil { if objectstore.IsNoSuchKeyError(err) { - return nil, false, nil + slog.WarnContext(ctx, "volume metadata not found", "volumeId", volumeID) + return ebsmetadata.Volume{}, errors.New(awserrors.ErrorInvalidVolumeNotFound) } - return nil, false, fmt.Errorf("get volume tags: %w", err) + return ebsmetadata.Volume{}, fmt.Errorf("get volume metadata: %w", err) } - defer getResult.Body.Close() - - body, err := io.ReadAll(getResult.Body) - if err != nil { - return nil, false, fmt.Errorf("read volume tags body: %w", err) - } - var tags map[string]string - if err := json.Unmarshal(body, &tags); err != nil { - return nil, false, fmt.Errorf("unmarshal volume tags: %w", err) - } - if tags == nil { - return nil, false, errors.New("unmarshal volume tags: expected JSON object") - } - return tags, true, nil -} - -// GetVolumeConfig reads the effective VolumeConfig for a given volume ID. -func (s *VolumeServiceImpl) GetVolumeConfig(volumeID string) (*viperblock.VolumeConfig, error) { - return s.getVolumeConfig(context.Background(), volumeID) -} - -// getVolumeConfig is GetVolumeConfig carrying the caller's context. -func (s *VolumeServiceImpl) getVolumeConfig(ctx context.Context, volumeID string) (*viperblock.VolumeConfig, error) { - cfg, _, err := s.getVolumeConfigAndEncryption(ctx, volumeID) - return cfg, err -} - -// getVolumeConfigAndEncryption reads config.json and overlays control-plane-owned -// state and tags. Pre-VBState blobs report encryptionEnabled=false. -func (s *VolumeServiceImpl) getVolumeConfigAndEncryption(ctx context.Context, volumeID string) (*viperblock.VolumeConfig, bool, error) { - vc, encryptionEnabled, err := s.getBaseVolumeConfigAndEncryption(ctx, volumeID) - if err != nil { - return nil, false, err - } - - // Overlay the control-plane-owned attachment state from state.json. The State - // embedded in config.json is rewritten by the live nbdkit VB (stale - // "available") and is not authoritative; state.json is. Absent state.json - // keeps the embedded value (volumes predating the split). - if rec, found, stateErr := s.getVolumeState(ctx, volumeID); stateErr != nil { - return nil, false, stateErr - } else if found { - vc.VolumeMetadata.State = rec.State - vc.VolumeMetadata.AttachedInstance = rec.AttachedInstance - vc.VolumeMetadata.DeviceName = rec.DeviceName - vc.VolumeMetadata.AttachedAt = rec.AttachedAt - } - - // tags.json is authoritative when present, including when it contains an - // empty map. Falling back to config.json preserves tags on older volumes and - // tags supplied at volume creation until their first tag mutation. - if tags, found, tagsErr := s.getVolumeTags(ctx, volumeID); tagsErr != nil { - return nil, false, tagsErr - } else if found { - vc.VolumeMetadata.Tags = tags - } - - return vc, encryptionEnabled, nil -} - -// getBaseVolumeConfigAndEncryption reads config.json without overlaying the -// control-plane-owned state and tags objects. -func (s *VolumeServiceImpl) getBaseVolumeConfigAndEncryption(ctx context.Context, volumeID string) (*viperblock.VolumeConfig, bool, error) { - configKey := volumeID + "/config.json" - - getResult, err := s.store.GetObject(ctx, &s3.GetObjectInput{ - Bucket: aws.String(s.bucketName), - Key: aws.String(configKey), - }) - if err != nil { - if objectstore.IsNoSuchKeyError(err) { - return nil, false, errors.New(awserrors.ErrorInvalidVolumeNotFound) - } - return nil, false, fmt.Errorf("failed to get config: %w", err) - } - defer getResult.Body.Close() - - body, err := io.ReadAll(getResult.Body) - if err != nil { - return nil, false, fmt.Errorf("failed to read config body: %w", err) - } - // Unwrap the at-rest encryption envelope (authenticated-but-plaintext - // metadata); no-op for unencrypted volumes. - body = viperblock.StateBody(body) - - // Try full VBState first (matches mergeVolumeConfig's careful decode - // pattern). A populated BlockSize is the marker that the blob is a full - // state rather than a wrapper-only fallback. - var vc *viperblock.VolumeConfig - var encryptionEnabled bool - - var state viperblock.VBState - if decodeErr := json.NewDecoder(bytes.NewReader(body)).Decode(&state); decodeErr == nil && state.BlockSize != 0 { - vc = &state.VolumeConfig - encryptionEnabled = state.EncryptionEnabled - } else { - var wrapper volumeConfigWrapper - if err := json.Unmarshal(body, &wrapper); err != nil { - return nil, false, fmt.Errorf("failed to unmarshal config: %w", err) - } - vc = &wrapper.VolumeConfig - } - - return vc, encryptionEnabled, nil -} - -// putVolumeConfig writes a VolumeConfig back to S3 as config.json. -// It performs a read-modify-write to preserve full VBState if viperblock -// has already written state (BlockSize, SeqNum, WALNum, etc.) to config.json. -// Callers are the safe non-live writers only (CreateVolume pre-mount and -// ModifyVolume stopped-instance); control-plane-owned state and tags go to -// state.json and tags.json, never here. -func (s *VolumeServiceImpl) putVolumeConfig(ctx context.Context, volumeID string, cfg *viperblock.VolumeConfig) error { - configKey := volumeID + "/config.json" - - // config.json for an encrypted volume is a sealed VBState whose AES-GCM tag - // and StateSeqNum-derived nonce can only be advanced by the master-key holder. - // With no live owner, hand it to a viperblockd worker that opens the detached - // volume exclusively and reseals — a detached volume has no concurrent writer. - encrypted, err := s.configIsEncrypted(ctx, volumeID) - if err != nil { - return err - } - if encrypted { - return s.putVolumeConfigViaDetached(ctx, volumeID, cfg) - } - - data, err := s.mergeVolumeConfig(ctx, configKey, cfg) - if err != nil { - return err - } - - _, err = s.store.PutObject(ctx, &s3.PutObjectInput{ - Bucket: aws.String(s.bucketName), - Key: aws.String(configKey), - Body: bytes.NewReader(data), - }) - if err != nil { - return fmt.Errorf("failed to write config to S3: %w", err) - } - - return nil -} - -// configIsEncrypted reports whether the persisted config.json for volumeID is a -// sealed (encrypted) VBState. A missing object (new volume) reports false. -func (s *VolumeServiceImpl) configIsEncrypted(ctx context.Context, volumeID string) (bool, error) { - getResult, err := s.store.GetObject(ctx, &s3.GetObjectInput{ - Bucket: aws.String(s.bucketName), - Key: aws.String(volumeID + "/config.json"), - }) - if err != nil { - if objectstore.IsNoSuchKeyError(err) { - return false, nil - } - return false, fmt.Errorf("failed to read config for encryption check: %w", err) - } - defer getResult.Body.Close() - - body, err := io.ReadAll(getResult.Body) - if err != nil { - return false, fmt.Errorf("failed to read config body: %w", err) - } - - var state viperblock.VBState - if decodeErr := json.Unmarshal(viperblock.StateBody(body), &state); decodeErr == nil && state.BlockSize != 0 { - return state.EncryptionEnabled, nil - } - return false, nil -} - -// putVolumeConfigViaDetached hands an encrypted-volume config update to the -// ebs.config queue group, where a viperblockd worker opens the detached volume -// exclusively and reseals. Safe only when no node owns the volume: a detached -// volume has no concurrent writer, so the reopen is nonce-safe. -func (s *VolumeServiceImpl) putVolumeConfigViaDetached(ctx context.Context, volumeID string, cfg *viperblock.VolumeConfig) error { - if s.natsConn == nil { - return fmt.Errorf("encrypted volume %s requires NATS to reach the viperblock keyholder, but no connection is configured", volumeID) - } - cfgData, err := json.Marshal(cfg) - if err != nil { - return fmt.Errorf("marshal VolumeConfig: %w", err) - } - req := types.EBSConfigUpdateRequest{Volume: volumeID, VolumeConfig: cfgData} - resp, err := utils.NATSRequest[types.EBSConfigUpdateResponse](ctx, s.natsConn, "ebs.config", req, 30*time.Second, "") - if err != nil { - return fmt.Errorf("ebs.config request for %s: %w", volumeID, err) - } - if !resp.Success || resp.Error != "" { - return fmt.Errorf("ebs.config update for %s failed: %s", volumeID, resp.Error) - } - return nil -} - -// mergeVolumeConfig reads existing config.json from S3 and merges the new VolumeConfig -// into it, preserving full VBState when present. Refuses to merge encrypted VBState — -// re-marshaling without the master key would corrupt the on-disk AES-GCM tag. -func (s *VolumeServiceImpl) mergeVolumeConfig(ctx context.Context, configKey string, cfg *viperblock.VolumeConfig) ([]byte, error) { - getResult, err := s.store.GetObject(ctx, &s3.GetObjectInput{ - Bucket: aws.String(s.bucketName), - Key: aws.String(configKey), - }) - if err != nil { - if objectstore.IsNoSuchKeyError(err) { - // No existing config.json -- write wrapper for new volume - return json.Marshal(volumeConfigWrapper{VolumeConfig: *cfg}) - } - return nil, fmt.Errorf("failed to read existing config for merge: %w", err) - } - defer getResult.Body.Close() - - body, err := io.ReadAll(getResult.Body) - if err != nil { - return nil, fmt.Errorf("failed to read existing config: %w", err) - } - - // StateBody unwraps the at-rest encryption envelope so the guard below sees - // the real VBState. Without it the wrapper decodes to a zero-valued state - // (BlockSize==0), routing to the wrapper-only path and bricking the volume. - var state viperblock.VBState - if decodeErr := json.Unmarshal(viperblock.StateBody(body), &state); decodeErr != nil || state.BlockSize == 0 { - // Not a full VBState (new volume or wrapper-only) -- write wrapper - return json.Marshal(volumeConfigWrapper{VolumeConfig: *cfg}) - } - - if state.EncryptionEnabled { - return nil, fmt.Errorf("mergeVolumeConfig: refusing to merge encrypted VBState for %s without master key (would strip AES-GCM tag and brick volume)", configKey) - } - - // Full VBState exists -- update VolumeConfig and reconcile VolumeSize - state.VolumeConfig = *cfg - configSizeBytes := cfg.VolumeMetadata.SizeGiB * 1024 * 1024 * 1024 - if configSizeBytes > 0 && configSizeBytes > state.VolumeSize { - state.VolumeSize = configSizeBytes - } - - slog.Info("putVolumeConfig: preserved VBState", "volumeId", strings.TrimSuffix(configKey, "/config.json"), - "blockSize", state.BlockSize, "seqNum", state.SeqNum) - - return json.Marshal(state) + return meta, nil } // UpdateVolumeState updates the control-plane-owned attachment state (state, -// attachment, device) by writing the per-volume state.json. It does NOT write -// config.json: the live nbdkit VB owns that object and its next SaveState -// clobbers any State written out-of-band (and for an encrypted volume a second -// writer reuses the AES-GCM nonce). Readers overlay state.json in -// getVolumeConfigAndEncryption. The config.json read here is a presence/ownership -// gate so a missing volume still errors. +// attachment, device) on the volume's ebsmetadata document. func (s *VolumeServiceImpl) UpdateVolumeState(volumeID, state, attachedInstance, deviceName string) error { - if _, err := s.GetVolumeConfig(volumeID); err != nil { - return fmt.Errorf("failed to get volume config for state update: %w", err) + ctx := context.Background() + meta, err := s.metadata.GetVolume(ctx, volumeID) + if err != nil { + return fmt.Errorf("failed to get volume metadata: %w", err) } // A detached volume is "available": never persist an empty State for an @@ -1370,18 +902,16 @@ func (s *VolumeServiceImpl) UpdateVolumeState(volumeID, state, attachedInstance, if state == "" && attachedInstance == "" { state = "available" } - - rec := volumestate.Record{ - State: state, - AttachedInstance: attachedInstance, - DeviceName: deviceName, - } + meta.State = state + meta.AttachedInstance = attachedInstance + meta.DeviceName = deviceName if attachedInstance != "" { - rec.AttachedAt = time.Now() + meta.AttachedAt = time.Now() + } else { + meta.AttachedAt = time.Time{} } - - if err := s.putVolumeState(context.Background(), volumeID, rec); err != nil { - return fmt.Errorf("failed to write volume state: %w", err) + if err := s.metadata.PutVolume(ctx, meta); err != nil { + return fmt.Errorf("failed to write volume metadata: %w", err) } slog.Info("Updated volume state", "volumeId", volumeID, "state", state, "attachedInstance", attachedInstance, "deviceName", deviceName) @@ -1390,94 +920,77 @@ func (s *VolumeServiceImpl) UpdateVolumeState(volumeID, state, attachedInstance, // ModifyVolume modifies an EBS volume (grow-only, requires stopped instance). func (s *VolumeServiceImpl) ModifyVolume(ctx context.Context, input *ec2.ModifyVolumeInput, accountID string) (*ec2.ModifyVolumeOutput, error) { - if input.VolumeId == nil || *input.VolumeId == "" { + if input == nil || input.VolumeId == nil || *input.VolumeId == "" { return nil, errors.New(awserrors.ErrorInvalidVolumeIDMalformed) } volumeID := *input.VolumeId slog.InfoContext(ctx, "ModifyVolume request", "volumeId", volumeID) - cfg, err := s.getVolumeConfig(ctx, volumeID) - if err != nil { - slog.ErrorContext(ctx, "ModifyVolume failed to get volume config", "volumeId", volumeID, "err", err) + if err := s.requireProvider(ctx, "ModifyVolume"); err != nil { return nil, err } - // Verify caller owns this volume - if cfg.VolumeMetadata.TenantID != accountID { + meta, err := s.metadata.GetVolume(ctx, volumeID) + if err != nil || meta.TenantID != accountID { return nil, errors.New(awserrors.ErrorInvalidVolumeNotFound) } - - volMeta := &cfg.VolumeMetadata - - // Record original values before modification - originalSize := utils.SafeUint64ToInt64(volMeta.SizeGiB) - originalType := volMeta.VolumeType + originalSize := utils.SafeUint64ToInt64(meta.CapacityGiB) + if input.Size == nil || *input.Size <= originalSize { + return nil, errors.New(awserrors.ErrorInvalidParameterValue) + } + if meta.AttachedInstance != "" && meta.State == "in-use" { + return nil, errors.New(awserrors.ErrorIncorrectState) + } + originalType := meta.VolumeType if originalType == "" { originalType = "gp3" } - originalIOPS := int64(volMeta.IOPS) + originalIOPS := int64(meta.IOPS) - // Validate: grow only (new size must be greater than current) - if input.Size != nil && *input.Size <= originalSize { - return nil, errors.New(awserrors.ErrorInvalidParameterValue) - } - - // Validate: if volume is attached, instance must not be in-use (must be stopped) - if volMeta.AttachedInstance != "" && volMeta.State == "in-use" { - return nil, errors.New(awserrors.ErrorIncorrectState) - } - - // Apply modifications - if input.Size != nil { - volMeta.SizeGiB = utils.SafeInt64ToUint64(*input.Size) + expanded, err := s.provider.ExpandVolume(ctx, ebsprovider.ExpandVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, Handle: meta.ProviderHandle, + CapacityRange: ebsprovider.CapacityRange{RequiredBytes: *input.Size * 1024 * 1024 * 1024}, + }) + if err != nil || expanded == nil { + return nil, errors.New(awserrors.ErrorServerInternal) } + meta.CapacityGiB = utils.SafeInt64ToUint64(*input.Size) if input.VolumeType != nil { - volMeta.VolumeType = *input.VolumeType + meta.VolumeType = *input.VolumeType } if input.Iops != nil { - volMeta.IOPS = int(*input.Iops) + meta.IOPS = int(*input.Iops) } - - // Build target values (after modification) - targetSize := utils.SafeUint64ToInt64(volMeta.SizeGiB) - targetType := volMeta.VolumeType + targetType := meta.VolumeType if targetType == "" { targetType = "gp3" } - targetIOPS := int64(volMeta.IOPS) - // Persist the modification record so DescribeVolumesModifications can read it back. - // Modifications are synchronous, so state is always completed/100. + // Persist the modification record on the volume document so a subsequent + // DescribeVolumesModifications can read it back. Modifications are + // synchronous, so state is always completed/100. now := time.Now() - cfg.Modification = &viperblock.VolumeModification{ - VolumeID: volumeID, + meta.Modification = &ebsmetadata.VolumeModification{ ModificationState: "completed", Progress: 100, OriginalSize: originalSize, - OriginalIops: originalIOPS, + OriginalIOPS: originalIOPS, OriginalVolumeType: originalType, - TargetSize: targetSize, - TargetIops: targetIOPS, + TargetSize: utils.SafeUint64ToInt64(meta.CapacityGiB), + TargetIOPS: int64(meta.IOPS), TargetVolumeType: targetType, StartTime: now, EndTime: now, } - - // Persist updated config - if err := s.putVolumeConfig(ctx, volumeID, cfg); err != nil { - slog.ErrorContext(ctx, "ModifyVolume failed to write config", "volumeId", volumeID, "err", err) + if err := s.metadata.PutVolume(ctx, meta); err != nil { return nil, errors.New(awserrors.ErrorServerInternal) } - modification := vbModificationToEC2(cfg.Modification) - slog.InfoContext(ctx, "ModifyVolume completed", "volumeId", volumeID, - "originalSize", originalSize, "targetSize", targetSize) + "originalSize", originalSize, "targetSize", meta.Modification.TargetSize) - return &ec2.ModifyVolumeOutput{ - VolumeModification: modification, - }, nil + return &ec2.ModifyVolumeOutput{VolumeModification: ebsModificationToEC2(volumeID, meta.Modification)}, nil } // DeleteVolumeOnTerminate deletes a DeleteOnTermination volume as part of an @@ -1510,7 +1023,8 @@ func (s *VolumeServiceImpl) DetachVolumeOnTerminate(_ context.Context, volumeID, return s.UpdateVolumeState(volumeID, "available", "", "") } -// DeleteVolume deletes an EBS volume: validates state, notifies viperblockd, and removes S3 data. +// DeleteVolume deletes an EBS volume: validates state, asks the EBS provider to +// destroy the backing data, then removes the control-plane metadata document. func (s *VolumeServiceImpl) DeleteVolume(ctx context.Context, input *ec2.DeleteVolumeInput, accountID string) (*ec2.DeleteVolumeOutput, error) { if input == nil || input.VolumeId == nil || *input.VolumeId == "" { return nil, errors.New(awserrors.ErrorInvalidParameterValue) @@ -1519,65 +1033,41 @@ func (s *VolumeServiceImpl) DeleteVolume(ctx context.Context, input *ec2.DeleteV volumeID := *input.VolumeId slog.InfoContext(ctx, "DeleteVolume request", "volumeId", volumeID) - // Fetch volume config to validate state. AWS-faithful: an absent volume - // returns InvalidVolume.NotFound (the provider tolerates it on destroy); - // destroy orchestration tolerates it too. - cfg, err := s.getVolumeConfig(ctx, volumeID) - if err != nil { - slog.ErrorContext(ctx, "DeleteVolume failed to get volume config", "volumeId", volumeID, "err", err) + if err := s.requireProvider(ctx, "DeleteVolume"); err != nil { return nil, err } - // Verify caller owns this volume - if cfg.VolumeMetadata.TenantID != accountID { + // AWS-faithful: an absent volume returns InvalidVolume.NotFound. + meta, err := s.metadata.GetVolume(ctx, volumeID) + if err != nil { + return nil, errors.New(awserrors.ErrorInvalidVolumeNotFound) + } + if meta.TenantID != accountID { return nil, errors.New(awserrors.ErrorInvalidVolumeNotFound) } - // Validate: an unattached volume is deletable. State must be "available" OR - // empty: a detach/terminate that failed to write back "available" leaves the - // State drifted to empty with no attachment, and gating on State=="available" - // exactly would return VolumeInUse for a volume nothing is using, stranding it - // undeletable and blocking stack teardown. - state := cfg.VolumeMetadata.State - if cfg.VolumeMetadata.AttachedInstance != "" || (state != "available" && state != "") { - slog.ErrorContext(ctx, "DeleteVolume: volume is in use", "volumeId", volumeID, "state", state, "attachedInstance", cfg.VolumeMetadata.AttachedInstance) + // An unattached volume is deletable. State must be "available" OR empty: a + // detach/terminate that failed to write back "available" leaves the State + // drifted to empty with no attachment, and gating on State=="available" + // exactly would strand it undeletable and block stack teardown. + if meta.AttachedInstance != "" || (meta.State != "available" && meta.State != "") { + slog.ErrorContext(ctx, "DeleteVolume: volume is in use", "volumeId", volumeID, "state", meta.State, "attachedInstance", meta.AttachedInstance) return nil, errors.New(awserrors.ErrorVolumeInUse) } - // Check if any snapshots reference this volume via JetStream KV. - // Snapshot-backed clones read chunk files from the source volume's - // S3 prefix via ReadFrom(). Deleting the source would break all clones. + // Snapshot-backed clones read chunk files from the source volume's S3 + // prefix, so deleting a volume that still has snapshots would break them. if err := s.checkVolumeHasNoSnapshots(ctx, volumeID); err != nil { return nil, err } - - // Notify viperblockd to stop nbdkit/WAL syncer (best-effort) - if s.natsConn != nil { - deleteReq := types.EBSDeleteRequest{Volume: volumeID} - deleteResp, err := utils.NATSRequest[types.EBSDeleteResponse](ctx, s.natsConn, "ebs.delete", deleteReq, 5*time.Second, accountID) - if err != nil { - slog.WarnContext(ctx, "ebs.delete notification failed (volume may not be mounted)", "volumeId", volumeID, "err", err) - } else if deleteResp.Error != "" { - slog.ErrorContext(ctx, "ebs.delete returned error", "volumeId", volumeID, "err", deleteResp.Error) - return nil, errors.New(awserrors.ErrorServerInternal) - } - } else { - slog.WarnContext(ctx, "DeleteVolume: natsConn is nil, skipping viperblockd notification", "volumeId", volumeID) + if err := s.provider.DeleteVolume(ctx, ebsprovider.DeleteVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, Handle: meta.ProviderHandle, + }); err != nil { + slog.ErrorContext(ctx, "DeleteVolume: provider deletion failed", "volumeId", volumeID, "err", err) + return nil, errors.New(awserrors.ErrorServerInternal) } - - // Delete all S3 objects for this volume and its sub-volumes. - // Auxiliary prefixes are deleted first so the main config.json remains - // available for retry if an auxiliary deletion fails. - prefixes := []string{ - volumeID + "-efi/", - volumeID + "/", - } - - for _, prefix := range prefixes { - if err := s.deleteS3Prefix(ctx, prefix); err != nil { - slog.ErrorContext(ctx, "DeleteVolume failed to delete S3 prefix", "prefix", prefix, "err", err) - return nil, errors.New(awserrors.ErrorServerInternal) - } + if err := s.metadata.DeleteVolume(ctx, volumeID); err != nil { + return nil, errors.New(awserrors.ErrorServerInternal) } slog.InfoContext(ctx, "DeleteVolume completed", "volumeId", volumeID) @@ -1585,46 +1075,6 @@ func (s *VolumeServiceImpl) DeleteVolume(ctx context.Context, input *ec2.DeleteV return &ec2.DeleteVolumeOutput{}, nil } -// deleteS3Prefix deletes all S3 objects under the given prefix. -func (s *VolumeServiceImpl) deleteS3Prefix(ctx context.Context, prefix string) error { - bucket := s.bucketName - - var continuationToken *string - for { - listOutput, err := s.store.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ - Bucket: aws.String(bucket), - Prefix: aws.String(prefix), - ContinuationToken: continuationToken, - }) - if err != nil { - return fmt.Errorf("failed to list objects with prefix %s: %w", prefix, err) - } - - if len(listOutput.Contents) == 0 { - break - } - - for _, obj := range listOutput.Contents { - _, err := s.store.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: aws.String(bucket), - Key: obj.Key, - }) - // A concurrent sweep may have deleted the object between the list - // and here. It is gone either way, which is what this asked for. - if err != nil && !objectstore.IsNoSuchKeyError(err) { - return fmt.Errorf("failed to delete object %s: %w", *obj.Key, err) - } - } - - if !aws.BoolValue(listOutput.IsTruncated) { - break - } - continuationToken = listOutput.NextContinuationToken - } - - return nil -} - // snapshotMetadata holds the subset of snapshot metadata needed by CreateVolume. // Matches the JSON written by the snapshot service's SnapshotConfig. type snapshotMetadata struct { @@ -1719,40 +1169,39 @@ func (s *VolumeServiceImpl) RemoveRecordTags(input *ec2.DeleteTagsInput, account return s.mirrorVolumeTags(context.Background(), input.Resources, accountID, utils.RemoveTagsMut(input)) } -// mirrorVolumeTags read-modify-writes tags.json for each vol- id. The base -// config supplies the ownership gate and create-time tag fallback. Mismatch or -// absence is a no-op. +// mirrorVolumeTags read-modify-writes the ebsmetadata document for each vol- +// id. The document supplies the ownership gate. Mismatch or absence is a no-op. func (s *VolumeServiceImpl) mirrorVolumeTags(ctx context.Context, resources []*string, accountID string, mut func(map[string]string)) error { for _, res := range resources { if res == nil || !strings.HasPrefix(*res, "vol-") { continue } - cfg, _, err := s.getBaseVolumeConfigAndEncryption(ctx, *res) - if err != nil { - if err.Error() == awserrors.ErrorInvalidVolumeNotFound { - continue - } - return err - } - if cfg.VolumeMetadata.TenantID != accountID { - slog.Debug("mirrorVolumeTags: skipping volume not owned by caller", "volumeId", *res) - continue - } - - tags, found, err := s.getVolumeTags(ctx, *res) - if err != nil { - return err - } - if !found { - tags = cfg.VolumeMetadata.Tags - } - if tags == nil { - tags = map[string]string{} - } - mut(tags) - if err := s.putVolumeTags(ctx, *res, tags); err != nil { + if err := s.mirrorVolumeTagsOne(ctx, *res, accountID, mut); err != nil { return err } } return nil } + +// mirrorVolumeTagsOne read-modify-writes one ebsmetadata.Volume's Tags field. +func (s *VolumeServiceImpl) mirrorVolumeTagsOne(ctx context.Context, volumeID, accountID string, mut func(map[string]string)) error { + meta, err := s.metadata.GetVolume(ctx, volumeID) + if err != nil { + if objectstore.IsNoSuchKeyError(err) { + return nil + } + return err + } + if meta.TenantID != accountID { + slog.Debug("mirrorVolumeTags: skipping volume not owned by caller", "volumeId", volumeID) + return nil + } + + tags := meta.Tags + if tags == nil { + tags = map[string]string{} + } + mut(tags) + meta.Tags = tags + return s.metadata.PutVolume(ctx, meta) +} diff --git a/spinifex/handlers/ec2/volume/service_impl_encryption_test.go b/spinifex/handlers/ec2/volume/service_impl_encryption_test.go index 8907644cb..fc3f19031 100644 --- a/spinifex/handlers/ec2/volume/service_impl_encryption_test.go +++ b/spinifex/handlers/ec2/volume/service_impl_encryption_test.go @@ -2,78 +2,33 @@ package handlers_ec2_volume import ( "context" - "encoding/json" "path/filepath" - "strings" "testing" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" - "github.com/aws/aws-sdk-go/service/s3" "github.com/mulgadc/spinifex/spinifex/awserrors" "github.com/mulgadc/spinifex/spinifex/config" + "github.com/mulgadc/spinifex/spinifex/ebsmetadata" + "github.com/mulgadc/spinifex/spinifex/ebsprovider" "github.com/mulgadc/spinifex/spinifex/objectstore" - "github.com/mulgadc/viperblock/viperblock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestGetVolumeConfig_CorruptJSON(t *testing.T) { +// A document that does not decode must surface as an error rather than +// resolving to an empty volume, which would read as a volume that exists with +// no owner, no size and no attachment. +func TestGetVolumeMetadata_CorruptJSON(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) const volumeID = "vol-bad-json" - // Body is not a valid VBState (BlockSize stays 0 → falls through) and is - // not a valid volumeConfigWrapper either (unmarshal fails) — must surface - // "failed to unmarshal config" from getVolumeConfigAndEncryption. - _, err := store.PutObject(context.Background(), &s3.PutObjectInput{ - Bucket: aws.String("test-bucket"), - Key: aws.String(volumeID + "/config.json"), - Body: strings.NewReader("not valid json {{{"), - }) - require.NoError(t, err) + putRawVolumeDocument(t, store, volumeID, "not valid json {{{") - _, err = svc.GetVolumeConfig(volumeID) + _, err := svc.GetVolumeMetadata(volumeID) require.Error(t, err) - assert.Contains(t, err.Error(), "failed to unmarshal config") -} - -func TestMergeVolumeConfig_RefusesEncryptedVBState(t *testing.T) { - store := objectstore.NewMemoryObjectStore() - svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - - const volumeID = "vol-encrypted" - const configKey = volumeID + "/config.json" - - // Seed an encrypted-at-rest VBState. mergeVolumeConfig must refuse to - // re-marshal it because spinifex does not currently hold the master key - // and a tag-less rewrite would brick the volume. - state := viperblock.VBState{ - VolumeName: volumeID, - VolumeSize: 1024 * 1024 * 1024, - BlockSize: 4096, - EncryptionEnabled: true, - VolumeConfig: viperblock.VolumeConfig{ - VolumeMetadata: viperblock.VolumeMetadata{ - VolumeID: volumeID, - SizeGiB: 1, - }, - }, - } - data, err := json.Marshal(state) - require.NoError(t, err) - _, err = store.PutObject(context.Background(), &s3.PutObjectInput{ - Bucket: aws.String("test-bucket"), - Key: aws.String(configKey), - Body: strings.NewReader(string(data)), - }) - require.NoError(t, err) - - _, err = svc.mergeVolumeConfig(context.Background(), configKey, &viperblock.VolumeConfig{ - VolumeMetadata: viperblock.VolumeMetadata{VolumeID: volumeID, SizeGiB: 2}, - }) - require.Error(t, err) - assert.Contains(t, err.Error(), "refusing to merge encrypted VBState") + assert.ErrorIs(t, err, ebsmetadata.ErrCorruptDocument) } // newTestVolumeServiceWithEncryptionKey wires CreateVolume to a configured @@ -91,13 +46,15 @@ func newTestVolumeServiceWithEncryptionKey(az, keyFile string) *VolumeServiceImp Viperblock: config.ViperblockConfig{EncryptionKeyFile: keyFile}, WalDir: "/tmp/test-wal", } - return NewVolumeServiceImplWithStore(cfg, objectstore.NewMemoryObjectStore(), nil) + svc := NewVolumeServiceImplWithStore(cfg, objectstore.NewMemoryObjectStore(), nil) + svc.SetEBSProvider(ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})) + return svc } func TestCreateVolume_EncryptionKeyLoadError(t *testing.T) { // Point EncryptionKeyFile at a non-existent path so LoadViperblockMasterKey - // fails — CreateVolume must abort with ServerInternal, before any backend - // init / SaveState call. + // fails — CreateVolume must abort with ServerInternal and roll the + // provider's allocation back rather than leave an orphaned volume. missing := filepath.Join(t.TempDir(), "absent.key") svc := newTestVolumeServiceWithEncryptionKey("ap-southeast-2a", missing) @@ -107,4 +64,8 @@ func TestCreateVolume_EncryptionKeyLoadError(t *testing.T) { }, "") require.Error(t, err) assert.Equal(t, awserrors.ErrorServerInternal, err.Error()) + + out, err := svc.DescribeVolumes(context.Background(), &ec2.DescribeVolumesInput{}, "") + require.NoError(t, err) + assert.Empty(t, out.Volumes, "a failed create must not leave a volume behind") } diff --git a/spinifex/handlers/ec2/volume/service_impl_test.go b/spinifex/handlers/ec2/volume/service_impl_test.go index 3ae8dbd16..8c76129b0 100644 --- a/spinifex/handlers/ec2/volume/service_impl_test.go +++ b/spinifex/handlers/ec2/volume/service_impl_test.go @@ -3,7 +3,7 @@ package handlers_ec2_volume import ( "context" "encoding/json" - "io" + "errors" "strings" "testing" "time" @@ -13,30 +13,38 @@ import ( "github.com/aws/aws-sdk-go/service/s3" "github.com/mulgadc/spinifex/spinifex/awserrors" "github.com/mulgadc/spinifex/spinifex/config" + "github.com/mulgadc/spinifex/spinifex/ebsmetadata" + "github.com/mulgadc/spinifex/spinifex/ebsprovider" + "github.com/mulgadc/spinifex/spinifex/filterutil" "github.com/mulgadc/spinifex/spinifex/objectstore" "github.com/mulgadc/spinifex/spinifex/testutil" - "github.com/mulgadc/spinifex/spinifex/types" - "github.com/mulgadc/viperblock/viperblock" - "github.com/nats-io/nats-server/v2/server" - "github.com/nats-io/nats.go" + "github.com/mulgadc/spinifex/spinifex/testutil/ebsfake" + "github.com/mulgadc/spinifex/spinifex/utils" "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func newTestVolumeService(az string) *VolumeServiceImpl { - cfg := &config.Config{ - AZ: az, - Predastore: config.PredastoreConfig{ - Bucket: "test-bucket", - Region: "ap-southeast-2", - Host: fakeS3Host, - AccessKey: "testkey", - SecretKey: "testsecret", - }, - WalDir: "/tmp/test-wal", + return newTestVolumeServiceWithStore(az, objectstore.NewMemoryObjectStore()) +} + +// seedTestProviderVolume gives the service's provider a volume to operate on. +// A control-plane document alone is not a volume: the provider owns the blocks +// and refuses to expand one it has never allocated. +func seedTestProviderVolume(t *testing.T, svc *VolumeServiceImpl, volumeID string, sizeGiB int64) { + t.Helper() + // A fixture with no ID or no size is deliberately unusable and exists to + // exercise a rejection path, so there is nothing to allocate. + if svc == nil || svc.provider == nil || volumeID == "" || sizeGiB <= 0 { + return } - return NewVolumeServiceImplWithStore(cfg, objectstore.NewMemoryObjectStore(), nil) + _, err := svc.provider.CreateVolume(context.Background(), ebsprovider.CreateVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, + CapacityRange: ebsprovider.CapacityRange{RequiredBytes: sizeGiB * 1024 * 1024 * 1024}, + AvailabilityZone: "ap-southeast-2a", + }) + require.NoError(t, err) } func TestCreateVolume_Validation(t *testing.T) { @@ -290,38 +298,348 @@ func TestCreateVolume_PassesValidation(t *testing.T) { } } -// TestCreateVolume_BuildVBConfig_GCEnabled asserts that CreateVolume's -// viperblock config carries GCEnabled through from spinifex.toml's -// [viperblock] gc_enabled key (config.Config.Viperblock.GCEnabled). This is -// the config-level seam: GCEnabled isn't part of viperblock.VBState, so it -// never persists to config.json and can't be observed by reading a created -// volume back — the only place it's checkable is the config CreateVolume -// hands to viperblock.New, before construction ever reaches the backend. -// A regression that unwires GC in buildVBConfig, or a future refactor of -// CreateVolume that stops routing through buildVBConfig, fails this test. -func TestCreateVolume_BuildVBConfig_GCEnabled(t *testing.T) { - tests := []struct { - name string - gcEnabled *bool - want bool - }{ - {name: "NilDefaultsToDisabled", gcEnabled: nil, want: false}, - {name: "ExplicitFalse", gcEnabled: aws.Bool(false), want: false}, - {name: "ExplicitTrue", gcEnabled: aws.Bool(true), want: true}, +func TestCreateVolume_UsesInjectedProvider(t *testing.T) { + svc := newTestVolumeService("ap-southeast-2a") + svc.snapshotKV = setupTestVolumeKV(t) + + vol, err := svc.CreateVolume(context.Background(), &ec2.CreateVolumeInput{ + Size: aws.Int64(8), + AvailabilityZone: aws.String("ap-southeast-2a"), + }, "acct-1") + require.NoError(t, err) + require.NotNil(t, vol) + require.NotEmpty(t, vol.VolumeId) + + metadata, err := svc.metadata.GetVolume(context.Background(), aws.StringValue(vol.VolumeId)) + require.NoError(t, err) + assert.Equal(t, "acct-1", metadata.TenantID) + assert.Equal(t, uint64(8), metadata.CapacityGiB) + assert.Equal(t, "memory://volume/"+aws.StringValue(vol.VolumeId), metadata.ProviderHandle) + described, err := svc.DescribeVolumes(context.Background(), &ec2.DescribeVolumesInput{VolumeIds: []*string{vol.VolumeId}}, "acct-1") + require.NoError(t, err) + require.Len(t, described.Volumes, 1) + assert.Equal(t, int64(8), aws.Int64Value(described.Volumes[0].Size)) + require.NoError(t, svc.UpdateVolumeState(aws.StringValue(vol.VolumeId), "in-use", "i-123", "/dev/sda1")) + _, err = svc.DeleteVolume(context.Background(), &ec2.DeleteVolumeInput{VolumeId: vol.VolumeId}, "acct-1") + assert.EqualError(t, err, awserrors.ErrorVolumeInUse) + require.NoError(t, svc.UpdateVolumeState(aws.StringValue(vol.VolumeId), "available", "", "")) + _, err = svc.DeleteVolume(context.Background(), &ec2.DeleteVolumeInput{VolumeId: vol.VolumeId}, "acct-1") + require.NoError(t, err) + _, err = svc.metadata.GetVolume(context.Background(), aws.StringValue(vol.VolumeId)) + assert.Error(t, err) +} + +// TestDescribeVolumes_Provider_FilterAndTenantIsolation covers DescribeVolumes' +// provider-metadata branch: a filter must narrow results, an unknown +// requested volume ID must come back InvalidVolume.NotFound, and one +// tenant must never see another tenant's volumes. +func TestDescribeVolumes_Provider_FilterAndTenantIsolation(t *testing.T) { + svc := newTestVolumeService("ap-southeast-2a") + svc.SetEBSProvider(ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})) + ctx := context.Background() + + volA, err := svc.CreateVolume(ctx, &ec2.CreateVolumeInput{Size: aws.Int64(8), AvailabilityZone: aws.String("ap-southeast-2a")}, "acct-1") + require.NoError(t, err) + _, err = svc.CreateVolume(ctx, &ec2.CreateVolumeInput{Size: aws.Int64(16), AvailabilityZone: aws.String("ap-southeast-2a")}, "acct-2") + require.NoError(t, err) + + t.Run("tenant isolation", func(t *testing.T) { + out, err := svc.DescribeVolumes(ctx, &ec2.DescribeVolumesInput{}, "acct-1") + require.NoError(t, err) + require.Len(t, out.Volumes, 1) + assert.Equal(t, aws.StringValue(volA.VolumeId), aws.StringValue(out.Volumes[0].VolumeId)) + }) + + t.Run("filter match", func(t *testing.T) { + out, err := svc.DescribeVolumes(ctx, &ec2.DescribeVolumesInput{ + Filters: []*ec2.Filter{{Name: aws.String("size"), Values: []*string{aws.String("8")}}}, + }, "acct-1") + require.NoError(t, err) + require.Len(t, out.Volumes, 1) + assert.Equal(t, aws.StringValue(volA.VolumeId), aws.StringValue(out.Volumes[0].VolumeId)) + }) + + t.Run("unknown volume id is InvalidVolume.NotFound", func(t *testing.T) { + _, err := svc.DescribeVolumes(ctx, &ec2.DescribeVolumesInput{VolumeIds: []*string{aws.String("vol-does-not-exist")}}, "acct-1") + require.EqualError(t, err, awserrors.ErrorInvalidVolumeNotFound) + }) +} + +// TestModifyVolume_Provider covers ModifyVolume's provider-metadata branch: +// a grow succeeds and persists the new capacity, a shrink is rejected +// (grow-only), and an in-use volume is rejected regardless of size. +func TestModifyVolume_Provider(t *testing.T) { + svc := newTestVolumeService("ap-southeast-2a") + svc.SetEBSProvider(ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{OnlineExpansion: true})) + ctx := context.Background() + + vol, err := svc.CreateVolume(ctx, &ec2.CreateVolumeInput{Size: aws.Int64(8), AvailabilityZone: aws.String("ap-southeast-2a")}, "acct-1") + require.NoError(t, err) + volumeID := aws.StringValue(vol.VolumeId) + + t.Run("grow succeeds", func(t *testing.T) { + out, err := svc.ModifyVolume(ctx, &ec2.ModifyVolumeInput{VolumeId: vol.VolumeId, Size: aws.Int64(16)}, "acct-1") + require.NoError(t, err) + require.NotNil(t, out.VolumeModification) + assert.Equal(t, int64(8), aws.Int64Value(out.VolumeModification.OriginalSize)) + assert.Equal(t, int64(16), aws.Int64Value(out.VolumeModification.TargetSize)) + + meta, err := svc.metadata.GetVolume(ctx, volumeID) + require.NoError(t, err) + assert.Equal(t, uint64(16), meta.CapacityGiB) + }) + + t.Run("shrink is rejected", func(t *testing.T) { + _, err := svc.ModifyVolume(ctx, &ec2.ModifyVolumeInput{VolumeId: vol.VolumeId, Size: aws.Int64(8)}, "acct-1") + require.EqualError(t, err, awserrors.ErrorInvalidParameterValue) + }) + + t.Run("in-use is rejected", func(t *testing.T) { + require.NoError(t, svc.UpdateVolumeState(volumeID, "in-use", "i-123", "/dev/sda1")) + _, err := svc.ModifyVolume(ctx, &ec2.ModifyVolumeInput{VolumeId: vol.VolumeId, Size: aws.Int64(32)}, "acct-1") + require.EqualError(t, err, awserrors.ErrorIncorrectState) + }) + + t.Run("unknown volume id is InvalidVolume.NotFound", func(t *testing.T) { + _, err := svc.ModifyVolume(ctx, &ec2.ModifyVolumeInput{VolumeId: aws.String("vol-does-not-exist"), Size: aws.Int64(32)}, "acct-1") + require.EqualError(t, err, awserrors.ErrorInvalidVolumeNotFound) + }) +} + +// TestModifyVolume_Provider_PersistsAndDescribesModification covers 2a/2b: +// a ModifyVolume grow under the provider path must persist its modification +// on the ebsmetadata.Volume document, and a subsequent +// DescribeVolumesModifications (both the fast VolumeIds path and the slow +// list-everything path) must be able to read it back. +func TestModifyVolume_Provider_PersistsAndDescribesModification(t *testing.T) { + svc := newTestVolumeService("ap-southeast-2a") + svc.SetEBSProvider(ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{OnlineExpansion: true})) + ctx := context.Background() + + vol, err := svc.CreateVolume(ctx, &ec2.CreateVolumeInput{Size: aws.Int64(8), AvailabilityZone: aws.String("ap-southeast-2a")}, "acct-1") + require.NoError(t, err) + volumeID := aws.StringValue(vol.VolumeId) + + _, err = svc.ModifyVolume(ctx, &ec2.ModifyVolumeInput{VolumeId: vol.VolumeId, Size: aws.Int64(16), Iops: aws.Int64(4000)}, "acct-1") + require.NoError(t, err) + + meta, err := svc.metadata.GetVolume(ctx, volumeID) + require.NoError(t, err) + require.NotNil(t, meta.Modification, "ModifyVolume must persist the modification on the ebsmetadata.Volume document") + assert.Equal(t, int64(8), meta.Modification.OriginalSize) + assert.Equal(t, int64(16), meta.Modification.TargetSize) + assert.Equal(t, int64(4000), meta.Modification.TargetIOPS) + + t.Run("fast path", func(t *testing.T) { + out, err := svc.DescribeVolumesModifications(ctx, &ec2.DescribeVolumesModificationsInput{ + VolumeIds: []*string{vol.VolumeId}, + }, "acct-1") + require.NoError(t, err) + require.Len(t, out.VolumesModifications, 1) + assert.Equal(t, volumeID, aws.StringValue(out.VolumesModifications[0].VolumeId)) + assert.Equal(t, int64(16), aws.Int64Value(out.VolumesModifications[0].TargetSize)) + }) + + t.Run("slow path", func(t *testing.T) { + out, err := svc.DescribeVolumesModifications(ctx, &ec2.DescribeVolumesModificationsInput{}, "acct-1") + require.NoError(t, err) + require.Len(t, out.VolumesModifications, 1) + assert.Equal(t, volumeID, aws.StringValue(out.VolumesModifications[0].VolumeId)) + }) + + t.Run("unmodified volume has no modification record", func(t *testing.T) { + other, err := svc.CreateVolume(ctx, &ec2.CreateVolumeInput{Size: aws.Int64(8), AvailabilityZone: aws.String("ap-southeast-2a")}, "acct-1") + require.NoError(t, err) + out, err := svc.DescribeVolumesModifications(ctx, &ec2.DescribeVolumesModificationsInput{ + VolumeIds: []*string{other.VolumeId}, + }, "acct-1") + require.NoError(t, err) + assert.Empty(t, out.VolumesModifications) + }) +} + +// TestDescribeVolumeStatus_Provider covers the DescribeVolumeStatus provider +// gap found during the survey: neither the fast nor the slow path had a +// provider branch, so both silently returned nothing for provider-managed +// volumes. +func TestDescribeVolumeStatus_Provider(t *testing.T) { + svc := newTestVolumeService("ap-southeast-2a") + svc.SetEBSProvider(ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})) + ctx := context.Background() + + vol, err := svc.CreateVolume(ctx, &ec2.CreateVolumeInput{Size: aws.Int64(8), AvailabilityZone: aws.String("ap-southeast-2a")}, "acct-1") + require.NoError(t, err) + + t.Run("fast path", func(t *testing.T) { + out, err := svc.DescribeVolumeStatus(ctx, &ec2.DescribeVolumeStatusInput{VolumeIds: []*string{vol.VolumeId}}, "acct-1") + require.NoError(t, err) + require.Len(t, out.VolumeStatuses, 1) + assert.Equal(t, aws.StringValue(vol.VolumeId), aws.StringValue(out.VolumeStatuses[0].VolumeId)) + }) + + t.Run("slow path", func(t *testing.T) { + out, err := svc.DescribeVolumeStatus(ctx, &ec2.DescribeVolumeStatusInput{}, "acct-1") + require.NoError(t, err) + require.Len(t, out.VolumeStatuses, 1) + }) + + t.Run("cross tenant is not found", func(t *testing.T) { + _, err := svc.DescribeVolumeStatus(ctx, &ec2.DescribeVolumeStatusInput{VolumeIds: []*string{vol.VolumeId}}, "acct-2") + require.EqualError(t, err, awserrors.ErrorInvalidVolumeNotFound) + }) +} + +// TestCreateVolume_Provider_EncryptedFollowsConfig covers 1b: the Encrypted +// bit on a provider-managed volume must follow config.Viperblock.EncryptionKeyFile, +// the same knob the legacy path uses, and must be visible via both +// CreateVolume's response and a subsequent DescribeVolumes. +func TestCreateVolume_Provider_EncryptedFollowsConfig(t *testing.T) { + svc := newTestVolumeService("ap-southeast-2a") + svc.SetEBSProvider(ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})) + ctx := context.Background() + + vol, err := svc.CreateVolume(ctx, &ec2.CreateVolumeInput{Size: aws.Int64(8), AvailabilityZone: aws.String("ap-southeast-2a")}, "acct-1") + require.NoError(t, err) + assert.False(t, aws.BoolValue(vol.Encrypted), "no encryption key file configured, so the volume must not report encrypted") + + described, err := svc.DescribeVolumes(ctx, &ec2.DescribeVolumesInput{VolumeIds: []*string{vol.VolumeId}}, "acct-1") + require.NoError(t, err) + require.Len(t, described.Volumes, 1) + assert.False(t, aws.BoolValue(described.Volumes[0].Encrypted)) +} + +// TestVolumeLeakReaper_Provider covers 2d: the reaper must read attachment +// and tags from ebsmetadata and mark orphaned volumes there when a provider +// is configured, and the mark must be visible via DescribeVolumes. +func TestVolumeLeakReaper_Provider(t *testing.T) { + svc := newTestVolumeService("ap-southeast-2a") + svc.SetEBSProvider(ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})) + ctx := context.Background() + + vol, err := svc.CreateVolume(ctx, &ec2.CreateVolumeInput{Size: aws.Int64(8), AvailabilityZone: aws.String("ap-southeast-2a")}, "acct-1") + require.NoError(t, err) + volumeID := aws.StringValue(vol.VolumeId) + require.NoError(t, svc.UpdateVolumeState(volumeID, "in-use", "i-gone0000000000", "/dev/sda1")) + + reaper := svc.NewVolumeLeakReaper(func() (map[string]bool, error) { + return map[string]bool{"i-gone0000000000": true}, nil + }) + + marked, err := reaper.Sweep(ctx) + require.NoError(t, err) + assert.Equal(t, 1, marked) + + meta, err := svc.metadata.GetVolume(ctx, volumeID) + require.NoError(t, err) + assert.NotEmpty(t, meta.Tags[orphanTagKey], "the reaper must mark the provider-managed volume orphaned in ebsmetadata") + assert.Equal(t, "available", meta.State, "the reaper must reconcile the stale attachment") + assert.Empty(t, meta.AttachedInstance) + + described, err := svc.DescribeVolumes(ctx, &ec2.DescribeVolumesInput{VolumeIds: []*string{vol.VolumeId}}, "acct-1") + require.NoError(t, err) + require.Len(t, described.Volumes, 1) + assert.NotEmpty(t, filterutil.EC2TagsToMap(described.Volumes[0].Tags)[orphanTagKey], + "the orphan mark must be visible via DescribeVolumes, which reads ebsmetadata under the provider path") + + // Idempotent: a second sweep re-marks nothing. + marked, err = reaper.Sweep(ctx) + require.NoError(t, err) + assert.Equal(t, 0, marked) +} + +// TestApplyRecordTags_Provider_VisibleViaDescribeVolumes covers 2e: a tag +// applied post-create under the provider path must be visible via +// DescribeVolumes, which reads ebsmetadata.Volume.Tags directly (there is no +// tags.json for it to fall back to). +func TestApplyRecordTags_Provider_VisibleViaDescribeVolumes(t *testing.T) { + svc := newTestVolumeService("ap-southeast-2a") + svc.SetEBSProvider(ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})) + ctx := context.Background() + + vol, err := svc.CreateVolume(ctx, &ec2.CreateVolumeInput{Size: aws.Int64(8), AvailabilityZone: aws.String("ap-southeast-2a")}, "acct-1") + require.NoError(t, err) + + require.NoError(t, svc.ApplyRecordTags(&ec2.CreateTagsInput{ + Resources: []*string{vol.VolumeId}, + Tags: []*ec2.Tag{{Key: aws.String("owner"), Value: aws.String("control-plane")}}, + }, "acct-1")) + + described, err := svc.DescribeVolumes(ctx, &ec2.DescribeVolumesInput{VolumeIds: []*string{vol.VolumeId}}, "acct-1") + require.NoError(t, err) + require.Len(t, described.Volumes, 1) + assert.Equal(t, "control-plane", filterutil.EC2TagsToMap(described.Volumes[0].Tags)["owner"], + "a tag applied post-create must be visible via DescribeVolumes under the provider path") + + require.NoError(t, svc.RemoveRecordTags(&ec2.DeleteTagsInput{ + Resources: []*string{vol.VolumeId}, + Tags: []*ec2.Tag{{Key: aws.String("owner"), Value: aws.String("control-plane")}}, + }, "acct-1")) + + described, err = svc.DescribeVolumes(ctx, &ec2.DescribeVolumesInput{VolumeIds: []*string{vol.VolumeId}}, "acct-1") + require.NoError(t, err) + require.Len(t, described.Volumes, 1) + _, hasOwnerTag := filterutil.EC2TagsToMap(described.Volumes[0].Tags)["owner"] + assert.False(t, hasOwnerTag, "RemoveRecordTags must remove the tag under the provider path too") + + t.Run("wrong tenant is a no-op", func(t *testing.T) { + require.NoError(t, svc.ApplyRecordTags(&ec2.CreateTagsInput{ + Resources: []*string{vol.VolumeId}, + Tags: []*ec2.Tag{{Key: aws.String("owner"), Value: aws.String("intruder")}}, + }, "acct-2")) + described, err := svc.DescribeVolumes(ctx, &ec2.DescribeVolumesInput{VolumeIds: []*string{vol.VolumeId}}, "acct-1") + require.NoError(t, err) + _, hasOwnerTag := filterutil.EC2TagsToMap(described.Volumes[0].Tags)["owner"] + assert.False(t, hasOwnerTag, "a caller who does not own the volume must not be able to tag it") + }) + + t.Run("unknown volume is a no-op", func(t *testing.T) { + require.NoError(t, svc.ApplyRecordTags(&ec2.CreateTagsInput{ + Resources: []*string{aws.String("vol-does-not-exist")}, + Tags: []*ec2.Tag{{Key: aws.String("owner"), Value: aws.String("control-plane")}}, + }, "acct-1")) + }) +} + +// prefixFailingObjectStore fails PutObject for any key under failPrefix and +// records the last such key, so a test can recover a randomly generated +// resource ID from the write CreateVolume/CreateSnapshot attempted, without +// needing to predict utils.GenerateResourceID's output up front. +type prefixFailingObjectStore struct { + objectstore.ObjectStore + + failPrefix string + attemptedKey string +} + +func (s *prefixFailingObjectStore) PutObject(ctx context.Context, input *s3.PutObjectInput) (*s3.PutObjectOutput, error) { + key := aws.StringValue(input.Key) + if strings.HasPrefix(key, s.failPrefix) { + s.attemptedKey = key + return nil, errors.New("simulated metadata write failure") } + return s.ObjectStore.PutObject(ctx, input) +} - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - svc := newTestVolumeService("ap-southeast-2a") - svc.config.Viperblock.GCEnabled = tt.gcEnabled +// TestCreateVolume_Provider_RollbackOnMetadataWriteFailure covers CreateVolume's +// rollback path: when the control-plane metadata write fails after the +// provider has already allocated the volume, CreateVolume must delete the +// just-created provider volume rather than orphan it. +func TestCreateVolume_Provider_RollbackOnMetadataWriteFailure(t *testing.T) { + store := &prefixFailingObjectStore{ObjectStore: objectstore.NewMemoryObjectStore(), failPrefix: "spinifex/ebsmetadata/v1/volumes/"} + cfg := &config.Config{AZ: "ap-southeast-2a", Predastore: config.PredastoreConfig{Bucket: "test-bucket"}} + svc := NewVolumeServiceImplWithStore(cfg, store, nil) + provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{}) + svc.SetEBSProvider(provider) - vbconfig := svc.buildVBConfig("vol-gc-test", 10*1024*1024*1024, - viperblock.VolumeConfig{}, nil, "", "") + _, err := svc.CreateVolume(context.Background(), &ec2.CreateVolumeInput{ + Size: aws.Int64(8), AvailabilityZone: aws.String("ap-southeast-2a"), + }, "acct-1") + require.EqualError(t, err, awserrors.ErrorServerInternal) + require.NotEmpty(t, store.attemptedKey, "the metadata write must have been attempted") - assert.Equal(t, tt.want, vbconfig.GCEnabled, - "buildVBConfig GCEnabled must follow config.Viperblock.GCEnabled") - }) - } + volumeID := strings.TrimSuffix(strings.TrimPrefix(store.attemptedKey, store.failPrefix), ".json") + _, err = provider.GetVolume(context.Background(), ebsprovider.GetVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID}) + require.ErrorIs(t, err, ebsprovider.ErrNotFound, "rollback must delete the just-created provider volume") } func TestDeleteVolume_Validation(t *testing.T) { @@ -403,7 +721,9 @@ func newTestVolumeServiceWithStore(az string, store *objectstore.MemoryObjectSto }, WalDir: "/tmp/test-wal", } - return NewVolumeServiceImplWithStore(cfg, store, nil) + svc := NewVolumeServiceImplWithStore(cfg, store, nil) + svc.SetEBSProvider(ebsfake.New(store, "test-bucket")) + return svc } // putTestSnapshotMetadata writes snapshot metadata in the store, matching the @@ -595,26 +915,13 @@ func setupTestVolumeKV(t *testing.T) jetstream.KeyValue { return kv } -func createVolumeInStore(t *testing.T, store *objectstore.MemoryObjectStore, volumeID string) { +func createVolumeInStore(t *testing.T, svc *VolumeServiceImpl, store *objectstore.MemoryObjectStore, volumeID string) { t.Helper() - volumeState := viperblock.VBState{ - VolumeConfig: viperblock.VolumeConfig{ - VolumeMetadata: viperblock.VolumeMetadata{ - VolumeID: volumeID, - SizeGiB: 10, - State: "available", - }, - }, - } - data, err := json.Marshal(volumeState) - require.NoError(t, err) - - _, err = store.PutObject(context.Background(), &s3.PutObjectInput{ - Bucket: aws.String("test-bucket"), - Key: aws.String(volumeID + "/config.json"), - Body: strings.NewReader(string(data)), + createVolumeInStoreWithMeta(t, svc, store, volumeID, ebsmetadata.Volume{ + VolumeID: volumeID, + CapacityGiB: 10, + State: "available", }) - require.NoError(t, err) } func TestDeleteVolume_BlockedByKV(t *testing.T) { @@ -624,7 +931,7 @@ func TestDeleteVolume_BlockedByKV(t *testing.T) { svc.snapshotKV = kv volumeID := "vol-kvblocked" - createVolumeInStore(t, store, volumeID) + createVolumeInStore(t, svc, store, volumeID) // Put a snapshot ref in KV data, err := json.Marshal([]string{"snap-001"}) @@ -647,7 +954,8 @@ func TestDeleteVolume_AllowedByKV(t *testing.T) { svc.snapshotKV = kv volumeID := "vol-kvallowed" - createVolumeInStore(t, store, volumeID) + createVolumeInStore(t, svc, store, volumeID) + seedTestProviderVolume(t, svc, volumeID, 10) // No KV entry → delete allowed _, err := svc.DeleteVolume(context.Background(), &ec2.DeleteVolumeInput{ @@ -662,7 +970,7 @@ func TestDeleteVolume_ErrorWhenKVNil(t *testing.T) { // snapshotKV is nil by default volumeID := "vol-nokvtest" - createVolumeInStore(t, store, volumeID) + createVolumeInStore(t, svc, store, volumeID) // Should fail because snapshotKV is nil _, err := svc.DeleteVolume(context.Background(), &ec2.DeleteVolumeInput{ @@ -672,59 +980,50 @@ func TestDeleteVolume_ErrorWhenKVNil(t *testing.T) { assert.Contains(t, err.Error(), awserrors.ErrorServerInternal) } -// createVolumeInStoreWithMeta seeds a volume config.json with custom metadata. -func createVolumeInStoreWithMeta(t *testing.T, store *objectstore.MemoryObjectStore, volumeID string, meta viperblock.VolumeMetadata) { +// createVolumeInStoreWithMeta seeds a volume on both sides: the document the +// control plane reads and the provider volume behind it. +func createVolumeInStoreWithMeta(t *testing.T, svc *VolumeServiceImpl, store *objectstore.MemoryObjectStore, volumeID string, meta ebsmetadata.Volume) { t.Helper() - wrapper := volumeConfigWrapper{ - VolumeConfig: viperblock.VolumeConfig{ - VolumeMetadata: meta, - }, + if meta.VolumeID == "" { + meta.VolumeID = volumeID } - data, err := json.Marshal(wrapper) - require.NoError(t, err) + seedVolumeDocument(t, store, meta) + seedTestProviderVolume(t, svc, meta.VolumeID, utils.SafeUint64ToInt64(meta.CapacityGiB)) +} +// putRawVolumeDocument writes document bytes verbatim, for cases a typed seed +// cannot express: a corrupt document, or one predating a field. +func putRawVolumeDocument(t *testing.T, store *objectstore.MemoryObjectStore, volumeID, body string) { + t.Helper() + key, err := ebsmetadata.VolumeKey(volumeID) + require.NoError(t, err) _, err = store.PutObject(context.Background(), &s3.PutObjectInput{ Bucket: aws.String("test-bucket"), - Key: aws.String(volumeID + "/config.json"), - Body: strings.NewReader(string(data)), + Key: aws.String(key), + Body: strings.NewReader(body), }) require.NoError(t, err) } -// createVolumeInStoreWithVBState seeds a volume config.json as a full VBState -// (with BlockSize > 0) so that mergeVolumeConfig preserves VBState fields. -func createVolumeInStoreWithVBState(t *testing.T, store *objectstore.MemoryObjectStore, volumeID string, meta viperblock.VolumeMetadata, blockSize uint32, seqNum uint64) { +// --- Group 1: single-volume read and projection tests --- + +// volumeByID reads a volume's control-plane document and renders it the way +// DescribeVolumes does, so the read and the projection are covered together. +func volumeByID(t *testing.T, svc *VolumeServiceImpl, volumeID string) *ec2.Volume { t.Helper() - state := viperblock.VBState{ - VolumeName: volumeID, - VolumeSize: meta.SizeGiB * 1024 * 1024 * 1024, - BlockSize: blockSize, - SeqNum: seqNum, - VolumeConfig: viperblock.VolumeConfig{ - VolumeMetadata: meta, - }, - } - data, err := json.Marshal(state) - require.NoError(t, err) - - _, err = store.PutObject(context.Background(), &s3.PutObjectInput{ - Bucket: aws.String("test-bucket"), - Key: aws.String(volumeID + "/config.json"), - Body: strings.NewReader(string(data)), - }) + meta, err := svc.GetVolumeMetadata(volumeID) require.NoError(t, err) + return metadataVolumeToEC2(meta) } -// --- Group 1: getVolumeByID tests --- - func TestGetVolumeByID_FullMetadata(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) now := time.Now() - meta := viperblock.VolumeMetadata{ + meta := ebsmetadata.Volume{ VolumeID: "vol-full", - SizeGiB: 20, + CapacityGiB: 20, State: "in-use", CreatedAt: now, AvailabilityZone: "ap-southeast-2a", @@ -738,27 +1037,10 @@ func TestGetVolumeByID_FullMetadata(t *testing.T) { AttachedAt: now, Tags: map[string]string{"Name": "test-vol", "env": "dev"}, } - // Seed as a full VBState with EncryptionEnabled=true so getVolumeByID - // reports Encrypted via the authoritative VBState.EncryptionEnabled path. - state := viperblock.VBState{ - VolumeName: "vol-full", - VolumeSize: meta.SizeGiB * 1024 * 1024 * 1024, - BlockSize: 4096, - EncryptionEnabled: true, - VolumeConfig: viperblock.VolumeConfig{VolumeMetadata: meta}, - } - data, err := json.Marshal(state) - require.NoError(t, err) - _, err = store.PutObject(context.Background(), &s3.PutObjectInput{ - Bucket: aws.String("test-bucket"), - Key: aws.String("vol-full/config.json"), - Body: strings.NewReader(string(data)), - }) - require.NoError(t, err) + meta.Encrypted = true + seedVolumeDocument(t, store, meta) - result, err := svc.getVolumeByID(context.Background(), "vol-full") - require.NoError(t, err) - vol := result.volume + vol := volumeByID(t, svc, "vol-full") assert.Equal(t, "vol-full", *vol.VolumeId) assert.Equal(t, int64(20), *vol.Size) @@ -786,98 +1068,71 @@ func TestGetVolumeByID_AttachmentDetached(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - meta := viperblock.VolumeMetadata{ + meta := ebsmetadata.Volume{ VolumeID: "vol-detach", - SizeGiB: 10, + CapacityGiB: 10, State: "available", AttachedInstance: "i-99999", DeviceName: "/dev/nbd1", } - createVolumeInStoreWithMeta(t, store, "vol-detach", meta) + createVolumeInStoreWithMeta(t, svc, store, "vol-detach", meta) - result, err := svc.getVolumeByID(context.Background(), "vol-detach") - require.NoError(t, err) + vol := volumeByID(t, svc, "vol-detach") - require.Len(t, result.volume.Attachments, 1) - assert.Equal(t, "detached", *result.volume.Attachments[0].State) + require.Len(t, vol.Attachments, 1) + assert.Equal(t, "detached", *vol.Attachments[0].State) } func TestGetVolumeByID_DefaultStateAndType(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - meta := viperblock.VolumeMetadata{ - VolumeID: "vol-defaults", - SizeGiB: 5, - State: "", + meta := ebsmetadata.Volume{ + VolumeID: "vol-defaults", + CapacityGiB: 5, + State: "", } - createVolumeInStoreWithMeta(t, store, "vol-defaults", meta) + createVolumeInStoreWithMeta(t, svc, store, "vol-defaults", meta) - result, err := svc.getVolumeByID(context.Background(), "vol-defaults") - require.NoError(t, err) + vol := volumeByID(t, svc, "vol-defaults") - assert.Equal(t, "available", *result.volume.State) - assert.Equal(t, "gp3", *result.volume.VolumeType) + assert.Equal(t, "available", *vol.State) + assert.Equal(t, "gp3", *vol.VolumeType) } // TestGetVolumeByID_ThroughputOmitted_PreFieldVolume covers a volume written // before Throughput existed on VolumeMetadata: json.Unmarshal leaves the new -// int field at its zero value, and getVolumeByID must omit Throughput from +// int field at its zero value, and the projection must omit Throughput from // the response rather than surface a misleading 0 MiB/s. func TestGetVolumeByID_ThroughputOmitted_PreFieldVolume(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - // Simulate a pre-field volume by writing raw JSON with no Throughput key. - rawConfig := `{"VolumeConfig":{"VolumeMetadata":{"VolumeID":"vol-prefield","SizeGiB":5,"State":"available","VolumeType":"gp3","IOPS":3000}}}` - _, err := store.PutObject(context.Background(), &s3.PutObjectInput{ - Bucket: aws.String("test-bucket"), - Key: aws.String("vol-prefield/config.json"), - Body: strings.NewReader(rawConfig), + // Simulate a pre-field volume by writing a document with no throughput key. + putRawVolumeDocument(t, store, "vol-prefield", + `{"schema_version":1,"volume_id":"vol-prefield","capacity_gib":5,"state":"available","volume_type":"gp3","iops":3000}`) + + vol := volumeByID(t, svc, "vol-prefield") + + assert.Nil(t, vol.Throughput) +} + +// A volume with no ID is not representable: the document store refuses to key +// it, so an ID-less volume can never be written and later read back as one. +func TestPutVolume_EmptyVolumeIDRejected(t *testing.T) { + store := objectstore.NewMemoryObjectStore() + + err := ebsmetadata.NewStore(store, "test-bucket").PutVolume(context.Background(), ebsmetadata.Volume{ + CapacityGiB: 10, }) - require.NoError(t, err) - - result, err := svc.getVolumeByID(context.Background(), "vol-prefield") - require.NoError(t, err) - - assert.Nil(t, result.volume.Throughput) -} - -func TestGetVolumeByID_EmptyVolumeID(t *testing.T) { - store := objectstore.NewMemoryObjectStore() - svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - - meta := viperblock.VolumeMetadata{ - VolumeID: "", - SizeGiB: 10, - } - createVolumeInStoreWithMeta(t, store, "vol-emptyid", meta) - - _, err := svc.getVolumeByID(context.Background(), "vol-emptyid") require.Error(t, err) - assert.Contains(t, err.Error(), "volume ID is empty") -} - -func TestGetVolumeByID_ZeroSize(t *testing.T) { - store := objectstore.NewMemoryObjectStore() - svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - - meta := viperblock.VolumeMetadata{ - VolumeID: "vol-zerosize", - SizeGiB: 0, - } - createVolumeInStoreWithMeta(t, store, "vol-zerosize", meta) - - _, err := svc.getVolumeByID(context.Background(), "vol-zerosize") - require.Error(t, err) - assert.Contains(t, err.Error(), "zero size") } func TestGetVolumeByID_NotFound(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - _, err := svc.getVolumeByID(context.Background(), "vol-nonexistent") + _, err := svc.GetVolumeMetadata("vol-nonexistent") require.Error(t, err) assert.Equal(t, awserrors.ErrorInvalidVolumeNotFound, err.Error()) } @@ -889,8 +1144,8 @@ func TestDescribeVolumes_NilInput(t *testing.T) { svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) // Seed one volume so slow path has something to find - createVolumeInStoreWithMeta(t, store, "vol-nil1", viperblock.VolumeMetadata{ - VolumeID: "vol-nil1", SizeGiB: 10, State: "available", + createVolumeInStoreWithMeta(t, svc, store, "vol-nil1", ebsmetadata.Volume{ + VolumeID: "vol-nil1", CapacityGiB: 10, State: "available", }) output, err := svc.DescribeVolumes(context.Background(), nil, "") @@ -912,8 +1167,8 @@ func TestDescribeVolumes_SlowPath_MultipleVolumes(t *testing.T) { svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) for _, id := range []string{"vol-a", "vol-b", "vol-c"} { - createVolumeInStoreWithMeta(t, store, id, viperblock.VolumeMetadata{ - VolumeID: id, SizeGiB: 10, State: "available", + createVolumeInStoreWithMeta(t, svc, store, id, ebsmetadata.Volume{ + VolumeID: id, CapacityGiB: 10, State: "available", }) } @@ -927,8 +1182,8 @@ func TestDescribeVolumes_FastPath_SpecificIDs(t *testing.T) { svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) for _, id := range []string{"vol-x", "vol-y", "vol-z"} { - createVolumeInStoreWithMeta(t, store, id, viperblock.VolumeMetadata{ - VolumeID: id, SizeGiB: 10, State: "available", + createVolumeInStoreWithMeta(t, svc, store, id, ebsmetadata.Volume{ + VolumeID: id, CapacityGiB: 10, State: "available", }) } @@ -950,8 +1205,8 @@ func TestDescribeVolumes_FastPath_MixedExistingAndMissing(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - createVolumeInStoreWithMeta(t, store, "vol-exists", viperblock.VolumeMetadata{ - VolumeID: "vol-exists", SizeGiB: 10, State: "available", + createVolumeInStoreWithMeta(t, svc, store, "vol-exists", ebsmetadata.Volume{ + VolumeID: "vol-exists", CapacityGiB: 10, State: "available", }) // AWS returns InvalidVolume.NotFound when any requested ID is missing @@ -966,8 +1221,8 @@ func TestDescribeVolumes_FastPath_NilVolumeID(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - createVolumeInStoreWithMeta(t, store, "vol-ok", viperblock.VolumeMetadata{ - VolumeID: "vol-ok", SizeGiB: 10, State: "available", + createVolumeInStoreWithMeta(t, svc, store, "vol-ok", ebsmetadata.Volume{ + VolumeID: "vol-ok", CapacityGiB: 10, State: "available", }) output, err := svc.DescribeVolumes(context.Background(), &ec2.DescribeVolumesInput{ @@ -984,11 +1239,11 @@ func TestDescribeVolumes_AccountScoping_SlowPath(t *testing.T) { svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) // Create volumes for two different accounts - createVolumeInStoreWithMeta(t, store, "vol-acctA", viperblock.VolumeMetadata{ - VolumeID: "vol-acctA", SizeGiB: 10, State: "available", TenantID: "111111111111", + createVolumeInStoreWithMeta(t, svc, store, "vol-acctA", ebsmetadata.Volume{ + VolumeID: "vol-acctA", CapacityGiB: 10, State: "available", TenantID: "111111111111", }) - createVolumeInStoreWithMeta(t, store, "vol-acctB", viperblock.VolumeMetadata{ - VolumeID: "vol-acctB", SizeGiB: 10, State: "available", TenantID: "222222222222", + createVolumeInStoreWithMeta(t, svc, store, "vol-acctB", ebsmetadata.Volume{ + VolumeID: "vol-acctB", CapacityGiB: 10, State: "available", TenantID: "222222222222", }) // Account A sees only its own volume @@ -1016,11 +1271,11 @@ func TestDescribeVolumes_AccountScoping_FastPath(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - createVolumeInStoreWithMeta(t, store, "vol-mine", viperblock.VolumeMetadata{ - VolumeID: "vol-mine", SizeGiB: 10, State: "available", TenantID: "111111111111", + createVolumeInStoreWithMeta(t, svc, store, "vol-mine", ebsmetadata.Volume{ + VolumeID: "vol-mine", CapacityGiB: 10, State: "available", TenantID: "111111111111", }) - createVolumeInStoreWithMeta(t, store, "vol-other", viperblock.VolumeMetadata{ - VolumeID: "vol-other", SizeGiB: 10, State: "available", TenantID: "222222222222", + createVolumeInStoreWithMeta(t, svc, store, "vol-other", ebsmetadata.Volume{ + VolumeID: "vol-other", CapacityGiB: 10, State: "available", TenantID: "222222222222", }) // Requesting another account's volume by ID returns NotFound @@ -1044,8 +1299,8 @@ func TestDeleteVolume_AccountScoping(t *testing.T) { svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) svc.snapshotKV = setupTestVolumeKV(t) - createVolumeInStoreWithMeta(t, store, "vol-owned", viperblock.VolumeMetadata{ - VolumeID: "vol-owned", SizeGiB: 10, State: "available", TenantID: "111111111111", + createVolumeInStoreWithMeta(t, svc, store, "vol-owned", ebsmetadata.Volume{ + VolumeID: "vol-owned", CapacityGiB: 10, State: "available", TenantID: "111111111111", }) // Another account cannot delete @@ -1066,8 +1321,8 @@ func TestModifyVolume_AccountScoping(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - createVolumeInStoreWithMeta(t, store, "vol-modify", viperblock.VolumeMetadata{ - VolumeID: "vol-modify", SizeGiB: 10, State: "available", TenantID: "111111111111", + createVolumeInStoreWithMeta(t, svc, store, "vol-modify", ebsmetadata.Volume{ + VolumeID: "vol-modify", CapacityGiB: 10, State: "available", TenantID: "111111111111", }) // Another account cannot modify @@ -1091,11 +1346,11 @@ func TestDescribeVolumeStatus_AccountScoping(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - createVolumeInStoreWithMeta(t, store, "vol-statusA", viperblock.VolumeMetadata{ - VolumeID: "vol-statusA", SizeGiB: 10, State: "available", TenantID: "111111111111", + createVolumeInStoreWithMeta(t, svc, store, "vol-statusA", ebsmetadata.Volume{ + VolumeID: "vol-statusA", CapacityGiB: 10, State: "available", TenantID: "111111111111", }) - createVolumeInStoreWithMeta(t, store, "vol-statusB", viperblock.VolumeMetadata{ - VolumeID: "vol-statusB", SizeGiB: 10, State: "available", TenantID: "222222222222", + createVolumeInStoreWithMeta(t, svc, store, "vol-statusB", ebsmetadata.Volume{ + VolumeID: "vol-statusB", CapacityGiB: 10, State: "available", TenantID: "222222222222", }) // Slow path: Account A only sees its own volume status @@ -1164,8 +1419,8 @@ func TestModifyVolume_ShrinkRejected(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - createVolumeInStoreWithMeta(t, store, "vol-shrink", viperblock.VolumeMetadata{ - VolumeID: "vol-shrink", SizeGiB: 10, State: "available", + createVolumeInStoreWithMeta(t, svc, store, "vol-shrink", ebsmetadata.Volume{ + VolumeID: "vol-shrink", CapacityGiB: 10, State: "available", }) _, err := svc.ModifyVolume(context.Background(), &ec2.ModifyVolumeInput{ @@ -1180,8 +1435,8 @@ func TestModifyVolume_SameSizeRejected(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - createVolumeInStoreWithMeta(t, store, "vol-same", viperblock.VolumeMetadata{ - VolumeID: "vol-same", SizeGiB: 10, State: "available", + createVolumeInStoreWithMeta(t, svc, store, "vol-same", ebsmetadata.Volume{ + VolumeID: "vol-same", CapacityGiB: 10, State: "available", }) _, err := svc.ModifyVolume(context.Background(), &ec2.ModifyVolumeInput{ @@ -1196,9 +1451,9 @@ func TestModifyVolume_AttachedInUse(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - createVolumeInStoreWithMeta(t, store, "vol-inuse", viperblock.VolumeMetadata{ + createVolumeInStoreWithMeta(t, svc, store, "vol-inuse", ebsmetadata.Volume{ VolumeID: "vol-inuse", - SizeGiB: 10, + CapacityGiB: 10, State: "in-use", AttachedInstance: "i-12345", }) @@ -1215,12 +1470,12 @@ func TestModifyVolume_SuccessfulGrow(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - createVolumeInStoreWithMeta(t, store, "vol-grow", viperblock.VolumeMetadata{ - VolumeID: "vol-grow", - SizeGiB: 10, - State: "available", - VolumeType: "gp3", - IOPS: 3000, + createVolumeInStoreWithMeta(t, svc, store, "vol-grow", ebsmetadata.Volume{ + VolumeID: "vol-grow", + CapacityGiB: 10, + State: "available", + VolumeType: "gp3", + IOPS: 3000, }) output, err := svc.ModifyVolume(context.Background(), &ec2.ModifyVolumeInput{ @@ -1237,21 +1492,21 @@ func TestModifyVolume_SuccessfulGrow(t *testing.T) { assert.Equal(t, int64(100), *mod.Progress) // Verify persisted config - cfg, err := svc.GetVolumeConfig("vol-grow") + meta, err := svc.GetVolumeMetadata("vol-grow") require.NoError(t, err) - assert.Equal(t, uint64(20), cfg.VolumeMetadata.SizeGiB) + assert.Equal(t, uint64(20), meta.CapacityGiB) } func TestModifyVolume_ModifyTypeAndIOPS(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - createVolumeInStoreWithMeta(t, store, "vol-typemod", viperblock.VolumeMetadata{ - VolumeID: "vol-typemod", - SizeGiB: 10, - State: "available", - VolumeType: "gp3", - IOPS: 3000, + createVolumeInStoreWithMeta(t, svc, store, "vol-typemod", ebsmetadata.Volume{ + VolumeID: "vol-typemod", + CapacityGiB: 10, + State: "available", + VolumeType: "gp3", + IOPS: 3000, }) output, err := svc.ModifyVolume(context.Background(), &ec2.ModifyVolumeInput{ @@ -1274,9 +1529,9 @@ func TestModifyVolume_AvailableWithAttachment(t *testing.T) { svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) // Volume attached but state is "available" (stopped instance) -- allowed - createVolumeInStoreWithMeta(t, store, "vol-stopinst", viperblock.VolumeMetadata{ + createVolumeInStoreWithMeta(t, svc, store, "vol-stopinst", ebsmetadata.Volume{ VolumeID: "vol-stopinst", - SizeGiB: 10, + CapacityGiB: 10, State: "available", AttachedInstance: "i-stopped", }) @@ -1295,28 +1550,28 @@ func TestUpdateVolumeState_AttachVolume(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - createVolumeInStoreWithMeta(t, store, "vol-attach", viperblock.VolumeMetadata{ - VolumeID: "vol-attach", SizeGiB: 10, State: "available", + createVolumeInStoreWithMeta(t, svc, store, "vol-attach", ebsmetadata.Volume{ + VolumeID: "vol-attach", CapacityGiB: 10, State: "available", }) err := svc.UpdateVolumeState("vol-attach", "in-use", "i-abc123", "/dev/nbd0") require.NoError(t, err) - cfg, err := svc.GetVolumeConfig("vol-attach") + meta, err := svc.GetVolumeMetadata("vol-attach") require.NoError(t, err) - assert.Equal(t, "in-use", cfg.VolumeMetadata.State) - assert.Equal(t, "i-abc123", cfg.VolumeMetadata.AttachedInstance) - assert.Equal(t, "/dev/nbd0", cfg.VolumeMetadata.DeviceName) - assert.False(t, cfg.VolumeMetadata.AttachedAt.IsZero()) + assert.Equal(t, "in-use", meta.State) + assert.Equal(t, "i-abc123", meta.AttachedInstance) + assert.Equal(t, "/dev/nbd0", meta.DeviceName) + assert.False(t, meta.AttachedAt.IsZero()) } func TestUpdateVolumeState_DetachVolume(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - createVolumeInStoreWithMeta(t, store, "vol-detach2", viperblock.VolumeMetadata{ + createVolumeInStoreWithMeta(t, svc, store, "vol-detach2", ebsmetadata.Volume{ VolumeID: "vol-detach2", - SizeGiB: 10, + CapacityGiB: 10, State: "in-use", AttachedInstance: "i-xyz789", DeviceName: "/dev/nbd1", @@ -1325,11 +1580,11 @@ func TestUpdateVolumeState_DetachVolume(t *testing.T) { err := svc.UpdateVolumeState("vol-detach2", "available", "", "") require.NoError(t, err) - cfg, err := svc.GetVolumeConfig("vol-detach2") + meta, err := svc.GetVolumeMetadata("vol-detach2") require.NoError(t, err) - assert.Equal(t, "available", cfg.VolumeMetadata.State) - assert.Empty(t, cfg.VolumeMetadata.AttachedInstance) - assert.Empty(t, cfg.VolumeMetadata.DeviceName) + assert.Equal(t, "available", meta.State) + assert.Empty(t, meta.AttachedInstance) + assert.Empty(t, meta.DeviceName) } func TestUpdateVolumeState_VolumeNotFound(t *testing.T) { @@ -1337,46 +1592,31 @@ func TestUpdateVolumeState_VolumeNotFound(t *testing.T) { err := svc.UpdateVolumeState("vol-missing", "available", "", "") require.Error(t, err) - assert.Contains(t, err.Error(), "failed to get volume config") + assert.Contains(t, err.Error(), "failed to get volume metadata") } -func TestUpdateVolumeState_PreservesVBState(t *testing.T) { +func TestUpdateVolumeState_PreservesProviderConfig(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - meta := viperblock.VolumeMetadata{ - VolumeID: "vol-vbstate", SizeGiB: 10, State: "available", - } - createVolumeInStoreWithVBState(t, store, "vol-vbstate", meta, 4096, 5) + createVolumeInStoreWithMeta(t, svc, store, "vol-vbstate", ebsmetadata.Volume{ + VolumeID: "vol-vbstate", CapacityGiB: 10, State: "available", + }) + seedProviderConfig(t, store, "vol-vbstate") + before := getStoredConfig(t, store, "vol-vbstate") err := svc.UpdateVolumeState("vol-vbstate", "in-use", "i-preserve", "/dev/nbd0") require.NoError(t, err) - // config.json is owned by the live VB and must be left untouched: VBState - // fields survive and its embedded State is NOT rewritten (the control plane's - // attachment state lives in state.json now). - getResult, err := store.GetObject(context.Background(), &s3.GetObjectInput{ - Bucket: aws.String("test-bucket"), - Key: aws.String("vol-vbstate/config.json"), - }) + // config.json belongs to the live VB, which rewrites it from its own state. + // The attachment lives on the document instead, so the bytes must survive. + assert.Equal(t, string(before), string(getStoredConfig(t, store, "vol-vbstate")), + "UpdateVolumeState must not rewrite provider-owned config.json") + + readback, err := svc.GetVolumeMetadata("vol-vbstate") require.NoError(t, err) - - body, err := io.ReadAll(getResult.Body) - require.NoError(t, err) - - var state viperblock.VBState - require.NoError(t, json.Unmarshal(body, &state)) - - assert.Equal(t, uint32(4096), state.BlockSize) - assert.Equal(t, uint64(5), state.SeqNum) - assert.Equal(t, "available", state.VolumeConfig.VolumeMetadata.State, - "UpdateVolumeState must not rewrite config.json's embedded State") - - // The attachment state is read back through the state.json overlay. - cfg, err := svc.GetVolumeConfig("vol-vbstate") - require.NoError(t, err) - assert.Equal(t, "in-use", cfg.VolumeMetadata.State) - assert.Equal(t, "i-preserve", cfg.VolumeMetadata.AttachedInstance) + assert.Equal(t, "in-use", readback.State) + assert.Equal(t, "i-preserve", readback.AttachedInstance) } // --- Group 6: listAllVolumeIDs tests --- @@ -1385,15 +1625,14 @@ func TestListAllVolumeIDs_FiltersCorrectly(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - // Seed objects with various prefixes - for _, key := range []string{ - "vol-abc/config.json", - "vol-def/config.json", - "vol-abc-efi/config.json", - "vol-abc-cloudinit/config.json", - "ami-123/metadata.json", - "snap-456/metadata.json", - } { + for _, id := range []string{"vol-abc", "vol-def"} { + createVolumeInStoreWithMeta(t, svc, store, id, ebsmetadata.Volume{ + VolumeID: id, CapacityGiB: 10, State: "available", + }) + } + // Auxiliary volumes and other resources hold blocks but no document, which + // is exactly what keeps them out of the listing. + for _, key := range []string{"vol-abc-efi/config.json", "vol-abc-cloudinit/config.json", "ami-123/metadata.json", "snap-456/metadata.json"} { _, err := store.PutObject(context.Background(), &s3.PutObjectInput{ Bucket: aws.String("test-bucket"), Key: aws.String(key), @@ -1429,12 +1668,9 @@ func TestListAllVolumeIDs_NilPrefix(t *testing.T) { svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) // Seed a single volume to ensure the loop runs - _, err := store.PutObject(context.Background(), &s3.PutObjectInput{ - Bucket: aws.String("test-bucket"), - Key: aws.String("vol-only/config.json"), - Body: strings.NewReader("{}"), + createVolumeInStoreWithMeta(t, svc, store, "vol-only", ebsmetadata.Volume{ + VolumeID: "vol-only", CapacityGiB: 10, State: "available", }) - require.NoError(t, err) ids, err := svc.listAllVolumeIDs(context.Background()) require.NoError(t, err) @@ -1450,9 +1686,9 @@ func TestDeleteVolume_VolumeInUse(t *testing.T) { svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) svc.snapshotKV = kv - createVolumeInStoreWithMeta(t, store, "vol-busy", viperblock.VolumeMetadata{ + createVolumeInStoreWithMeta(t, svc, store, "vol-busy", ebsmetadata.Volume{ VolumeID: "vol-busy", - SizeGiB: 10, + CapacityGiB: 10, State: "in-use", AttachedInstance: "i-running", }) @@ -1473,9 +1709,9 @@ func TestDeleteVolume_VolumeAttachedButAvailable(t *testing.T) { // State != "available" triggers the check even without "in-use" // Actually: the code checks `State != "available" || AttachedInstance != ""` // So having AttachedInstance set while state is "available" still triggers VolumeInUse - createVolumeInStoreWithMeta(t, store, "vol-attached", viperblock.VolumeMetadata{ + createVolumeInStoreWithMeta(t, svc, store, "vol-attached", ebsmetadata.Volume{ VolumeID: "vol-attached", - SizeGiB: 10, + CapacityGiB: 10, State: "available", AttachedInstance: "i-stopped", }) @@ -1495,10 +1731,10 @@ func TestDeleteVolume_EmptyStateUnattachedDeletable(t *testing.T) { // Drift: a detach/terminate left State empty with no attachment. The volume // is not in use and must be deletable, not VolumeInUse. - createVolumeInStoreWithMeta(t, store, "vol-drift", viperblock.VolumeMetadata{ - VolumeID: "vol-drift", - SizeGiB: 10, - State: "", + createVolumeInStoreWithMeta(t, svc, store, "vol-drift", ebsmetadata.Volume{ + VolumeID: "vol-drift", + CapacityGiB: 10, + State: "", }) _, err := svc.DeleteVolume(context.Background(), &ec2.DeleteVolumeInput{ @@ -1522,9 +1758,9 @@ func TestDeleteVolumeOnTerminate_ClearsAttachmentThenDeletes(t *testing.T) { svc.snapshotKV = kv volumeID := "vol-stopped-root" - createVolumeInStoreWithMeta(t, store, volumeID, viperblock.VolumeMetadata{ + createVolumeInStoreWithMeta(t, svc, store, volumeID, ebsmetadata.Volume{ VolumeID: volumeID, - SizeGiB: 10, + CapacityGiB: 10, State: "available", AttachedInstance: "i-stopped", }) @@ -1532,7 +1768,7 @@ func TestDeleteVolumeOnTerminate_ClearsAttachmentThenDeletes(t *testing.T) { err := svc.DeleteVolumeOnTerminate(context.Background(), volumeID, "") require.NoError(t, err, "terminate implies detach: a stale attachment must not block the terminate delete") - _, err = svc.GetVolumeConfig(volumeID) + _, err = svc.GetVolumeMetadata(volumeID) require.Error(t, err, "the volume must actually be deleted, not merely detached") assert.Contains(t, err.Error(), awserrors.ErrorInvalidVolumeNotFound) } @@ -1549,9 +1785,9 @@ func TestDeleteVolumeOnTerminate_SurfacesDeleteFailure(t *testing.T) { svc.snapshotKV = kv volumeID := "vol-snapshotted-root" - createVolumeInStoreWithMeta(t, store, volumeID, viperblock.VolumeMetadata{ + createVolumeInStoreWithMeta(t, svc, store, volumeID, ebsmetadata.Volume{ VolumeID: volumeID, - SizeGiB: 10, + CapacityGiB: 10, State: "available", AttachedInstance: "i-stopped", }) @@ -1566,9 +1802,9 @@ func TestDeleteVolumeOnTerminate_SurfacesDeleteFailure(t *testing.T) { require.Error(t, err, "a DeleteVolume failure must be surfaced, not swallowed") assert.Contains(t, err.Error(), awserrors.ErrorVolumeInUse) - cfg, getErr := svc.GetVolumeConfig(volumeID) + meta, getErr := svc.GetVolumeMetadata(volumeID) require.NoError(t, getErr, "the volume must still exist after a failed delete") - assert.Empty(t, cfg.VolumeMetadata.AttachedInstance, "the attachment clear runs before delete and is not rolled back on a later delete failure") + assert.Empty(t, meta.AttachedInstance, "the attachment clear runs before delete and is not rolled back on a later delete failure") } func TestDescribeVolumes_EmptyStateDerivedFromAttachment(t *testing.T) { @@ -1598,60 +1834,10 @@ func TestUpdateVolumeState_EmptyUnattachedNormalizesToAvailable(t *testing.T) { // A detach writeback that clears the attachment without a state must not // strand the volume with an empty State. require.NoError(t, svc.UpdateVolumeState("vol-norm", "", "", "")) - cfg, err := svc.GetVolumeConfig("vol-norm") + meta, err := svc.GetVolumeMetadata("vol-norm") require.NoError(t, err) - assert.Equal(t, "available", cfg.VolumeMetadata.State) - assert.Empty(t, cfg.VolumeMetadata.AttachedInstance) -} - -func TestDeleteVolume_WithNATSNotification(t *testing.T) { - kv := setupTestVolumeKV(t) - store := objectstore.NewMemoryObjectStore() - - // Set up NATS server and connection for this test - opts := &server.Options{ - Host: "127.0.0.1", - Port: -1, - JetStream: true, - StoreDir: t.TempDir(), - NoLog: true, - NoSigs: true, - } - ns, err := server.NewServer(opts) - require.NoError(t, err) - go ns.Start() - require.True(t, ns.ReadyForConnections(5*time.Second)) - t.Cleanup(func() { ns.Shutdown() }) - - nc, err := nats.Connect(ns.ClientURL()) - require.NoError(t, err) - t.Cleanup(func() { nc.Close() }) - - svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - svc.snapshotKV = kv - svc.natsConn = nc - - volumeID := "vol-natsok" - createVolumeInStoreWithMeta(t, store, volumeID, viperblock.VolumeMetadata{ - VolumeID: volumeID, SizeGiB: 10, State: "available", - }) - - // Subscribe to ebs.delete and reply with success - sub, err := nc.Subscribe("ebs.delete", func(msg *nats.Msg) { - resp := types.EBSDeleteResponse{Volume: volumeID, Success: true} - data, _ := json.Marshal(resp) - msg.Respond(data) - }) - require.NoError(t, err) - defer sub.Unsubscribe() - - _, err = svc.DeleteVolume(context.Background(), &ec2.DeleteVolumeInput{ - VolumeId: aws.String(volumeID), - }, "") - require.NoError(t, err) - - // Verify all objects deleted - assert.Equal(t, 0, store.Count()) + assert.Equal(t, "available", meta.State) + assert.Empty(t, meta.AttachedInstance) } func TestDescribeVolumeStatus_SlowPath_WithVolumes(t *testing.T) { @@ -1659,9 +1845,9 @@ func TestDescribeVolumeStatus_SlowPath_WithVolumes(t *testing.T) { svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) for _, id := range []string{"vol-s1", "vol-s2"} { - createVolumeInStoreWithMeta(t, store, id, viperblock.VolumeMetadata{ + createVolumeInStoreWithMeta(t, svc, store, id, ebsmetadata.Volume{ VolumeID: id, - SizeGiB: 10, + CapacityGiB: 10, State: "available", AvailabilityZone: "ap-southeast-2a", }) @@ -1682,9 +1868,9 @@ func TestDescribeVolumeStatus_FastPath_WithVolumes(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - createVolumeInStoreWithMeta(t, store, "vol-status1", viperblock.VolumeMetadata{ + createVolumeInStoreWithMeta(t, svc, store, "vol-status1", ebsmetadata.Volume{ VolumeID: "vol-status1", - SizeGiB: 10, + CapacityGiB: 10, State: "in-use", AvailabilityZone: "ap-southeast-2a", }) @@ -1698,138 +1884,17 @@ func TestDescribeVolumeStatus_FastPath_WithVolumes(t *testing.T) { assert.Equal(t, "ok", *output.VolumeStatuses[0].VolumeStatus.Status) } -func TestDescribeVolumes_SlowPath_SkipsBrokenConfig(t *testing.T) { - store := objectstore.NewMemoryObjectStore() - svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - - // Good volume - createVolumeInStoreWithMeta(t, store, "vol-good", viperblock.VolumeMetadata{ - VolumeID: "vol-good", SizeGiB: 10, State: "available", - }) - // Bad volume: zero size triggers error in getVolumeByID - createVolumeInStoreWithMeta(t, store, "vol-bad", viperblock.VolumeMetadata{ - VolumeID: "vol-bad", SizeGiB: 0, - }) - - output, err := svc.DescribeVolumes(context.Background(), &ec2.DescribeVolumesInput{}, "") - require.NoError(t, err) - // Only the good volume should be returned - assert.Len(t, output.Volumes, 1) - assert.Equal(t, "vol-good", *output.Volumes[0].VolumeId) -} - -func TestDescribeVolumeStatus_SlowPath_SkipsBrokenConfig(t *testing.T) { - store := objectstore.NewMemoryObjectStore() - svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - - createVolumeInStoreWithMeta(t, store, "vol-ok", viperblock.VolumeMetadata{ - VolumeID: "vol-ok", SizeGiB: 10, State: "available", AvailabilityZone: "ap-southeast-2a", - }) - createVolumeInStoreWithMeta(t, store, "vol-broken", viperblock.VolumeMetadata{ - VolumeID: "vol-broken", SizeGiB: 0, - }) - - output, err := svc.DescribeVolumeStatus(context.Background(), nil, "") - require.NoError(t, err) - assert.Len(t, output.VolumeStatuses, 1) -} - -func TestDeleteVolume_NATSErrorResponse(t *testing.T) { - kv := setupTestVolumeKV(t) - store := objectstore.NewMemoryObjectStore() - - opts := &server.Options{ - Host: "127.0.0.1", - Port: -1, - JetStream: true, - StoreDir: t.TempDir(), - NoLog: true, - NoSigs: true, - } - ns, err := server.NewServer(opts) - require.NoError(t, err) - go ns.Start() - require.True(t, ns.ReadyForConnections(5*time.Second)) - t.Cleanup(func() { ns.Shutdown() }) - - nc, err := nats.Connect(ns.ClientURL()) - require.NoError(t, err) - t.Cleanup(func() { nc.Close() }) - - svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - svc.snapshotKV = kv - svc.natsConn = nc - - volumeID := "vol-natserr" - createVolumeInStoreWithMeta(t, store, volumeID, viperblock.VolumeMetadata{ - VolumeID: volumeID, SizeGiB: 10, State: "available", - }) - - // Subscribe and respond with an error - sub, err := nc.Subscribe("ebs.delete", func(msg *nats.Msg) { - resp := types.EBSDeleteResponse{Volume: volumeID, Error: "volume still mounted"} - data, _ := json.Marshal(resp) - msg.Respond(data) - }) - require.NoError(t, err) - defer sub.Unsubscribe() - - _, err = svc.DeleteVolume(context.Background(), &ec2.DeleteVolumeInput{ - VolumeId: aws.String(volumeID), - }, "") - require.Error(t, err) - assert.Equal(t, awserrors.ErrorServerInternal, err.Error()) -} - -func TestDeleteVolume_NATSTimeout(t *testing.T) { - kv := setupTestVolumeKV(t) - store := objectstore.NewMemoryObjectStore() - - opts := &server.Options{ - Host: "127.0.0.1", - Port: -1, - JetStream: true, - StoreDir: t.TempDir(), - NoLog: true, - NoSigs: true, - } - ns, err := server.NewServer(opts) - require.NoError(t, err) - go ns.Start() - require.True(t, ns.ReadyForConnections(5*time.Second)) - t.Cleanup(func() { ns.Shutdown() }) - - nc, err := nats.Connect(ns.ClientURL()) - require.NoError(t, err) - t.Cleanup(func() { nc.Close() }) - - svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - svc.snapshotKV = kv - svc.natsConn = nc - - volumeID := "vol-natstimeout" - createVolumeInStoreWithMeta(t, store, volumeID, viperblock.VolumeMetadata{ - VolumeID: volumeID, SizeGiB: 10, State: "available", - }) - - // No subscriber → NATS request will timeout, but delete proceeds (best-effort) - _, err = svc.DeleteVolume(context.Background(), &ec2.DeleteVolumeInput{ - VolumeId: aws.String(volumeID), - }, "") - require.NoError(t, err) -} - // --- DescribeVolumes filter tests --- func TestDescribeVolumes_FilterByStatus(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - createVolumeInStoreWithMeta(t, store, "vol-avail", viperblock.VolumeMetadata{ - VolumeID: "vol-avail", SizeGiB: 10, State: "available", TenantID: "acct1", + createVolumeInStoreWithMeta(t, svc, store, "vol-avail", ebsmetadata.Volume{ + VolumeID: "vol-avail", CapacityGiB: 10, State: "available", TenantID: "acct1", }) - createVolumeInStoreWithMeta(t, store, "vol-inuse", viperblock.VolumeMetadata{ - VolumeID: "vol-inuse", SizeGiB: 20, State: "in-use", TenantID: "acct1", + createVolumeInStoreWithMeta(t, svc, store, "vol-inuse", ebsmetadata.Volume{ + VolumeID: "vol-inuse", CapacityGiB: 20, State: "in-use", TenantID: "acct1", }) out, err := svc.DescribeVolumes(context.Background(), &ec2.DescribeVolumesInput{ @@ -1846,11 +1911,11 @@ func TestDescribeVolumes_FilterByVolumeType(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - createVolumeInStoreWithMeta(t, store, "vol-gp3", viperblock.VolumeMetadata{ - VolumeID: "vol-gp3", SizeGiB: 10, State: "available", VolumeType: "gp3", TenantID: "acct1", + createVolumeInStoreWithMeta(t, svc, store, "vol-gp3", ebsmetadata.Volume{ + VolumeID: "vol-gp3", CapacityGiB: 10, State: "available", VolumeType: "gp3", TenantID: "acct1", }) - createVolumeInStoreWithMeta(t, store, "vol-io1", viperblock.VolumeMetadata{ - VolumeID: "vol-io1", SizeGiB: 10, State: "available", VolumeType: "io1", TenantID: "acct1", + createVolumeInStoreWithMeta(t, svc, store, "vol-io1", ebsmetadata.Volume{ + VolumeID: "vol-io1", CapacityGiB: 10, State: "available", VolumeType: "io1", TenantID: "acct1", }) out, err := svc.DescribeVolumes(context.Background(), &ec2.DescribeVolumesInput{ @@ -1867,11 +1932,11 @@ func TestDescribeVolumes_FilterBySize(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - createVolumeInStoreWithMeta(t, store, "vol-small", viperblock.VolumeMetadata{ - VolumeID: "vol-small", SizeGiB: 10, State: "available", TenantID: "acct1", + createVolumeInStoreWithMeta(t, svc, store, "vol-small", ebsmetadata.Volume{ + VolumeID: "vol-small", CapacityGiB: 10, State: "available", TenantID: "acct1", }) - createVolumeInStoreWithMeta(t, store, "vol-big", viperblock.VolumeMetadata{ - VolumeID: "vol-big", SizeGiB: 100, State: "available", TenantID: "acct1", + createVolumeInStoreWithMeta(t, svc, store, "vol-big", ebsmetadata.Volume{ + VolumeID: "vol-big", CapacityGiB: 100, State: "available", TenantID: "acct1", }) out, err := svc.DescribeVolumes(context.Background(), &ec2.DescribeVolumesInput{ @@ -1888,12 +1953,12 @@ func TestDescribeVolumes_FilterByAttachmentInstanceId(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - createVolumeInStoreWithMeta(t, store, "vol-att", viperblock.VolumeMetadata{ - VolumeID: "vol-att", SizeGiB: 10, State: "in-use", + createVolumeInStoreWithMeta(t, svc, store, "vol-att", ebsmetadata.Volume{ + VolumeID: "vol-att", CapacityGiB: 10, State: "in-use", AttachedInstance: "i-12345", DeviceName: "/dev/nbd0", TenantID: "acct1", }) - createVolumeInStoreWithMeta(t, store, "vol-free", viperblock.VolumeMetadata{ - VolumeID: "vol-free", SizeGiB: 10, State: "available", TenantID: "acct1", + createVolumeInStoreWithMeta(t, svc, store, "vol-free", ebsmetadata.Volume{ + VolumeID: "vol-free", CapacityGiB: 10, State: "available", TenantID: "acct1", }) out, err := svc.DescribeVolumes(context.Background(), &ec2.DescribeVolumesInput{ @@ -1910,12 +1975,12 @@ func TestDescribeVolumes_FilterByAttachmentDevice(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - createVolumeInStoreWithMeta(t, store, "vol-nbd0", viperblock.VolumeMetadata{ - VolumeID: "vol-nbd0", SizeGiB: 10, State: "in-use", + createVolumeInStoreWithMeta(t, svc, store, "vol-nbd0", ebsmetadata.Volume{ + VolumeID: "vol-nbd0", CapacityGiB: 10, State: "in-use", AttachedInstance: "i-12345", DeviceName: "/dev/nbd0", TenantID: "acct1", }) - createVolumeInStoreWithMeta(t, store, "vol-nbd1", viperblock.VolumeMetadata{ - VolumeID: "vol-nbd1", SizeGiB: 10, State: "in-use", + createVolumeInStoreWithMeta(t, svc, store, "vol-nbd1", ebsmetadata.Volume{ + VolumeID: "vol-nbd1", CapacityGiB: 10, State: "in-use", AttachedInstance: "i-12345", DeviceName: "/dev/nbd1", TenantID: "acct1", }) @@ -1933,12 +1998,12 @@ func TestDescribeVolumes_FilterByAZ(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - createVolumeInStoreWithMeta(t, store, "vol-az1", viperblock.VolumeMetadata{ - VolumeID: "vol-az1", SizeGiB: 10, State: "available", + createVolumeInStoreWithMeta(t, svc, store, "vol-az1", ebsmetadata.Volume{ + VolumeID: "vol-az1", CapacityGiB: 10, State: "available", AvailabilityZone: "ap-southeast-2a", TenantID: "acct1", }) - createVolumeInStoreWithMeta(t, store, "vol-az2", viperblock.VolumeMetadata{ - VolumeID: "vol-az2", SizeGiB: 10, State: "available", + createVolumeInStoreWithMeta(t, svc, store, "vol-az2", ebsmetadata.Volume{ + VolumeID: "vol-az2", CapacityGiB: 10, State: "available", AvailabilityZone: "ap-southeast-2b", TenantID: "acct1", }) @@ -1956,15 +2021,15 @@ func TestDescribeVolumes_FilterMultipleValues_OR(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - createVolumeInStoreWithMeta(t, store, "vol-avail", viperblock.VolumeMetadata{ - VolumeID: "vol-avail", SizeGiB: 10, State: "available", TenantID: "acct1", + createVolumeInStoreWithMeta(t, svc, store, "vol-avail", ebsmetadata.Volume{ + VolumeID: "vol-avail", CapacityGiB: 10, State: "available", TenantID: "acct1", }) - createVolumeInStoreWithMeta(t, store, "vol-inuse", viperblock.VolumeMetadata{ - VolumeID: "vol-inuse", SizeGiB: 10, State: "in-use", + createVolumeInStoreWithMeta(t, svc, store, "vol-inuse", ebsmetadata.Volume{ + VolumeID: "vol-inuse", CapacityGiB: 10, State: "in-use", AttachedInstance: "i-1", DeviceName: "/dev/nbd0", TenantID: "acct1", }) - createVolumeInStoreWithMeta(t, store, "vol-del", viperblock.VolumeMetadata{ - VolumeID: "vol-del", SizeGiB: 10, State: "deleted", TenantID: "acct1", + createVolumeInStoreWithMeta(t, svc, store, "vol-del", ebsmetadata.Volume{ + VolumeID: "vol-del", CapacityGiB: 10, State: "deleted", TenantID: "acct1", }) out, err := svc.DescribeVolumes(context.Background(), &ec2.DescribeVolumesInput{ @@ -1980,12 +2045,12 @@ func TestDescribeVolumes_FilterMultipleFilters_AND(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - createVolumeInStoreWithMeta(t, store, "vol-match", viperblock.VolumeMetadata{ - VolumeID: "vol-match", SizeGiB: 10, State: "available", + createVolumeInStoreWithMeta(t, svc, store, "vol-match", ebsmetadata.Volume{ + VolumeID: "vol-match", CapacityGiB: 10, State: "available", VolumeType: "gp3", TenantID: "acct1", }) - createVolumeInStoreWithMeta(t, store, "vol-nomatch", viperblock.VolumeMetadata{ - VolumeID: "vol-nomatch", SizeGiB: 10, State: "in-use", + createVolumeInStoreWithMeta(t, svc, store, "vol-nomatch", ebsmetadata.Volume{ + VolumeID: "vol-nomatch", CapacityGiB: 10, State: "in-use", VolumeType: "gp3", AttachedInstance: "i-1", DeviceName: "/dev/nbd0", TenantID: "acct1", }) @@ -2017,8 +2082,8 @@ func TestDescribeVolumes_FilterNoResults(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - createVolumeInStoreWithMeta(t, store, "vol-one", viperblock.VolumeMetadata{ - VolumeID: "vol-one", SizeGiB: 10, State: "available", TenantID: "acct1", + createVolumeInStoreWithMeta(t, svc, store, "vol-one", ebsmetadata.Volume{ + VolumeID: "vol-one", CapacityGiB: 10, State: "available", TenantID: "acct1", }) out, err := svc.DescribeVolumes(context.Background(), &ec2.DescribeVolumesInput{ @@ -2034,11 +2099,11 @@ func TestDescribeVolumes_FilterNoFilters(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - createVolumeInStoreWithMeta(t, store, "vol-a", viperblock.VolumeMetadata{ - VolumeID: "vol-a", SizeGiB: 10, State: "available", TenantID: "acct1", + createVolumeInStoreWithMeta(t, svc, store, "vol-a", ebsmetadata.Volume{ + VolumeID: "vol-a", CapacityGiB: 10, State: "available", TenantID: "acct1", }) - createVolumeInStoreWithMeta(t, store, "vol-b", viperblock.VolumeMetadata{ - VolumeID: "vol-b", SizeGiB: 20, State: "available", TenantID: "acct1", + createVolumeInStoreWithMeta(t, svc, store, "vol-b", ebsmetadata.Volume{ + VolumeID: "vol-b", CapacityGiB: 20, State: "available", TenantID: "acct1", }) out, err := svc.DescribeVolumes(context.Background(), &ec2.DescribeVolumesInput{}, "acct1") @@ -2050,12 +2115,12 @@ func TestDescribeVolumes_FilterWildcard(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - createVolumeInStoreWithMeta(t, store, "vol-az1", viperblock.VolumeMetadata{ - VolumeID: "vol-az1", SizeGiB: 10, State: "available", + createVolumeInStoreWithMeta(t, svc, store, "vol-az1", ebsmetadata.Volume{ + VolumeID: "vol-az1", CapacityGiB: 10, State: "available", AvailabilityZone: "ap-southeast-2a", TenantID: "acct1", }) - createVolumeInStoreWithMeta(t, store, "vol-az2", viperblock.VolumeMetadata{ - VolumeID: "vol-az2", SizeGiB: 10, State: "available", + createVolumeInStoreWithMeta(t, svc, store, "vol-az2", ebsmetadata.Volume{ + VolumeID: "vol-az2", CapacityGiB: 10, State: "available", AvailabilityZone: "us-east-1a", TenantID: "acct1", }) @@ -2073,12 +2138,12 @@ func TestDescribeVolumes_FilterByTag(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - createVolumeInStoreWithMeta(t, store, "vol-tagged", viperblock.VolumeMetadata{ - VolumeID: "vol-tagged", SizeGiB: 10, State: "available", TenantID: "acct1", + createVolumeInStoreWithMeta(t, svc, store, "vol-tagged", ebsmetadata.Volume{ + VolumeID: "vol-tagged", CapacityGiB: 10, State: "available", TenantID: "acct1", Tags: map[string]string{"Environment": "prod"}, }) - createVolumeInStoreWithMeta(t, store, "vol-untagged", viperblock.VolumeMetadata{ - VolumeID: "vol-untagged", SizeGiB: 10, State: "available", TenantID: "acct1", + createVolumeInStoreWithMeta(t, svc, store, "vol-untagged", ebsmetadata.Volume{ + VolumeID: "vol-untagged", CapacityGiB: 10, State: "available", TenantID: "acct1", }) out, err := svc.DescribeVolumes(context.Background(), &ec2.DescribeVolumesInput{ @@ -2095,11 +2160,11 @@ func TestDescribeVolumes_FilterWithVolumeIds(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - createVolumeInStoreWithMeta(t, store, "vol-a", viperblock.VolumeMetadata{ - VolumeID: "vol-a", SizeGiB: 10, State: "available", TenantID: "acct1", + createVolumeInStoreWithMeta(t, svc, store, "vol-a", ebsmetadata.Volume{ + VolumeID: "vol-a", CapacityGiB: 10, State: "available", TenantID: "acct1", }) - createVolumeInStoreWithMeta(t, store, "vol-b", viperblock.VolumeMetadata{ - VolumeID: "vol-b", SizeGiB: 10, State: "in-use", + createVolumeInStoreWithMeta(t, svc, store, "vol-b", ebsmetadata.Volume{ + VolumeID: "vol-b", CapacityGiB: 10, State: "in-use", AttachedInstance: "i-1", DeviceName: "/dev/nbd0", TenantID: "acct1", }) @@ -2119,11 +2184,11 @@ func TestDescribeVolumeStatus_FilterByVolumeId(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - createVolumeInStoreWithMeta(t, store, "vol-vs1", viperblock.VolumeMetadata{ - VolumeID: "vol-vs1", SizeGiB: 10, State: "available", AvailabilityZone: "ap-southeast-2a", + createVolumeInStoreWithMeta(t, svc, store, "vol-vs1", ebsmetadata.Volume{ + VolumeID: "vol-vs1", CapacityGiB: 10, State: "available", AvailabilityZone: "ap-southeast-2a", }) - createVolumeInStoreWithMeta(t, store, "vol-vs2", viperblock.VolumeMetadata{ - VolumeID: "vol-vs2", SizeGiB: 20, State: "available", AvailabilityZone: "ap-southeast-2a", + createVolumeInStoreWithMeta(t, svc, store, "vol-vs2", ebsmetadata.Volume{ + VolumeID: "vol-vs2", CapacityGiB: 20, State: "available", AvailabilityZone: "ap-southeast-2a", }) out, err := svc.DescribeVolumeStatus(context.Background(), &ec2.DescribeVolumeStatusInput{ @@ -2140,8 +2205,8 @@ func TestDescribeVolumeStatus_FilterByStatus(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - createVolumeInStoreWithMeta(t, store, "vol-vss1", viperblock.VolumeMetadata{ - VolumeID: "vol-vss1", SizeGiB: 10, State: "available", AvailabilityZone: "ap-southeast-2a", + createVolumeInStoreWithMeta(t, svc, store, "vol-vss1", ebsmetadata.Volume{ + VolumeID: "vol-vss1", CapacityGiB: 10, State: "available", AvailabilityZone: "ap-southeast-2a", }) // Status is always "ok" in Spinifex @@ -2166,8 +2231,8 @@ func TestDescribeVolumeStatus_FilterByAZ(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - createVolumeInStoreWithMeta(t, store, "vol-vsaz", viperblock.VolumeMetadata{ - VolumeID: "vol-vsaz", SizeGiB: 10, State: "available", AvailabilityZone: "ap-southeast-2a", + createVolumeInStoreWithMeta(t, svc, store, "vol-vsaz", ebsmetadata.Volume{ + VolumeID: "vol-vsaz", CapacityGiB: 10, State: "available", AvailabilityZone: "ap-southeast-2a", }) out, err := svc.DescribeVolumeStatus(context.Background(), &ec2.DescribeVolumeStatusInput{ @@ -2191,14 +2256,14 @@ func TestDescribeVolumeStatus_FilterMultipleValues_OR(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - createVolumeInStoreWithMeta(t, store, "vol-vsor1", viperblock.VolumeMetadata{ - VolumeID: "vol-vsor1", SizeGiB: 10, State: "available", AvailabilityZone: "ap-southeast-2a", + createVolumeInStoreWithMeta(t, svc, store, "vol-vsor1", ebsmetadata.Volume{ + VolumeID: "vol-vsor1", CapacityGiB: 10, State: "available", AvailabilityZone: "ap-southeast-2a", }) - createVolumeInStoreWithMeta(t, store, "vol-vsor2", viperblock.VolumeMetadata{ - VolumeID: "vol-vsor2", SizeGiB: 20, State: "available", AvailabilityZone: "ap-southeast-2a", + createVolumeInStoreWithMeta(t, svc, store, "vol-vsor2", ebsmetadata.Volume{ + VolumeID: "vol-vsor2", CapacityGiB: 20, State: "available", AvailabilityZone: "ap-southeast-2a", }) - createVolumeInStoreWithMeta(t, store, "vol-vsor3", viperblock.VolumeMetadata{ - VolumeID: "vol-vsor3", SizeGiB: 30, State: "available", AvailabilityZone: "ap-southeast-2a", + createVolumeInStoreWithMeta(t, svc, store, "vol-vsor3", ebsmetadata.Volume{ + VolumeID: "vol-vsor3", CapacityGiB: 30, State: "available", AvailabilityZone: "ap-southeast-2a", }) out, err := svc.DescribeVolumeStatus(context.Background(), &ec2.DescribeVolumeStatusInput{ @@ -2214,8 +2279,8 @@ func TestDescribeVolumeStatus_FilterMultipleFilters_AND(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - createVolumeInStoreWithMeta(t, store, "vol-vsand", viperblock.VolumeMetadata{ - VolumeID: "vol-vsand", SizeGiB: 10, State: "available", AvailabilityZone: "ap-southeast-2a", + createVolumeInStoreWithMeta(t, svc, store, "vol-vsand", ebsmetadata.Volume{ + VolumeID: "vol-vsand", CapacityGiB: 10, State: "available", AvailabilityZone: "ap-southeast-2a", }) // Both match @@ -2255,8 +2320,8 @@ func TestDescribeVolumeStatus_FilterWildcard(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - createVolumeInStoreWithMeta(t, store, "vol-vswild", viperblock.VolumeMetadata{ - VolumeID: "vol-vswild", SizeGiB: 10, State: "available", AvailabilityZone: "ap-southeast-2a", + createVolumeInStoreWithMeta(t, svc, store, "vol-vswild", ebsmetadata.Volume{ + VolumeID: "vol-vswild", CapacityGiB: 10, State: "available", AvailabilityZone: "ap-southeast-2a", }) out, err := svc.DescribeVolumeStatus(context.Background(), &ec2.DescribeVolumeStatusInput{ @@ -2272,8 +2337,8 @@ func TestDescribeVolumeStatus_FilterNoResults(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - createVolumeInStoreWithMeta(t, store, "vol-vsnr", viperblock.VolumeMetadata{ - VolumeID: "vol-vsnr", SizeGiB: 10, State: "available", AvailabilityZone: "ap-southeast-2a", + createVolumeInStoreWithMeta(t, svc, store, "vol-vsnr", ebsmetadata.Volume{ + VolumeID: "vol-vsnr", CapacityGiB: 10, State: "available", AvailabilityZone: "ap-southeast-2a", }) out, err := svc.DescribeVolumeStatus(context.Background(), &ec2.DescribeVolumeStatusInput{ @@ -2289,11 +2354,11 @@ func TestDescribeVolumeStatus_FilterWithVolumeIds(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - createVolumeInStoreWithMeta(t, store, "vol-vsf1", viperblock.VolumeMetadata{ - VolumeID: "vol-vsf1", SizeGiB: 10, State: "available", AvailabilityZone: "ap-southeast-2a", + createVolumeInStoreWithMeta(t, svc, store, "vol-vsf1", ebsmetadata.Volume{ + VolumeID: "vol-vsf1", CapacityGiB: 10, State: "available", AvailabilityZone: "ap-southeast-2a", }) - createVolumeInStoreWithMeta(t, store, "vol-vsf2", viperblock.VolumeMetadata{ - VolumeID: "vol-vsf2", SizeGiB: 20, State: "available", AvailabilityZone: "us-east-1a", + createVolumeInStoreWithMeta(t, svc, store, "vol-vsf2", ebsmetadata.Volume{ + VolumeID: "vol-vsf2", CapacityGiB: 20, State: "available", AvailabilityZone: "us-east-1a", }) // Fast path with VolumeIds + filter: should apply filter to requested IDs @@ -2311,14 +2376,14 @@ func TestDescribeVolumeStatus_FilterWithVolumeIds(t *testing.T) { // --- Group: DescribeVolumesModifications tests --- // TestDescribeVolumesModifications_RoundTrip proves ModifyVolume persists the -// modification record into cfg.Modification AND that DescribeVolumesModifications +// modification record into meta.Modification AND that DescribeVolumesModifications // reads it back. Guards the load-bearing wiring between the two APIs. func TestDescribeVolumesModifications_RoundTrip(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - createVolumeInStoreWithMeta(t, store, "vol-rt", viperblock.VolumeMetadata{ - VolumeID: "vol-rt", SizeGiB: 10, State: "available", + createVolumeInStoreWithMeta(t, svc, store, "vol-rt", ebsmetadata.Volume{ + VolumeID: "vol-rt", CapacityGiB: 10, State: "available", VolumeType: "gp3", IOPS: 3000, TenantID: "111111111111", }) @@ -2329,11 +2394,11 @@ func TestDescribeVolumesModifications_RoundTrip(t *testing.T) { require.NoError(t, err) // Confirm Modification was persisted on cfg. - cfg, err := svc.GetVolumeConfig("vol-rt") + meta, err := svc.GetVolumeMetadata("vol-rt") require.NoError(t, err) - require.NotNil(t, cfg.Modification) - assert.Equal(t, int64(10), cfg.Modification.OriginalSize) - assert.Equal(t, int64(20), cfg.Modification.TargetSize) + require.NotNil(t, meta.Modification) + assert.Equal(t, int64(10), meta.Modification.OriginalSize) + assert.Equal(t, int64(20), meta.Modification.TargetSize) out, err := svc.DescribeVolumesModifications(context.Background(), &ec2.DescribeVolumesModificationsInput{ VolumeIds: []*string{aws.String("vol-rt")}, @@ -2355,8 +2420,8 @@ func TestDescribeVolumesModifications_OverwriteSemantics(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - createVolumeInStoreWithMeta(t, store, "vol-ow", viperblock.VolumeMetadata{ - VolumeID: "vol-ow", SizeGiB: 10, State: "available", + createVolumeInStoreWithMeta(t, svc, store, "vol-ow", ebsmetadata.Volume{ + VolumeID: "vol-ow", CapacityGiB: 10, State: "available", VolumeType: "gp3", IOPS: 3000, TenantID: "111111111111", }) @@ -2387,8 +2452,8 @@ func TestDescribeVolumesModifications_CrossTenantFastPath(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - createVolumeInStoreWithMeta(t, store, "vol-tenantA", viperblock.VolumeMetadata{ - VolumeID: "vol-tenantA", SizeGiB: 10, State: "available", + createVolumeInStoreWithMeta(t, svc, store, "vol-tenantA", ebsmetadata.Volume{ + VolumeID: "vol-tenantA", CapacityGiB: 10, State: "available", VolumeType: "gp3", IOPS: 3000, TenantID: "111111111111", }) _, err := svc.ModifyVolume(context.Background(), &ec2.ModifyVolumeInput{ @@ -2410,16 +2475,16 @@ func TestDescribeVolumesModifications_SlowPathScoping(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - createVolumeInStoreWithMeta(t, store, "vol-modA", viperblock.VolumeMetadata{ - VolumeID: "vol-modA", SizeGiB: 10, State: "available", + createVolumeInStoreWithMeta(t, svc, store, "vol-modA", ebsmetadata.Volume{ + VolumeID: "vol-modA", CapacityGiB: 10, State: "available", VolumeType: "gp3", IOPS: 3000, TenantID: "111111111111", }) - createVolumeInStoreWithMeta(t, store, "vol-unmodA", viperblock.VolumeMetadata{ - VolumeID: "vol-unmodA", SizeGiB: 10, State: "available", + createVolumeInStoreWithMeta(t, svc, store, "vol-unmodA", ebsmetadata.Volume{ + VolumeID: "vol-unmodA", CapacityGiB: 10, State: "available", VolumeType: "gp3", IOPS: 3000, TenantID: "111111111111", }) - createVolumeInStoreWithMeta(t, store, "vol-modB", viperblock.VolumeMetadata{ - VolumeID: "vol-modB", SizeGiB: 10, State: "available", + createVolumeInStoreWithMeta(t, svc, store, "vol-modB", ebsmetadata.Volume{ + VolumeID: "vol-modB", CapacityGiB: 10, State: "available", VolumeType: "gp3", IOPS: 3000, TenantID: "222222222222", }) @@ -2445,12 +2510,12 @@ func TestDescribeVolumesModifications_FilterMatching(t *testing.T) { store := objectstore.NewMemoryObjectStore() svc := newTestVolumeServiceWithStore("ap-southeast-2a", store) - createVolumeInStoreWithMeta(t, store, "vol-fa", viperblock.VolumeMetadata{ - VolumeID: "vol-fa", SizeGiB: 10, State: "available", + createVolumeInStoreWithMeta(t, svc, store, "vol-fa", ebsmetadata.Volume{ + VolumeID: "vol-fa", CapacityGiB: 10, State: "available", VolumeType: "gp3", IOPS: 3000, TenantID: "111111111111", }) - createVolumeInStoreWithMeta(t, store, "vol-fb", viperblock.VolumeMetadata{ - VolumeID: "vol-fb", SizeGiB: 50, State: "available", + createVolumeInStoreWithMeta(t, svc, store, "vol-fb", ebsmetadata.Volume{ + VolumeID: "vol-fb", CapacityGiB: 50, State: "available", VolumeType: "gp3", IOPS: 3000, TenantID: "111111111111", }) _, err := svc.ModifyVolume(context.Background(), &ec2.ModifyVolumeInput{ diff --git a/spinifex/handlers/ec2/volume/testhelpers_test.go b/spinifex/handlers/ec2/volume/testhelpers_test.go new file mode 100644 index 000000000..143f269fb --- /dev/null +++ b/spinifex/handlers/ec2/volume/testhelpers_test.go @@ -0,0 +1,86 @@ +package handlers_ec2_volume + +import ( + "context" + "io" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/mulgadc/spinifex/spinifex/ebsmetadata" + "github.com/mulgadc/spinifex/spinifex/objectstore" + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" + "github.com/stretchr/testify/require" +) + +// startTestNATS spins up an in-process NATS server and returns a connected client. +func startTestNATS(t *testing.T) *nats.Conn { + t.Helper() + opts := &server.Options{ + Host: "127.0.0.1", + Port: -1, + JetStream: true, + StoreDir: t.TempDir(), + NoLog: true, + NoSigs: true, + } + ns, err := server.NewServer(opts) + require.NoError(t, err) + go ns.Start() + require.True(t, ns.ReadyForConnections(5*time.Second)) + t.Cleanup(func() { ns.Shutdown() }) + + nc, err := nats.Connect(ns.ClientURL()) + require.NoError(t, err) + t.Cleanup(func() { nc.Close() }) + return nc +} + +// seedVolumeDocument writes the control-plane document that makes a volume +// exist as far as the control plane is concerned. +func seedVolumeDocument(t *testing.T, store objectstore.ObjectStore, volume ebsmetadata.Volume) { + t.Helper() + require.NoError(t, ebsmetadata.NewStore(store, "test-bucket").PutVolume(context.Background(), volume)) +} + +// seedProviderConfig writes provider-owned state at the volume's config.json. +// Its contents are deliberately opaque here: the control plane must neither +// read nor rewrite them, so the tests only care that the bytes survive. +func seedProviderConfig(t *testing.T, store *objectstore.MemoryObjectStore, volumeID string) { + t.Helper() + putProviderConfig(t, store, volumeID, `{"VolumeName":"`+volumeID+`","VolumeSize":10737418240,"BlockSize":4096,"SeqNum":7,"State":"available"}`) +} + +// seedEncryptedConfig writes a sealed provider config.json: an opaque +// ciphertext blob a rewrite would corrupt rather than merely overwrite. +func seedEncryptedConfig(t *testing.T, store *objectstore.MemoryObjectStore, volumeID string) { + t.Helper() + putProviderConfig(t, store, volumeID, `{"EncryptionEnabled":true,"StateSeqNum":7,"Sealed":"3q2+7wAAAAAAAAAAAAAAAA=="}`) +} + +func putProviderConfig(t *testing.T, store *objectstore.MemoryObjectStore, volumeID, body string) { + t.Helper() + _, err := store.PutObject(context.Background(), &s3.PutObjectInput{ + Bucket: aws.String("test-bucket"), + Key: aws.String(volumeID + "/config.json"), + Body: strings.NewReader(body), + }) + require.NoError(t, err) +} + +// getStoredConfig reads the raw config.json bytes from the memory store. +func getStoredConfig(t *testing.T, store *objectstore.MemoryObjectStore, volumeID string) []byte { + t.Helper() + res, err := store.GetObject(context.Background(), &s3.GetObjectInput{ + Bucket: aws.String("test-bucket"), + Key: aws.String(volumeID + "/config.json"), + }) + require.NoError(t, err) + defer res.Body.Close() + data, err := io.ReadAll(res.Body) + require.NoError(t, err) + return data +} diff --git a/spinifex/handlers/ec2/volume/volume_leak_reaper.go b/spinifex/handlers/ec2/volume/volume_leak_reaper.go index 1cad9a571..672f61c95 100644 --- a/spinifex/handlers/ec2/volume/volume_leak_reaper.go +++ b/spinifex/handlers/ec2/volume/volume_leak_reaper.go @@ -6,7 +6,6 @@ import ( "time" "github.com/mulgadc/spinifex/spinifex/vm" - "github.com/mulgadc/viperblock/viperblock" ) // orphanTagKey marks a volume the GC has surfaced as orphaned. It is a @@ -67,26 +66,27 @@ func (r *VolumeLeakReaper) Sweep(ctx context.Context) (int, error) { default: } - cfg, err := r.svc.getVolumeConfig(ctx, id) + meta, err := r.svc.metadata.GetVolume(ctx, id) if err != nil { continue } - md := &cfg.VolumeMetadata - if md.AttachedInstance == "" || !leaked[md.AttachedInstance] { + attachedInstance, tags, sizeGiB := meta.AttachedInstance, meta.Tags, meta.CapacityGiB + + if attachedInstance == "" || !leaked[attachedInstance] { continue } - if md.Tags[orphanTagKey] != "" { + if tags[orphanTagKey] != "" { continue // already surfaced; idempotent } - if err := r.svc.markVolumeOrphaned(ctx, id, cfg); err != nil { + if err := r.svc.markVolumeOrphaned(ctx, id, tags, attachedInstance); err != nil { slog.Error("volume-leak: failed to mark orphaned volume", "volumeId", id, "err", err) continue } // ALARM — surface the retained orphan to the operator. NEVER delete: // reclamation is the explicit --purge / operator path (ADR-0005 §3). slog.Warn("DATA-SAFETY ALARM: orphaned volume retained, not deleted", - "volumeId", id, "attachedInstance", md.AttachedInstance, "sizeGiB", md.SizeGiB) + "volumeId", id, "attachedInstance", attachedInstance, "sizeGiB", sizeGiB) // Reconcile the attachment record only — never the volume data. The // owning instance is definitively terminated (this node's leaked @@ -100,13 +100,20 @@ func (r *VolumeLeakReaper) Sweep(ctx context.Context) (int, error) { return marked, nil } -// markVolumeOrphaned tags the volume as orphaned in the control-plane-owned -// tags.json. The volume's data and state remain untouched. -func (s *VolumeServiceImpl) markVolumeOrphaned(ctx context.Context, volumeID string, cfg *viperblock.VolumeConfig) error { - if cfg.VolumeMetadata.Tags == nil { - cfg.VolumeMetadata.Tags = make(map[string]string) +// markVolumeOrphaned tags the volume as orphaned. tags is the volume's +// current, already-authoritative tag set; the volume's data and state remain +// untouched. +func (s *VolumeServiceImpl) markVolumeOrphaned(ctx context.Context, volumeID string, tags map[string]string, attachedInstance string) error { + if tags == nil { + tags = make(map[string]string) } - cfg.VolumeMetadata.Tags[orphanTagKey] = time.Now().UTC().Format(time.RFC3339) - cfg.VolumeMetadata.Tags[orphanInstanceTagKey] = cfg.VolumeMetadata.AttachedInstance - return s.putVolumeTags(ctx, volumeID, cfg.VolumeMetadata.Tags) + tags[orphanTagKey] = time.Now().UTC().Format(time.RFC3339) + tags[orphanInstanceTagKey] = attachedInstance + + meta, err := s.metadata.GetVolume(ctx, volumeID) + if err != nil { + return err + } + meta.Tags = tags + return s.metadata.PutVolume(ctx, meta) } diff --git a/spinifex/handlers/ec2/volume/volume_leak_reaper_test.go b/spinifex/handlers/ec2/volume/volume_leak_reaper_test.go index ef5db3cfd..a06e61b19 100644 --- a/spinifex/handlers/ec2/volume/volume_leak_reaper_test.go +++ b/spinifex/handlers/ec2/volume/volume_leak_reaper_test.go @@ -28,12 +28,12 @@ func TestVolumeLeakReaper_ReconcilesAttachmentNeverDeletesData(t *testing.T) { require.NoError(t, err) assert.Equal(t, 1, marked) - cfg, err := svc.GetVolumeConfig("vol-leaked-reconcile") + meta, err := svc.GetVolumeMetadata("vol-leaked-reconcile") require.NoError(t, err, "the reaper must never delete the volume, only mark + reconcile it") - assert.NotEmpty(t, cfg.VolumeMetadata.Tags[orphanTagKey], "the volume must still be marked orphaned") - assert.Equal(t, "available", cfg.VolumeMetadata.State, + assert.NotEmpty(t, meta.Tags[orphanTagKey], "the volume must still be marked orphaned") + assert.Equal(t, "available", meta.State, "the reaper must reconcile the attachment record to available so DeleteVolume can proceed") - assert.Empty(t, cfg.VolumeMetadata.AttachedInstance, + assert.Empty(t, meta.AttachedInstance, "the reaper must clear the stale attachment left by the terminated instance") // The reconcile only clears control-plane attachment state — this is @@ -41,7 +41,7 @@ func TestVolumeLeakReaper_ReconcilesAttachmentNeverDeletesData(t *testing.T) { // AttachedInstance=="" and State in {"available",""}. Confirm the reaper // leaves the volume passing that guard, without exercising DeleteVolume's // full path here (which needs a snapshotKV this unit fixture omits). - assert.True(t, cfg.VolumeMetadata.AttachedInstance == "" && (cfg.VolumeMetadata.State == "available" || cfg.VolumeMetadata.State == ""), + assert.True(t, meta.AttachedInstance == "" && (meta.State == "available" || meta.State == ""), "the reconciled volume must pass DeleteVolume's in-use guard so an operator can delete it") } @@ -62,8 +62,8 @@ func TestVolumeLeakReaper_LiveOwnerAttachmentUntouched(t *testing.T) { require.NoError(t, err) assert.Equal(t, 0, marked) - cfg, err := svc.GetVolumeConfig("vol-live-reconcile") + meta, err := svc.GetVolumeMetadata("vol-live-reconcile") require.NoError(t, err) - assert.Equal(t, "in-use", cfg.VolumeMetadata.State, "a live-owner volume's attachment must never be reconciled") - assert.Equal(t, "i-running00000000", cfg.VolumeMetadata.AttachedInstance) + assert.Equal(t, "in-use", meta.State, "a live-owner volume's attachment must never be reconciled") + assert.Equal(t, "i-running00000000", meta.AttachedInstance) } diff --git a/spinifex/handlers/ec2/vpc/eni.go b/spinifex/handlers/ec2/vpc/eni.go index 4f74adbd8..be72caf12 100644 --- a/spinifex/handlers/ec2/vpc/eni.go +++ b/spinifex/handlers/ec2/vpc/eni.go @@ -23,6 +23,12 @@ const ( KVBucketENIsVersion = 1 ) +// AutoENIDescriptionPrefix marks an ENI the launch path created for an +// instance, as opposed to one a caller created through the API. It is the only +// thing distinguishing the two before the attachment lands, which is what lets +// a sweep tell an abandoned launch from a deliberate standalone ENI. +const AutoENIDescriptionPrefix = "Primary network interface for " + // ENIRecord represents a stored Elastic Network Interface. type ENIRecord struct { NetworkInterfaceId string `json:"network_interface_id"` @@ -760,6 +766,70 @@ func (s *VPCServiceImpl) listInstanceENIs(ctx context.Context, accountID, instan return out, nil } +// AccountENI pairs a record with the account its key is scoped by, which a +// cluster-wide sweep needs because it has no single account to scope to. +type AccountENI struct { + AccountID string + Record ENIRecord +} + +// ListAbandonedInstanceENIs returns auto-created instance ENIs that never +// reached an attachment and are older than minAge. +// +// A launch attaches within seconds of creating the ENI, and stopping an +// instance leaves the attachment in place, so an unattached one this old is +// the residue of a launch that died in between. Such a record keeps its +// security group undeletable forever, since the SG dependency check counts it. +func (s *VPCServiceImpl) ListAbandonedInstanceENIs(ctx context.Context, minAge time.Duration) ([]AccountENI, error) { + keys, err := s.eniKV.Keys(ctx) + if err != nil { + if errors.Is(err, jetstream.ErrNoKeysFound) { + return nil, nil + } + return nil, errors.New(awserrors.ErrorServerInternal) + } + + var out []AccountENI + for _, key := range keys { + if key == utils.VersionKey { + continue + } + accountID, _, found := strings.Cut(key, ".") + if !found || accountID == "" { + continue + } + entry, err := s.eniKV.Get(ctx, key) + if err != nil { + continue + } + var record ENIRecord + if err := json.Unmarshal(entry.Value(), &record); err != nil { + continue + } + if !eniIsAbandonedLaunch(&record, minAge) { + continue + } + out = append(out, AccountENI{AccountID: accountID, Record: record}) + } + return out, nil +} + +// eniIsAbandonedLaunch reports whether a record is an instance ENI left behind +// by a launch that never attached it. A zero CreatedAt is not reapable: age is +// the only guard against deleting an ENI a launch is still using. +func eniIsAbandonedLaunch(record *ENIRecord, minAge time.Duration) bool { + if !strings.HasPrefix(record.Description, AutoENIDescriptionPrefix) { + return false + } + if record.InstanceId != "" || record.AttachmentId != "" || record.Status == "in-use" { + return false + } + if record.CreatedAt.IsZero() { + return false + } + return time.Since(record.CreatedAt) >= minAge +} + // FindENIByAttachment scans the ENI bucket for the record with the given // AttachmentId. Used by DetachNetworkInterface which identifies by attachment ID. func (s *VPCServiceImpl) FindENIByAttachment(accountID, attachmentId string) (ENIRecord, error) { diff --git a/spinifex/handlers/ec2/vpc/eni_abandoned_test.go b/spinifex/handlers/ec2/vpc/eni_abandoned_test.go new file mode 100644 index 000000000..151fd827f --- /dev/null +++ b/spinifex/handlers/ec2/vpc/eni_abandoned_test.go @@ -0,0 +1,99 @@ +package handlers_ec2_vpc + +import ( + "context" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mulgadc/spinifex/spinifex/awserrors" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// createAutoENI creates an ENI the way the launch path does, then backdates it +// so the age guard treats it as settled rather than a launch still in flight. +func createAutoENI(t *testing.T, svc *VPCServiceImpl, subnetID, instanceID string, sgIDs []string, age time.Duration) string { + t.Helper() + groups := make([]*string, 0, len(sgIDs)) + for _, id := range sgIDs { + groups = append(groups, aws.String(id)) + } + out, err := svc.CreateNetworkInterface(context.Background(), &ec2.CreateNetworkInterfaceInput{ + SubnetId: aws.String(subnetID), + Description: aws.String(AutoENIDescriptionPrefix + instanceID), + Groups: groups, + }, testAccountID) + require.NoError(t, err) + + eniID := *out.NetworkInterface.NetworkInterfaceId + require.NoError(t, svc.UpdateENI(testAccountID, eniID, func(rec *ENIRecord) { + rec.CreatedAt = time.Now().Add(-age) + })) + return eniID +} + +func TestListAbandonedInstanceENIs(t *testing.T) { + svc := setupTestVPCService(t) + vpcID := createTestVPC(t, svc, "10.0.0.0/16") + subnetID := createTestSubnet(t, svc, vpcID, "10.0.1.0/24") + + abandoned := createAutoENI(t, svc, subnetID, "i-abandoned", nil, time.Hour) + createAutoENI(t, svc, subnetID, "i-inflight", nil, time.Minute) + + // A standalone ENI is indistinguishable by age or attachment; only the + // description separates one a caller asked for from launch residue. + standalone := createTestENI(t, svc, subnetID) + require.NoError(t, svc.UpdateENI(testAccountID, standalone, func(rec *ENIRecord) { + rec.CreatedAt = time.Now().Add(-time.Hour) + })) + + attached := createAutoENI(t, svc, subnetID, "i-running", nil, time.Hour) + _, err := svc.AttachENI(testAccountID, attached, "i-running", 0) + require.NoError(t, err) + + got, err := svc.ListAbandonedInstanceENIs(context.Background(), 15*time.Minute) + require.NoError(t, err) + + ids := make([]string, 0, len(got)) + for _, o := range got { + assert.Equal(t, testAccountID, o.AccountID) + ids = append(ids, o.Record.NetworkInterfaceId) + } + assert.Equal(t, []string{abandoned}, ids, + "only the aged, unattached, launch-created ENI is residue; the others are live or deliberate") +} + +// TestAbandonedENIBlocksSecurityGroupDelete pins the consequence that makes this +// worth sweeping: the SG dependency check counts an abandoned ENI, so the group +// stays undeletable for as long as the record survives. +func TestAbandonedENIBlocksSecurityGroupDelete(t *testing.T) { + svc := setupTestVPCService(t) + vpcID := createTestVPC(t, svc, "10.0.0.0/16") + subnetID := createTestSubnet(t, svc, vpcID, "10.0.1.0/24") + sgID := createTestSG(t, svc, vpcID, "runner-sg") + + eniID := createAutoENI(t, svc, subnetID, "i-abandoned", []string{sgID}, time.Hour) + + _, err := svc.DeleteSecurityGroup(context.Background(), &ec2.DeleteSecurityGroupInput{ + GroupId: aws.String(sgID), + }, testAccountID) + require.Error(t, err) + assert.Equal(t, awserrors.ErrorDependencyViolation, err.Error()) + + orphans, err := svc.ListAbandonedInstanceENIs(context.Background(), 15*time.Minute) + require.NoError(t, err) + require.Len(t, orphans, 1) + require.Equal(t, eniID, orphans[0].Record.NetworkInterfaceId) + + _, err = svc.DeleteNetworkInterface(context.Background(), &ec2.DeleteNetworkInterfaceInput{ + NetworkInterfaceId: aws.String(eniID), + }, testAccountID) + require.NoError(t, err) + + _, err = svc.DeleteSecurityGroup(context.Background(), &ec2.DeleteSecurityGroupInput{ + GroupId: aws.String(sgID), + }, testAccountID) + require.NoError(t, err, "with the residue gone the group must delete") +} diff --git a/spinifex/migrate/VERSIONS.md b/spinifex/migrate/VERSIONS.md index 5ac6cbee5..c3e4f25d5 100644 --- a/spinifex/migrate/VERSIONS.md +++ b/spinifex/migrate/VERSIONS.md @@ -21,13 +21,24 @@ Single source of truth for the current schema version of every config file Spini |---|---|---| | `spinifex.toml` | `3` → `4`: rename `[nodes.*.predastore]` `node_id` → `host_id` | `spinifex/migrate/005_spinifex_predastore_host_id.go` | +## Registered object-store migrations + +These migrate data in Predastore rather than a config file. Predastore has no conditional write, so their version is stamped in a shared JetStream KV bucket instead of in an object; see `RunObject`. + +| Target | Migration | File | Version bucket | +|---|---|---|---| +| `ebsmetadata` | `0` → `1`: backfill `spinifex/ebsmetadata/v1` documents from legacy viperblock `config.json` | `spinifex/migrate/ebsmetadatabackfill/006_ebsmetadata_backfill.go` | `spinifex-ebsmetadata-migrate` | + +Unlike config migrations, these do not run from `spx admin upgrade`. The ebsmetadata backfill runs from `Daemon.configureEBSProvider`, and only when `[ebs] provider = "viperblockd"` is selected: backfilling while the embedded engine is still authoritative would produce documents that immediately go stale. + ## Where migrations live Register a `ConfigMigration` against `DefaultRegistry` in a new numbered file under `spinifex/migrate/`, and bump the version in both the template and the table above. ## Framework -The framework (`migrate.go`, `version_readers.go`) covers both kinds of target: +The framework (`migrate.go`, `version_readers.go`) covers three kinds of target: - `RunKV` is called from service-startup paths to stamp NATS KV bucket versions. - `RunConfig` / `RunAllConfig` / `PendingConfig` back `spx admin upgrade`, which `scripts/setup.sh` runs with `--yes` while the services are stopped. +- `RunObject` migrates object-store data, stamping its version in a caller-supplied JetStream KV bucket. The stamp is a read-then-write, not a compare-and-swap, so two nodes can race to run the same step — every `ObjectMigration.Run` must be safe to re-execute. diff --git a/spinifex/migrate/migrate.go b/spinifex/migrate/migrate.go index 6f78df1cd..4c8d781c9 100644 --- a/spinifex/migrate/migrate.go +++ b/spinifex/migrate/migrate.go @@ -10,6 +10,7 @@ import ( "time" "github.com/mulgadc/spinifex/spinifex/kvutil" + "github.com/mulgadc/spinifex/spinifex/objectstore" "github.com/nats-io/nats.go/jetstream" ) @@ -38,6 +39,25 @@ type ConfigMigration struct { Run func(ctx ConfigContext) error } +// ObjectMigration represents a versioned transformation of object-store data +// (Predastore/S3 objects), such as backfilling a new document shape from an +// older one written by a different subsystem. +type ObjectMigration struct { + FromVersion int + ToVersion int + Description string + Run func(ctx context.Context, octx ObjectContext) error +} + +// ObjectContext provides object-store migration functions access to the +// bucket being migrated. Objects is the same object-store abstraction +// handlers use, so a migration reads/writes exactly as a handler would. +type ObjectContext struct { + Objects objectstore.ObjectStore + Bucket string + Logger *slog.Logger +} + // ConfigContext provides config migration functions access to the filesystem. type ConfigContext struct { ConfigDir string // path to /etc/spinifex or equivalent @@ -64,6 +84,7 @@ type Registry struct { kvMigrations map[string][]KVMigration configMigrations map[string][]ConfigMigration configTargets map[string]configTarget + objectMigrations map[string][]ObjectMigration } // DefaultRegistry is the global migration registry. Migrations self-register @@ -76,6 +97,7 @@ func NewRegistry() *Registry { kvMigrations: make(map[string][]KVMigration), configMigrations: make(map[string][]ConfigMigration), configTargets: make(map[string]configTarget), + objectMigrations: make(map[string][]ObjectMigration), } } @@ -101,6 +123,82 @@ func (r *Registry) RegisterConfig(target string, m ConfigMigration) { }) } +// RegisterObject adds an object-store migration. Migrations are kept sorted by FromVersion. +func (r *Registry) RegisterObject(target string, m ObjectMigration) { + r.objectMigrations[target] = append(r.objectMigrations[target], m) + sort.Slice(r.objectMigrations[target], func(i, j int) bool { + return r.objectMigrations[target][i].FromVersion < r.objectMigrations[target][j].FromVersion + }) +} + +// RunObject applies pending object-store migrations for target up to +// targetVersion. Predastore has no conditional write, so progress is stamped +// in JetStream KV instead, exactly as RunKV stamps a KV bucket's own version. +// +// Callers must pass a KV bucket shared by every node. The stamp is a +// read-then-write rather than a compare-and-swap, so two nodes can still race +// to run a step; each Run must be safe to re-execute for that to be harmless. +func (r *Registry) RunObject(ctx context.Context, target string, objects objectstore.ObjectStore, bucket string, versionKV jetstream.KeyValue, targetVersion int) error { + current, err := kvutil.ReadVersion(ctx, versionKV) + if err != nil { + return fmt.Errorf("read version for %s: %w", target, err) + } + + if current >= targetVersion { + return nil + } + + all := r.objectMigrations[target] + + // Fresh target, no migrations: stamp directly (common first-init path). + if current == 0 && len(all) == 0 { + return kvutil.WriteVersion(ctx, versionKV, targetVersion) + } + + // Fresh target with migrations: no v0 schema by convention; start at chain bottom. + if current == 0 { + current = all[0].FromVersion + } + + // Require a complete chain from current to target. + var pending []ObjectMigration + for _, m := range all { + if m.FromVersion >= current && m.ToVersion <= targetVersion { + pending = append(pending, m) + } + } + + if len(pending) == 0 { + return fmt.Errorf("no migrations registered for %s from version %d to %d", target, current, targetVersion) + } + + // Validate contiguous chain. + expected := current + for _, m := range pending { + if m.FromVersion != expected { + return fmt.Errorf("migration chain gap for %s: expected from %d, got from %d", target, expected, m.FromVersion) + } + expected = m.ToVersion + } + if expected != targetVersion { + return fmt.Errorf("migration chain for %s ends at version %d, target is %d", target, expected, targetVersion) + } + + logger := slog.Default() + for _, m := range pending { + logger.Info("Running object migration", "target", target, "from", m.FromVersion, "to", m.ToVersion, "description", m.Description) + octx := ObjectContext{Objects: objects, Bucket: bucket, Logger: logger} + if err := m.Run(ctx, octx); err != nil { + return fmt.Errorf("object migration %s %d→%d failed: %w", target, m.FromVersion, m.ToVersion, err) + } + if err := kvutil.WriteVersion(ctx, versionKV, m.ToVersion); err != nil { + return fmt.Errorf("stamp version %d on %s: %w", m.ToVersion, target, err) + } + } + + return nil +} + // RunKVWithJetStream is RunKV with a JetStream handle attached to each // migration's KVContext, enabling cross-bucket reads (e.g. owner-attribution // during a backfill). Prefer plain RunKV when the migration is self-contained. diff --git a/spinifex/nbd/nbd.go b/spinifex/nbd/nbd.go index 970f8be08..b7e2f46d9 100644 --- a/spinifex/nbd/nbd.go +++ b/spinifex/nbd/nbd.go @@ -44,8 +44,26 @@ type NBDKitConfig struct { // plugin's gc_enabled param, gating chunk garbage collection on the VB // this nbdkit process constructs. Default false, matching ShardWAL. GCEnabled bool `json:"gc_enabled"` + + // ReadOnly serves the export read-only via nbdkit -r, which both sets the + // NBD read-only transmission flag and passes readonly=true to the plugin's + // Open. The plugin refuses writes on that flag. + ReadOnly bool `json:"read_only"` } +// Block size constraints advertised to clients through nbdkit's +// blocksize-policy filter. The plugin itself has no way to advertise them: +// the nbdkit Go binding has no block-size callback. +// +// The minimum is 1 because viperblock's WriteAt handles unaligned requests +// with a read-modify-write; the preferred size is its 4KB block, which is the +// size that avoids that cycle. The maximum matches what qemu-nbd advertises. +const ( + blockSizeMinimum = 1 + blockSizePreferred = 4096 + blockSizeMaximum = 32 << 20 +) + // buildArgs constructs the nbdkit command-line arguments from the config. func (cfg *NBDKitConfig) buildArgs() ([]string, error) { args := []string{ @@ -53,6 +71,12 @@ func (cfg *NBDKitConfig) buildArgs() ([]string, error) { "--pidfile", cfg.PidFile, } + if cfg.ReadOnly { + args = append(args, "-r") + } + + args = append(args, "--filter=blocksize-policy") + // Add transport-specific arguments if cfg.UseTCP { // TCP transport - for remote/DPU scenarios @@ -75,6 +99,9 @@ func (cfg *NBDKitConfig) buildArgs() ([]string, error) { // they are passed via cmd.Env in Execute so they never appear in argv, // which is world-readable via /proc//cmdline. pluginArgs := []string{ + fmt.Sprintf("blocksize-minimum=%d", blockSizeMinimum), + fmt.Sprintf("blocksize-preferred=%d", blockSizePreferred), + fmt.Sprintf("blocksize-maximum=%d", blockSizeMaximum), fmt.Sprintf("size=%d", cfg.Size), fmt.Sprintf("volume=%s", cfg.Volume), fmt.Sprintf("bucket=%s", cfg.Bucket), diff --git a/spinifex/nbd/nbd_test.go b/spinifex/nbd/nbd_test.go index f2132097c..1570e99a9 100644 --- a/spinifex/nbd/nbd_test.go +++ b/spinifex/nbd/nbd_test.go @@ -33,8 +33,12 @@ func TestBuildArgs_TCPTransport(t *testing.T) { expected := []string{ "-f", "--pidfile", "/tmp/nbd.pid", + "--filter=blocksize-policy", "-p", "10809", "/usr/lib/nbdkit/plugins/vb.so", + "blocksize-minimum=1", + "blocksize-preferred=4096", + "blocksize-maximum=33554432", "size=1073741824", "volume=vol-abc123", "bucket=my-bucket", @@ -76,8 +80,12 @@ func TestBuildArgs_UnixSocketTransport(t *testing.T) { expected := []string{ "-f", "--pidfile", "/tmp/nbd.pid", + "--filter=blocksize-policy", "--unix", "/tmp/nbd.sock", "/usr/lib/nbdkit/plugins/vb.so", + "blocksize-minimum=1", + "blocksize-preferred=4096", + "blocksize-maximum=33554432", "size=536870912", "volume=vol-def456", "bucket=bucket-2", @@ -378,3 +386,44 @@ func assertArgs(t *testing.T, expected, got []string) { } } } + +// TestBuildArgs_ReadOnly checks -r reaches nbdkit before the plugin path, so +// it is parsed as an nbdkit flag rather than a plugin parameter. +func TestBuildArgs_ReadOnly(t *testing.T) { + cfg := &NBDKitConfig{ + Socket: "/tmp/nbd.sock", + PidFile: "/tmp/nbd.pid", + PluginPath: "/plugin.so", + ReadOnly: true, + } + + args, err := cfg.buildArgs() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + readOnlyIdx := slices.Index(args, "-r") + if readOnlyIdx < 0 { + t.Fatalf("expected -r in args, got: %v", args) + } + if pluginIdx := slices.Index(args, "/plugin.so"); readOnlyIdx > pluginIdx { + t.Errorf("-r must precede the plugin path, got: %v", args) + } +} + +func TestBuildArgs_ReadOnlyOmittedByDefault(t *testing.T) { + cfg := &NBDKitConfig{ + Socket: "/tmp/nbd.sock", + PidFile: "/tmp/nbd.pid", + PluginPath: "/plugin.so", + } + + args, err := cfg.buildArgs() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if slices.Index(args, "-r") >= 0 { + t.Errorf("-r must be absent unless ReadOnly is set, got: %v", args) + } +} diff --git a/spinifex/service/service.go b/spinifex/service/service.go index 21dc14a79..3562a19de 100644 --- a/spinifex/service/service.go +++ b/spinifex/service/service.go @@ -7,6 +7,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/services/nats" "github.com/mulgadc/spinifex/spinifex/services/northstar" "github.com/mulgadc/spinifex/spinifex/services/predastore" + "github.com/mulgadc/spinifex/spinifex/services/qemunbdd" "github.com/mulgadc/spinifex/spinifex/services/qmpcollector" "github.com/mulgadc/spinifex/spinifex/services/spinifex" "github.com/mulgadc/spinifex/spinifex/services/spinifexui" @@ -27,6 +28,7 @@ var ( _ Service = (*northstar.Service)(nil) _ Service = (*predastore.Service)(nil) _ Service = (*viperblockd.Service)(nil) + _ Service = (*qemunbdd.Service)(nil) _ Service = (*spinifex.Service)(nil) _ Service = (*awsgw.Service)(nil) _ Service = (*spinifexui.Service)(nil) @@ -48,6 +50,9 @@ func New(btype string, config any) (Service, error) { case "viperblock": return viperblockd.New(config) + case "qemunbd": + return qemunbdd.New(config) + case "spinifex": return spinifex.New(config) diff --git a/spinifex/services/qemunbdd/bench_test.go b/spinifex/services/qemunbdd/bench_test.go new file mode 100644 index 000000000..79e9c3b77 --- /dev/null +++ b/spinifex/services/qemunbdd/bench_test.go @@ -0,0 +1,42 @@ +package qemunbdd_test + +// The control for the A/B. qemunbdd shares no code with viperblock and stores +// real qcow2 files, so what it costs above the null provider is what a +// straightforward local implementation of these verbs costs. + +import ( + "testing" + "time" + + "github.com/mulgadc/spinifex/spinifex/ebsprovider" + "github.com/mulgadc/spinifex/spinifex/ebsprovider/conformance" + "github.com/mulgadc/spinifex/spinifex/ebsprovider/natsserve" + "github.com/mulgadc/spinifex/spinifex/testutil" + "github.com/stretchr/testify/require" +) + +// benchVolumeBytes matches the size the live viperblockd benchmark uses, so +// the two sets of numbers are comparable rather than merely adjacent. +const benchVolumeBytes int64 = 64 << 20 + +func BenchmarkQEMUNBDProvider_InProcess(b *testing.B) { + requireQEMUTools(b) + conformance.RunBenchSuite(b, newQEMUProviderTB(b), conformance.BenchConfig{ + VolumeBytes: benchVolumeBytes, + }) +} + +// BenchmarkQEMUNBDProvider_OverNATS is the shape the control plane actually +// calls: same verbs, same transport, a provider that does real filesystem +// work. The gap to the in-process run should be the seam and nothing else. +func BenchmarkQEMUNBDProvider_OverNATS(b *testing.B) { + requireQEMUTools(b) + _, conn := testutil.StartTestNATS(b) + stop, err := natsserve.Serve(b.Context(), conn, newQEMUProviderTB(b), natsserve.Options{}) + require.NoError(b, err) + b.Cleanup(stop) + + conformance.RunBenchSuite(b, ebsprovider.NewNATSProvider(conn, 120*time.Second), conformance.BenchConfig{ + VolumeBytes: benchVolumeBytes, + }) +} diff --git a/spinifex/services/qemunbdd/conformance_test.go b/spinifex/services/qemunbdd/conformance_test.go new file mode 100644 index 000000000..b64ecfae3 --- /dev/null +++ b/spinifex/services/qemunbdd/conformance_test.go @@ -0,0 +1,87 @@ +package qemunbdd_test + +// The point of this provider: the same contract suite that covers +// MemoryProvider, run unmodified against an implementation that shares no +// code with viperblock and stores real qcow2 files. + +import ( + "os" + "os/exec" + "testing" + "time" + + "github.com/mulgadc/spinifex/spinifex/ebsprovider" + "github.com/mulgadc/spinifex/spinifex/ebsprovider/conformance" + "github.com/mulgadc/spinifex/spinifex/ebsprovider/natsserve" + "github.com/mulgadc/spinifex/spinifex/services/qemunbdd" + "github.com/mulgadc/spinifex/spinifex/testutil" + "github.com/stretchr/testify/require" +) + +// requireQEMUTools skips when the host has no qemu, so the suite stays +// runnable on a machine that cannot exercise the real binaries. CI sets +// SPINIFEX_REQUIRE_CONFORMANCE_TOOLS so a runner image that loses qemu fails +// loudly instead of quietly leaving MemoryProvider as the only implementation +// the contract suite judges. +func requireQEMUTools(t testing.TB) { + t.Helper() + for _, bin := range []string{"qemu-img", "qemu-nbd", "qemu-io"} { + if _, err := exec.LookPath(bin); err != nil { + if os.Getenv("SPINIFEX_REQUIRE_CONFORMANCE_TOOLS") != "" { + t.Fatalf("%s not installed, but SPINIFEX_REQUIRE_CONFORMANCE_TOOLS is set", bin) + } + t.Skipf("%s not installed", bin) + } + } +} + +// shortTempDir is used instead of t.TempDir because the test name is part of +// that path, and AF_UNIX truncates a socket path at 108 bytes. Real base +// directories are short; only the test names are not. +func shortTempDir(t testing.TB) string { + t.Helper() + dir, err := os.MkdirTemp("", "qnbd") //nolint:usetesting // t.TempDir embeds the test name and overflows the 108-byte socket path limit + require.NoError(t, err) + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + return dir +} + +func newQEMUProvider(t *testing.T) ebsprovider.EBSProvider { + t.Helper() + return newQEMUProviderTB(t) +} + +func newQEMUProviderTB(t testing.TB) ebsprovider.EBSProvider { + t.Helper() + provider, err := qemunbdd.NewProvider(shortTempDir(t)) + require.NoError(t, err) + return provider +} + +func TestQEMUNBDProviderConformance(t *testing.T) { + requireQEMUTools(t) + conformance.RunSuite(t, newQEMUProvider) +} + +// TestQEMUNBDExternalNBDClient checks the half of the boundary our own Go +// client can never check: whether the export we publish is usable by an NBD +// client that knows nothing about this codebase. +func TestQEMUNBDExternalNBDClient(t *testing.T) { + requireQEMUTools(t) + conformance.RunNBDClientSuite(t, newQEMUProvider) +} + +// TestQEMUNBDProviderNATSConformance is the swappability claim itself: the +// same wire contract, the same neutral server, a provider with no viperblock +// in it, and a client that cannot tell the difference. +func TestQEMUNBDProviderNATSConformance(t *testing.T) { + requireQEMUTools(t) + conformance.RunSuite(t, func(t *testing.T) ebsprovider.EBSProvider { + t.Helper() + _, conn := testutil.StartTestNATS(t) + stop, err := natsserve.Serve(t.Context(), conn, newQEMUProvider(t), natsserve.Options{}) + require.NoError(t, err) + t.Cleanup(stop) + return ebsprovider.NewNATSProvider(conn, 15*time.Second) + }) +} diff --git a/spinifex/services/qemunbdd/live_lifecycle_test.go b/spinifex/services/qemunbdd/live_lifecycle_test.go new file mode 100644 index 000000000..1e8e9afc4 --- /dev/null +++ b/spinifex/services/qemunbdd/live_lifecycle_test.go @@ -0,0 +1,201 @@ +// This test proves the qemu-nbd EBS provider works end to end against a +// real qemunbdd daemon over a live cluster's NATS. It is opt-in only: run it +// with SPINIFEX_QEMUNBD_LIVE=1 on the node the daemon is running on. +package qemunbdd_test + +import ( + "context" + "fmt" + "os" + "os/exec" + "strconv" + "strings" + "testing" + "time" + + "github.com/mulgadc/spinifex/spinifex/admin" + "github.com/mulgadc/spinifex/spinifex/config" + "github.com/mulgadc/spinifex/spinifex/ebsprovider" + "github.com/mulgadc/spinifex/spinifex/utils" + "github.com/stretchr/testify/require" +) + +// liveEnvVar opts this test into running against a real cluster; unset (the +// default) it always skips, so it never runs under make preflight. +const liveEnvVar = "SPINIFEX_QEMUNBD_LIVE" + +// defaultConfigPath mirrors the path the qemunbd systemd unit sets for +// SPINIFEX_CONFIG_PATH when the operator does not override it. +const defaultConfigPath = "/etc/spinifex/spinifex.toml" + +const liveVolumeBytes int64 = 64 * 1024 * 1024 + +// TestLive_QEMUNBDLifecycle drives a real qemunbdd daemon end to end through +// ebsprovider.NewNATSProvider, the same client shape the control plane uses, +// over the cluster's real NATS. +func TestLive_QEMUNBDLifecycle(t *testing.T) { + if os.Getenv(liveEnvVar) == "" { + t.Skipf("skipping: set %s=1 to run this test against a real qemunbdd daemon over a live cluster's NATS", liveEnvVar) + } + + cfgPath := os.Getenv("SPINIFEX_CONFIG_PATH") + if cfgPath == "" { + cfgPath = defaultConfigPath + } + t.Logf("loading cluster config from %s", cfgPath) + + clusterConfig, err := config.LoadConfig(cfgPath) + require.NoError(t, err, "load cluster config") + + nodeID := clusterConfig.Node + require.NotEmpty(t, nodeID, "cluster config has no node name set") + + nodeConfig, ok := clusterConfig.Nodes[nodeID] + require.True(t, ok, "cluster config has no entry for node %q", nodeID) + + t.Logf("node %q: base dir %s", nodeID, nodeConfig.Predastore.BaseDir) + + // NATS token and CA cert are read from config and handed straight to the + // connect helper. Never log them, and never let a failure message embed + // the config value that produced it. + nc, err := utils.ConnectNATSWithRetry(admin.DialTarget(nodeConfig.NATS.Host), nodeConfig.NATS.ACL.Token, nodeConfig.NATS.CACert) + require.NoError(t, err, "connect to NATS") + t.Cleanup(nc.Close) + t.Logf("connected to NATS") + + provider := ebsprovider.NewNATSProvider(nc, 60*time.Second) + + runID := strconv.FormatInt(time.Now().UnixNano(), 10) + volumeID := "live-qnbd-vol-" + runID + snapshotID := "live-qnbd-snap-" + runID + restoredVolumeID := "live-qnbd-restore-" + runID + pattern := byte(0xAB) + + t.Cleanup(func() { + cleanupCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + cleanupBestEffort(t, "unpublish restored volume", func() error { + return provider.UnpublishVolume(cleanupCtx, ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: restoredVolumeID, NodeID: nodeID}) + }) + cleanupBestEffort(t, "delete restored volume", func() error { + return provider.DeleteVolume(cleanupCtx, ebsprovider.DeleteVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: restoredVolumeID}) + }) + cleanupBestEffort(t, "unpublish volume", func() error { + return provider.UnpublishVolume(cleanupCtx, ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: nodeID}) + }) + cleanupBestEffort(t, "delete snapshot", func() error { + return provider.DeleteSnapshot(cleanupCtx, ebsprovider.DeleteSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID}) + }) + cleanupBestEffort(t, "delete volume", func() error { + return provider.DeleteVolume(cleanupCtx, ebsprovider.DeleteVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID}) + }) + }) + + ctx := t.Context() + + vol, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), + VolumeID: volumeID, + CapacityRange: ebsprovider.CapacityRange{RequiredBytes: liveVolumeBytes}, + }) + require.NoError(t, err, "create volume") + t.Logf("created volume %s: %d bytes, state=%s", vol.ID, vol.CapacityBytes, vol.State) + + pub, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: nodeID}) + require.NoError(t, err, "publish volume") + t.Logf("published volume %s to node %s: %s", pub.VolumeID, pub.NodeID, pub.NBDURI) + sockPath := requireUnixSocketPath(t, pub.NBDURI) + + writePattern(t, sockPath, pattern, liveVolumeBytes) + readAndVerifyPattern(t, sockPath, pattern, liveVolumeBytes) + + require.NoError(t, provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: nodeID}), "unpublish volume") + t.Logf("unpublished volume %s", volumeID) + + snap, err := provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, VolumeID: volumeID}) + require.NoError(t, err, "create snapshot") + t.Logf("created snapshot %s from volume %s: state=%s, size=%d", snap.ID, volumeID, snap.State, snap.SizeBytes) + require.Equal(t, ebsprovider.SnapshotStateCompleted, snap.State, "snapshot must report completed") + + restoredVol, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), + VolumeID: restoredVolumeID, + CapacityRange: ebsprovider.CapacityRange{RequiredBytes: liveVolumeBytes}, + SourceSnapshotID: snapshotID, + }) + require.NoError(t, err, "create volume from snapshot") + t.Logf("created volume %s from snapshot %s: %d bytes, state=%s", restoredVol.ID, snapshotID, restoredVol.CapacityBytes, restoredVol.State) + + restoredPub, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: restoredVolumeID, NodeID: nodeID}) + require.NoError(t, err, "publish restored volume") + t.Logf("published restored volume %s to node %s: %s", restoredPub.VolumeID, restoredPub.NodeID, restoredPub.NBDURI) + restoredSockPath := requireUnixSocketPath(t, restoredPub.NBDURI) + + // The real proof: bytes written before the snapshot survive both the + // snapshot and the restore into an independent volume. + readAndVerifyPattern(t, restoredSockPath, pattern, liveVolumeBytes) + + require.NoError(t, provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: restoredVolumeID, NodeID: nodeID}), "unpublish restored volume") + t.Logf("unpublished restored volume %s", restoredVolumeID) +} + +// cleanupBestEffort logs a cleanup step's outcome without failing the test: +// cleanup runs after the real assertions and must never mask their result. +func cleanupBestEffort(t *testing.T, step string, fn func() error) { + t.Helper() + if err := fn(); err != nil { + t.Logf("cleanup: %s: %v", step, err) + } else { + t.Logf("cleanup: %s: ok", step) + } +} + +// requireUnixSocketPath asserts nbdURI is a nbd:unix: URI and returns its +// socket path, parsed with the same helper the daemon uses to format it +// rather than by slicing the string. +func requireUnixSocketPath(t *testing.T, nbdURI string) string { + t.Helper() + serverType, path, _, _, err := utils.ParseNBDURI(nbdURI) + require.NoError(t, err, "parse NBD URI %q", nbdURI) + require.Equal(t, "unix", serverType, "published volume must be a nbd:unix: URI, got %q", nbdURI) + require.NotEmpty(t, path, "NBD URI %q has an empty socket path", nbdURI) + return path +} + +// qemuIOTarget builds the client-facing NBD URI qemu-io understands. +// qemu-nbd exports the guest view raw regardless of the qcow2 file backing +// it, so the client always dials with -f raw. +func qemuIOTarget(sockPath string) string { + return fmt.Sprintf("nbd+unix:///?socket=%s", sockPath) +} + +// runQEMUIO execs one qemu-io command against sockPath as its own process, +// which is itself a fresh NBD connection independent of any prior exec. +func runQEMUIO(t *testing.T, sockPath, ioCmd string) (string, error) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + out, err := exec.CommandContext(ctx, "qemu-io", "-f", "raw", "-c", ioCmd, qemuIOTarget(sockPath)).CombinedOutput() + return string(out), err +} + +// writePattern writes a byte pattern across the whole volume through NBD. +func writePattern(t *testing.T, sockPath string, pattern byte, size int64) { + t.Helper() + ioCmd := fmt.Sprintf("write -P 0x%02x 0 %d", pattern, size) + out, err := runQEMUIO(t, sockPath, ioCmd) + require.NoError(t, err, "qemu-io write failed: %s", strings.TrimSpace(out)) + t.Logf("wrote pattern 0x%02x across %d bytes via qemu-io", pattern, size) +} + +// readAndVerifyPattern reads the volume back through NBD and asserts every +// byte matches pattern, checking both qemu-io's exit status and its output +// for a mismatch complaint. +func readAndVerifyPattern(t *testing.T, sockPath string, pattern byte, size int64) { + t.Helper() + ioCmd := fmt.Sprintf("read -P 0x%02x 0 %d", pattern, size) + out, err := runQEMUIO(t, sockPath, ioCmd) + require.NoError(t, err, "qemu-io read failed: %s", strings.TrimSpace(out)) + require.NotContains(t, out, "Pattern verification failed", "qemu-io reported a pattern mismatch: %s", strings.TrimSpace(out)) + t.Logf("verified pattern 0x%02x across %d bytes via qemu-io", pattern, size) +} diff --git a/spinifex/services/qemunbdd/provider.go b/spinifex/services/qemunbdd/provider.go new file mode 100644 index 000000000..95ef13a17 --- /dev/null +++ b/spinifex/services/qemunbdd/provider.go @@ -0,0 +1,769 @@ +package qemunbdd + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "syscall" + + "github.com/mulgadc/spinifex/spinifex/ebsprovider" + "github.com/mulgadc/spinifex/spinifex/utils" +) + +// nbdSharedClients bounds how many simultaneous NBD connections qemu-nbd +// accepts against one export. A volume has exactly one attaching client, so +// this stays at qemu-nbd's own default rather than inviting concurrent writers through the same socket. +const nbdSharedClients = 1 + +// maxUnixSocketPath is the kernel's sun_path limit for an AF_UNIX address. +// A base directory deep enough to push a socket past it must fail with that +// reason rather than with qemu-nbd's raw exit status. +const maxUnixSocketPath = 108 + +// capabilities is fixed: this provider offers crash consistent snapshots and +// base:allocation extents but, unlike viperblockd, cannot expand a published +// volume or answer for a volume over an owner subject. +var capabilities = ebsprovider.Capabilities{ + CrashConsistentSnapshot: true, + OnlineExpansion: false, + SparseExtentReporting: true, + VolumeSeeding: true, + ReadOnlyPublish: true, + OwnerRouting: false, + VolumeEnumeration: true, + SnapshotEnumeration: true, + // Publication is tracked in this process's own map, so a second publish + // is refused here and nowhere else. Another node running against the same + // baseDir would not be seen, which is why this is node and not cluster. + Exclusion: ebsprovider.ExclusionSemantics{Scope: ebsprovider.ExclusionScopeNode}, +} + +// volumeMeta carries the request attributes CreateVolume's idempotency check +// needs but a qcow2 file has no field for: a bare image has no notion of +// availability zone or opaque caller parameters. Kept in memory only, so it does not survive a provider restart. +type volumeMeta struct { + availabilityZone string + parameters []byte +} + +// snapshotMeta records a snapshot's source volume. CreateSnapshot copies the +// volume file outright rather than leaving a backing-file pointer (the +// snapshot must survive the source volume's deletion), so this is the only place that link is recorded; it is memory-only like volumeMeta. +type snapshotMeta struct { + sourceVolumeID string +} + +// publication is a running qemu-nbd export: its socket, its node, and the PID +// captured from --pid-file so UnpublishVolume can stop it. qemu-nbd forks +// into the background on success (--fork), so this PID is the only handle this process retains on it. +type publication struct { + nodeID string + socketPath string + pid int +} + +// Provider is an EBSProvider backed by qcow2 files and qemu-nbd. One volume +// is one qcow2 file; a snapshot is an independent copy, not a chunk +// reference, so DeleteVolume never has to consult a snapshot the way viperblock does. +type Provider struct { + baseDir string + run runner + + mu sync.Mutex + published map[string]*publication + volumeMeta map[string]volumeMeta + snapshotMeta map[string]snapshotMeta +} + +var _ ebsprovider.EBSProvider = (*Provider)(nil) + +// NewProvider creates a Provider rooted at baseDir, creating its volumes, +// snapshots, sockets and tmp subdirectories if they do not already exist. +func NewProvider(baseDir string) (*Provider, error) { + return newProvider(baseDir, execRunner{}) +} + +func newProvider(baseDir string, run runner) (*Provider, error) { + if baseDir == "" { + return nil, fmt.Errorf("qemunbdd: base directory is required") + } + absBaseDir, err := filepath.Abs(baseDir) + if err != nil { + return nil, fmt.Errorf("qemunbdd: resolve base directory: %w", err) + } + for _, sub := range []string{"volumes", "snapshots", "sockets", "tmp"} { + if err := os.MkdirAll(filepath.Join(absBaseDir, sub), 0o750); err != nil { + return nil, fmt.Errorf("qemunbdd: create %s directory: %w", sub, err) + } + } + return &Provider{ + baseDir: absBaseDir, + run: run, + published: make(map[string]*publication), + volumeMeta: make(map[string]volumeMeta), + snapshotMeta: make(map[string]snapshotMeta), + }, nil +} + +func (p *Provider) volumePath(volumeID string) string { + return filepath.Join(p.baseDir, "volumes", volumeID+".qcow2") +} + +func (p *Provider) snapshotPath(snapshotID string) string { + return filepath.Join(p.baseDir, "snapshots", snapshotID+".qcow2") +} + +func (p *Provider) socketPath(volumeID string) string { + return filepath.Join(p.baseDir, "sockets", volumeID+".sock") +} + +func (p *Provider) pidFilePath(volumeID string) string { + return filepath.Join(p.baseDir, "sockets", volumeID+".pid") +} + +// checkVersion mirrors ebsprovider's own unexported check: the wire schema +// version must match exactly, since this provider does not translate between +// versions. +func checkVersion(version uint16) error { + if version != ebsprovider.SchemaVersion { + return fmt.Errorf("%w: got %d, want %d", ebsprovider.ErrUnsupportedVersion, version, ebsprovider.SchemaVersion) + } + return nil +} + +func validCapacityRange(cr ebsprovider.CapacityRange) bool { + return cr.RequiredBytes > 0 && (cr.LimitBytes == 0 || cr.RequiredBytes <= cr.LimitBytes) +} + +func (p *Provider) GetCapabilities(_ context.Context, req ebsprovider.GetCapabilitiesRequest) (*ebsprovider.GetCapabilitiesResponse, error) { + if err := checkVersion(req.SchemaVersion); err != nil { + return nil, err + } + return &ebsprovider.GetCapabilitiesResponse{Versioned: ebsprovider.NewVersioned(), Capabilities: capabilities}, nil +} + +// qemuImgInfo is the subset of `qemu-img info --output=json` this provider +// reads: the image's logical size, and the backing file a CoW clone points +// at, which stands in for a Go-side "created from snapshot X" record. +type qemuImgInfo struct { + VirtualSize int64 `json:"virtual-size"` + BackingFilename string `json:"backing-filename"` +} + +// imgInfo reads an image's metadata. --force-share is required because a +// published volume is held under qemu-nbd's write lock, and describing a +// volume must not depend on whether it is currently attached. +func (p *Provider) imgInfo(ctx context.Context, path string) (*qemuImgInfo, error) { + out, err := p.run.Run(ctx, "qemu-img", "info", "--output=json", "--force-share", path) + if err != nil { + return nil, fmt.Errorf("inspect %s: %w", path, err) + } + var info qemuImgInfo + if err := json.Unmarshal(out, &info); err != nil { + return nil, fmt.Errorf("parse qemu-img info for %s: %w", path, err) + } + return &info, nil +} + +func (p *Provider) CreateVolume(ctx context.Context, req ebsprovider.CreateVolumeRequest) (*ebsprovider.Volume, error) { + if err := checkVersion(req.SchemaVersion); err != nil { + return nil, err + } + if req.VolumeID == "" || !validCapacityRange(req.CapacityRange) { + return nil, fmt.Errorf("%w: invalid volume ID or capacity range", ebsprovider.ErrInvalidArgument) + } + if req.SourceSnapshotID != "" && req.SourceSnapshotVolumeID == "" { + return nil, fmt.Errorf("%w: source_snapshot_volume_id is required with source_snapshot_id", ebsprovider.ErrInvalidArgument) + } + if err := ebsprovider.ValidateSeedData(req.SeedData); err != nil { + return nil, err + } + if int64(len(req.SeedData)) > req.CapacityRange.RequiredBytes { + return nil, fmt.Errorf("%w: seed data is larger than the requested capacity", ebsprovider.ErrInvalidArgument) + } + + p.mu.Lock() + defer p.mu.Unlock() + + volPath := p.volumePath(req.VolumeID) + expectedBacking := "" + if req.SourceSnapshotID != "" { + expectedBacking = p.snapshotPath(req.SourceSnapshotID) + } + + if _, err := os.Stat(volPath); err == nil { + info, err := p.imgInfo(ctx, volPath) + if err != nil { + return nil, err + } + meta := p.volumeMeta[req.VolumeID] + if info.VirtualSize != req.CapacityRange.RequiredBytes || + meta.availabilityZone != req.AvailabilityZone || + info.BackingFilename != expectedBacking || + !bytes.Equal(meta.parameters, req.Parameters) { + return nil, fmt.Errorf("%w: volume %s", ebsprovider.ErrAlreadyExists, req.VolumeID) + } + return p.buildVolume(req.VolumeID, info.VirtualSize, meta.availabilityZone), nil + } else if !errors.Is(err, os.ErrNotExist) { + return nil, err + } + + sizeArg := strconv.FormatInt(req.CapacityRange.RequiredBytes, 10) + switch { + case req.SourceSnapshotID != "": + snapPath := p.snapshotPath(req.SourceSnapshotID) + snapInfo, err := p.statSnapshot(ctx, snapPath) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("%w: snapshot %s", ebsprovider.ErrNotFound, req.SourceSnapshotID) + } + return nil, err + } + if req.CapacityRange.RequiredBytes < snapInfo.VirtualSize { + return nil, fmt.Errorf("%w: requested capacity is smaller than snapshot %s", ebsprovider.ErrInvalidArgument, req.SourceSnapshotID) + } + // Only refuse an origin we can disprove: a snapshot file left by an + // earlier process has no snapshotMeta entry, so its origin is unknown + // rather than mismatched. + if snapMeta, ok := p.snapshotMeta[req.SourceSnapshotID]; ok && snapMeta.sourceVolumeID != req.SourceSnapshotVolumeID { + return nil, fmt.Errorf("%w: snapshot %s was taken from volume %q, not %q", + ebsprovider.ErrInvalidArgument, req.SourceSnapshotID, snapMeta.sourceVolumeID, req.SourceSnapshotVolumeID) + } + if _, err := p.run.Run(ctx, "qemu-img", "create", "-f", "qcow2", "-b", snapPath, "-F", "qcow2", volPath, sizeArg); err != nil { + return nil, fmt.Errorf("create volume %s from snapshot %s: %w", req.VolumeID, req.SourceSnapshotID, err) + } + + default: + if _, err := p.run.Run(ctx, "qemu-img", "create", "-f", "qcow2", volPath, sizeArg); err != nil { + return nil, fmt.Errorf("create volume %s: %w", req.VolumeID, err) + } + } + + if err := p.writeSeed(ctx, volPath, req.SeedData); err != nil { + _ = os.Remove(volPath) + return nil, err + } + + p.volumeMeta[req.VolumeID] = volumeMeta{availabilityZone: req.AvailabilityZone, parameters: bytes.Clone(req.Parameters)} + return p.buildVolume(req.VolumeID, req.CapacityRange.RequiredBytes, req.AvailabilityZone), nil +} + +// statSnapshot resolves os.ErrNotExist through os.Stat before shelling out, +// so a missing source snapshot is reported without invoking qemu-img on a +// path that cannot exist. +func (p *Provider) statSnapshot(ctx context.Context, snapPath string) (*qemuImgInfo, error) { + if _, err := os.Stat(snapPath); err != nil { + return nil, err + } + return p.imgInfo(ctx, snapPath) +} + +// writeSeed stages SeedData to a temporary file and writes it at offset 0 +// with qemu-io. qemu-img has no operation for arbitrary byte writes, and +// qemu-io's write command reads its buffer from a file (-s), not an inline argument, so the bytes must round-trip through disk first. +func (p *Provider) writeSeed(ctx context.Context, volPath string, seed []byte) error { + if len(seed) == 0 { + return nil + } + tmp, err := os.CreateTemp(filepath.Join(p.baseDir, "tmp"), "seed-*.bin") + if err != nil { + return fmt.Errorf("stage seed data: %w", err) + } + tmpPath := tmp.Name() + defer os.Remove(tmpPath) + if _, err := tmp.Write(seed); err != nil { + _ = tmp.Close() + return fmt.Errorf("stage seed data: %w", err) + } + if err := tmp.Close(); err != nil { + return fmt.Errorf("stage seed data: %w", err) + } + writeCmd := fmt.Sprintf("write -s %s 0 %d", tmpPath, len(seed)) + if _, err := p.run.Run(ctx, "qemu-io", "-f", "qcow2", "-c", writeCmd, volPath); err != nil { + return fmt.Errorf("write seed data: %w", err) + } + return nil +} + +// buildVolume must be called with p.mu held: it consults the published map. +func (p *Provider) buildVolume(volumeID string, size int64, az string) *ebsprovider.Volume { + state := ebsprovider.VolumeStateAvailable + if _, ok := p.published[volumeID]; ok { + state = ebsprovider.VolumeStateInUse + } + return &ebsprovider.Volume{ + ID: volumeID, + CapacityBytes: size, + State: state, + Handle: p.volumePath(volumeID), + AvailabilityZone: az, + } +} + +func (p *Provider) GetVolume(ctx context.Context, req ebsprovider.GetVolumeRequest) (*ebsprovider.Volume, error) { + if err := checkVersion(req.SchemaVersion); err != nil { + return nil, err + } + if req.VolumeID == "" { + return nil, fmt.Errorf("%w: volume ID is required", ebsprovider.ErrInvalidArgument) + } + volPath := p.volumePath(req.VolumeID) + if _, err := os.Stat(volPath); err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("%w: volume %s", ebsprovider.ErrNotFound, req.VolumeID) + } + return nil, err + } + if req.Handle != "" && req.Handle != volPath { + return nil, fmt.Errorf("%w: volume %s", ebsprovider.ErrNotFound, req.VolumeID) + } + info, err := p.imgInfo(ctx, volPath) + if err != nil { + return nil, err + } + + p.mu.Lock() + defer p.mu.Unlock() + meta := p.volumeMeta[req.VolumeID] + return p.buildVolume(req.VolumeID, info.VirtualSize, meta.availabilityZone), nil +} + +// ListVolumes reads the volumes directory rather than the in-memory maps. +// The qcow2 files are what actually exists: volumeMeta does not survive a +// restart, and a volume the provider has forgotten is exactly what a caller +// enumerating is looking for. +func (p *Provider) ListVolumes(_ context.Context, req ebsprovider.ListVolumesRequest) (*ebsprovider.ListVolumesResponse, error) { + if err := checkVersion(req.SchemaVersion); err != nil { + return nil, err + } + entries, err := os.ReadDir(filepath.Join(p.baseDir, "volumes")) + if err != nil { + return nil, fmt.Errorf("qemunbdd: read volumes directory: %w", err) + } + + pageSize := int(req.PageSize()) + response := &ebsprovider.ListVolumesResponse{Versioned: ebsprovider.NewVersioned()} + // ReadDir returns entries sorted by filename, so resuming after the token + // walks the same order the previous page ended on. + for _, entry := range entries { + volumeID := strings.TrimSuffix(entry.Name(), ".qcow2") + if entry.IsDir() || volumeID == entry.Name() || volumeID <= req.StartingToken { + continue + } + if len(response.Volumes) == pageSize { + response.NextToken = response.Volumes[pageSize-1].ID + break + } + response.Volumes = append(response.Volumes, ebsprovider.VolumeRef{ + ID: volumeID, + Handle: p.volumePath(volumeID), + }) + } + return response, nil +} + +func (p *Provider) ExpandVolume(ctx context.Context, req ebsprovider.ExpandVolumeRequest) (*ebsprovider.Volume, error) { + if err := checkVersion(req.SchemaVersion); err != nil { + return nil, err + } + if req.VolumeID == "" || !validCapacityRange(req.CapacityRange) { + return nil, fmt.Errorf("%w: invalid volume ID or capacity range", ebsprovider.ErrInvalidArgument) + } + + p.mu.Lock() + defer p.mu.Unlock() + + volPath := p.volumePath(req.VolumeID) + if _, err := os.Stat(volPath); err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("%w: volume %s", ebsprovider.ErrNotFound, req.VolumeID) + } + return nil, err + } + if req.Handle != "" && req.Handle != volPath { + return nil, fmt.Errorf("%w: volume %s", ebsprovider.ErrNotFound, req.VolumeID) + } + info, err := p.imgInfo(ctx, volPath) + if err != nil { + return nil, err + } + if req.CapacityRange.RequiredBytes < info.VirtualSize { + return nil, fmt.Errorf("%w: volume expansion is grow-only", ebsprovider.ErrInvalidArgument) + } + // ebsprovider/errors.go has no failed-precondition sentinel, so a + // published volume this provider cannot expand online is reported as + // ErrVolumeInUse, matching memory.go's identical refusal. + if _, published := p.published[req.VolumeID]; published && req.CapacityRange.RequiredBytes > info.VirtualSize { + return nil, fmt.Errorf("%w: provider does not support online expansion", ebsprovider.ErrVolumeInUse) + } + if req.CapacityRange.RequiredBytes != info.VirtualSize { + if _, err := p.run.Run(ctx, "qemu-img", "resize", volPath, strconv.FormatInt(req.CapacityRange.RequiredBytes, 10)); err != nil { + return nil, fmt.Errorf("expand volume %s: %w", req.VolumeID, err) + } + } + meta := p.volumeMeta[req.VolumeID] + return p.buildVolume(req.VolumeID, req.CapacityRange.RequiredBytes, meta.availabilityZone), nil +} + +func (p *Provider) DeleteVolume(_ context.Context, req ebsprovider.DeleteVolumeRequest) error { + if err := checkVersion(req.SchemaVersion); err != nil { + return err + } + if req.VolumeID == "" { + return fmt.Errorf("%w: volume ID is required", ebsprovider.ErrInvalidArgument) + } + + p.mu.Lock() + defer p.mu.Unlock() + + volPath := p.volumePath(req.VolumeID) + if _, err := os.Stat(volPath); err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } + return err + } + if req.Handle != "" && req.Handle != volPath { + return nil + } + if _, published := p.published[req.VolumeID]; published { + return fmt.Errorf("%w: volume %s", ebsprovider.ErrVolumeInUse, req.VolumeID) + } + if err := os.Remove(volPath); err != nil { + return fmt.Errorf("delete volume %s: %w", req.VolumeID, err) + } + delete(p.volumeMeta, req.VolumeID) + return nil +} + +func (p *Provider) buildSnapshot(ctx context.Context, snapshotID, sourceVolumeID, path string) (*ebsprovider.Snapshot, error) { + info, err := p.imgInfo(ctx, path) + if err != nil { + return nil, err + } + fi, err := os.Stat(path) + if err != nil { + return nil, fmt.Errorf("stat snapshot %s: %w", snapshotID, err) + } + return &ebsprovider.Snapshot{ + ID: snapshotID, + SourceVolumeID: sourceVolumeID, + SizeBytes: info.VirtualSize, + CreatedAt: fi.ModTime().UTC(), + State: ebsprovider.SnapshotStateCompleted, + Handle: path, + }, nil +} + +func (p *Provider) CreateSnapshot(ctx context.Context, req ebsprovider.CreateSnapshotRequest) (*ebsprovider.Snapshot, error) { + if err := checkVersion(req.SchemaVersion); err != nil { + return nil, err + } + if req.SnapshotID == "" || req.VolumeID == "" { + return nil, fmt.Errorf("%w: snapshot and volume IDs are required", ebsprovider.ErrInvalidArgument) + } + + p.mu.Lock() + defer p.mu.Unlock() + + snapPath := p.snapshotPath(req.SnapshotID) + if meta, ok := p.snapshotMeta[req.SnapshotID]; ok { + if meta.sourceVolumeID != req.VolumeID { + return nil, fmt.Errorf("%w: snapshot %s", ebsprovider.ErrAlreadyExists, req.SnapshotID) + } + return p.buildSnapshot(ctx, req.SnapshotID, meta.sourceVolumeID, snapPath) + } + // A file on disk with no matching snapshotMeta entry means the provider + // restarted since it was created: the source-volume link lived only in + // memory and cannot be recovered, so the ID is refused rather than silently overwritten or accepted without provenance. + if _, err := os.Stat(snapPath); err == nil { + return nil, fmt.Errorf("%w: snapshot %s", ebsprovider.ErrAlreadyExists, req.SnapshotID) + } else if !errors.Is(err, os.ErrNotExist) { + return nil, err + } + + volPath := p.volumePath(req.VolumeID) + if _, err := os.Stat(volPath); err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("%w: volume %s", ebsprovider.ErrNotFound, req.VolumeID) + } + return nil, err + } + if req.VolumeHandle != "" && req.VolumeHandle != volPath { + return nil, fmt.Errorf("%w: volume %s", ebsprovider.ErrNotFound, req.VolumeID) + } + + if _, err := p.run.Run(ctx, "qemu-img", "convert", "-O", "qcow2", volPath, snapPath); err != nil { + return nil, fmt.Errorf("create snapshot %s: %w", req.SnapshotID, err) + } + p.snapshotMeta[req.SnapshotID] = snapshotMeta{sourceVolumeID: req.VolumeID} + return p.buildSnapshot(ctx, req.SnapshotID, req.VolumeID, snapPath) +} + +// ListSnapshots walks the snapshots directory rather than snapshotMeta, so a +// snapshot that outlived the process still appears. Its SourceVolumeID comes +// back empty in that case: the link was only ever in memory. +func (p *Provider) ListSnapshots(_ context.Context, req ebsprovider.ListSnapshotsRequest) (*ebsprovider.ListSnapshotsResponse, error) { + if err := checkVersion(req.SchemaVersion); err != nil { + return nil, err + } + entries, err := os.ReadDir(filepath.Join(p.baseDir, "snapshots")) + if err != nil { + return nil, fmt.Errorf("qemunbdd: read snapshots directory: %w", err) + } + + p.mu.Lock() + defer p.mu.Unlock() + + pageSize := int(req.PageSize()) + response := &ebsprovider.ListSnapshotsResponse{Versioned: ebsprovider.NewVersioned()} + // ReadDir returns entries sorted by filename, so resuming after the token + // walks the same order the previous page ended on. + for _, entry := range entries { + snapshotID := strings.TrimSuffix(entry.Name(), ".qcow2") + if entry.IsDir() || snapshotID == entry.Name() || snapshotID <= req.StartingToken { + continue + } + if len(response.Snapshots) == pageSize { + response.NextToken = response.Snapshots[pageSize-1].ID + break + } + response.Snapshots = append(response.Snapshots, ebsprovider.SnapshotRef{ + ID: snapshotID, + SourceVolumeID: p.snapshotMeta[snapshotID].sourceVolumeID, + Handle: p.snapshotPath(snapshotID), + }) + } + return response, nil +} + +// snapshotInUse reports whether any volume file still points at snapPath as +// its qcow2 backing file, the same relationship CreateVolume establishes for +// a SourceSnapshotID clone. +func (p *Provider) snapshotInUse(ctx context.Context, snapPath string) (bool, error) { + volumesDir := filepath.Join(p.baseDir, "volumes") + entries, err := os.ReadDir(volumesDir) + if err != nil { + return false, fmt.Errorf("list volumes: %w", err) + } + for _, entry := range entries { + if entry.IsDir() { + continue + } + info, err := p.imgInfo(ctx, filepath.Join(volumesDir, entry.Name())) + if err != nil { + return false, err + } + if info.BackingFilename == snapPath { + return true, nil + } + } + return false, nil +} + +func (p *Provider) DeleteSnapshot(ctx context.Context, req ebsprovider.DeleteSnapshotRequest) error { + if err := checkVersion(req.SchemaVersion); err != nil { + return err + } + if req.SnapshotID == "" { + return fmt.Errorf("%w: snapshot ID is required", ebsprovider.ErrInvalidArgument) + } + + p.mu.Lock() + defer p.mu.Unlock() + + snapPath := p.snapshotPath(req.SnapshotID) + if _, err := os.Stat(snapPath); err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } + return err + } + if req.Handle != "" && req.Handle != snapPath { + return nil + } + inUse, err := p.snapshotInUse(ctx, snapPath) + if err != nil { + return err + } + if inUse { + // Same sentinel-substitution as ExpandVolume/DeleteVolume: no + // failed-precondition error exists, so a snapshot a volume still + // backs onto is reported as ErrVolumeInUse. + return fmt.Errorf("%w: snapshot %s is a backing file for another volume", ebsprovider.ErrVolumeInUse, req.SnapshotID) + } + if err := os.Remove(snapPath); err != nil { + return fmt.Errorf("delete snapshot %s: %w", req.SnapshotID, err) + } + delete(p.snapshotMeta, req.SnapshotID) + return nil +} + +func (p *Provider) CopySnapshot(ctx context.Context, req ebsprovider.CopySnapshotRequest) (*ebsprovider.Snapshot, error) { + if err := checkVersion(req.SchemaVersion); err != nil { + return nil, err + } + if req.SourceSnapshotID == "" || req.DestinationSnapshotID == "" || req.VolumeID == "" { + return nil, fmt.Errorf("%w: source snapshot, destination snapshot, and volume IDs are required", ebsprovider.ErrInvalidArgument) + } + if req.SourceSnapshotID == req.DestinationSnapshotID { + return nil, fmt.Errorf("%w: destination snapshot must differ from the source", ebsprovider.ErrInvalidArgument) + } + + p.mu.Lock() + defer p.mu.Unlock() + + srcMeta, ok := p.snapshotMeta[req.SourceSnapshotID] + if !ok { + return nil, fmt.Errorf("%w: snapshot %s", ebsprovider.ErrNotFound, req.SourceSnapshotID) + } + if srcMeta.sourceVolumeID != req.VolumeID { + return nil, fmt.Errorf("%w: snapshot %s belongs to volume %s, not %s", ebsprovider.ErrInvalidArgument, req.SourceSnapshotID, srcMeta.sourceVolumeID, req.VolumeID) + } + dstPath := p.snapshotPath(req.DestinationSnapshotID) + if _, ok := p.snapshotMeta[req.DestinationSnapshotID]; ok { + return nil, fmt.Errorf("%w: snapshot %s", ebsprovider.ErrAlreadyExists, req.DestinationSnapshotID) + } + // A stray file with no metadata entry is the same restart-orphan case + // CreateSnapshot refuses: without provenance it cannot be told apart + // from a legitimate destination, so it is treated as already existing. + if _, err := os.Stat(dstPath); err == nil { + return nil, fmt.Errorf("%w: snapshot %s", ebsprovider.ErrAlreadyExists, req.DestinationSnapshotID) + } else if !errors.Is(err, os.ErrNotExist) { + return nil, err + } + + srcPath := p.snapshotPath(req.SourceSnapshotID) + if err := copyFile(srcPath, dstPath); err != nil { + return nil, fmt.Errorf("copy snapshot %s to %s: %w", req.SourceSnapshotID, req.DestinationSnapshotID, err) + } + p.snapshotMeta[req.DestinationSnapshotID] = snapshotMeta{sourceVolumeID: req.VolumeID} + return p.buildSnapshot(ctx, req.DestinationSnapshotID, req.VolumeID, dstPath) +} + +// copyFile duplicates a snapshot file directly: CopySnapshot's job is to +// produce a second, independent file, not a CoW clone, so this is plain +// filesystem I/O and never touches the runner. +func copyFile(srcPath, dstPath string) error { + src, err := os.Open(srcPath) + if err != nil { + return err + } + defer src.Close() + dst, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return err + } + defer dst.Close() + if _, err := io.Copy(dst, src); err != nil { + return err + } + return dst.Close() +} + +func (p *Provider) PublishVolume(ctx context.Context, req ebsprovider.PublishVolumeRequest) (*ebsprovider.PublishedVolume, error) { + if err := checkVersion(req.SchemaVersion); err != nil { + return nil, err + } + if req.VolumeID == "" || req.NodeID == "" { + return nil, fmt.Errorf("%w: volume and node IDs are required", ebsprovider.ErrInvalidArgument) + } + + p.mu.Lock() + defer p.mu.Unlock() + + volPath := p.volumePath(req.VolumeID) + if _, err := os.Stat(volPath); err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("%w: volume %s", ebsprovider.ErrNotFound, req.VolumeID) + } + return nil, err + } + if req.Handle != "" && req.Handle != volPath { + return nil, fmt.Errorf("%w: volume %s", ebsprovider.ErrNotFound, req.VolumeID) + } + if pub, ok := p.published[req.VolumeID]; ok { + if pub.nodeID != req.NodeID { + return nil, fmt.Errorf("%w: volume %s is published to %s", ebsprovider.ErrVolumeInUse, req.VolumeID, pub.nodeID) + } + return &ebsprovider.PublishedVolume{VolumeID: req.VolumeID, NodeID: pub.nodeID, NBDURI: utils.FormatNBDSocketURI(pub.socketPath)}, nil + } + + sockPath := p.socketPath(req.VolumeID) + if len(sockPath) >= maxUnixSocketPath { + return nil, fmt.Errorf("%w: socket path %s is %d bytes, over the %d-byte kernel limit", + ebsprovider.ErrInvalidArgument, sockPath, len(sockPath), maxUnixSocketPath) + } + pidPath := p.pidFilePath(req.VolumeID) + _ = os.Remove(sockPath) + _ = os.Remove(pidPath) + + args := []string{ + "--socket", sockPath, + "--format", "qcow2", + "--persistent", + "--shared=" + strconv.Itoa(nbdSharedClients), + "--fork", + "--pid-file", pidPath, + } + if req.ReadOnly { + args = append(args, "-r") + } + args = append(args, volPath) + if _, err := p.run.Run(ctx, "qemu-nbd", args...); err != nil { + return nil, fmt.Errorf("publish volume %s: %w", req.VolumeID, err) + } + + pid := readPID(pidPath) + p.published[req.VolumeID] = &publication{nodeID: req.NodeID, socketPath: sockPath, pid: pid} + return &ebsprovider.PublishedVolume{VolumeID: req.VolumeID, NodeID: req.NodeID, NBDURI: utils.FormatNBDSocketURI(sockPath)}, nil +} + +// readPID best-effort reads the PID qemu-nbd's --fork wrote to pidPath. A +// missing or unparsable file (as in tests, where no real qemu-nbd runs) +// yields 0, meaning UnpublishVolume will skip signaling a process. +func readPID(pidPath string) int { + data, err := os.ReadFile(pidPath) + if err != nil { + return 0 + } + pid, err := strconv.Atoi(string(bytes.TrimSpace(data))) + if err != nil { + return 0 + } + return pid +} + +func (p *Provider) UnpublishVolume(_ context.Context, req ebsprovider.UnpublishVolumeRequest) error { + if err := checkVersion(req.SchemaVersion); err != nil { + return err + } + if req.VolumeID == "" || req.NodeID == "" { + return fmt.Errorf("%w: volume and node IDs are required", ebsprovider.ErrInvalidArgument) + } + + p.mu.Lock() + defer p.mu.Unlock() + + pub, ok := p.published[req.VolumeID] + if !ok || pub.nodeID != req.NodeID { + return nil + } + if pub.pid > 0 { + if proc, err := os.FindProcess(pub.pid); err == nil { + _ = proc.Signal(syscall.SIGTERM) + } + } + _ = os.Remove(pub.socketPath) + _ = os.Remove(p.pidFilePath(req.VolumeID)) + delete(p.published, req.VolumeID) + return nil +} diff --git a/spinifex/services/qemunbdd/provider_test.go b/spinifex/services/qemunbdd/provider_test.go new file mode 100644 index 000000000..cc46ce31f --- /dev/null +++ b/spinifex/services/qemunbdd/provider_test.go @@ -0,0 +1,662 @@ +package qemunbdd + +import ( + "context" + "os" + "path/filepath" + "sync" + "testing" + + "github.com/mulgadc/spinifex/spinifex/ebsprovider" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// call records one invocation the fake runner captured, so tests can assert +// the exact argv a provider method built. +type call struct { + name string + args []string +} + +// fakeRunner stands in for qemu-img/qemu-io/qemu-nbd. It mimics the side +// effect a real "create" or "convert" has (leaving a file behind) so +// existence checks with os.Stat behave the same as against a real qemu-img, without needing qemu installed. +type fakeRunner struct { + mu sync.Mutex + calls []call + + // infoResponses maps a path to the canned `qemu-img info --output=json` + // body returned when that path is queried. + infoResponses map[string]string + + // failOn, when non-empty, makes Run return err for every call whose + // command name matches. + failOn string + err error +} + +func newFakeRunner() *fakeRunner { + return &fakeRunner{infoResponses: make(map[string]string)} +} + +func (f *fakeRunner) Run(_ context.Context, name string, args ...string) ([]byte, error) { + f.mu.Lock() + f.calls = append(f.calls, call{name: name, args: append([]string(nil), args...)}) + f.mu.Unlock() + + if f.err != nil && name == f.failOn { + return nil, f.err + } + + if name == "qemu-img" && len(args) > 0 { + switch args[0] { + case "create": + // The size argument is always last; the target path is the + // element before it. + _ = os.WriteFile(args[len(args)-2], nil, 0o640) + case "convert": + _ = os.WriteFile(args[len(args)-1], nil, 0o640) + case "info": + path := args[len(args)-1] + f.mu.Lock() + resp, ok := f.infoResponses[path] + f.mu.Unlock() + if ok { + return []byte(resp), nil + } + return []byte(`{"virtual-size":0}`), nil + } + } + return []byte("ok"), nil +} + +func (f *fakeRunner) callsSnapshot() []call { + f.mu.Lock() + defer f.mu.Unlock() + return append([]call(nil), f.calls...) +} + +func newTestProvider(t *testing.T) (*Provider, *fakeRunner) { + t.Helper() + fr := newFakeRunner() + p, err := newProvider(t.TempDir(), fr) + require.NoError(t, err) + return p, fr +} + +func TestNewProvider_CreatesDirectories(t *testing.T) { + dir := t.TempDir() + _, err := newProvider(dir, newFakeRunner()) + require.NoError(t, err) + for _, sub := range []string{"volumes", "snapshots", "sockets", "tmp"} { + info, err := os.Stat(filepath.Join(dir, sub)) + require.NoError(t, err) + assert.True(t, info.IsDir()) + } +} + +func TestNewProvider_RequiresBaseDir(t *testing.T) { + _, err := newProvider("", newFakeRunner()) + require.Error(t, err) +} + +func versioned() ebsprovider.Versioned { return ebsprovider.NewVersioned() } + +func TestCreateVolume_DefaultArgv(t *testing.T) { + p, fr := newTestProvider(t) + ctx := context.Background() + + vol, err := p.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{ + Versioned: versioned(), + VolumeID: "vol-1", + CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}, + }) + require.NoError(t, err) + require.NotNil(t, vol) + assert.Equal(t, "vol-1", vol.ID) + assert.Equal(t, int64(1<<30), vol.CapacityBytes) + assert.Equal(t, ebsprovider.VolumeStateAvailable, vol.State) + assert.Equal(t, p.volumePath("vol-1"), vol.Handle) + + calls := fr.callsSnapshot() + require.Len(t, calls, 1) + assert.Equal(t, "qemu-img", calls[0].name) + assert.Equal(t, []string{"create", "-f", "qcow2", p.volumePath("vol-1"), "1073741824"}, calls[0].args) +} + +func TestCreateVolume_FromSnapshotArgv(t *testing.T) { + p, fr := newTestProvider(t) + ctx := context.Background() + + snapPath := p.snapshotPath("snap-1") + require.NoError(t, os.WriteFile(snapPath, nil, 0o640)) + fr.infoResponses[snapPath] = `{"virtual-size":536870912}` + + vol, err := p.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{ + Versioned: versioned(), + VolumeID: "vol-clone", + CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}, + SourceSnapshotID: "snap-1", + SourceSnapshotVolumeID: "vol-origin", + }) + require.NoError(t, err) + require.NotNil(t, vol) + + volPath := p.volumePath("vol-clone") + calls := fr.callsSnapshot() + require.Len(t, calls, 2) + assert.Equal(t, []string{"info", "--output=json", "--force-share", snapPath}, calls[0].args) + assert.Equal(t, []string{"create", "-f", "qcow2", "-b", snapPath, "-F", "qcow2", volPath, "1073741824"}, calls[1].args) +} + +func TestCreateVolume_FromMissingSnapshotNotFound(t *testing.T) { + p, _ := newTestProvider(t) + _, err := p.CreateVolume(context.Background(), ebsprovider.CreateVolumeRequest{ + Versioned: versioned(), + VolumeID: "vol-orphan", + CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}, + SourceSnapshotID: "snap-missing", + SourceSnapshotVolumeID: "vol-origin", + }) + require.ErrorIs(t, err, ebsprovider.ErrNotFound) +} + +// TestCreateVolume_SnapshotSourceRequiresOriginVolume covers the contract rule +// that a snapshot source must name the volume it was taken from, so a provider +// needing it to resolve blocks gets it rather than guessing. +func TestCreateVolume_SnapshotSourceRequiresOriginVolume(t *testing.T) { + p, _ := newTestProvider(t) + _, err := p.CreateVolume(context.Background(), ebsprovider.CreateVolumeRequest{ + Versioned: versioned(), + VolumeID: "vol-noorigin", + CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}, + SourceSnapshotID: "snap-any", + }) + require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument) +} + +func TestCreateVolume_IdempotentRepeatSameParameters(t *testing.T) { + p, fr := newTestProvider(t) + ctx := context.Background() + req := ebsprovider.CreateVolumeRequest{ + Versioned: versioned(), + VolumeID: "vol-idem", + CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}, + } + + first, err := p.CreateVolume(ctx, req) + require.NoError(t, err) + fr.infoResponses[p.volumePath("vol-idem")] = `{"virtual-size":1073741824}` + + second, err := p.CreateVolume(ctx, req) + require.NoError(t, err) + assert.Equal(t, first, second) + + calls := fr.callsSnapshot() + require.Len(t, calls, 2, "the repeat must only inspect the file, not recreate it") + assert.Equal(t, "create", calls[0].args[0]) + assert.Equal(t, "info", calls[1].args[0]) +} + +func TestCreateVolume_ConflictingCapacityIsAlreadyExists(t *testing.T) { + p, fr := newTestProvider(t) + ctx := context.Background() + req := ebsprovider.CreateVolumeRequest{ + Versioned: versioned(), + VolumeID: "vol-conflict", + CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}, + } + _, err := p.CreateVolume(ctx, req) + require.NoError(t, err) + fr.infoResponses[p.volumePath("vol-conflict")] = `{"virtual-size":1073741824}` + + req2 := req + req2.CapacityRange.RequiredBytes = 2 << 30 + _, err = p.CreateVolume(ctx, req2) + require.ErrorIs(t, err, ebsprovider.ErrAlreadyExists) +} + +func TestCreateVolume_InvalidArgumentOnEmptyID(t *testing.T) { + p, _ := newTestProvider(t) + _, err := p.CreateVolume(context.Background(), ebsprovider.CreateVolumeRequest{ + Versioned: versioned(), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}, + }) + require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument) +} + +func TestCreateVolume_UnsupportedVersion(t *testing.T) { + p, _ := newTestProvider(t) + _, err := p.CreateVolume(context.Background(), ebsprovider.CreateVolumeRequest{ + VolumeID: "vol-noversion", CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}, + }) + require.ErrorIs(t, err, ebsprovider.ErrUnsupportedVersion) +} + +func TestCreateVolume_SeedDataWritesViaQemuIO(t *testing.T) { + p, fr := newTestProvider(t) + ctx := context.Background() + seed := []byte("firmware-vars-template") + + _, err := p.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{ + Versioned: versioned(), + VolumeID: "vol-seeded", + CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 4096}, + SeedData: seed, + }) + require.NoError(t, err) + + calls := fr.callsSnapshot() + require.Len(t, calls, 2) + assert.Equal(t, "qemu-img", calls[0].name) + seedCall := calls[1] + assert.Equal(t, "qemu-io", seedCall.name) + require.Len(t, seedCall.args, 5) + assert.Equal(t, []string{"-f", "qcow2"}, seedCall.args[0:2]) + assert.Equal(t, "-c", seedCall.args[2]) + assert.Contains(t, seedCall.args[3], "write -s ") + assert.Contains(t, seedCall.args[3], " 0 22") + assert.Equal(t, p.volumePath("vol-seeded"), seedCall.args[4]) +} + +func TestCreateVolume_SeedDataExceedsMax(t *testing.T) { + p, _ := newTestProvider(t) + _, err := p.CreateVolume(context.Background(), ebsprovider.CreateVolumeRequest{ + Versioned: versioned(), + VolumeID: "vol-seed-toobig", + CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}, + SeedData: make([]byte, ebsprovider.MaxSeedBytes+1), + }) + require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument) +} + +func TestCreateVolume_SeedDataExceedsCapacity(t *testing.T) { + p, _ := newTestProvider(t) + _, err := p.CreateVolume(context.Background(), ebsprovider.CreateVolumeRequest{ + Versioned: versioned(), + VolumeID: "vol-seed-overcap", + CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 512}, + SeedData: make([]byte, 4096), + }) + require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument) +} + +func TestGetVolume_Success(t *testing.T) { + p, fr := newTestProvider(t) + volPath := p.volumePath("vol-get") + require.NoError(t, os.WriteFile(volPath, nil, 0o640)) + fr.infoResponses[volPath] = `{"virtual-size":268435456}` + + vol, err := p.GetVolume(context.Background(), ebsprovider.GetVolumeRequest{Versioned: versioned(), VolumeID: "vol-get"}) + require.NoError(t, err) + assert.Equal(t, int64(268435456), vol.CapacityBytes) + assert.Equal(t, ebsprovider.VolumeStateAvailable, vol.State) +} + +func TestGetVolume_NotFound(t *testing.T) { + p, _ := newTestProvider(t) + _, err := p.GetVolume(context.Background(), ebsprovider.GetVolumeRequest{Versioned: versioned(), VolumeID: "vol-never-existed"}) + require.ErrorIs(t, err, ebsprovider.ErrNotFound) +} + +func TestGetVolume_InvalidArgumentOnEmptyID(t *testing.T) { + p, _ := newTestProvider(t) + _, err := p.GetVolume(context.Background(), ebsprovider.GetVolumeRequest{Versioned: versioned()}) + require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument) +} + +func TestExpandVolume_Argv(t *testing.T) { + p, fr := newTestProvider(t) + volPath := p.volumePath("vol-expand") + require.NoError(t, os.WriteFile(volPath, nil, 0o640)) + fr.infoResponses[volPath] = `{"virtual-size":1073741824}` + + vol, err := p.ExpandVolume(context.Background(), ebsprovider.ExpandVolumeRequest{ + Versioned: versioned(), VolumeID: "vol-expand", CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 2 << 30}, + }) + require.NoError(t, err) + assert.Equal(t, int64(2<<30), vol.CapacityBytes) + + calls := fr.callsSnapshot() + require.Len(t, calls, 2) + assert.Equal(t, []string{"resize", volPath, "2147483648"}, calls[1].args) +} + +func TestExpandVolume_ShrinkIsInvalidArgument(t *testing.T) { + p, fr := newTestProvider(t) + volPath := p.volumePath("vol-shrink") + require.NoError(t, os.WriteFile(volPath, nil, 0o640)) + fr.infoResponses[volPath] = `{"virtual-size":2147483648}` + + _, err := p.ExpandVolume(context.Background(), ebsprovider.ExpandVolumeRequest{ + Versioned: versioned(), VolumeID: "vol-shrink", CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}, + }) + require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument) +} + +func TestExpandVolume_PublishedRefusesOnlineExpansion(t *testing.T) { + p, fr := newTestProvider(t) + volPath := p.volumePath("vol-expand-inuse") + require.NoError(t, os.WriteFile(volPath, nil, 0o640)) + fr.infoResponses[volPath] = `{"virtual-size":1073741824}` + + _, err := p.PublishVolume(context.Background(), ebsprovider.PublishVolumeRequest{Versioned: versioned(), VolumeID: "vol-expand-inuse", NodeID: "node-1"}) + require.NoError(t, err) + + _, err = p.ExpandVolume(context.Background(), ebsprovider.ExpandVolumeRequest{ + Versioned: versioned(), VolumeID: "vol-expand-inuse", CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 2 << 30}, + }) + require.ErrorIs(t, err, ebsprovider.ErrVolumeInUse) +} + +func TestExpandVolume_NotFound(t *testing.T) { + p, _ := newTestProvider(t) + _, err := p.ExpandVolume(context.Background(), ebsprovider.ExpandVolumeRequest{ + Versioned: versioned(), VolumeID: "vol-never-existed", CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}, + }) + require.ErrorIs(t, err, ebsprovider.ErrNotFound) +} + +func TestDeleteVolume_Success(t *testing.T) { + p, _ := newTestProvider(t) + volPath := p.volumePath("vol-del") + require.NoError(t, os.WriteFile(volPath, nil, 0o640)) + + require.NoError(t, p.DeleteVolume(context.Background(), ebsprovider.DeleteVolumeRequest{Versioned: versioned(), VolumeID: "vol-del"})) + _, err := os.Stat(volPath) + require.True(t, os.IsNotExist(err)) +} + +func TestDeleteVolume_AbsentIsIdempotent(t *testing.T) { + p, _ := newTestProvider(t) + require.NoError(t, p.DeleteVolume(context.Background(), ebsprovider.DeleteVolumeRequest{Versioned: versioned(), VolumeID: "vol-never-existed"})) + // A second delete of the same, already-absent target must also succeed. + require.NoError(t, p.DeleteVolume(context.Background(), ebsprovider.DeleteVolumeRequest{Versioned: versioned(), VolumeID: "vol-never-existed"})) +} + +func TestDeleteVolume_PublishedRefuses(t *testing.T) { + p, fr := newTestProvider(t) + volPath := p.volumePath("vol-del-inuse") + require.NoError(t, os.WriteFile(volPath, nil, 0o640)) + fr.infoResponses[volPath] = `{"virtual-size":1073741824}` + + _, err := p.PublishVolume(context.Background(), ebsprovider.PublishVolumeRequest{Versioned: versioned(), VolumeID: "vol-del-inuse", NodeID: "node-1"}) + require.NoError(t, err) + + err = p.DeleteVolume(context.Background(), ebsprovider.DeleteVolumeRequest{Versioned: versioned(), VolumeID: "vol-del-inuse"}) + require.ErrorIs(t, err, ebsprovider.ErrVolumeInUse) +} + +func TestCreateSnapshot_Argv(t *testing.T) { + p, fr := newTestProvider(t) + volPath := p.volumePath("vol-snap-src") + require.NoError(t, os.WriteFile(volPath, nil, 0o640)) + + snap, err := p.CreateSnapshot(context.Background(), ebsprovider.CreateSnapshotRequest{Versioned: versioned(), SnapshotID: "snap-1", VolumeID: "vol-snap-src"}) + require.NoError(t, err) + assert.Equal(t, ebsprovider.SnapshotStateCompleted, snap.State) + assert.Equal(t, "vol-snap-src", snap.SourceVolumeID) + + calls := fr.callsSnapshot() + snapPath := p.snapshotPath("snap-1") + require.Len(t, calls, 2) + assert.Equal(t, []string{"convert", "-O", "qcow2", volPath, snapPath}, calls[0].args) + assert.Equal(t, []string{"info", "--output=json", "--force-share", snapPath}, calls[1].args) +} + +func TestCreateSnapshot_NotFoundSourceVolume(t *testing.T) { + p, _ := newTestProvider(t) + _, err := p.CreateSnapshot(context.Background(), ebsprovider.CreateSnapshotRequest{Versioned: versioned(), SnapshotID: "snap-orphan", VolumeID: "vol-never-existed"}) + require.ErrorIs(t, err, ebsprovider.ErrNotFound) +} + +func TestCreateSnapshot_IdempotentSameVolume(t *testing.T) { + p, _ := newTestProvider(t) + volPath := p.volumePath("vol-snap-idem") + require.NoError(t, os.WriteFile(volPath, nil, 0o640)) + + first, err := p.CreateSnapshot(context.Background(), ebsprovider.CreateSnapshotRequest{Versioned: versioned(), SnapshotID: "snap-idem", VolumeID: "vol-snap-idem"}) + require.NoError(t, err) + second, err := p.CreateSnapshot(context.Background(), ebsprovider.CreateSnapshotRequest{Versioned: versioned(), SnapshotID: "snap-idem", VolumeID: "vol-snap-idem"}) + require.NoError(t, err) + assert.Equal(t, first.SourceVolumeID, second.SourceVolumeID) +} + +func TestCreateSnapshot_ConflictingSourceVolumeIsAlreadyExists(t *testing.T) { + p, _ := newTestProvider(t) + volA := p.volumePath("vol-snap-a") + volB := p.volumePath("vol-snap-b") + require.NoError(t, os.WriteFile(volA, nil, 0o640)) + require.NoError(t, os.WriteFile(volB, nil, 0o640)) + + _, err := p.CreateSnapshot(context.Background(), ebsprovider.CreateSnapshotRequest{Versioned: versioned(), SnapshotID: "snap-conflict", VolumeID: "vol-snap-a"}) + require.NoError(t, err) + _, err = p.CreateSnapshot(context.Background(), ebsprovider.CreateSnapshotRequest{Versioned: versioned(), SnapshotID: "snap-conflict", VolumeID: "vol-snap-b"}) + require.ErrorIs(t, err, ebsprovider.ErrAlreadyExists) +} + +func TestDeleteSnapshot_Success(t *testing.T) { + p, _ := newTestProvider(t) + volPath := p.volumePath("vol-delsnap-src") + require.NoError(t, os.WriteFile(volPath, nil, 0o640)) + _, err := p.CreateSnapshot(context.Background(), ebsprovider.CreateSnapshotRequest{Versioned: versioned(), SnapshotID: "snap-delete-ok", VolumeID: "vol-delsnap-src"}) + require.NoError(t, err) + + require.NoError(t, p.DeleteSnapshot(context.Background(), ebsprovider.DeleteSnapshotRequest{Versioned: versioned(), SnapshotID: "snap-delete-ok"})) + _, err = os.Stat(p.snapshotPath("snap-delete-ok")) + require.True(t, os.IsNotExist(err)) +} + +func TestDeleteSnapshot_AbsentIsIdempotent(t *testing.T) { + p, _ := newTestProvider(t) + require.NoError(t, p.DeleteSnapshot(context.Background(), ebsprovider.DeleteSnapshotRequest{Versioned: versioned(), SnapshotID: "snap-never-existed"})) + require.NoError(t, p.DeleteSnapshot(context.Background(), ebsprovider.DeleteSnapshotRequest{Versioned: versioned(), SnapshotID: "snap-never-existed"})) +} + +func TestDeleteSnapshot_RefusedWhenBackingFileInUse(t *testing.T) { + p, fr := newTestProvider(t) + snapPath := p.snapshotPath("snap-backing") + require.NoError(t, os.WriteFile(snapPath, nil, 0o640)) + // The snapshotInUse scan lists every file under volumes/ and checks its + // reported backing-filename, so a bare marker file with a canned info + // response is enough to simulate a CoW clone without a real qemu-img. + clonePath := p.volumePath("vol-clone") + require.NoError(t, os.WriteFile(clonePath, nil, 0o640)) + fr.infoResponses[clonePath] = `{"virtual-size":1073741824,"backing-filename":"` + snapPath + `"}` + + err := p.DeleteSnapshot(context.Background(), ebsprovider.DeleteSnapshotRequest{Versioned: versioned(), SnapshotID: "snap-backing"}) + require.ErrorIs(t, err, ebsprovider.ErrVolumeInUse) + _, statErr := os.Stat(snapPath) + require.NoError(t, statErr, "a refused delete must leave the snapshot file in place") +} + +func TestCopySnapshot_Success(t *testing.T) { + p, _ := newTestProvider(t) + volPath := p.volumePath("vol-copysnap-src") + require.NoError(t, os.WriteFile(volPath, []byte("volume-bytes"), 0o640)) + _, err := p.CreateSnapshot(context.Background(), ebsprovider.CreateSnapshotRequest{Versioned: versioned(), SnapshotID: "snap-copysnap-src", VolumeID: "vol-copysnap-src"}) + require.NoError(t, err) + + copied, err := p.CopySnapshot(context.Background(), ebsprovider.CopySnapshotRequest{ + Versioned: versioned(), SourceSnapshotID: "snap-copysnap-src", DestinationSnapshotID: "snap-copysnap-dst", VolumeID: "vol-copysnap-src", + }) + require.NoError(t, err) + assert.Equal(t, "snap-copysnap-dst", copied.ID) + assert.Equal(t, "vol-copysnap-src", copied.SourceVolumeID) + + srcBytes, err := os.ReadFile(p.snapshotPath("snap-copysnap-src")) + require.NoError(t, err) + dstBytes, err := os.ReadFile(p.snapshotPath("snap-copysnap-dst")) + require.NoError(t, err) + assert.Equal(t, srcBytes, dstBytes) +} + +func TestCopySnapshot_NotFoundSource(t *testing.T) { + p, _ := newTestProvider(t) + _, err := p.CopySnapshot(context.Background(), ebsprovider.CopySnapshotRequest{ + Versioned: versioned(), SourceSnapshotID: "snap-missing", DestinationSnapshotID: "snap-dst", VolumeID: "vol-a", + }) + require.ErrorIs(t, err, ebsprovider.ErrNotFound) +} + +func TestCopySnapshot_AlreadyExistsDestination(t *testing.T) { + p, _ := newTestProvider(t) + volPath := p.volumePath("vol-copysnap-conflict") + require.NoError(t, os.WriteFile(volPath, nil, 0o640)) + _, err := p.CreateSnapshot(context.Background(), ebsprovider.CreateSnapshotRequest{Versioned: versioned(), SnapshotID: "snap-a", VolumeID: "vol-copysnap-conflict"}) + require.NoError(t, err) + _, err = p.CreateSnapshot(context.Background(), ebsprovider.CreateSnapshotRequest{Versioned: versioned(), SnapshotID: "snap-b", VolumeID: "vol-copysnap-conflict"}) + require.NoError(t, err) + + _, err = p.CopySnapshot(context.Background(), ebsprovider.CopySnapshotRequest{ + Versioned: versioned(), SourceSnapshotID: "snap-a", DestinationSnapshotID: "snap-b", VolumeID: "vol-copysnap-conflict", + }) + require.ErrorIs(t, err, ebsprovider.ErrAlreadyExists) +} + +func TestCopySnapshot_InvalidArgumentWhenIDsMatch(t *testing.T) { + p, _ := newTestProvider(t) + _, err := p.CopySnapshot(context.Background(), ebsprovider.CopySnapshotRequest{ + Versioned: versioned(), SourceSnapshotID: "snap-same", DestinationSnapshotID: "snap-same", VolumeID: "vol-a", + }) + require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument) +} + +func TestCopySnapshot_InvalidArgumentWrongOwner(t *testing.T) { + p, _ := newTestProvider(t) + volPath := p.volumePath("vol-owner") + require.NoError(t, os.WriteFile(volPath, nil, 0o640)) + _, err := p.CreateSnapshot(context.Background(), ebsprovider.CreateSnapshotRequest{Versioned: versioned(), SnapshotID: "snap-owned", VolumeID: "vol-owner"}) + require.NoError(t, err) + + _, err = p.CopySnapshot(context.Background(), ebsprovider.CopySnapshotRequest{ + Versioned: versioned(), SourceSnapshotID: "snap-owned", DestinationSnapshotID: "snap-owned-dst", VolumeID: "vol-foreign", + }) + require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument) +} + +func TestPublishVolume_ArgvReadWrite(t *testing.T) { + p, fr := newTestProvider(t) + volPath := p.volumePath("vol-pub") + require.NoError(t, os.WriteFile(volPath, nil, 0o640)) + + pub, err := p.PublishVolume(context.Background(), ebsprovider.PublishVolumeRequest{Versioned: versioned(), VolumeID: "vol-pub", NodeID: "node-1"}) + require.NoError(t, err) + assert.Equal(t, "vol-pub", pub.VolumeID) + assert.Equal(t, "node-1", pub.NodeID) + assert.Equal(t, "nbd+unix:///?socket="+p.socketPath("vol-pub"), pub.NBDURI) + + calls := fr.callsSnapshot() + require.Len(t, calls, 1) + assert.Equal(t, "qemu-nbd", calls[0].name) + assert.Equal(t, []string{ + "--socket", p.socketPath("vol-pub"), + "--format", "qcow2", + "--persistent", + "--shared=1", + "--fork", + "--pid-file", p.pidFilePath("vol-pub"), + volPath, + }, calls[0].args) + assert.NotContains(t, calls[0].args, "-r") +} + +func TestPublishVolume_ReadOnlyAddsFlag(t *testing.T) { + p, fr := newTestProvider(t) + volPath := p.volumePath("vol-pub-ro") + require.NoError(t, os.WriteFile(volPath, nil, 0o640)) + + _, err := p.PublishVolume(context.Background(), ebsprovider.PublishVolumeRequest{Versioned: versioned(), VolumeID: "vol-pub-ro", NodeID: "node-1", ReadOnly: true}) + require.NoError(t, err) + + calls := fr.callsSnapshot() + require.Len(t, calls, 1) + assert.Contains(t, calls[0].args, "-r") + // -r must precede the trailing volume path argument. + assert.Equal(t, "-r", calls[0].args[len(calls[0].args)-2]) +} + +func TestPublishVolume_IdempotentRepublishSameNode(t *testing.T) { + p, fr := newTestProvider(t) + volPath := p.volumePath("vol-pub-idem") + require.NoError(t, os.WriteFile(volPath, nil, 0o640)) + + first, err := p.PublishVolume(context.Background(), ebsprovider.PublishVolumeRequest{Versioned: versioned(), VolumeID: "vol-pub-idem", NodeID: "node-1"}) + require.NoError(t, err) + second, err := p.PublishVolume(context.Background(), ebsprovider.PublishVolumeRequest{Versioned: versioned(), VolumeID: "vol-pub-idem", NodeID: "node-1"}) + require.NoError(t, err) + assert.Equal(t, first, second) + + calls := fr.callsSnapshot() + require.Len(t, calls, 1, "a republish to the same node must not start a second qemu-nbd") +} + +func TestPublishVolume_ConflictDifferentNode(t *testing.T) { + p, _ := newTestProvider(t) + volPath := p.volumePath("vol-pub-conflict") + require.NoError(t, os.WriteFile(volPath, nil, 0o640)) + + _, err := p.PublishVolume(context.Background(), ebsprovider.PublishVolumeRequest{Versioned: versioned(), VolumeID: "vol-pub-conflict", NodeID: "node-1"}) + require.NoError(t, err) + _, err = p.PublishVolume(context.Background(), ebsprovider.PublishVolumeRequest{Versioned: versioned(), VolumeID: "vol-pub-conflict", NodeID: "node-2"}) + require.ErrorIs(t, err, ebsprovider.ErrVolumeInUse) +} + +func TestPublishVolume_NotFound(t *testing.T) { + p, _ := newTestProvider(t) + _, err := p.PublishVolume(context.Background(), ebsprovider.PublishVolumeRequest{Versioned: versioned(), VolumeID: "vol-never-existed", NodeID: "node-1"}) + require.ErrorIs(t, err, ebsprovider.ErrNotFound) +} + +func TestUnpublishVolume_Success(t *testing.T) { + p, _ := newTestProvider(t) + volPath := p.volumePath("vol-unpub") + require.NoError(t, os.WriteFile(volPath, nil, 0o640)) + _, err := p.PublishVolume(context.Background(), ebsprovider.PublishVolumeRequest{Versioned: versioned(), VolumeID: "vol-unpub", NodeID: "node-1"}) + require.NoError(t, err) + + require.NoError(t, p.UnpublishVolume(context.Background(), ebsprovider.UnpublishVolumeRequest{Versioned: versioned(), VolumeID: "vol-unpub", NodeID: "node-1"})) + _, err = os.Stat(p.socketPath("vol-unpub")) + require.True(t, os.IsNotExist(err)) +} + +func TestUnpublishVolume_AbsentIsIdempotent(t *testing.T) { + p, _ := newTestProvider(t) + require.NoError(t, p.UnpublishVolume(context.Background(), ebsprovider.UnpublishVolumeRequest{Versioned: versioned(), VolumeID: "vol-never-existed", NodeID: "node-1"})) +} + +func TestUnpublishVolume_RepeatIsIdempotent(t *testing.T) { + p, fr := newTestProvider(t) + volPath := p.volumePath("vol-unpub-repeat") + require.NoError(t, os.WriteFile(volPath, nil, 0o640)) + _, err := p.PublishVolume(context.Background(), ebsprovider.PublishVolumeRequest{Versioned: versioned(), VolumeID: "vol-unpub-repeat", NodeID: "node-1"}) + require.NoError(t, err) + + require.NoError(t, p.UnpublishVolume(context.Background(), ebsprovider.UnpublishVolumeRequest{Versioned: versioned(), VolumeID: "vol-unpub-repeat", NodeID: "node-1"})) + require.NoError(t, p.UnpublishVolume(context.Background(), ebsprovider.UnpublishVolumeRequest{Versioned: versioned(), VolumeID: "vol-unpub-repeat", NodeID: "node-1"})) + + // Republishing after unpublish must start a fresh qemu-nbd. + _, err = p.PublishVolume(context.Background(), ebsprovider.PublishVolumeRequest{Versioned: versioned(), VolumeID: "vol-unpub-repeat", NodeID: "node-1"}) + require.NoError(t, err) + calls := fr.callsSnapshot() + nbdCalls := 0 + for _, c := range calls { + if c.name == "qemu-nbd" { + nbdCalls++ + } + } + assert.Equal(t, 2, nbdCalls) +} + +func TestGetCapabilities_Fixed(t *testing.T) { + p, _ := newTestProvider(t) + resp, err := p.GetCapabilities(context.Background(), ebsprovider.GetCapabilitiesRequest{Versioned: versioned()}) + require.NoError(t, err) + assert.Equal(t, capabilities, resp.Capabilities) + assert.True(t, resp.Capabilities.CrashConsistentSnapshot) + assert.True(t, resp.Capabilities.VolumeSeeding) + assert.True(t, resp.Capabilities.SparseExtentReporting, "qemu-nbd advertises base:allocation") + assert.True(t, resp.Capabilities.ReadOnlyPublish, "PublishVolume passes -r to qemu-nbd") + assert.False(t, resp.Capabilities.OnlineExpansion) + assert.False(t, resp.Capabilities.OwnerRouting) +} diff --git a/spinifex/services/qemunbdd/runner.go b/spinifex/services/qemunbdd/runner.go new file mode 100644 index 000000000..e0100a4cf --- /dev/null +++ b/spinifex/services/qemunbdd/runner.go @@ -0,0 +1,30 @@ +// Package qemunbdd implements ebsprovider.EBSProvider over qcow2 files and +// qemu-nbd. It shares no code with viperblock: every operation shells out to +// qemu-img, qemu-io or qemu-nbd, and a volume's only state is its file. +package qemunbdd + +import ( + "context" + "fmt" + "os/exec" + "strings" +) + +// runner executes a host command and returns its combined output. Every +// qemu-img, qemu-io and qemu-nbd invocation goes through this seam so unit +// tests can assert argv without a qemu installation. +type runner interface { + Run(ctx context.Context, name string, args ...string) ([]byte, error) +} + +// execRunner is the production runner, shelling out via os/exec. +type execRunner struct{} + +func (execRunner) Run(ctx context.Context, name string, args ...string) ([]byte, error) { + cmd := exec.CommandContext(ctx, name, args...) + out, err := cmd.CombinedOutput() + if err != nil { + return out, fmt.Errorf("%s %s: %w: %s", name, strings.Join(args, " "), err, strings.TrimSpace(string(out))) + } + return out, nil +} diff --git a/spinifex/services/qemunbdd/service.go b/spinifex/services/qemunbdd/service.go new file mode 100644 index 000000000..9a1e02dcb --- /dev/null +++ b/spinifex/services/qemunbdd/service.go @@ -0,0 +1,102 @@ +package qemunbdd + +import ( + "context" + "fmt" + "log/slog" + "os" + "os/signal" + "syscall" + + "github.com/mulgadc/spinifex/spinifex/admin" + "github.com/mulgadc/spinifex/spinifex/ebsprovider/natsserve" + "github.com/mulgadc/spinifex/spinifex/utils" +) + +// serviceName roots this daemon's PID file: baseDir/qemunbd.pid. +var serviceName = "qemunbd" + +// Config is the qemunbdd service's startup configuration, mirroring +// viperblockd.Config's NATS and base-directory fields. NodeName scopes the +// natsserve PublishVolume/UnpublishVolume subjects to this node. +type Config struct { + BaseDir string + NatsHost string + NatsToken string + NatsCACert string + NodeName string + Debug bool +} + +// Service runs the qcow2/qemu-nbd EBSProvider behind natsserve. It and +// viperblockd both answer ebs.provider.v1.*, so only one of the two may be +// pointed at the same NATS cluster at a time. +type Service struct { + Config *Config +} + +func New(config any) (svc *Service, err error) { + cfg, ok := config.(*Config) + if !ok { + return nil, fmt.Errorf("invalid config type for qemunbdd service") + } + return &Service{Config: cfg}, nil +} + +func (svc *Service) Start() (int, error) { + if err := utils.WritePidFileTo(svc.Config.BaseDir, serviceName, os.Getpid()); err != nil { + return 0, fmt.Errorf("write pid file: %w", err) + } + if err := launchService(svc.Config); err != nil { + slog.Error("Failed to launch service", "err", err) + return 0, err + } + return os.Getpid(), nil +} + +func (svc *Service) Stop() (err error) { + return utils.StopProcessAt(svc.Config.BaseDir, serviceName) +} + +func (svc *Service) Status() (string, error) { + return utils.ServiceStatus(svc.Config.BaseDir, serviceName) +} + +func (svc *Service) Shutdown() (err error) { + return svc.Stop() +} + +func (svc *Service) Reload() (err error) { + return nil +} + +// launchService connects to NATS, roots a qcow2 provider at cfg.BaseDir and +// serves ebs.provider.v1.* until SIGINT/SIGTERM, then unsubscribes. It blocks +// for the life of the process. +func launchService(cfg *Config) error { + nc, err := utils.ConnectNATSWithRetry(admin.DialTarget(cfg.NatsHost), cfg.NatsToken, cfg.NatsCACert) + if err != nil { + return fmt.Errorf("connect to NATS: %w", err) + } + defer nc.Close() + + provider, err := NewProvider(cfg.BaseDir) + if err != nil { + return fmt.Errorf("construct qemunbd provider: %w", err) + } + + stop, err := natsserve.Serve(context.Background(), nc, provider, natsserve.Options{NodeID: cfg.NodeName}) + if err != nil { + return fmt.Errorf("serve ebs.provider.v1: %w", err) + } + + slog.Info("qemunbdd: waiting for EBS provider events", "node", cfg.NodeName) + + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) + <-sigChan + slog.Info("qemunbdd: shutting down gracefully...") + stop() + + return nil +} diff --git a/spinifex/services/qemunbdd/service_test.go b/spinifex/services/qemunbdd/service_test.go new file mode 100644 index 000000000..d7e4f9c67 --- /dev/null +++ b/spinifex/services/qemunbdd/service_test.go @@ -0,0 +1,86 @@ +package qemunbdd + +import ( + "encoding/json" + "testing" + "time" + + "github.com/mulgadc/spinifex/spinifex/ebsprovider" + "github.com/mulgadc/spinifex/spinifex/testutil" + "github.com/nats-io/nats.go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestLaunchService_ServesRealProviderOverNATS is launchService's one piece +// of real logic: it must root an actual qcow2 Provider at cfg.BaseDir and +// answer capabilities with that provider's own, not a stub's. +func TestLaunchService_ServesRealProviderOverNATS(t *testing.T) { + ns, client := testutil.StartTestNATS(t) + + cfg := &Config{ + NatsHost: ns.ClientURL(), + BaseDir: t.TempDir(), + NodeName: "test-node", + } + go func() { _ = launchService(cfg) }() + + reqBody, err := json.Marshal(ebsprovider.GetCapabilitiesRequest{Versioned: ebsprovider.NewVersioned()}) + require.NoError(t, err) + + var msg *nats.Msg + require.Eventually(t, func() bool { + m, reqErr := client.Request(ebsprovider.CapabilitiesSubject, reqBody, time.Second) + if reqErr != nil { + return false + } + msg = m + return true + }, 5*time.Second, 50*time.Millisecond, "qemunbdd never answered %s", ebsprovider.CapabilitiesSubject) + + var resp ebsprovider.GetCapabilitiesResponse + require.NoError(t, json.Unmarshal(msg.Data, &resp)) + require.Nil(t, resp.Error) + assert.Equal(t, capabilities, resp.Capabilities) +} + +// TestLaunchService_ScopesPublishToNodeID confirms cfg.NodeName reaches +// natsserve.Options.NodeID: PublishVolume must be reachable on this node's +// own subject, not the wildcard every-node form. +func TestLaunchService_ScopesPublishToNodeID(t *testing.T) { + ns, client := testutil.StartTestNATS(t) + + cfg := &Config{ + NatsHost: ns.ClientURL(), + BaseDir: t.TempDir(), + NodeName: "node-a", + } + go func() { _ = launchService(cfg) }() + + publishSubject, err := ebsprovider.PublishSubject(cfg.NodeName) + require.NoError(t, err) + + reqBody, err := json.Marshal(ebsprovider.PublishVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), + VolumeID: "vol-does-not-exist", + NodeID: cfg.NodeName, + }) + require.NoError(t, err) + + var msg *nats.Msg + require.Eventually(t, func() bool { + m, reqErr := client.Request(publishSubject, reqBody, time.Second) + if reqErr != nil { + return false + } + msg = m + return true + }, 5*time.Second, 50*time.Millisecond, "qemunbdd never answered %s", publishSubject) + + var resp ebsprovider.PublishVolumeResponse + require.NoError(t, json.Unmarshal(msg.Data, &resp)) + // The volume does not exist, so the request fails, but a reply at all + // proves the node-scoped subject (not the wildcard) is what's subscribed. + require.NotNil(t, resp.Error) + assert.Equal(t, ebsprovider.ErrorCodeNotFound, resp.Error.Code) +} diff --git a/spinifex/services/viperblockd/config_update_test.go b/spinifex/services/viperblockd/config_update_test.go index 5ffb09805..764193df0 100644 --- a/spinifex/services/viperblockd/config_update_test.go +++ b/spinifex/services/viperblockd/config_update_test.go @@ -35,7 +35,7 @@ func TestApplyConfigUpdate_GrowsVolumeSize(t *testing.T) { t.Parallel() vb := createTestVBWithState(t, "vol-apply-grow") - require.NoError(t, applyConfigUpdate(vb, configReq(t, "vol-apply-grow", 5))) + require.NoError(t, applyConfigUpdate(t.Context(), vb, configReq(t, "vol-apply-grow", 5))) assert.Equal(t, uint64(5), vb.VolumeConfig.VolumeMetadata.SizeGiB) assert.Equal(t, 5*gib, vb.VolumeSize) @@ -46,7 +46,7 @@ func TestApplyConfigUpdate_ShrinkKeepsSize(t *testing.T) { vb := createTestVBWithState(t, "vol-apply-shrink") vb.VolumeSize = 10 * gib - require.NoError(t, applyConfigUpdate(vb, configReq(t, "vol-apply-shrink", 2))) + require.NoError(t, applyConfigUpdate(t.Context(), vb, configReq(t, "vol-apply-shrink", 2))) // Config metadata applied, but VolumeSize is grow-only. assert.Equal(t, uint64(2), vb.VolumeConfig.VolumeMetadata.SizeGiB) @@ -58,7 +58,7 @@ func TestApplyConfigUpdate_InvalidVolumeConfig(t *testing.T) { vb := createTestVBWithState(t, "vol-apply-bad") req := types.EBSConfigUpdateRequest{Volume: "vol-apply-bad", VolumeConfig: json.RawMessage(`"not-an-object"`)} - err := applyConfigUpdate(vb, req) + err := applyConfigUpdate(t.Context(), vb, req) require.Error(t, err) assert.Contains(t, err.Error(), "unmarshal VolumeConfig") } diff --git a/spinifex/services/viperblockd/conformance_test.go b/spinifex/services/viperblockd/conformance_test.go new file mode 100644 index 000000000..d4885ef68 --- /dev/null +++ b/spinifex/services/viperblockd/conformance_test.go @@ -0,0 +1,124 @@ +package viperblockd + +// The contract suite run against the provider production actually uses, over +// a real predastore and a real nbdkit export. Until this existed viperblockd +// was judged only by a live cluster run, so CI's only external witness to the +// contract was qemunbdd. + +import ( + "os" + "os/exec" + "strconv" + "testing" + "time" + + "github.com/mulgadc/spinifex/spinifex/ebsprovider" + "github.com/mulgadc/spinifex/spinifex/ebsprovider/conformance" + "github.com/mulgadc/spinifex/spinifex/types" + testpredastore "github.com/mulgadc/spinifex/tests/fixtures/predastore" +) + +// pluginPathEnvVar names the built nbdkit plugin (viperblock's +// lib/nbdkit-viperblock-plugin.so, `make plugin` in that repo). It has no +// default: the path depends on where viperblock is checked out. +const pluginPathEnvVar = "VIPERBLOCK_NBDKIT_PLUGIN" + +// requireExportTools skips unless this host can actually publish an export. +// SPINIFEX_REQUIRE_CONFORMANCE_TOOLS turns the skip into a failure so a CI +// image that loses nbdkit cannot quietly stop running this suite. +func requireExportTools(t *testing.T) string { + t.Helper() + required := os.Getenv("SPINIFEX_REQUIRE_CONFORMANCE_TOOLS") != "" + fail := func(format string, args ...any) { + t.Helper() + if required { + t.Fatalf(format, args...) + } + t.Skipf(format, args...) + } + + if _, err := exec.LookPath("nbdkit"); err != nil { + fail("nbdkit not installed") + } + pluginPath := os.Getenv(pluginPathEnvVar) + if pluginPath == "" { + fail("%s is not set to the built nbdkit plugin", pluginPathEnvVar) + } + if _, err := os.Stat(pluginPath); err != nil { + fail("%s=%s is not readable: %v", pluginPathEnvVar, pluginPath, err) + } + return pluginPath +} + +// inProcessProvider stands two nodes of the provider handlers up over one +// embedded NATS and one predastore, and returns a client for them plus both +// node names. Everything is torn down with the test. +// +// Two nodes rather than one because the exclusion this provider advertises is +// cluster-scoped, and a single node cannot exercise it: the second publish +// would go to a subject nobody serves and time out instead of being refused. +// Both share the NATS the volume leases live in and the bucket the volumes +// live in, so a second opener is refused by the lease, not by the harness. +func inProcessProvider(t *testing.T) (func(t *testing.T) ebsprovider.EBSProvider, string, string) { + t.Helper() + pluginPath := requireExportTools(t) + + fixture := testpredastore.Start(t) + _, natsURL := setupEmbeddedNATS(t) + + newNode := func(name string) *Config { + cfg := setupTestConfig(t, natsURL) + cfg.S3Host = "https://" + fixture.Host + cfg.Bucket = testpredastore.DefaultBucket + cfg.Region = fixture.Region + cfg.AccessKey = fixture.AccessKey + cfg.SecretKey = fixture.SecretKey + cfg.PluginPath = pluginPath + cfg.NBDTransport = types.NBDTransportTCP + cfg.NodeName = name + // The lease owner is derived from NodeName, and setupTestConfig built + // the store before this function could set it. Rebuild it, or both + // nodes claim as the same owner and each reclaims the other's lease + // as if it were its own stale entry. + installTestVolumeLeases(t, cfg, natsURL) + return cfg + } + + first := newNode("node-1") + second := newNode("node-2") + nc := startProviderSubjects(t, first, natsURL) + startProviderSubjects(t, second, natsURL) + + return func(t *testing.T) ebsprovider.EBSProvider { + t.Helper() + return ebsprovider.NewNATSProvider(nc, 60*time.Second) + }, first.NodeName, second.NodeName +} + +// runPrefix names one run's volumes so it cannot meet an earlier run's +// leftovers: the predastore fixture's object store outlives a single test. +func runPrefix() string { + return "ci" + strconv.FormatInt(time.Now().UnixNano(), 10) + "-" +} + +// TestViperblockdConformance judges the provider handlers by the same suite +// MemoryProvider and qemunbdd answer to. +func TestViperblockdConformance(t *testing.T) { + newProvider, nodeID, otherNodeID := inProcessProvider(t) + conformance.RunSuiteWithConfig(t, newProvider, conformance.SuiteConfig{ + NamePrefix: runPrefix(), + NodeID: nodeID, + OtherNodeID: otherNodeID, + }) +} + +// TestViperblockdNBDClient checks the half of the boundary our own Go client +// cannot: whether the nbdkit export viperblockd publishes is usable by libnbd, +// which knows nothing about this codebase. +func TestViperblockdNBDClient(t *testing.T) { + newProvider, nodeID, _ := inProcessProvider(t) + conformance.RunNBDClientSuiteWithConfig(t, newProvider, conformance.NBDClientConfig{ + NodeID: nodeID, + VolumePrefix: "vol-" + runPrefix() + "nbd", + }) +} diff --git a/spinifex/services/viperblockd/create_local_state_test.go b/spinifex/services/viperblockd/create_local_state_test.go new file mode 100644 index 000000000..bcd952347 --- /dev/null +++ b/spinifex/services/viperblockd/create_local_state_test.go @@ -0,0 +1,46 @@ +package viperblockd + +import ( + "encoding/json" + "os" + "testing" + + "github.com/mulgadc/spinifex/spinifex/ebsprovider" + testpredastore "github.com/mulgadc/spinifex/tests/fixtures/predastore" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestProviderHandlers_CreateVolume_LeavesNoLocalState pins that a create +// leaves nothing behind on the node that served it. A state read prefers a +// local copy over the backend, so a leftover here answers describe as if the +// volume still existed after another node deleted it. +func TestProviderHandlers_CreateVolume_LeavesNoLocalState(t *testing.T) { + fixture := testpredastore.Start(t) + _, natsURL := setupEmbeddedNATS(t) + + cfg := setupTestConfig(t, natsURL) + cfg.S3Host = "https://" + fixture.Host + cfg.Bucket = testpredastore.DefaultBucket + cfg.Region = fixture.Region + cfg.AccessKey = fixture.AccessKey + cfg.SecretKey = fixture.SecretKey + nc := startProviderSubjects(t, cfg, natsURL) + + const volumeName = "vol-createlocalstate01" + body := marshalRequest(t, map[string]any{ + "schema_version": ebsprovider.SchemaVersion, + "volume_id": volumeName, + "capacity_range": map[string]any{"required_bytes": int64(1) << 30}, + }) + msg := requestProvider(t, nc, ebsprovider.CreateVolumeSubject, body) + + var resp ebsprovider.CreateVolumeResponse + require.NoError(t, json.Unmarshal(msg.Data, &resp)) + require.Nil(t, resp.Error) + + localPath, err := localVolumeDir(cfg.BaseDir, volumeName) + require.NoError(t, err) + _, err = os.Stat(localPath) + assert.True(t, os.IsNotExist(err), "create must leave no local state at %s, got err=%v", localPath, err) +} diff --git a/spinifex/handlers/ec2/volume/delete_prefix_test.go b/spinifex/services/viperblockd/delete_prefix_test.go similarity index 50% rename from spinifex/handlers/ec2/volume/delete_prefix_test.go rename to spinifex/services/viperblockd/delete_prefix_test.go index c2a213b56..c49960ef4 100644 --- a/spinifex/handlers/ec2/volume/delete_prefix_test.go +++ b/spinifex/services/viperblockd/delete_prefix_test.go @@ -1,4 +1,4 @@ -package handlers_ec2_volume +package viperblockd import ( "context" @@ -6,28 +6,13 @@ import ( "strings" "testing" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/mulgadc/spinifex/spinifex/config" + awssdk "github.com/aws/aws-sdk-go/aws" + awss3 "github.com/aws/aws-sdk-go/service/s3" "github.com/mulgadc/spinifex/spinifex/objectstore" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func newPrefixTestConfig() *config.Config { - return &config.Config{ - AZ: "ap-southeast-2a", - Predastore: config.PredastoreConfig{ - Bucket: "test-bucket", - Region: "ap-southeast-2", - Host: fakeS3Host, - AccessKey: "testkey", - SecretKey: "testsecret", - }, - WalDir: "/tmp/test-wal", - } -} - // racingDeleteStore answers every delete the way a backend does once a // concurrent sweep has already removed the key. type racingDeleteStore struct { @@ -37,7 +22,7 @@ type racingDeleteStore struct { err error } -func (r *racingDeleteStore) DeleteObject(_ context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) { +func (r *racingDeleteStore) DeleteObject(_ context.Context, _ *awss3.DeleteObjectInput) (*awss3.DeleteObjectOutput, error) { r.deletes++ return nil, r.err } @@ -46,9 +31,9 @@ func newPrefixTestStore(t *testing.T, keys ...string) *objectstore.MemoryObjectS t.Helper() store := objectstore.NewMemoryObjectStore() for _, key := range keys { - _, err := store.PutObject(context.Background(), &s3.PutObjectInput{ - Bucket: aws.String("test-bucket"), - Key: aws.String(key), + _, err := store.PutObject(context.Background(), &awss3.PutObjectInput{ + Bucket: awssdk.String("test-bucket"), + Key: awssdk.String(key), Body: strings.NewReader("chunk"), }) require.NoError(t, err) @@ -56,40 +41,37 @@ func newPrefixTestStore(t *testing.T, keys ...string) *objectstore.MemoryObjectS return store } -func TestDeleteS3PrefixTreatsMissingObjectAsDeleted(t *testing.T) { +func TestDeleteObjectPrefixTreatsMissingObjectAsDeleted(t *testing.T) { backing := newPrefixTestStore(t, "vol-abc/chunks/chunk.00000028.bin", "vol-abc/config.json") store := &racingDeleteStore{ ObjectStore: backing, err: &objectstore.NoSuchKeyError{Key: "vol-abc/chunks/chunk.00000028.bin"}, } - svc := NewVolumeServiceImplWithStore(newPrefixTestConfig(), store, nil) - err := svc.deleteS3Prefix(context.Background(), "vol-abc/") + err := deleteObjectPrefix(context.Background(), store, "test-bucket", "vol-abc/") require.NoError(t, err, "an object another sweep already removed must not fail the delete") assert.Equal(t, 2, store.deletes, "the sweep must continue past a missing object") } -func TestDeleteS3PrefixPropagatesBackendFailure(t *testing.T) { +func TestDeleteObjectPrefixPropagatesBackendFailure(t *testing.T) { backing := newPrefixTestStore(t, "vol-abc/chunks/chunk.00000028.bin") store := &racingDeleteStore{ObjectStore: backing, err: errors.New("connection refused")} - svc := NewVolumeServiceImplWithStore(newPrefixTestConfig(), store, nil) - err := svc.deleteS3Prefix(context.Background(), "vol-abc/") + err := deleteObjectPrefix(context.Background(), store, "test-bucket", "vol-abc/") require.Error(t, err) assert.Contains(t, err.Error(), "connection refused") } -func TestDeleteS3PrefixRemovesEveryObject(t *testing.T) { +func TestDeleteObjectPrefixRemovesEveryObject(t *testing.T) { store := newPrefixTestStore(t, "vol-abc/chunks/chunk.00000000.bin", "vol-abc/config.json") - svc := NewVolumeServiceImplWithStore(newPrefixTestConfig(), store, nil) - require.NoError(t, svc.deleteS3Prefix(context.Background(), "vol-abc/")) + require.NoError(t, deleteObjectPrefix(context.Background(), store, "test-bucket", "vol-abc/")) - out, err := store.ListObjectsV2(context.Background(), &s3.ListObjectsV2Input{ - Bucket: aws.String("test-bucket"), - Prefix: aws.String("vol-abc/"), + out, err := store.ListObjectsV2(context.Background(), &awss3.ListObjectsV2Input{ + Bucket: awssdk.String("test-bucket"), + Prefix: awssdk.String("vol-abc/"), }) require.NoError(t, err) assert.Empty(t, out.Contents) diff --git a/spinifex/services/viperblockd/describe_read_only_test.go b/spinifex/services/viperblockd/describe_read_only_test.go new file mode 100644 index 000000000..b81350b8e --- /dev/null +++ b/spinifex/services/viperblockd/describe_read_only_test.go @@ -0,0 +1,171 @@ +package viperblockd + +import ( + "crypto/rand" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/mulgadc/bluebottle/pkg/masterkey" + "github.com/mulgadc/viperblock/viperblock" + "github.com/mulgadc/viperblock/viperblock/backends/file" + "github.com/stretchr/testify/require" +) + +const describeTestVolumeSize = 64 << 20 + +// recordingS3 serves one volume's config.json and records every request it is +// asked for, so what a describe costs is asserted from the wire. +type recordingS3 struct { + *httptest.Server + + mu sync.Mutex + requests []string + conns map[net.Conn]struct{} +} + +func newRecordingS3(t *testing.T, configPath string, config []byte) *recordingS3 { + t.Helper() + r := &recordingS3{conns: map[net.Conn]struct{}{}} + r.Server = httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + r.mu.Lock() + r.requests = append(r.requests, req.Method+" "+req.URL.RequestURI()) + r.mu.Unlock() + + if req.Method == http.MethodGet && strings.HasSuffix(req.URL.Path, configPath) { + _, _ = w.Write(config) + return + } + // Lists and writes are answered rather than refused so a describe that + // issues them gets far enough for every assertion below to be reached. + // Refusing them would fail this test on the first one and leave the + // rest unproven. + if req.URL.Query().Get("list-type") == "2" { + w.Header().Set("Content-Type", "application/xml") + _, _ = w.Write([]byte(`test-bucket0false`)) + return + } + if req.Method == http.MethodPut { + return + } + w.WriteHeader(http.StatusNotFound) + })) + r.Config.ConnState = func(c net.Conn, state http.ConnState) { + if state != http.StateNew { + return + } + r.mu.Lock() + r.conns[c] = struct{}{} + r.mu.Unlock() + } + r.Start() + t.Cleanup(r.Close) + return r +} + +func (r *recordingS3) connCount() int { + r.mu.Lock() + defer r.mu.Unlock() + return len(r.conns) +} + +func (r *recordingS3) matching(pred func(string) bool) []string { + r.mu.Lock() + defer r.mu.Unlock() + var out []string + for _, req := range r.requests { + if pred(req) { + out = append(out, req) + } + } + return out +} + +// seedVolumeConfig writes a real encrypted volume's state with a real engine +// and returns the persisted config.json. Hand-rolling the envelope would test +// the test's idea of the format rather than viperblock's. +func seedVolumeConfig(t *testing.T, volume string, key *masterkey.Key) []byte { + t.Helper() + store := t.TempDir() + vb, err := viperblock.New(&viperblock.VB{ + VolumeName: volume, + VolumeSize: describeTestVolumeSize, + BaseDir: t.TempDir(), + MasterKey: key, + EncryptionEnabled: true, + }, "file", file.FileConfig{BaseDir: store, VolumeName: volume}) + require.NoError(t, err) + defer func() { + vb.StopChunkUploader() + vb.StopWALSyncer() + }() + require.NoError(t, vb.Backend.Init()) + require.NoError(t, vb.SaveState()) + + config, err := os.ReadFile(filepath.Join(store, volume, "config.json")) + require.NoError(t, err) + return config +} + +func testMasterKey(t *testing.T) *masterkey.Key { + t.Helper() + raw := make([]byte, 32) + _, err := rand.Read(raw) + require.NoError(t, err) + key, err := masterkey.New(raw) + require.NoError(t, err) + return key +} + +// TestDescribeVolume_ReadsWithoutOpeningTheVolume pins what a describe is +// allowed to cost. Opening an engine to answer it paid a reachability list and, +// on an encrypted volume, a PutObject of the volume's own config.json — a +// write issued by whichever worker took the request rather than by the node +// that owns the volume. +func TestDescribeVolume_ReadsWithoutOpeningTheVolume(t *testing.T) { + const volume = "vol-describeonly001" + key := testMasterKey(t) + server := newRecordingS3(t, "/"+volume+"/config.json", seedVolumeConfig(t, volume, key)) + + _, natsURL := setupEmbeddedNATS(t) + cfg := setupTestConfig(t, natsURL) + cfg.S3Host = server.URL + cfg.masterKey = key + + vol, err := describeVolumeEngine(t.Context(), cfg, volume) + require.NoError(t, err) + require.Equal(t, int64(describeTestVolumeSize), vol.CapacityBytes) + + require.Empty(t, server.matching(func(r string) bool { return strings.HasPrefix(r, "PUT ") }), + "a describe wrote to the volume it was only asked about") + require.Empty(t, server.matching(func(r string) bool { return strings.Contains(r, "list-type=2") }), + "a describe paid for a reachability list it did not need") +} + +// TestDescribeVolume_ReusesOneConnection is what makes the single request a +// describe now issues cheap. Each viperblock backend builds its own HTTP client +// unless handed one, and a client per describe is a connection pool per +// describe — so the one request pays for the handshake every time. +func TestDescribeVolume_ReusesOneConnection(t *testing.T) { + const volume = "vol-describeconn001" + key := testMasterKey(t) + server := newRecordingS3(t, "/"+volume+"/config.json", seedVolumeConfig(t, volume, key)) + + _, natsURL := setupEmbeddedNATS(t) + cfg := setupTestConfig(t, natsURL) + cfg.S3Host = server.URL + cfg.masterKey = key + + for range 3 { + _, err := describeVolumeEngine(t.Context(), cfg, volume) + require.NoError(t, err) + } + + require.Equal(t, 1, server.connCount(), + "three describes opened three connections, so each one paid its own setup") +} diff --git a/spinifex/services/viperblockd/engine_tracing_test.go b/spinifex/services/viperblockd/engine_tracing_test.go new file mode 100644 index 000000000..4a642d38c --- /dev/null +++ b/spinifex/services/viperblockd/engine_tracing_test.go @@ -0,0 +1,64 @@ +package viperblockd + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/mulgadc/spinifex/spinifex/testutil" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/trace" +) + +// emptyBucketS3Host answers ListObjectsV2 with an empty bucket and everything +// else with 404. Init therefore succeeds and LoadState is reached and fails, +// which is what puts both legs of an engine open on the wire. +func emptyBucketS3Host(t *testing.T) string { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("list-type") == "2" { + w.Header().Set("Content-Type", "application/xml") + _, _ = w.Write([]byte(`test-bucket0false`)) + return + } + w.WriteHeader(http.StatusNotFound) + })) + t.Cleanup(srv.Close) + return srv.URL +} + +// TestOpenVolumeVB_S3TimeJoinsTheCallersTrace is why openVolumeVB takes a +// context. Both S3 clients only emit a span when the request already carries +// one, so opening an engine through the no-context Init and LoadState leaves +// every byte of that time unattributable. +// +// Two spans is the assertion that matters: one leg traced and the other not +// would still pass a "some span exists" check while hiding half the cost. +func TestOpenVolumeVB_S3TimeJoinsTheCallersTrace(t *testing.T) { + recorder := testutil.RecordSpans(t) + _, natsURL := setupEmbeddedNATS(t) + cfg := setupTestConfig(t, natsURL) + cfg.S3Host = emptyBucketS3Host(t) + + ctx, caller := otel.Tracer("test").Start(t.Context(), "caller") + // The open is expected to fail on a bucket with no state in it. What is + // under test is which spans it emitted on the way, not whether it opened. + vb, lease, err := openVolumeVB(ctx, cfg, "vol-tracedopen000001") + require.Error(t, err) + require.Nil(t, vb) + require.Nil(t, lease) + caller.End() + + var joined int + for _, span := range recorder.Ended() { + if span.SpanKind() != trace.SpanKindClient { + continue + } + if span.SpanContext().TraceID() == caller.SpanContext().TraceID() { + joined++ + } + } + require.GreaterOrEqual(t, joined, 2, + "an engine open must put both its backend init and its state load in the caller's trace; got %d S3 client spans", joined) +} diff --git a/spinifex/services/viperblockd/list_volumes_test.go b/spinifex/services/viperblockd/list_volumes_test.go new file mode 100644 index 000000000..28c4bdeca --- /dev/null +++ b/spinifex/services/viperblockd/list_volumes_test.go @@ -0,0 +1,61 @@ +package viperblockd + +import ( + "bytes" + "testing" + + awssdk "github.com/aws/aws-sdk-go/aws" + awss3 "github.com/aws/aws-sdk-go/service/s3" + "github.com/mulgadc/spinifex/spinifex/objectstore" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const listTestBucket = "test-bucket" + +func putListObject(t *testing.T, store objectstore.ObjectStore, key string) { + t.Helper() + _, err := store.PutObject(t.Context(), &awss3.PutObjectInput{ + Bucket: awssdk.String(listTestBucket), + Key: awssdk.String(key), + Body: bytes.NewReader([]byte("{}")), + }) + require.NoError(t, err) +} + +// TestListVolumePrefixes_SkipsWhatIsNotAVolume locks what a live env19 run +// found: the bucket's top level holds more than volumes, and reporting a +// snapshot, the key store or the control plane's own metadata as a volume +// makes every one of them look like an orphan. +func TestListVolumePrefixes_SkipsWhatIsNotAVolume(t *testing.T) { + store := objectstore.NewMemoryObjectStore() + putListObject(t, store, "vol-0123456789abc/config.json") + putListObject(t, store, "ami-dd7f063caded1ff82/config.json") + putListObject(t, store, "snap-ami-dd7f063caded1ff82/config.json") + putListObject(t, store, "keys/cluster.key") + putListObject(t, store, "spinifex/ebsmetadata/v1/volumes/vol-0123456789abc.json") + + ids, err := listVolumePrefixes(t.Context(), store, listTestBucket) + require.NoError(t, err) + + assert.Equal(t, []string{"ami-dd7f063caded1ff82", "vol-0123456789abc"}, ids, + "only volumes may be enumerated: a snapshot is a separate resource, and keys/ and spinifex/ are not volumes at all") +} + +// An AMI's blocks are a real provider volume, so enumeration must report it. +// Excusing it belongs to the orphan report, which knows an AMI is documented +// as an image; the provider must not hide storage it genuinely holds. +func TestListVolumePrefixes_ReportsAMIBackingVolumes(t *testing.T) { + store := objectstore.NewMemoryObjectStore() + putListObject(t, store, "ami-dd7f063caded1ff82/config.json") + + ids, err := listVolumePrefixes(t.Context(), store, listTestBucket) + require.NoError(t, err) + assert.Equal(t, []string{"ami-dd7f063caded1ff82"}, ids) +} + +func TestListVolumePrefixes_EmptyBucket(t *testing.T) { + ids, err := listVolumePrefixes(t.Context(), objectstore.NewMemoryObjectStore(), listTestBucket) + require.NoError(t, err) + assert.Empty(t, ids) +} diff --git a/spinifex/services/viperblockd/live_bench_test.go b/spinifex/services/viperblockd/live_bench_test.go new file mode 100644 index 000000000..f0400f111 --- /dev/null +++ b/spinifex/services/viperblockd/live_bench_test.go @@ -0,0 +1,25 @@ +// This prices the provider verbs against a real viperblockd over a live +// cluster's NATS. Opt-in: SPINIFEX_VIPERBLOCK_LIVE=1, on the node itself. +package viperblockd_test + +import ( + "testing" + + "github.com/mulgadc/spinifex/spinifex/ebsprovider/conformance" +) + +// BenchmarkLive_Viperblockd runs the same suite the null and qemunbdd +// providers run, so the three sets of numbers differ only in what answers. +// +// It measures control operations. Guest I/O never crosses these verbs, so +// nothing here says anything about viperblock's data-path throughput. +func BenchmarkLive_Viperblockd(b *testing.B) { + provider, nodeID := liveProviderTB(b) + conformance.RunBenchSuite(b, provider, conformance.BenchConfig{ + SuiteConfig: conformance.SuiteConfig{ + NamePrefix: runPrefix("bench"), + NodeID: nodeID, + }, + VolumeBytes: liveVolumeBytes, + }) +} diff --git a/spinifex/services/viperblockd/live_datapath_test.go b/spinifex/services/viperblockd/live_datapath_test.go new file mode 100644 index 000000000..65b51bc19 --- /dev/null +++ b/spinifex/services/viperblockd/live_datapath_test.go @@ -0,0 +1,23 @@ +// This prices guest I/O against a real viperblockd over a live cluster's +// NATS, which the provider-verb benchmarks cannot: guest reads and writes +// never cross those verbs. Opt-in: SPINIFEX_VIPERBLOCK_LIVE=1, on the node. +package viperblockd_test + +import ( + "strconv" + "testing" + "time" + + "github.com/mulgadc/spinifex/spinifex/ebsprovider/conformance" +) + +// TestLive_ViperblockdDataPath drives qemu-img bench against the published +// export. It is a measurement, not a contract: it asserts only that the I/O +// completes, and logs the table. +func TestLive_ViperblockdDataPath(t *testing.T) { + newProvider, nodeID := liveProvider(t) + conformance.RunDataPathSuite(t, newProvider, conformance.NBDClientConfig{ + NodeID: nodeID, + VolumePrefix: "vol-dpath-" + strconv.FormatInt(time.Now().UnixNano(), 10), + }) +} diff --git a/spinifex/services/viperblockd/live_nbdclient_test.go b/spinifex/services/viperblockd/live_nbdclient_test.go new file mode 100644 index 000000000..a8dfbd1f4 --- /dev/null +++ b/spinifex/services/viperblockd/live_nbdclient_test.go @@ -0,0 +1,101 @@ +// This runs the external NBD client suite against a real viperblockd over a +// live cluster's NATS, so the nbdkit export is judged by libnbd rather than +// by our own client. Opt-in: SPINIFEX_VIPERBLOCK_LIVE=1, on the node itself. +package viperblockd_test + +import ( + "os" + "strconv" + "testing" + "time" + + "github.com/mulgadc/spinifex/spinifex/admin" + "github.com/mulgadc/spinifex/spinifex/config" + "github.com/mulgadc/spinifex/spinifex/ebsprovider" + "github.com/mulgadc/spinifex/spinifex/ebsprovider/conformance" + "github.com/mulgadc/spinifex/spinifex/utils" + "github.com/stretchr/testify/require" +) + +const liveEnvVar = "SPINIFEX_VIPERBLOCK_LIVE" + +// defaultConfigPath mirrors the path the systemd unit sets for +// SPINIFEX_CONFIG_PATH when the operator does not override it. +const defaultConfigPath = "/etc/spinifex/spinifex.toml" + +// liveVolumeBytes is one nbdkit chunk-friendly size that is still quick to +// round trip over the real store. +const liveVolumeBytes int64 = 64 << 20 + +// liveProvider dials the running cluster's NATS and returns a provider client +// pointed at the local viperblockd, plus the node it is running on. +func liveProvider(t *testing.T) (func(t *testing.T) ebsprovider.EBSProvider, string) { + t.Helper() + provider, nodeID := liveProviderTB(t) + return func(t *testing.T) ebsprovider.EBSProvider { + t.Helper() + return provider + }, nodeID +} + +// liveProviderTB is liveProvider for any test kind. Benchmarks need the same +// connection and the same node, and cannot be handed a *testing.T. +func liveProviderTB(t testing.TB) (ebsprovider.EBSProvider, string) { + t.Helper() + if os.Getenv(liveEnvVar) == "" { + t.Skipf("skipping: set %s=1 to run this against a real viperblockd over a live cluster's NATS", liveEnvVar) + } + + cfgPath := os.Getenv("SPINIFEX_CONFIG_PATH") + if cfgPath == "" { + cfgPath = defaultConfigPath + } + + clusterConfig, err := config.LoadConfig(cfgPath) + require.NoError(t, err, "load cluster config from %s", cfgPath) + + nodeID := clusterConfig.Node + require.NotEmpty(t, nodeID, "cluster config has no node name set") + nodeConfig, ok := clusterConfig.Nodes[nodeID] + require.True(t, ok, "cluster config has no entry for node %q", nodeID) + t.Logf("node %q, config %s", nodeID, cfgPath) + + // NATS token and CA cert go straight from config into the connect + // helper. Never log them, and never let a failure message embed them. + nc, err := utils.ConnectNATSWithRetry(admin.DialTarget(nodeConfig.NATS.Host), nodeConfig.NATS.ACL.Token, nodeConfig.NATS.CACert) + require.NoError(t, err, "connect to NATS") + t.Cleanup(nc.Close) + + return ebsprovider.NewNATSProvider(nc, 120*time.Second), nodeID +} + +// runPrefix names one run's volumes so it cannot collide with an earlier run's +// leftovers on a store that keeps them. +func runPrefix(kind string) string { + return kind + strconv.FormatInt(time.Now().UnixNano(), 10) + "-" +} + +// TestLive_ViperblockdExternalNBDClient points libnbd at whatever nbdkit +// export viperblockd publishes, through the same NATS client shape the +// control plane uses. +func TestLive_ViperblockdExternalNBDClient(t *testing.T) { + newProvider, nodeID := liveProvider(t) + conformance.RunNBDClientSuiteWithConfig(t, newProvider, + conformance.NBDClientConfig{ + NodeID: nodeID, + VolumePrefix: "vol-nbdlive-" + strconv.FormatInt(time.Now().UnixNano(), 10), + VolumeBytes: liveVolumeBytes, + }) +} + +// TestLive_ViperblockdConformance runs the contract suite against the real +// viperblockd. Until now it had only ever run against MemoryProvider and +// qemunbdd, so the provider production actually uses was the one implementation +// the suite never judged. +func TestLive_ViperblockdConformance(t *testing.T) { + newProvider, nodeID := liveProvider(t) + conformance.RunSuiteWithConfig(t, newProvider, conformance.SuiteConfig{ + NamePrefix: runPrefix("live"), + NodeID: nodeID, + }) +} diff --git a/spinifex/services/viperblockd/mount_idempotency_test.go b/spinifex/services/viperblockd/mount_idempotency_test.go new file mode 100644 index 000000000..8bbee824e --- /dev/null +++ b/spinifex/services/viperblockd/mount_idempotency_test.go @@ -0,0 +1,58 @@ +package viperblockd + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestMountVolume_AlreadyMountedReturnsExistingExport pins the guard at +// mountVolume rather than at one handler. The legacy ebs..mount subject +// is the route production takes for instance launch and hot-attach, so a guard +// that only sits in handlePublishVolume leaves a retried attach free to start a +// second nbdkit against a volume this node already exports. +func TestMountVolume_AlreadyMountedReturnsExistingExport(t *testing.T) { + _, natsURL := setupEmbeddedNATS(t) + cfg := setupTestConfig(t, natsURL) + + const volumeName = "vol-mountidempotent01" + const existingURI = "nbd://127.0.0.1:10809" + cfg.MountedVolumes = append(cfg.MountedVolumes, MountedVolume{ + Name: volumeName, + NBDURI: existingURI, + PID: 4242, + ReadOnly: false, + }) + + resp, err := mountVolume(context.Background(), cfg, nil, volumeName, false) + + require.NoError(t, err) + assert.True(t, resp.Mounted) + assert.Equal(t, existingURI, resp.URI, "the running export must be handed back, not replaced") + assert.Len(t, cfg.MountedVolumes, 1, "a second entry means a second nbdkit was started against one volume") +} + +// TestMountVolume_AlreadyMountedOtherModeRefused covers the half the idempotent +// return cannot answer: nbdkit's access mode is fixed when it starts, so a +// remount asking for the other mode has to be refused rather than served with +// the running export. +func TestMountVolume_AlreadyMountedOtherModeRefused(t *testing.T) { + _, natsURL := setupEmbeddedNATS(t) + cfg := setupTestConfig(t, natsURL) + + const volumeName = "vol-mountidempotent02" + cfg.MountedVolumes = append(cfg.MountedVolumes, MountedVolume{ + Name: volumeName, + NBDURI: "nbd://127.0.0.1:10809", + ReadOnly: true, + }) + + resp, err := mountVolume(context.Background(), cfg, nil, volumeName, false) + + require.Error(t, err) + assert.False(t, resp.Mounted) + assert.Contains(t, err.Error(), "already mounted read_only=true") + assert.Len(t, cfg.MountedVolumes, 1) +} diff --git a/spinifex/services/viperblockd/nbdkit_ready.go b/spinifex/services/viperblockd/nbdkit_ready.go new file mode 100644 index 000000000..9f7aa0ce9 --- /dev/null +++ b/spinifex/services/viperblockd/nbdkit_ready.go @@ -0,0 +1,77 @@ +package viperblockd + +import ( + "context" + "fmt" + "net" + "os" + "time" +) + +const ( + // nbdkitReadyPollInterval is short because readiness normally arrives in a + // few milliseconds; the cost of asking again is a syscall. + nbdkitReadyPollInterval = 5 * time.Millisecond + + // nbdkitReadyDeadline bounds a startup that never completes. It is longer + // than the second this replaced because it is now a ceiling that a healthy + // mount never reaches, rather than a delay every mount pays. + nbdkitReadyDeadline = 10 * time.Second + + nbdkitDialTimeout = 500 * time.Millisecond +) + +// pollUntil retries check until it succeeds, nbdkit exits, the context is +// cancelled or the deadline passes. exitChan is watched throughout so a +// process that dies during startup fails the mount immediately instead of +// waiting out the deadline. +func pollUntil(ctx context.Context, check func() error, exitChan <-chan int, deadline time.Time, sleep func(time.Duration)) error { + var lastErr error + for { + select { + case code := <-exitChan: + return fmt.Errorf("nbdkit exited unexpectedly (code=%d)", code) + case <-ctx.Done(): + return ctx.Err() + default: + } + + lastErr = check() + if lastErr == nil { + return nil + } + + if !time.Now().Before(deadline) { + return fmt.Errorf("nbdkit did not become ready: %w", lastErr) + } + sleep(nbdkitReadyPollInterval) + } +} + +// waitForSocketFile blocks until nbdkit has created its unix socket. The file +// has to exist before it can be chmod'd, and it has to be chmod'd before a +// client running as another user can connect. +func waitForSocketFile(ctx context.Context, path string, exitChan <-chan int, deadline time.Time) error { + return pollUntil(ctx, func() error { + _, err := os.Stat(path) + return err + }, exitChan, deadline, time.Sleep) +} + +// waitForNBDDial blocks until nbdkit accepts a connection on its endpoint. +// This is the question the caller actually needs answered: it is about to hand +// the URI to something that will connect to it. The previous check asked +// whether the process was still alive one second after spawn, which is both +// weaker and slower than asking directly. +func waitForNBDDial(ctx context.Context, network, address string, exitChan <-chan int, deadline time.Time) error { + return pollUntil(ctx, func() error { + conn, err := net.DialTimeout(network, address, nbdkitDialTimeout) + if err != nil { + return err + } + // Drop the probe connection straight away. nbdkit sees a client that + // disconnects before the handshake, which is the same thing it sees + // whenever a client goes away. + return conn.Close() + }, exitChan, deadline, time.Sleep) +} diff --git a/spinifex/services/viperblockd/nbdkit_ready_test.go b/spinifex/services/viperblockd/nbdkit_ready_test.go new file mode 100644 index 000000000..0395a931a --- /dev/null +++ b/spinifex/services/viperblockd/nbdkit_ready_test.go @@ -0,0 +1,95 @@ +package viperblockd + +import ( + "context" + "errors" + "net" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// acceptingSocket stands in for nbdkit: something bound to a socket and +// accepting connections on it. +func acceptingSocket(t *testing.T) string { + t.Helper() + // Socket paths are capped near 108 bytes, so this cannot use t.TempDir(). + path := filepath.Join(t.TempDir(), "s") + listener, err := net.Listen("unix", path) + require.NoError(t, err) + t.Cleanup(func() { _ = listener.Close() }) + go func() { + for { + conn, err := listener.Accept() + if err != nil { + return + } + _ = conn.Close() + } + }() + return path +} + +// TestWaitForNBDDial_ReturnsAsSoonAsItCanConnect is the point of the change. +// The check it replaced slept a fixed second and then asked only whether the +// process was still alive, so every mount paid a second nbdkit did not need. +func TestWaitForNBDDial_ReturnsAsSoonAsItCanConnect(t *testing.T) { + path := acceptingSocket(t) + + start := time.Now() + err := waitForNBDDial(t.Context(), "unix", path, make(chan int), time.Now().Add(nbdkitReadyDeadline)) + require.NoError(t, err) + require.Less(t, time.Since(start), 200*time.Millisecond, + "waiting for a listener that was already up took as long as the fixed delay it replaced") +} + +// TestWaitForNBDDial_FailsWhenTheProcessExits keeps the fast-failure behaviour +// the old loop did have. A nbdkit that dies during startup must fail the mount +// straight away rather than waiting out the deadline. +func TestWaitForNBDDial_FailsWhenTheProcessExits(t *testing.T) { + exitChan := make(chan int, 1) + exitChan <- 3 + + start := time.Now() + err := waitForNBDDial(t.Context(), "unix", filepath.Join(t.TempDir(), "absent"), exitChan, time.Now().Add(time.Minute)) + require.ErrorContains(t, err, "code=3") + require.Less(t, time.Since(start), time.Second, "an exited nbdkit was waited on") +} + +// TestWaitForNBDDial_FailsOnDeadline covers the nbdkit that neither serves nor +// exits, which would otherwise hang the mount forever. +func TestWaitForNBDDial_FailsOnDeadline(t *testing.T) { + err := waitForNBDDial(t.Context(), "unix", filepath.Join(t.TempDir(), "absent"), make(chan int), time.Now().Add(50*time.Millisecond)) + require.ErrorContains(t, err, "did not become ready") +} + +// TestWaitForSocketFile_WaitsForCreation covers the ordering the unix path +// depends on: the socket has to exist before it can be chmod'd, and the chmod +// has to happen before a client running as another user can reach it. +func TestWaitForSocketFile_WaitsForCreation(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "late.sock") + + go func() { + time.Sleep(30 * time.Millisecond) + listener, err := net.Listen("unix", path) + if err == nil { + t.Cleanup(func() { _ = listener.Close() }) + } + }() + + require.NoError(t, waitForSocketFile(t.Context(), path, make(chan int), time.Now().Add(5*time.Second))) +} + +// TestPollUntil_HonoursContextCancellation stops a mount whose caller has +// already given up from holding a goroutine until the deadline. +func TestPollUntil_HonoursContextCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + err := pollUntil(ctx, func() error { return errors.New("never ready") }, + make(chan int), time.Now().Add(time.Minute), time.Sleep) + require.ErrorIs(t, err, context.Canceled) +} diff --git a/spinifex/services/viperblockd/nbdkit_recovery.go b/spinifex/services/viperblockd/nbdkit_recovery.go new file mode 100644 index 000000000..e9e6c42b2 --- /dev/null +++ b/spinifex/services/viperblockd/nbdkit_recovery.go @@ -0,0 +1,194 @@ +package viperblockd + +// Rebuilds cfg.MountedVolumes from nbdkit processes that survived a +// viperblockd restart, so findMountedVolume can find them again (see +// orphan_scan.go for the equivalent problem/fix on the QEMU side). + +import ( + "bytes" + "context" + "fmt" + "log/slog" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/mulgadc/spinifex/spinifex/utils" + "github.com/nats-io/nats.go" +) + +// nbdkitComm is the exact /proc//comm value for an nbdkit child process +// (short enough that Linux's 15-byte comm truncation never applies). +const nbdkitComm = "nbdkit" + +// vbPluginSuffix identifies our own plugin in a discovered argv, so an nbdkit +// serving something else is never adopted. +const vbPluginSuffix = "nbdkit-viperblock-plugin.so" + +// discoveredNbdkit is one nbdkit process found in /proc, before corroboration. +// Exactly one of Socket/Port is set, matching whichever transport +// nbd.buildArgs put on this process's argv. +type discoveredNbdkit struct { + PID int + Volume string + Socket string + Port int + Plugin string + BaseDir string +} + +// scanNbdkitProcs walks procRoot for nbdkit-comm processes, parsing each +// survivor's cmdline for volume= and its NBD endpoint (--unix or +// -p ; see nbd.buildArgs). procRoot lets tests use a fabricated dir. +func scanNbdkitProcs(procRoot string) []discoveredNbdkit { + entries, err := os.ReadDir(procRoot) + if err != nil { + slog.Warn("recovery: failed to read proc root", "path", procRoot, "err", err) + return nil + } + + var found []discoveredNbdkit + for _, entry := range entries { + pid, err := strconv.Atoi(entry.Name()) + if err != nil || pid <= 0 { + continue + } + + comm, err := os.ReadFile(filepath.Join(procRoot, entry.Name(), "comm")) + if err != nil || strings.TrimSpace(string(comm)) != nbdkitComm { + continue + } + + cmdline, err := os.ReadFile(filepath.Join(procRoot, entry.Name(), "cmdline")) + if err != nil { + continue + } + + disc, ok := parseNbdkitCmdline(cmdline) + if !ok { + continue + } + disc.PID = pid + found = append(found, disc) + } + return found +} + +// parseNbdkitCmdline extracts the volume ID and NBD endpoint from a nbdkit +// process's NUL-separated /proc//cmdline (matching nbd.buildArgs). ok is +// false when no volume= argument was found: not one of ours. +func parseNbdkitCmdline(cmdline []byte) (disc discoveredNbdkit, ok bool) { + args := bytes.Split(bytes.TrimRight(cmdline, "\x00"), []byte{0}) + for i := 0; i < len(args); i++ { + arg := string(args[i]) + switch { + case arg == "--unix" && i+1 < len(args): + disc.Socket = string(args[i+1]) + i++ + case arg == "-p" && i+1 < len(args): + if port, err := strconv.Atoi(string(args[i+1])); err == nil { + disc.Port = port + } + i++ + case strings.HasPrefix(arg, "volume="): + disc.Volume = strings.TrimPrefix(arg, "volume=") + case strings.HasPrefix(arg, "base_dir="): + disc.BaseDir = strings.TrimPrefix(arg, "base_dir=") + case strings.HasSuffix(arg, ".so"): + disc.Plugin = arg + } + } + return disc, disc.Volume != "" +} + +// corroborateNbdkit reports whether disc is this daemon's own nbdkit: our +// plugin, our base dir, and a live endpoint. nbdkit runs with -f and never +// writes the --pidfile it is given, so argv is the only evidence there is. +func corroborateNbdkit(cfg *Config, disc discoveredNbdkit) bool { + if !strings.HasSuffix(disc.Plugin, vbPluginSuffix) { + return false + } + if filepath.Clean(disc.BaseDir) != filepath.Clean(cfg.BaseDir) { + return false + } + + // A socket mount must still have its socket; a TCP mount leaves no + // filesystem trace, so the parsed port is all there is to check. + if disc.Socket == "" { + return disc.Port > 0 + } + info, err := os.Stat(disc.Socket) + return err == nil && info.Mode()&os.ModeSocket != 0 +} + +// rebuildMountedVolume re-derives disc's NBD URI, builds its daemon-side VB +// (cfg.buildVB, mountVolume's construction) and registers the same +// config/owner subscriptions a fresh mount would. Never touches the process. +func rebuildMountedVolume(ctx context.Context, cfg *Config, nc *nats.Conn, disc discoveredNbdkit) (MountedVolume, error) { + vb, _, lease, err := cfg.buildVB(ctx, disc.Volume) + if err != nil { + return MountedVolume{}, fmt.Errorf("construct VB: %w", err) + } + + var nbdURI string + if disc.Socket != "" { + nbdURI = utils.FormatNBDSocketURI(disc.Socket) + } else { + nbdURI = utils.FormatNBDTCPURI("127.0.0.1", disc.Port) + } + + configSub, err := nc.Subscribe(fmt.Sprintf("ebs.config.%s", disc.Volume), makeConfigUpdateHandler(vb, disc.Volume)) + if err != nil { + slog.ErrorContext(ctx, "recovery: failed to subscribe to volume config topic", "volume", disc.Volume, "err", err) + } + + ownerSubs := subscribeOwnerSubjects(ctx, cfg, nc, disc.Volume) + + return MountedVolume{ + Name: disc.Volume, + Port: disc.Port, + Socket: disc.Socket, + NBDURI: nbdURI, + PID: disc.PID, + VB: vb, + ConfigSub: configSub, + OwnerSubs: ownerSubs, + Lease: lease, + }, nil +} + +// recoverMountedVolumes discovers, corroborates and rebuilds +// cfg.MountedVolumes from procRoot's surviving nbdkit processes. Never +// signals/kills/starts a process; failures are logged and skipped. +func recoverMountedVolumes(ctx context.Context, cfg *Config, nc *nats.Conn, procRoot string) { + for _, disc := range scanNbdkitProcs(procRoot) { + if !corroborateNbdkit(cfg, disc) { + slog.Warn("recovery: nbdkit process is not this daemon's, skipping", + "pid", disc.PID, "volume", disc.Volume, "base_dir", disc.BaseDir, "plugin", disc.Plugin) + continue + } + + // Two live nbdkits for one volume is the double-mount hazard itself: + // adopt the first and leave the second visible in the log. + if _, ok := findMountedVolume(cfg, disc.Volume); ok { + slog.Warn("recovery: volume already recovered from another nbdkit process, skipping", + "pid", disc.PID, "volume", disc.Volume) + continue + } + + mv, err := rebuildMountedVolume(ctx, cfg, nc, disc) + if err != nil { + slog.Error("recovery: failed to rebuild mounted volume registry entry, skipping", + "pid", disc.PID, "volume", disc.Volume, "err", err) + continue + } + + cfg.mu.Lock() + cfg.MountedVolumes = append(cfg.MountedVolumes, mv) + cfg.mu.Unlock() + + slog.Info("recovery: rebuilt mounted volume registry entry from surviving nbdkit process", + "pid", mv.PID, "volume", mv.Name, "socket", mv.Socket, "port", mv.Port) + } +} diff --git a/spinifex/services/viperblockd/nbdkit_recovery_test.go b/spinifex/services/viperblockd/nbdkit_recovery_test.go new file mode 100644 index 000000000..2538b8844 --- /dev/null +++ b/spinifex/services/viperblockd/nbdkit_recovery_test.go @@ -0,0 +1,332 @@ +package viperblockd + +// Tests for startup recovery: rebuilding cfg.MountedVolumes from nbdkit +// processes that survived a restart, exercised against a fabricated proc +// directory rather than the real host /proc. + +import ( + "context" + "encoding/json" + "fmt" + "net" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/mulgadc/spinifex/spinifex/ebsprovider" + "github.com/mulgadc/viperblock/viperblock" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// nbdkitArgv builds a realistic nbdkit argv (matching nbd.buildArgs' shape) +// for volume, using either socket or port depending on which is non-zero. +// The --pidfile is present but never written, exactly as nbdkit -f leaves it. +func nbdkitArgv(baseDir, socket string, port int, volume string) []string { + args := []string{"-f", "--pidfile", "/tmp/nbdkit-vol-" + volume + ".pid"} + if socket != "" { + args = append(args, "--unix", socket) + } else { + args = append(args, "-p", strconv.Itoa(port)) + } + args = append(args, + "/plugins/"+vbPluginSuffix, + "size=1073741824", + "volume="+volume, + "bucket=test-bucket", + "region=us-east-1", + "base_dir="+baseDir, + "host=https://s3.mock.local", + "cache_size=0", + "shardwal=false", + "gc_enabled=false", + ) + return args +} + +// listenUnix creates a real listening socket at path, so corroboration's +// "the socket this argv names still exists" check has something to find. +func listenUnix(t *testing.T, path string) { + t.Helper() + ln, err := net.Listen("unix", path) + require.NoError(t, err) + t.Cleanup(func() { ln.Close() }) +} + +// writeProcFixture creates procRoot//{comm,cmdline}, mimicking what the +// kernel exposes for a running process, without spawning anything real. +func writeProcFixture(t *testing.T, procRoot string, pid int, comm string, argv []string) { + t.Helper() + dir := filepath.Join(procRoot, strconv.Itoa(pid)) + require.NoError(t, os.MkdirAll(dir, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "comm"), []byte(comm+"\n"), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "cmdline"), []byte(strings.Join(argv, "\x00")+"\x00"), 0644)) +} + +// --- argv parsing --- + +func TestParseNbdkitCmdline_SocketTransport(t *testing.T) { + argv := nbdkitArgv("/var/lib/vb", "/run/spinifex/nbd/vol-argv001.sock", 0, "vol-argv001") + cmdline := []byte(strings.Join(argv, "\x00") + "\x00") + + disc, ok := parseNbdkitCmdline(cmdline) + require.True(t, ok) + assert.Equal(t, "vol-argv001", disc.Volume) + assert.Equal(t, "/run/spinifex/nbd/vol-argv001.sock", disc.Socket) + assert.Zero(t, disc.Port) + assert.Equal(t, "/var/lib/vb", disc.BaseDir) + assert.Equal(t, "/plugins/"+vbPluginSuffix, disc.Plugin) +} + +func TestParseNbdkitCmdline_TCPTransport(t *testing.T) { + argv := nbdkitArgv("/var/lib/vb", "", 10812, "vol-argv002") + cmdline := []byte(strings.Join(argv, "\x00") + "\x00") + + disc, ok := parseNbdkitCmdline(cmdline) + require.True(t, ok) + assert.Equal(t, "vol-argv002", disc.Volume) + assert.Equal(t, 10812, disc.Port) + assert.Empty(t, disc.Socket) +} + +func TestParseNbdkitCmdline_NoVolumeArgument_NotOK(t *testing.T) { + cmdline := []byte(strings.Join([]string{"-f", "--pidfile", "/tmp/x.pid", "--unix", "/tmp/x.sock", "/plugin.so"}, "\x00") + "\x00") + + _, ok := parseNbdkitCmdline(cmdline) + assert.False(t, ok, "cmdline with no volume= token must not be treated as one of ours") +} + +// --- /proc scanning --- + +func TestScanNbdkitProcs_OnlyMatchesNbdkitComm(t *testing.T) { + procRoot := t.TempDir() + + writeProcFixture(t, procRoot, 100, "nbdkit", + nbdkitArgv("/var/lib/vb", "/run/spinifex/nbd/vol-scan001.sock", 0, "vol-scan001")) + + // Same volume= shape on argv, but comm is not nbdkit: scanNbdkitProcs + // must reject this on comm alone, proving cmdline content is not enough. + writeProcFixture(t, procRoot, 200, "bash", + nbdkitArgv("/var/lib/vb", "/run/spinifex/nbd/vol-scan002.sock", 0, "vol-scan002")) + + found := scanNbdkitProcs(procRoot) + require.Len(t, found, 1, "only the nbdkit-comm process may be discovered") + assert.Equal(t, 100, found[0].PID) + assert.Equal(t, "vol-scan001", found[0].Volume) + assert.Equal(t, "/run/spinifex/nbd/vol-scan001.sock", found[0].Socket) +} + +// --- corroboration --- + +// corroborationFixture returns a Config and a discovery that agree on every +// checked field, with a real socket in place, so each test below can spoil +// exactly one of them and attribute the rejection to that field. +func corroborationFixture(t *testing.T) (*Config, discoveredNbdkit) { + t.Helper() + baseDir := t.TempDir() + socket := filepath.Join(t.TempDir(), "vol-corrob.sock") + listenUnix(t, socket) + + return &Config{BaseDir: baseDir}, discoveredNbdkit{ + PID: 4242, + Volume: "vol-corrob001", + Socket: socket, + Plugin: "/plugins/" + vbPluginSuffix, + BaseDir: baseDir, + } +} + +func TestCorroborateNbdkit_OurPluginAndBaseDirAccepted(t *testing.T) { + cfg, disc := corroborationFixture(t) + assert.True(t, corroborateNbdkit(cfg, disc)) +} + +func TestCorroborateNbdkit_ForeignPluginRejected(t *testing.T) { + cfg, disc := corroborationFixture(t) + disc.Plugin = "/usr/lib/nbdkit/plugins/nbdkit-file-plugin.so" + assert.False(t, corroborateNbdkit(cfg, disc), "an nbdkit serving someone else's plugin is not ours to adopt") +} + +func TestCorroborateNbdkit_ForeignBaseDirRejected(t *testing.T) { + cfg, disc := corroborationFixture(t) + disc.BaseDir = filepath.Join(t.TempDir(), "other") + assert.False(t, corroborateNbdkit(cfg, disc), "an nbdkit for a different data dir belongs to a different daemon") +} + +func TestCorroborateNbdkit_MissingSocketRejected(t *testing.T) { + cfg, disc := corroborationFixture(t) + disc.Socket = filepath.Join(t.TempDir(), "gone.sock") + assert.False(t, corroborateNbdkit(cfg, disc), "a socket mount whose socket is gone is not still serving") +} + +func TestCorroborateNbdkit_TCPMountAcceptedOnPort(t *testing.T) { + cfg, disc := corroborationFixture(t) + disc.Socket = "" + disc.Port = 10812 + assert.True(t, corroborateNbdkit(cfg, disc), "a TCP mount leaves no filesystem trace, so the port is all there is") +} + +// --- -efi cache sizing shared with mountVolume --- + +// TestRebuildMountedVolume_EFIVolumeDisablesCache proves recovery takes the +// same -efi cache-disabled branch mountVolume does (both share +// constructMountedVB), observed via its log line before the forced failure. +func TestRebuildMountedVolume_EFIVolumeDisablesCache(t *testing.T) { + _, natsURL := setupEmbeddedNATS(t) + cfg := setupTestConfig(t, natsURL) + cfg.S3Host = fastFailingS3Host(t) + nc := startProviderSubjects(t, cfg, natsURL) + + logs := captureLogs(t) + + disc := discoveredNbdkit{PID: 4242, Volume: "vol-efirecov001-efi", Socket: filepath.Join(t.TempDir(), "efi.sock")} + _, err := rebuildMountedVolume(context.Background(), cfg, nc, disc) + require.Error(t, err, "the fast-failing backend must fail construction, or this test proves nothing about which path it took") + + assert.Contains(t, logs.String(), `msg="Disabling cache for auxiliary volume" volume=vol-efirecov001-efi`, + "an -efi volume recovered from a surviving nbdkit process must take the cache-disabled branch") + assert.NotContains(t, logs.String(), "Enabling 128MB cache for main volume") +} + +// --- end-to-end: discoverable mount registers and is reused, not reopened --- + +// fileBackedConstructVB returns a Config.constructVB override that builds a +// real, file-backed VB (no real predastore needed) and stops its chunk +// uploader, mirroring what constructMountedVB does for production. +func fileBackedConstructVB(t *testing.T) func(ctx context.Context, volumeName string) (*viperblock.VB, int, error) { + t.Helper() + return func(_ context.Context, volumeName string) (*viperblock.VB, int, error) { + vb := createTestVBWithState(t, volumeName) + vb.StopChunkUploader() + return vb, 0, nil + } +} + +// TestRecoverMountedVolumes_DiscoverableMount_ResolvesAndSkipsReopen is the +// invariant recovery exists for: findMountedVolume must resolve a recovered +// volume, and a subsequent snapshot must reuse it, never open a second engine. +func TestRecoverMountedVolumes_DiscoverableMount_ResolvesAndSkipsReopen(t *testing.T) { + _, natsURL := setupEmbeddedNATS(t) + cfg := setupTestConfig(t, natsURL) + cfg.constructVB = fileBackedConstructVB(t) + nc := startProviderSubjects(t, cfg, natsURL) + + const volumeName = "vol-recoverable001" + const srcSnapshotID = "snap-recoverablesrc1" + const dstSnapshotID = "snap-recoverabledst1" + + procRoot := t.TempDir() + const pid = 424242 + socket := filepath.Join(t.TempDir(), "vol-recoverable001.sock") + listenUnix(t, socket) + + writeProcFixture(t, procRoot, pid, "nbdkit", nbdkitArgv(cfg.BaseDir, socket, 0, volumeName)) + + recoverMountedVolumes(context.Background(), cfg, nc, procRoot) + + mv, ok := findMountedVolume(cfg, volumeName) + require.True(t, ok, "recovery must register the surviving nbdkit process in MountedVolumes") + assert.Equal(t, pid, mv.PID) + assert.Equal(t, socket, mv.Socket) + t.Cleanup(func() { + if mv.VB != nil { + mv.VB.StopWALSyncer() + } + }) + + // Capture logs only from here: recovery's own construction legitimately + // opens the volume once, which is not the invariant under test. + logs := captureLogs(t) + + wantCompletionSubject, err := ebsprovider.SnapshotCompletionSubject(srcSnapshotID) + require.NoError(t, err) + completionSub, err := nc.SubscribeSync(wantCompletionSubject) + require.NoError(t, err) + require.NoError(t, nc.Flush()) + + createBody := marshalRequest(t, map[string]any{ + "schema_version": ebsprovider.SchemaVersion, + "volume_id": volumeName, + "snapshot_id": srcSnapshotID, + }) + requestProvider(t, nc, ebsprovider.SnapshotCreateSubjectPrefix+volumeName, createBody) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + completionMsg, err := completionSub.NextMsgWithContext(ctx) + require.NoError(t, err) + var completed ebsprovider.CreateSnapshotResponse + require.NoError(t, json.Unmarshal(completionMsg.Data, &completed)) + require.Nil(t, completed.Error, "snapshot create on a recovered volume must succeed with no S3 backend involved") + + copyBody := marshalRequest(t, map[string]any{ + "schema_version": ebsprovider.SchemaVersion, + "source_snapshot_id": srcSnapshotID, + "destination_snapshot_id": dstSnapshotID, + "volume_id": volumeName, + }) + copyMsg := requestProvider(t, nc, ebsprovider.CopySnapshotSubject, copyBody) + var copyResp ebsprovider.CopySnapshotResponse + require.NoError(t, json.Unmarshal(copyMsg.Data, ©Resp)) + require.Nil(t, copyResp.Error, "snapshot copy on a recovered volume must succeed with no S3 backend involved") + + assert.Equal(t, 0, volumeOpenCount(logs.String(), volumeName), + "snapshot create+copy on a volume recovered from a surviving nbdkit process must reuse the live VB, never open a second engine") +} + +// TestRecoverMountedVolumes_UncorroboratedProcessSkipped proves recovery +// leaves MountedVolumes empty for an nbdkit belonging to a different daemon, +// rather than adopting anything whose comm and volume= argument look right. +func TestRecoverMountedVolumes_UncorroboratedProcessSkipped(t *testing.T) { + _, natsURL := setupEmbeddedNATS(t) + cfg := setupTestConfig(t, natsURL) + cfg.constructVB = fileBackedConstructVB(t) + nc := startProviderSubjects(t, cfg, natsURL) + + const volumeName = "vol-uncorroborated1" + procRoot := t.TempDir() + socket := filepath.Join(t.TempDir(), "vol-uncorroborated1.sock") + listenUnix(t, socket) + + writeProcFixture(t, procRoot, 55555, "nbdkit", + nbdkitArgv(filepath.Join(t.TempDir(), "someone-elses-data-dir"), socket, 0, volumeName)) + + recoverMountedVolumes(context.Background(), cfg, nc, procRoot) + + _, ok := findMountedVolume(cfg, volumeName) + assert.False(t, ok, "an nbdkit serving another daemon's base dir must never be adopted into MountedVolumes") +} + +// TestRecoverMountedVolumes_DuplicateVolumeAdoptedOnce proves two live nbdkits +// for one volume — the double-mount hazard itself — yield a single registry +// entry rather than two conflicting ones. +func TestRecoverMountedVolumes_DuplicateVolumeAdoptedOnce(t *testing.T) { + _, natsURL := setupEmbeddedNATS(t) + cfg := setupTestConfig(t, natsURL) + cfg.constructVB = fileBackedConstructVB(t) + nc := startProviderSubjects(t, cfg, natsURL) + + const volumeName = "vol-duplicatemount1" + procRoot := t.TempDir() + socketDir := t.TempDir() + + for i, pid := range []int{6001, 6002} { + socket := filepath.Join(socketDir, fmt.Sprintf("dup-%d.sock", i)) + listenUnix(t, socket) + writeProcFixture(t, procRoot, pid, "nbdkit", nbdkitArgv(cfg.BaseDir, socket, 0, volumeName)) + } + + recoverMountedVolumes(context.Background(), cfg, nc, procRoot) + + count := 0 + for _, mv := range cfg.MountedVolumes { + if mv.Name == volumeName { + count++ + t.Cleanup(func() { mv.VB.StopWALSyncer() }) + } + } + assert.Equal(t, 1, count, "two nbdkits for one volume must produce one registry entry, not two") +} diff --git a/spinifex/services/viperblockd/provider_handlers.go b/spinifex/services/viperblockd/provider_handlers.go new file mode 100644 index 000000000..d1b53eb11 --- /dev/null +++ b/spinifex/services/viperblockd/provider_handlers.go @@ -0,0 +1,1794 @@ +package viperblockd + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "net" + "os" + "sort" + "strconv" + "strings" + "time" + + awssdk "github.com/aws/aws-sdk-go/aws" + awss3 "github.com/aws/aws-sdk-go/service/s3" + "github.com/mulgadc/spinifex/spinifex/admin" + "github.com/mulgadc/spinifex/spinifex/ebsprovider" + "github.com/mulgadc/spinifex/spinifex/nbd" + "github.com/mulgadc/spinifex/spinifex/objectstore" + "github.com/mulgadc/spinifex/spinifex/types" + "github.com/mulgadc/spinifex/spinifex/utils" + vbtypes "github.com/mulgadc/viperblock/types" + "github.com/mulgadc/viperblock/viperblock" + "github.com/nats-io/nats.go" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +// bytesPerGiB converts between the GiB units viperblock.VolumeMetadata +// persists and the byte units the ebsprovider wire contract uses. +const bytesPerGiB = 1024 * 1024 * 1024 + +// providerObjectStoreFactory builds the objectstore.ObjectStore DeleteVolume +// and DeleteSnapshot use to remove S3 object prefixes. Tests override this to +// inject objectstore.NewMemoryObjectStore(), keeping the unit tests free of +// any network dependency. +var providerObjectStoreFactory = func(cfg *Config) objectstore.ObjectStore { + return objectstore.NewS3ObjectStoreFromConfig(admin.DialTarget(cfg.S3Host), cfg.Region, cfg.AccessKey, cfg.SecretKey) +} + +// registerProviderSubjects subscribes the ebs.provider.v1.* handlers that +// serve the ebsprovider.EBSProvider NATS contract from this daemon, moving +// viperblock engine construction out of the EC2 control-plane handlers and +// into the storage daemon that owns BaseDir and the mounted-volume registry. +// +// PublishVolume/UnpublishVolume are node-addressed (ebsprovider.PublishSubject +// / UnpublishSubject already route to one node), so they are only registered +// when cfg.NodeName is set; there is no queue-group fallback the way the +// legacy ebs.mount/ebs.unmount subjects have. +// RegisterProviderSubjects serves the provider contract from cfg on nc without +// launching the rest of the daemon. It exists for harnesses that need a real +// provider behind the control plane, which otherwise has none to call. +func RegisterProviderSubjects(cfg *Config, nc *nats.Conn) error { + return registerProviderSubjects(cfg, nc) +} + +func registerProviderSubjects(cfg *Config, nc *nats.Conn) error { + // The lease store has to exist before any subject is served: a handler + // that reaches an engine open without one refuses, and refusing every + // publish is a worse failure than not starting. + if cfg.leases == nil { + leases, err := newVolumeLeases(context.Background(), nc, cfg.leaseOwner()) + if err != nil { + return fmt.Errorf("volume leases: %w", err) + } + cfg.leases = leases + } + + subs := []struct { + subject string + handler providerMsgHandler + }{ + {ebsprovider.CapabilitiesSubject, handleProviderCapabilities}, + {ebsprovider.CreateVolumeSubject, func(ctx context.Context, msg *nats.Msg) { handleCreateVolume(ctx, cfg, msg) }}, + {ebsprovider.GetVolumeSubject, func(ctx context.Context, msg *nats.Msg) { handleGetVolume(ctx, cfg, msg) }}, + {ebsprovider.ListVolumesSubject, func(ctx context.Context, msg *nats.Msg) { handleListVolumes(ctx, cfg, msg) }}, + {ebsprovider.ExpandVolumeSubject, func(ctx context.Context, msg *nats.Msg) { handleExpandVolume(ctx, cfg, msg) }}, + {ebsprovider.DeleteVolumeSubject, func(ctx context.Context, msg *nats.Msg) { handleDeleteVolume(ctx, cfg, nc, msg) }}, + {ebsprovider.DeleteSnapshotSubject, func(ctx context.Context, msg *nats.Msg) { handleDeleteSnapshot(ctx, cfg, msg) }}, + {ebsprovider.CopySnapshotSubject, func(ctx context.Context, msg *nats.Msg) { handleCopySnapshot(ctx, cfg, msg) }}, + {ebsprovider.ListSnapshotsSubject, func(ctx context.Context, msg *nats.Msg) { handleListSnapshots(ctx, cfg, msg) }}, + } + for _, s := range subs { + if _, err := nc.QueueSubscribe(s.subject, "spinifex-workers", tracedProviderHandler(s.handler)); err != nil { + return fmt.Errorf("subscribe to %s: %w", s.subject, err) + } + } + + snapshotCreateWildcard := ebsprovider.SnapshotCreateSubjectPrefix + "*" + if _, err := nc.QueueSubscribe(snapshotCreateWildcard, "spinifex-workers", tracedProviderHandler(func(ctx context.Context, msg *nats.Msg) { + handleCreateSnapshot(ctx, cfg, nc, msg) + })); err != nil { + return fmt.Errorf("subscribe to %s: %w", snapshotCreateWildcard, err) + } + + if cfg.NodeName == "" { + slog.Warn("ebs.provider: NodeName is empty, this node cannot serve PublishVolume/UnpublishVolume attachments") + return nil + } + + publishSubject, err := ebsprovider.PublishSubject(cfg.NodeName) + if err != nil { + return fmt.Errorf("build publish subject: %w", err) + } + if _, err := nc.Subscribe(publishSubject, tracedProviderHandler(func(ctx context.Context, msg *nats.Msg) { + handlePublishVolume(ctx, cfg, nc, msg) + })); err != nil { + return fmt.Errorf("subscribe to %s: %w", publishSubject, err) + } + + unpublishSubject, err := ebsprovider.UnpublishSubject(cfg.NodeName) + if err != nil { + return fmt.Errorf("build unpublish subject: %w", err) + } + if _, err := nc.Subscribe(unpublishSubject, tracedProviderHandler(func(ctx context.Context, msg *nats.Msg) { + handleUnpublishVolume(ctx, cfg, msg) + })); err != nil { + return fmt.Errorf("subscribe to %s: %w", unpublishSubject, err) + } + + return nil +} + +// providerMsgHandler takes the per-message context so the server span opened +// for the message reaches the object-store and engine work the handler does. +type providerMsgHandler func(ctx context.Context, msg *nats.Msg) + +// tracedProviderHandler opens a server span per message, joining the caller's +// trace from its headers, and ends it when the handler returns. +func tracedProviderHandler(handler providerMsgHandler) nats.MsgHandler { + return func(msg *nats.Msg) { + ctx, span := ebsprovider.StartServerSpan(context.Background(), msg) + defer span.End() + handler(ctx, msg) + } +} + +// respondProvider marshals a wire response, marks this message's span with +// any error it carries, and sends it. A failed verb must not look like a +// successful round trip in the trace. +func respondProvider(ctx context.Context, msg *nats.Msg, data any) { + response, err := json.Marshal(data) + if err != nil { + slog.Error("Failed to marshal response", "type", fmt.Sprintf("%T", data), "err", err) + _ = msg.Respond([]byte(`{"Error":"internal marshal failure"}`)) + return + } + ebsprovider.RecordResponseError(trace.SpanFromContext(ctx), response) + if err := msg.Respond(response); err != nil { + slog.Error("Failed to respond to NATS request", "err", err) + } +} + +// badRequestError maps a JSON decode failure to the wire error shape. +func badRequestError(err error) *ebsprovider.ProviderError { + return &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeInvalidArgument, Message: fmt.Sprintf("bad request: %v", err)} +} + +func unsupportedVersionError(got uint16) *ebsprovider.ProviderError { + return &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnsupportedVersion, Message: fmt.Sprintf("unsupported schema version %d, want %d", got, ebsprovider.SchemaVersion)} +} + +func invalidArgumentError(format string, args ...any) *ebsprovider.ProviderError { + return &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeInvalidArgument, Message: fmt.Sprintf(format, args...)} +} + +func notFoundError(format string, args ...any) *ebsprovider.ProviderError { + return &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeNotFound, Message: fmt.Sprintf(format, args...)} +} + +func alreadyExistsError(format string, args ...any) *ebsprovider.ProviderError { + return &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeAlreadyExists, Message: fmt.Sprintf(format, args...)} +} + +func volumeInUseError(format string, args ...any) *ebsprovider.ProviderError { + return &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeVolumeInUse, Message: fmt.Sprintf(format, args...)} +} + +func internalError(format string, args ...any) *ebsprovider.ProviderError { + return &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeInternal, Message: fmt.Sprintf(format, args...)} +} + +// unavailableError reports that the request could not be answered now and may +// succeed later. Reserved for conditions that genuinely pass: a peer that did +// not reply, not a request this provider will always refuse. +func unavailableError(format string, args ...any) *ebsprovider.ProviderError { + return &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnavailable, Message: fmt.Sprintf(format, args...)} +} + +// errSnapshotDestinationExists lets handleCopySnapshot map a pre-existing +// destination to ErrorCodeAlreadyExists without string-matching +// viperblock.CopySnapshotMeta's error text, which carries no sentinel of +// its own. +var errSnapshotDestinationExists = errors.New("snapshot destination already exists") + +// errSnapshotExistsElsewhere marks a snapshot ID already taken against a +// different volume. That is a conflicting recreate, not the idempotent repeat +// a matching source volume would make it. +var errSnapshotExistsElsewhere = errors.New("snapshot already exists on another volume") + +// volumeHandle and snapshotHandle produce the opaque handle strings this +// provider returns, mirroring memory.go's "memory://..." convention so +// callers never need a provider-specific branch to interpret them. +func volumeHandle(volumeID string) string { return "viperblock://volume/" + volumeID } +func snapshotHandle(snapshotID string) string { return "viperblock://snapshot/" + snapshotID } + +// findMountedVolume returns volumeID's MountedVolume entry if this node has +// it mounted, so handlers can prefer the live engine over opening a second +// one on the same volume (the double-writer bug this decouple removes). +func findMountedVolume(cfg *Config, volumeID string) (MountedVolume, bool) { + cfg.mu.Lock() + defer cfg.mu.Unlock() + for _, volume := range cfg.MountedVolumes { + if volume.Name == volumeID { + return volume, true + } + } + return MountedVolume{}, false +} + +func handleProviderCapabilities(ctx context.Context, msg *nats.Msg) { + var req ebsprovider.GetCapabilitiesRequest + if err := json.Unmarshal(msg.Data, &req); err != nil { + slog.Error("ebs.provider.capabilities: bad request", "err", err) + respondProvider(ctx, msg, ebsprovider.GetCapabilitiesResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)}) + return + } + if req.SchemaVersion != ebsprovider.SchemaVersion { + slog.Error("ebs.provider.capabilities: unsupported schema version", "version", req.SchemaVersion) + respondProvider(ctx, msg, ebsprovider.GetCapabilitiesResponse{Versioned: ebsprovider.NewVersioned(), Error: unsupportedVersionError(req.SchemaVersion)}) + return + } + respondProvider(ctx, msg, ebsprovider.GetCapabilitiesResponse{ + Versioned: ebsprovider.NewVersioned(), + Capabilities: ebsprovider.Capabilities{ + // Every snapshot freezes the live checkpoint rather than + // depending on a guest-coordinated quiesce. + CrashConsistentSnapshot: true, + // CreateVolume can write a caller-supplied seed at offset 0 in the + // engine it already builds, which is how an EFI variable store gets + // its firmware VARS template without a control-plane engine. + VolumeSeeding: true, + // ExpandVolume refuses a volume that is mounted with a live VB + // (see handleExpandVolume). The nbdkit Go binding exposes no + // extents callback, so the export cannot report base:allocation. + OnlineExpansion: false, + SparseExtentReporting: false, + // A read-only publish starts nbdkit with -r, which sets the NBD + // read-only transmission flag and makes the plugin refuse writes. + ReadOnlyPublish: true, + // mountVolume registers ebs.provider.v1.owner.{volumeID}.* for + // every mounted volume (see subscribeOwnerSubjects). + OwnerRouting: true, + // Volumes and snapshots are both top-level prefixes in the + // bucket, so the object store can be walked for what exists + // without any control-plane metadata to consult. + VolumeEnumeration: true, + SnapshotEnumeration: true, + // Every engine open takes a JetStream KV lease keyed by volume and + // the KV create is the compare-and-swap, so a second opener on any + // node is refused rather than racing. Losing the lease is logged + // and not enforced: until the object store can refuse a stale + // writer's PUT, a partitioned node keeps writing. + Exclusion: ebsprovider.ExclusionSemantics{ + Scope: ebsprovider.ExclusionScopeCluster, + ClaimTTLSeconds: int(volumeLeaseTTL / time.Second), + FencesLostClaim: false, + }, + }, + }) +} + +// reservedListPrefixes are top-level prefixes in the bucket that are not +// volumes: the control plane's own metadata, and the cluster key store. +var reservedListPrefixes = map[string]bool{"spinifex": true, "keys": true} + +// snapshotIDPrefix marks a snapshot's objects. A snapshot sits at the bucket's +// top level beside volumes and has a config of its own, so the name is the +// only thing separating the two; SnapPrefix builds it and the AMI removal path +// already reads it back the same way. +const snapshotIDPrefix = "snap-" + +// listVolumePrefixes returns every top-level prefix in the bucket that names a +// volume, sorted. It reports what storage actually holds, which is the point: +// a volume whose control-plane document is gone still appears here. +func listVolumePrefixes(ctx context.Context, store objectstore.ObjectStore, bucket string) ([]string, error) { + return listTopLevelPrefixes(ctx, store, bucket, func(id string) bool { + return !strings.HasPrefix(id, snapshotIDPrefix) + }) +} + +// listSnapshotPrefixes is the snapshot half of the same walk. A snapshot whose +// control-plane document is gone still appears here, which is what makes the +// index reconcilable against storage rather than against itself. +func listSnapshotPrefixes(ctx context.Context, store objectstore.ObjectStore, bucket string) ([]string, error) { + return listTopLevelPrefixes(ctx, store, bucket, func(id string) bool { + return strings.HasPrefix(id, snapshotIDPrefix) + }) +} + +// listTopLevelPrefixes walks the bucket's top level and returns the sorted IDs +// keep accepts. Reserved prefixes and names that are not safe as a path +// component are dropped before keep ever sees them. +func listTopLevelPrefixes(ctx context.Context, store objectstore.ObjectStore, bucket string, keep func(string) bool) ([]string, error) { + var ids []string + var continuationToken *string + for { + listOutput, err := store.ListObjectsV2(ctx, &awss3.ListObjectsV2Input{ + Bucket: awssdk.String(bucket), + Delimiter: awssdk.String("/"), + ContinuationToken: continuationToken, + }) + if err != nil { + return nil, fmt.Errorf("list bucket prefixes: %w", err) + } + for _, commonPrefix := range listOutput.CommonPrefixes { + id := strings.TrimSuffix(awssdk.StringValue(commonPrefix.Prefix), "/") + if !validVolumeName(id) || reservedListPrefixes[id] || !keep(id) { + continue + } + ids = append(ids, id) + } + if !awssdk.BoolValue(listOutput.IsTruncated) { + break + } + continuationToken = listOutput.NextContinuationToken + } + sort.Strings(ids) + return ids, nil +} + +func handleListVolumes(ctx context.Context, cfg *Config, msg *nats.Msg) { + var req ebsprovider.ListVolumesRequest + if err := json.Unmarshal(msg.Data, &req); err != nil { + slog.Error("ebs.provider.volume.list: bad request", "err", err) + respondProvider(ctx, msg, ebsprovider.ListVolumesResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)}) + return + } + if req.SchemaVersion != ebsprovider.SchemaVersion { + respondProvider(ctx, msg, ebsprovider.ListVolumesResponse{Versioned: ebsprovider.NewVersioned(), Error: unsupportedVersionError(req.SchemaVersion)}) + return + } + + ids, err := listVolumePrefixes(ctx, providerObjectStoreFactory(cfg), cfg.Bucket) + if err != nil { + slog.Error("ebs.provider.volume.list: failed to enumerate volumes", "err", err) + respondProvider(ctx, msg, ebsprovider.ListVolumesResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("enumerate volumes: %v", err)}) + return + } + + response := ebsprovider.ListVolumesResponse{Versioned: ebsprovider.NewVersioned()} + pageSize := int(req.PageSize()) + for _, id := range ids { + if id <= req.StartingToken { + continue + } + if len(response.Volumes) == pageSize { + response.NextToken = response.Volumes[pageSize-1].ID + break + } + response.Volumes = append(response.Volumes, ebsprovider.VolumeRef{ID: id, Handle: volumeHandle(id)}) + } + respondProvider(ctx, msg, response) +} + +// handleListSnapshots answers with the snapshot prefixes storage holds. +// SourceVolumeID is left empty: the link lives inside the snapshot's own +// encrypted state, and reading it would mean a fetch per snapshot listed. +func handleListSnapshots(ctx context.Context, cfg *Config, msg *nats.Msg) { + var req ebsprovider.ListSnapshotsRequest + if err := json.Unmarshal(msg.Data, &req); err != nil { + slog.Error("ebs.provider.snapshot.list: bad request", "err", err) + respondProvider(ctx, msg, ebsprovider.ListSnapshotsResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)}) + return + } + if req.SchemaVersion != ebsprovider.SchemaVersion { + respondProvider(ctx, msg, ebsprovider.ListSnapshotsResponse{Versioned: ebsprovider.NewVersioned(), Error: unsupportedVersionError(req.SchemaVersion)}) + return + } + + ids, err := listSnapshotPrefixes(ctx, providerObjectStoreFactory(cfg), cfg.Bucket) + if err != nil { + slog.Error("ebs.provider.snapshot.list: failed to enumerate snapshots", "err", err) + respondProvider(ctx, msg, ebsprovider.ListSnapshotsResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("enumerate snapshots: %v", err)}) + return + } + + response := ebsprovider.ListSnapshotsResponse{Versioned: ebsprovider.NewVersioned()} + pageSize := int(req.PageSize()) + for _, id := range ids { + if id <= req.StartingToken { + continue + } + if len(response.Snapshots) == pageSize { + response.NextToken = response.Snapshots[pageSize-1].ID + break + } + response.Snapshots = append(response.Snapshots, ebsprovider.SnapshotRef{ID: id, Handle: snapshotHandle(id)}) + } + respondProvider(ctx, msg, response) +} + +// buildProviderVBConfig assembles the viperblock.VB config CreateVolume hands +// to viperblock.New. Only storage-owned facts go into VolumeMetadata here: +// TenantID, Tags, VolumeType, IOPS, Throughput and AvailabilityZone are +// control-plane facts owned by ebsmetadata now, not viperblock's own state. +func buildProviderVBConfig(cfg *Config, volumeID string, volumeSizeBytes uint64, sourceSnapshotID, sourceVolumeID string) *viperblock.VB { + vbconfig := &viperblock.VB{ + VolumeName: volumeID, + VolumeSize: volumeSizeBytes, + BaseDir: cfg.BaseDir, + Cache: viperblock.Cache{Config: viperblock.CacheConfig{Size: 0}}, + VolumeConfig: viperblock.VolumeConfig{ + VolumeMetadata: viperblock.VolumeMetadata{ + VolumeID: volumeID, + SizeGiB: volumeSizeBytes / bytesPerGiB, + State: "available", + CreatedAt: time.Now().UTC(), + }, + }, + MasterKey: cfg.masterKey, + EncryptionEnabled: cfg.masterKey != nil, + GCEnabled: cfg.GCEnabled, + } + if sourceSnapshotID != "" { + vbconfig.SnapshotID = sourceSnapshotID + vbconfig.SourceVolumeName = sourceVolumeID + } + return vbconfig +} + +func handleCreateVolume(ctx context.Context, cfg *Config, msg *nats.Msg) { + var req ebsprovider.CreateVolumeRequest + if err := json.Unmarshal(msg.Data, &req); err != nil { + slog.Error("ebs.provider.volume.create: bad request", "err", err) + respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)}) + return + } + if req.SchemaVersion != ebsprovider.SchemaVersion { + slog.Error("ebs.provider.volume.create: unsupported schema version", "version", req.SchemaVersion) + respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: unsupportedVersionError(req.SchemaVersion)}) + return + } + if !validVolumeName(req.VolumeID) { + slog.Error("ebs.provider.volume.create: invalid volume id", "volume", req.VolumeID) + respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("invalid volume id %q", req.VolumeID)}) + return + } + if req.CapacityRange.RequiredBytes <= 0 || (req.CapacityRange.LimitBytes > 0 && req.CapacityRange.RequiredBytes > req.CapacityRange.LimitBytes) { + respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("invalid capacity range")}) + return + } + if req.SourceSnapshotID != "" && req.SourceSnapshotVolumeID == "" { + respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("source_snapshot_volume_id is required with source_snapshot_id")}) + return + } + if err := ebsprovider.ValidateSeedData(req.SeedData); err != nil { + slog.Error("ebs.provider.volume.create: seed data rejected", "volume", req.VolumeID, "err", err) + respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("%v", err)}) + return + } + if int64(len(req.SeedData)) > req.CapacityRange.RequiredBytes { + respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("seed data is larger than the requested capacity")}) + return + } + if len(req.SeedData) > 0 && req.SourceSnapshotID != "" { + respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("seed data cannot be combined with a source snapshot")}) + return + } + + existing, err := describeVolumeEngine(ctx, cfg, req.VolumeID) + switch { + case err == nil: + if existing.CapacityBytes != req.CapacityRange.RequiredBytes { + slog.Error("ebs.provider.volume.create: volume exists with different capacity", "volume", req.VolumeID) + respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: alreadyExistsError("volume %s already exists with a different capacity", req.VolumeID)}) + return + } + respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Volume: existing}) + return + case errors.Is(err, viperblock.ErrStateNotFound): + // Volume does not exist yet: fall through and create it. + default: + slog.Error("ebs.provider.volume.create: existence check failed", "volume", req.VolumeID, "err", err) + respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("check existing volume: %v", err)}) + return + } + + vbconfig := buildProviderVBConfig(cfg, req.VolumeID, utils.SafeInt64ToUint64(req.CapacityRange.RequiredBytes), req.SourceSnapshotID, req.SourceSnapshotVolumeID) + s3cfg := cfg.volumeS3Config(req.VolumeID) + vb, err := viperblock.New(vbconfig, "s3", s3cfg) + if err != nil { + slog.Error("ebs.provider.volume.create: new viperblock failed", "volume", req.VolumeID, "err", err) + respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("new viperblock: %v", err)}) + return + } + defer func() { + vb.StopChunkUploader() + vb.StopWALSyncer() + }() + vb.SetDebug(false) + + if err := vb.Backend.InitCtx(ctx); err != nil { + slog.Error("ebs.provider.volume.create: backend init failed", "volume", req.VolumeID, "err", err) + respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("backend init: %v", err)}) + return + } + if req.SourceSnapshotID != "" { + if perr := verifySourceSnapshot(vb, req.SourceSnapshotID, req.SourceSnapshotVolumeID); perr != nil { + slog.Error("ebs.provider.volume.create: source snapshot rejected", "volume", req.VolumeID, "snapshot", req.SourceSnapshotID, "err", perr.Message) + respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: perr}) + return + } + } + if len(req.SeedData) > 0 { + if err := seedVolume(ctx, vb, req.SeedData); err != nil { + slog.Error("ebs.provider.volume.create: seed failed", "volume", req.VolumeID, "bytes", len(req.SeedData), "err", err) + respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("seed volume: %v", err)}) + return + } + } else { + if err := vb.SaveStateCtx(ctx); err != nil { + slog.Error("ebs.provider.volume.create: save state failed", "volume", req.VolumeID, "err", err) + respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("save state: %v", err)}) + return + } + // A state read prefers a local copy over the backend, so state left on + // whichever node served the create answers describe even after another + // node deletes the volume. The seeded path removes it for this reason. + if err := vb.RemoveLocalFiles(); err != nil { + slog.Warn("ebs.provider.volume.create: could not remove local files", "volume", req.VolumeID, "err", err) + } + } + + slog.Info("ebs.provider.volume.create: created", "volume", req.VolumeID, "capacityBytes", req.CapacityRange.RequiredBytes, "seedBytes", len(req.SeedData)) + respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{ + Versioned: ebsprovider.NewVersioned(), + Volume: &ebsprovider.Volume{ + ID: req.VolumeID, + CapacityBytes: req.CapacityRange.RequiredBytes, + State: ebsprovider.VolumeStateAvailable, + Handle: volumeHandle(req.VolumeID), + }, + }) +} + +// verifySourceSnapshot reads the snapshot a new volume is being cloned from, +// before the volume is written. Without this a missing snapshot yields a +// blank volume and a wrong source volume is discovered only at mount. +// Snapshot metadata authenticates under its own recorded identity, so any VB +// sharing the backend and master key can read it. +func verifySourceSnapshot(vb *viperblock.VB, snapshotID, sourceVolumeID string) *ebsprovider.ProviderError { + exists, err := snapshotConfigExists(vb, snapshotID) + if err != nil { + return internalError("probe source snapshot %s: %v", snapshotID, err) + } + if !exists { + return notFoundError("source snapshot %s not found", snapshotID) + } + _, ident, err := vb.LoadSnapshotBlockMap(snapshotID) + if err != nil { + return internalError("read source snapshot %s: %v", snapshotID, err) + } + if ident.SourceVolumeName != sourceVolumeID { + return invalidArgumentError("source snapshot %s was taken from volume %q, not %q", snapshotID, ident.SourceVolumeName, sourceVolumeID) + } + return nil +} + +// seedVolume writes seed at offset 0 of a freshly created volume. Only +// CreateVolume calls it, and only when the existence check above found no +// volume, so it can never overwrite bytes a guest has already written. +func seedVolume(ctx context.Context, vb *viperblock.VB, seed []byte) error { + var err error + if vb.UseShardedWAL { + err = vb.OpenShardedWAL() + } else { + err = vb.OpenWAL(&vb.WAL, fmt.Sprintf("%s/%s", vb.WAL.BaseDir, vbtypes.GetFilePath(vbtypes.FileTypeWALChunk, vb.WAL.WallNum.Load(), vb.GetVolume()))) + } + if err != nil { + return fmt.Errorf("open WAL: %w", err) + } + if err := vb.OpenWAL(&vb.BlockToObjectWAL, fmt.Sprintf("%s/%s", vb.WAL.BaseDir, vbtypes.GetFilePath(vbtypes.FileTypeWALBlock, vb.BlockToObjectWAL.WallNum.Load(), vb.GetVolume()))); err != nil { + return fmt.Errorf("open block WAL: %w", err) + } + if err := vb.WriteAt(0, seed); err != nil { + return fmt.Errorf("write seed: %w", err) + } + + // Close is the durability boundary, not Flush: a partially persisted seed + // leaves the caller believing bytes it never got are on the volume. + if err := vb.CloseCtx(ctx); err != nil { + return fmt.Errorf("close: %w", err) + } + if err := vb.RemoveLocalFiles(); err != nil { + slog.Warn("ebs.provider.volume.create: could not remove local files after seeding", "volume", vb.GetVolume(), "err", err) + } + return nil +} + +// describeVolumeEngine resolves volumeID to a provider-neutral Volume, +// preferring a live mounted VB and otherwise reading persisted state rather +// than opening a second engine on the same volume. The returned error +// satisfies errors.Is(err, viperblock.ErrStateNotFound) when the volume does +// not exist. +func describeVolumeEngine(ctx context.Context, cfg *Config, volumeID string) (*ebsprovider.Volume, error) { + if mv, ok := findMountedVolume(cfg, volumeID); ok && mv.VB != nil { + return &ebsprovider.Volume{ + ID: volumeID, + CapacityBytes: utils.SafeUint64ToInt64(mv.VB.GetVolumeSize()), + State: ebsprovider.VolumeStateInUse, + Handle: volumeHandle(volumeID), + }, nil + } + + state, err := readVolumeState(ctx, cfg, volumeID) + if err != nil { + return nil, err + } + return &ebsprovider.Volume{ + ID: volumeID, + CapacityBytes: utils.SafeUint64ToInt64(state.VolumeSize), + State: ebsprovider.VolumeStateAvailable, + Handle: volumeHandle(volumeID), + }, nil +} + +func handleGetVolume(ctx context.Context, cfg *Config, msg *nats.Msg) { + var req ebsprovider.GetVolumeRequest + if err := json.Unmarshal(msg.Data, &req); err != nil { + slog.Error("ebs.provider.volume.describe: bad request", "err", err) + respondProvider(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)}) + return + } + if req.SchemaVersion != ebsprovider.SchemaVersion { + respondProvider(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: unsupportedVersionError(req.SchemaVersion)}) + return + } + if !validVolumeName(req.VolumeID) { + respondProvider(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("invalid volume id %q", req.VolumeID)}) + return + } + + volume, err := describeVolumeEngine(ctx, cfg, req.VolumeID) + if err != nil { + if errors.Is(err, viperblock.ErrStateNotFound) { + respondProvider(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: notFoundError("volume %s not found", req.VolumeID)}) + return + } + slog.Error("ebs.provider.volume.describe: failed", "volume", req.VolumeID, "err", err) + respondProvider(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("describe volume: %v", err)}) + return + } + respondProvider(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: ebsprovider.NewVersioned(), Volume: volume}) +} + +func handleExpandVolume(ctx context.Context, cfg *Config, msg *nats.Msg) { + var req ebsprovider.ExpandVolumeRequest + if err := json.Unmarshal(msg.Data, &req); err != nil { + slog.Error("ebs.provider.volume.expand: bad request", "err", err) + respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)}) + return + } + if req.SchemaVersion != ebsprovider.SchemaVersion { + respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: unsupportedVersionError(req.SchemaVersion)}) + return + } + if !validVolumeName(req.VolumeID) { + respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("invalid volume id %q", req.VolumeID)}) + return + } + if req.CapacityRange.RequiredBytes <= 0 { + respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("invalid capacity range")}) + return + } + + if mv, ok := findMountedVolume(cfg, req.VolumeID); ok && mv.VB != nil { + slog.Error("ebs.provider.volume.expand: volume is mounted", "volume", req.VolumeID) + respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: volumeInUseError("volume %s is mounted; detach before expanding", req.VolumeID)}) + return + } + + vb, lease, err := openVolumeVB(ctx, cfg, req.VolumeID) + if err != nil { + if errors.Is(err, viperblock.ErrStateNotFound) { + respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: notFoundError("volume %s not found", req.VolumeID)}) + return + } + if errors.Is(err, errVolumeLeaseHeld) { + slog.Error("ebs.provider.volume.expand: volume is leased elsewhere", "volume", req.VolumeID, "err", err) + respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: volumeInUseError("volume %s is open elsewhere: %v", req.VolumeID, err)}) + return + } + slog.Error("ebs.provider.volume.expand: open volume failed", "volume", req.VolumeID, "err", err) + respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("open volume: %v", err)}) + return + } + defer func() { + vb.StopChunkUploader() + vb.StopWALSyncer() + cfg.releaseVolumeLease(ctx, lease) + }() + + currentBytes := utils.SafeUint64ToInt64(vb.GetVolumeSize()) + if req.CapacityRange.RequiredBytes < currentBytes { + respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("volume expansion is grow-only")}) + return + } + + vc := vb.VolumeConfig + vc.VolumeMetadata.SizeGiB = utils.SafeInt64ToUint64(req.CapacityRange.RequiredBytes) / bytesPerGiB + rawConfig, err := json.Marshal(vc) + if err != nil { + slog.Error("ebs.provider.volume.expand: marshal volume config failed", "volume", req.VolumeID, "err", err) + respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("marshal volume config: %v", err)}) + return + } + if err := applyConfigUpdate(ctx, vb, types.EBSConfigUpdateRequest{Volume: req.VolumeID, VolumeConfig: rawConfig}); err != nil { + slog.Error("ebs.provider.volume.expand: apply config update failed", "volume", req.VolumeID, "err", err) + respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("apply config update: %v", err)}) + return + } + + slog.Info("ebs.provider.volume.expand: expanded", "volume", req.VolumeID, "capacityBytes", req.CapacityRange.RequiredBytes) + respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{ + Versioned: ebsprovider.NewVersioned(), + Volume: &ebsprovider.Volume{ + ID: req.VolumeID, + CapacityBytes: utils.SafeUint64ToInt64(vb.GetVolumeSize()), + State: ebsprovider.VolumeStateAvailable, + Handle: volumeHandle(req.VolumeID), + }, + }) +} + +// ownerProbeTimeout bounds the wait for a volume's owner subject to answer. +// A mounted volume's owner is on the same NATS as the deleting node, so a +// reply is a round trip away; no reply within this means nobody holds it. +const ownerProbeTimeout = 2 * time.Second + +// volumeOwnership is what probing a volume's owner subject established. +// ownershipUnknown is the zero value so a path that forgets to set it refuses +// rather than proceeds. +type volumeOwnership int + +const ( + ownershipUnknown volumeOwnership = iota + ownershipUnmounted + ownershipMounted +) + +// probeVolumeOwner asks volumeID's owner subject whether anyone answers for +// it. Only a node with the volume mounted subscribes that subject, so a reply +// means a live mount on some node, not necessarily this one. +// +// No responders and no answer are different facts and are reported as such. +// NATS says the first explicitly; the second is a timeout, which a busy or +// partitioned owner is indistinguishable from. Reading a timeout as +// "unmounted" is what lets a delete run against a volume a guest is writing. +func probeVolumeOwner(nc *nats.Conn, volumeID string) volumeOwnership { + if nc == nil { + return ownershipUnknown + } + subject, err := ebsprovider.GetVolumeOwnerSubject(volumeID) + if err != nil { + return ownershipUnknown + } + payload, err := json.Marshal(ebsprovider.GetVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID}) + if err != nil { + return ownershipUnknown + } + + _, err = nc.Request(subject, payload, ownerProbeTimeout) + switch { + case err == nil: + return ownershipMounted + case errors.Is(err, nats.ErrNoResponders): + return ownershipUnmounted + default: + return ownershipUnknown + } +} + +// deleteObjectPrefix deletes every object under prefix in bucket, paginating +// through ListObjectsV2. Mirrors handlers/ec2/volume's deleteS3Prefix so +// DeleteVolume and DeleteSnapshot get the same idempotent-when-absent +// behaviour without a second copy of the pagination loop. +func deleteObjectPrefix(ctx context.Context, store objectstore.ObjectStore, bucket, prefix string) error { + var continuationToken *string + for { + listOutput, err := store.ListObjectsV2(ctx, &awss3.ListObjectsV2Input{ + Bucket: awssdk.String(bucket), + Prefix: awssdk.String(prefix), + ContinuationToken: continuationToken, + }) + if err != nil { + return fmt.Errorf("list objects under %s: %w", prefix, err) + } + if len(listOutput.Contents) == 0 { + break + } + for _, obj := range listOutput.Contents { + // A concurrent sweep may have deleted the object between the list + // and here. It is gone either way, which is what this asked for. + if _, err := store.DeleteObject(ctx, &awss3.DeleteObjectInput{Bucket: awssdk.String(bucket), Key: obj.Key}); err != nil && !objectstore.IsNoSuchKeyError(err) { + return fmt.Errorf("delete object %s: %w", awssdk.StringValue(obj.Key), err) + } + } + if !awssdk.BoolValue(listOutput.IsTruncated) { + break + } + continuationToken = listOutput.NextContinuationToken + } + return nil +} + +func handleDeleteVolume(ctx context.Context, cfg *Config, nc *nats.Conn, msg *nats.Msg) { + var req ebsprovider.DeleteVolumeRequest + if err := json.Unmarshal(msg.Data, &req); err != nil { + slog.Error("ebs.provider.volume.delete: bad request", "err", err) + respondProvider(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)}) + return + } + if req.SchemaVersion != ebsprovider.SchemaVersion { + respondProvider(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: unsupportedVersionError(req.SchemaVersion)}) + return + } + if !validVolumeName(req.VolumeID) { + respondProvider(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("invalid volume id %q", req.VolumeID)}) + return + } + + // A published volume must outlive a delete: the nbdkit export and the + // guest writing through it survive the metadata going away, so deleting + // under them turns a refusable API call into later corruption. + _, mountedHere := findMountedVolume(cfg, req.VolumeID) + ownership := ownershipUnmounted + if !mountedHere { + ownership = probeVolumeOwner(nc, req.VolumeID) + } + if mountedHere || ownership == ownershipMounted { + slog.Error("ebs.provider.volume.delete: volume is published", "volume", req.VolumeID) + respondProvider(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: volumeInUseError("volume %s is published; unpublish it before deleting", req.VolumeID)}) + return + } + // Ownership could not be established. Deleting is irreversible, so an + // unanswered probe is refused rather than assumed safe; the caller may + // retry once the owner is reachable. + if ownership == ownershipUnknown { + slog.Error("ebs.provider.volume.delete: could not establish whether the volume is published", "volume", req.VolumeID) + respondProvider(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: unavailableError("volume %s: could not establish whether it is published; retry when its owner is reachable", req.VolumeID)}) + return + } + + store := providerObjectStoreFactory(cfg) + if err := deleteObjectPrefix(ctx, store, cfg.Bucket, req.VolumeID+"-efi/"); err != nil { + slog.Error("ebs.provider.volume.delete: failed to delete aux objects", "volume", req.VolumeID, "err", err) + respondProvider(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("delete aux objects: %v", err)}) + return + } + if err := deleteObjectPrefix(ctx, store, cfg.Bucket, req.VolumeID+"/"); err != nil { + slog.Error("ebs.provider.volume.delete: failed to delete volume objects", "volume", req.VolumeID, "err", err) + respondProvider(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("delete volume objects: %v", err)}) + return + } + + // Delete is permanent: remove any on-disk WAL/checkpoint cache left on + // this node regardless of mount-tracking state, mirroring ebs.delete. + if localPath, err := localVolumeDir(cfg.BaseDir, req.VolumeID); err == nil { + if err := os.RemoveAll(localPath); err != nil { + slog.Error("ebs.provider.volume.delete: failed to remove local volume directory", "volume", req.VolumeID, "path", localPath, "err", err) + } + } + + slog.Info("ebs.provider.volume.delete: deleted", "volume", req.VolumeID) + respondProvider(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: ebsprovider.NewVersioned()}) +} + +func handleDeleteSnapshot(ctx context.Context, cfg *Config, msg *nats.Msg) { + var req ebsprovider.DeleteSnapshotRequest + if err := json.Unmarshal(msg.Data, &req); err != nil { + slog.Error("ebs.provider.snapshot.delete: bad request", "err", err) + respondProvider(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)}) + return + } + if req.SchemaVersion != ebsprovider.SchemaVersion { + respondProvider(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: unsupportedVersionError(req.SchemaVersion)}) + return + } + if !validVolumeName(req.SnapshotID) { + respondProvider(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("invalid snapshot id %q", req.SnapshotID)}) + return + } + + store := providerObjectStoreFactory(cfg) + if err := deleteObjectPrefix(ctx, store, cfg.Bucket, req.SnapshotID+"/"); err != nil { + slog.Error("ebs.provider.snapshot.delete: failed", "snapshot", req.SnapshotID, "err", err) + respondProvider(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("delete snapshot objects: %v", err)}) + return + } + + slog.Info("ebs.provider.snapshot.delete: deleted", "snapshot", req.SnapshotID) + respondProvider(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: ebsprovider.NewVersioned()}) +} + +// handleCopySnapshot serves ebs.provider.v1.snapshot.copy. Unlike +// handleCreateSnapshot, this is a plain synchronous request: duplicating a +// snapshot's metadata is a couple of small object writes, not a +// flush-drain-upload sequence slow enough to warrant the accept-then-publish +// pattern. +func handleCopySnapshot(ctx context.Context, cfg *Config, msg *nats.Msg) { + var req ebsprovider.CopySnapshotRequest + if err := json.Unmarshal(msg.Data, &req); err != nil { + slog.Error("ebs.provider.snapshot.copy: bad request", "err", err) + respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)}) + return + } + if req.SchemaVersion != ebsprovider.SchemaVersion { + respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: unsupportedVersionError(req.SchemaVersion)}) + return + } + if !validVolumeName(req.SourceSnapshotID) || !validVolumeName(req.DestinationSnapshotID) || !validVolumeName(req.VolumeID) { + respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("invalid source snapshot, destination snapshot, or volume id")}) + return + } + if req.SourceSnapshotID == req.DestinationSnapshotID { + respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("destination snapshot %q must differ from the source", req.DestinationSnapshotID)}) + return + } + + snapshot, err := copySnapshotOnVolume(ctx, cfg, req.VolumeID, req.SourceSnapshotID, req.DestinationSnapshotID) + if err != nil { + switch { + case errors.Is(err, errSnapshotDestinationExists): + respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: alreadyExistsError("snapshot %s already exists", req.DestinationSnapshotID)}) + case errors.Is(err, viperblock.ErrSnapshotVolumeMismatch): + respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("%v", err)}) + case errors.Is(err, viperblock.ErrStateNotFound), errors.Is(err, os.ErrNotExist): + respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: notFoundError("source snapshot %s not found on volume %s", req.SourceSnapshotID, req.VolumeID)}) + default: + slog.Error("ebs.provider.snapshot.copy: failed", "volume", req.VolumeID, "source", req.SourceSnapshotID, "destination", req.DestinationSnapshotID, "err", err) + respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("copy snapshot: %v", err)}) + } + return + } + + slog.Info("ebs.provider.snapshot.copy: copied", "volume", req.VolumeID, "source", req.SourceSnapshotID, "destination", req.DestinationSnapshotID) + respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: ebsprovider.NewVersioned(), Snapshot: snapshot}) +} + +// snapshotConfigExists probes whether snapshotID's config object is already +// on the backend, the same check viperblock.CopySnapshotMeta makes +// internally before it writes. Doing it here first lets the handler surface +// a precise already_exists error instead of a generic internal one. +func snapshotConfigExists(vb *viperblock.VB, snapshotID string) (bool, error) { + if _, err := vb.Backend.ReadFrom(snapshotID, vbtypes.FileTypeConfig, 0, 0, 0); err == nil { + return true, nil + } else if !errors.Is(err, os.ErrNotExist) { + return false, fmt.Errorf("probe destination snapshot %s: %w", snapshotID, err) + } + return false, nil +} + +// copySnapshotOnVolume duplicates srcSnapshotID as dstSnapshotID by opening +// (or reusing) a VB on volumeID and calling viperblock.CopySnapshotMeta, +// which requires the VB to be opened over the snapshot's own source volume. +// Mirrors snapshotVolumeEngine: prefer the already-mounted VB over opening a +// second engine on the same volume, which would be a second writer. +func copySnapshotOnVolume(ctx context.Context, cfg *Config, volumeID, srcSnapshotID, dstSnapshotID string) (*ebsprovider.Snapshot, error) { + if mv, ok := findMountedVolume(cfg, volumeID); ok && mv.VB != nil { + return copySnapshotMetaWithVB(mv.VB, volumeID, srcSnapshotID, dstSnapshotID) + } + + vb, lease, err := openVolumeVB(ctx, cfg, volumeID) + if err != nil { + return nil, err + } + defer func() { + vb.StopChunkUploader() + vb.StopWALSyncer() + cfg.releaseVolumeLease(ctx, lease) + }() + return copySnapshotMetaWithVB(vb, volumeID, srcSnapshotID, dstSnapshotID) +} + +// copySnapshotMetaWithVB runs the destination-exists probe and the actual +// copy against an already-resolved VB, shared by copySnapshotOnVolume's +// mounted and freshly-opened branches. +func copySnapshotMetaWithVB(vb *viperblock.VB, volumeID, srcSnapshotID, dstSnapshotID string) (*ebsprovider.Snapshot, error) { + exists, err := snapshotConfigExists(vb, dstSnapshotID) + if err != nil { + return nil, err + } + if exists { + return nil, errSnapshotDestinationExists + } + + dst, err := vb.CopySnapshotMeta(srcSnapshotID, dstSnapshotID) + if err != nil { + return nil, fmt.Errorf("copy snapshot meta: %w", err) + } + return &ebsprovider.Snapshot{ + ID: dstSnapshotID, + SourceVolumeID: volumeID, + SizeBytes: utils.SafeUint64ToInt64(vb.GetVolumeSize()), + CreatedAt: dst.CreatedAt.UTC(), + State: ebsprovider.SnapshotStateCompleted, + Handle: snapshotHandle(dstSnapshotID), + }, nil +} + +func handleCreateSnapshot(ctx context.Context, cfg *Config, nc *nats.Conn, msg *nats.Msg) { + var req ebsprovider.CreateSnapshotRequest + if err := json.Unmarshal(msg.Data, &req); err != nil { + slog.Error("ebs.provider.snapshot.create: bad request", "err", err) + respondProvider(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)}) + return + } + if req.SchemaVersion != ebsprovider.SchemaVersion { + respondProvider(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: unsupportedVersionError(req.SchemaVersion)}) + return + } + + // handleCreateSnapshot serves both the wildcard queue subject (volume ID + // is the whole suffix) and the owner subject (volume ID is the first + // token); try the queue shape first since it is the common case. + subjectVolumeID := strings.TrimPrefix(msg.Subject, ebsprovider.SnapshotCreateSubjectPrefix) + if subjectVolumeID == msg.Subject { + if ownerVolumeID, _, ok := ebsprovider.ParseOwnerSubject(msg.Subject); ok { + subjectVolumeID = ownerVolumeID + } + } + if !validVolumeName(subjectVolumeID) || subjectVolumeID != req.VolumeID { + slog.Error("ebs.provider.snapshot.create: subject/volume mismatch", "subject", msg.Subject, "volume", req.VolumeID) + respondProvider(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("volume id %q in subject does not match request volume id %q", subjectVolumeID, req.VolumeID)}) + return + } + if !validVolumeName(req.SnapshotID) { + respondProvider(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("invalid snapshot id %q", req.SnapshotID)}) + return + } + + completionSubject, err := ebsprovider.SnapshotCompletionSubject(req.SnapshotID) + if err != nil { + respondProvider(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("%s", err.Error())}) + return + } + + operationID := utils.GenerateResourceID("op") + respondProvider(ctx, msg, ebsprovider.CreateSnapshotResponse{ + Versioned: ebsprovider.NewVersioned(), + OperationID: operationID, + CompletionSubject: completionSubject, + Snapshot: &ebsprovider.Snapshot{ + ID: req.SnapshotID, + SourceVolumeID: req.VolumeID, + State: ebsprovider.SnapshotStatePending, + }, + }) + + // The caller is responsible for draining the volume (flushing guest + // writes to the live checkpoint) before requesting a snapshot; that + // coordination goes through the guest over ec2.cmd. and + // stays a control-plane concern this provider boundary does not reach. + // WithoutCancel keeps the trace context while dropping the deadline: the + // work has to outlive the request it was accepted by, and it still has to + // be attributable to it. + go completeCreateSnapshot(context.WithoutCancel(ctx), cfg, nc, req, operationID, completionSubject) +} + +// completeCreateSnapshot runs the snapshot work in the background and +// publishes the result to completionSubject, matching the accept-then-publish +// contract NATSProvider.CreateSnapshot waits on. +func completeCreateSnapshot(ctx context.Context, cfg *Config, nc *nats.Conn, req ebsprovider.CreateSnapshotRequest, operationID, completionSubject string) { + completion := ebsprovider.CreateSnapshotResponse{ + Versioned: ebsprovider.NewVersioned(), + OperationID: operationID, + CompletionSubject: completionSubject, + } + + snapshot, err := snapshotVolumeEngine(ctx, cfg, req.VolumeID, req.SnapshotID) + if err != nil { + slog.Error("ebs.provider.snapshot.create: snapshot failed", "volume", req.VolumeID, "snapshot", req.SnapshotID, "err", err) + code := ebsprovider.ErrorCodeInternal + switch { + case errors.Is(err, viperblock.ErrStateNotFound): + code = ebsprovider.ErrorCodeNotFound + case errors.Is(err, errSnapshotExistsElsewhere): + code = ebsprovider.ErrorCodeAlreadyExists + } + completion.Error = &ebsprovider.ProviderError{Code: code, Message: err.Error()} + } else { + completion.Snapshot = snapshot + } + + data, err := json.Marshal(completion) + if err != nil { + slog.Error("ebs.provider.snapshot.create: failed to marshal completion", "volume", req.VolumeID, "snapshot", req.SnapshotID, "err", err) + return + } + if err := nc.Publish(completionSubject, data); err != nil { + slog.Error("ebs.provider.snapshot.create: failed to publish completion", "subject", completionSubject, "err", err) + } +} + +// checkSnapshotIDFree refuses a snapshot ID that already exists unless this +// volume is the one it was taken from. An existing snapshot whose identity +// cannot be read is refused too: unprovable ownership must not overwrite it. +func checkSnapshotIDFree(vb *viperblock.VB, volumeID, snapshotID string) error { + exists, err := snapshotConfigExists(vb, snapshotID) + if err != nil { + return err + } + if !exists { + return nil + } + if _, ident, err := vb.LoadSnapshotBlockMap(snapshotID); err == nil && ident.SourceVolumeName == volumeID { + return nil + } + return fmt.Errorf("%w: %s", errSnapshotExistsElsewhere, snapshotID) +} + +// snapshotVolumeEngine creates snapshotID off volumeID's live checkpoint, +// preferring an already-mounted VB over opening a second engine on the same +// volume. Draining the volume so the checkpoint is current is the caller's +// responsibility (see handleCreateSnapshot). +func snapshotVolumeEngine(ctx context.Context, cfg *Config, volumeID, snapshotID string) (*ebsprovider.Snapshot, error) { + if mv, ok := findMountedVolume(cfg, volumeID); ok && mv.VB != nil { + if err := checkSnapshotIDFree(mv.VB, volumeID, snapshotID); err != nil { + return nil, err + } + if err := mv.VB.LoadLiveCheckpoint(); err != nil { + return nil, fmt.Errorf("load live checkpoint: %w", err) + } + if _, err := mv.VB.CreateSnapshot(snapshotID); err != nil { + return nil, fmt.Errorf("create snapshot: %w", err) + } + return &ebsprovider.Snapshot{ + ID: snapshotID, + SourceVolumeID: volumeID, + SizeBytes: utils.SafeUint64ToInt64(mv.VB.GetVolumeSize()), + CreatedAt: time.Now().UTC(), + State: ebsprovider.SnapshotStateCompleted, + Handle: snapshotHandle(snapshotID), + }, nil + } + + s3cfg := cfg.volumeS3Config(volumeID) + vbconfig := &viperblock.VB{ + VolumeName: volumeID, + VolumeSize: 1, // Recalculated on LoadState. + BaseDir: cfg.BaseDir, + Cache: viperblock.Cache{Config: viperblock.CacheConfig{Size: 0}}, + MasterKey: cfg.masterKey, + EncryptionEnabled: cfg.masterKey != nil, + GCEnabled: cfg.GCEnabled, + } + vb, err := viperblock.New(vbconfig, "s3", s3cfg) + if err != nil { + return nil, fmt.Errorf("new viperblock: %w", err) + } + defer func() { + vb.StopChunkUploader() + vb.StopWALSyncer() + }() + if err := vb.Backend.InitCtx(ctx); err != nil { + return nil, fmt.Errorf("backend init: %w", err) + } + if err := loadStateWithRetry(ctx, vb, volumeID); err != nil { + return nil, fmt.Errorf("load state: %w", err) + } + if err := checkSnapshotIDFree(vb, volumeID, snapshotID); err != nil { + return nil, err + } + if err := vb.LoadLiveCheckpoint(); err != nil { + return nil, fmt.Errorf("load live checkpoint: %w", err) + } + if _, err := vb.CreateSnapshot(snapshotID); err != nil { + return nil, fmt.Errorf("create snapshot: %w", err) + } + return &ebsprovider.Snapshot{ + ID: snapshotID, + SourceVolumeID: volumeID, + SizeBytes: utils.SafeUint64ToInt64(vb.GetVolumeSize()), + CreatedAt: time.Now().UTC(), + State: ebsprovider.SnapshotStateCompleted, + Handle: snapshotHandle(snapshotID), + }, nil +} + +// ownerVerbHandlers pairs each owner-routable verb's subject builder with +// the exact handler its queue subject already dispatches to, so mounted vs. +// unmounted routing differs only in which node answers, never in behavior. +func ownerVerbHandlers(cfg *Config, nc *nats.Conn) []struct { + subjectFn func(string) (string, error) + handler nats.MsgHandler +} { + return []struct { + subjectFn func(string) (string, error) + handler nats.MsgHandler + }{ + {ebsprovider.SnapshotCreateOwnerSubject, tracedProviderHandler(func(ctx context.Context, msg *nats.Msg) { + handleCreateSnapshot(ctx, cfg, nc, msg) + })}, + {ebsprovider.SnapshotCopyOwnerSubject, tracedProviderHandler(func(ctx context.Context, msg *nats.Msg) { + handleCopySnapshot(ctx, cfg, msg) + })}, + {ebsprovider.ExpandVolumeOwnerSubject, tracedProviderHandler(func(ctx context.Context, msg *nats.Msg) { + handleExpandVolume(ctx, cfg, msg) + })}, + {ebsprovider.GetVolumeOwnerSubject, tracedProviderHandler(func(ctx context.Context, msg *nats.Msg) { + handleGetVolume(ctx, cfg, msg) + })}, + } +} + +// subscribeOwnerSubjects registers volumeName's owner subjects as plain, +// non-queue subscriptions, so a request for a mounted volume reaches this node +// rather than a random worker. A failure is logged and skipped, as ConfigSub is. +func subscribeOwnerSubjects(ctx context.Context, cfg *Config, nc *nats.Conn, volumeName string) []*nats.Subscription { + var subs []*nats.Subscription + for _, o := range ownerVerbHandlers(cfg, nc) { + subject, err := o.subjectFn(volumeName) + if err != nil { + slog.ErrorContext(ctx, "Failed to build owner subject", "volume", volumeName, "err", err) + continue + } + sub, err := nc.Subscribe(subject, o.handler) + if err != nil { + slog.ErrorContext(ctx, "Failed to subscribe to owner subject", "subject", subject, "volume", volumeName, "err", err) + continue + } + subs = append(subs, sub) + } + return subs +} + +// unsubscribeOwnerSubjects tears down subs, logging (not failing) any +// individual Unsubscribe error, matching ConfigSub's teardown handling. +func unsubscribeOwnerSubjects(volumeName string, subs []*nats.Subscription) { + for _, sub := range subs { + if sub == nil { + continue + } + if err := sub.Unsubscribe(); err != nil { + slog.Error("Failed to unsubscribe owner subject", "volume", volumeName, "err", err) + } + } +} + +// constructMountedVB builds and opens the daemon-side viperblock engine: cache +// sizing (0 for -efi), StopChunkUploader (nbdkit owns the data path), +// Backend.Init, LoadState. Shared by mountVolume and startup recovery. +func constructMountedVB(ctx context.Context, cfg *Config, volumeName string) (*viperblock.VB, int, error) { + s3cfg := cfg.volumeS3Config(volumeName) + + // TODO: Improve based on system availability. Default 128MB cache + defaultCache := (128 * 1024 * 1024) / int(viperblock.DefaultBlockSize) + + vbconfig := viperblock.VB{ + VolumeName: volumeName, + VolumeSize: 1, // Workaround, calculated on LoadState() + BaseDir: cfg.BaseDir, + Cache: viperblock.Cache{ + Config: viperblock.CacheConfig{ + // TODO: Improve, based on system memory + Size: defaultCache, + }, + }, + VolumeConfig: viperblock.VolumeConfig{}, + MasterKey: cfg.masterKey, + EncryptionEnabled: cfg.masterKey != nil, + } + + // Checked before the cache sizing below: viperblock.New returns a nil VB + // alongside its error, so calling SetCacheSize first panics the caller. + vb, err := viperblock.New(&vbconfig, "s3", s3cfg) + if err != nil { + return nil, 0, fmt.Errorf("connect to viperblock store: %w", err) + } + + // New starts the chunk uploader and WAL syncer, so a failure below must + // release them: the caller gets no handle to stop them with. + opened := false + defer func() { + if !opened { + vb.StopChunkUploader() + vb.StopWALSyncer() + } + }() + + // Enable 128MB cache for main volumes, disable for efi (small, rarely read) + // This cacheSize is passed to nbdkit plugin (separate viperblock instance) + var nbdCacheSize int + if strings.HasSuffix(volumeName, "-efi") { + slog.InfoContext(ctx, "Disabling cache for auxiliary volume", "volume", volumeName) + if err := vb.SetCacheSize(0, 0); err != nil { + slog.ErrorContext(ctx, "Failed to set cache size", "err", err) + } + nbdCacheSize = 0 + } else { + slog.InfoContext(ctx, "Enabling 128MB cache for main volume", "volume", volumeName, "blocks", defaultCache) + if err := vb.SetCacheSize(defaultCache, 0); err != nil { + slog.ErrorContext(ctx, "Failed to set cache size", "err", err) + } + nbdCacheSize = defaultCache + } + + // This daemon-side VB tracks state only; the nbdkit plugin process owns + // the data path and its own uploader. Stop this VB's background uploader + // so it cannot overwrite the live checkpoint every 30s (AEAD corruption). + vb.StopChunkUploader() + + if cfg.Debug { + vb.SetDebug(true) + } + + if err := vb.Backend.InitCtx(ctx); err != nil { + return nil, 0, err + } + + // Retry on transient backend errors so daemon recovery doesn't tip a healthy volume into cleanup. + if err := loadStateWithRetry(ctx, vb, volumeName); err != nil { + return nil, 0, err + } + + opened = true + return vb, nbdCacheSize, nil +} + +// mountVolume is launchService's ebs.mount body, extracted so the legacy +// ebs.mount handler and the ebs.provider.v1.*.mount handler share one nbdkit +// mount path instead of risking two divergent implementations of it. It +// starts (or fails to start) nbdkit for volumeName and registers the result +// in cfg.MountedVolumes; it does not publish a response, that stays with the +// caller. +func mountVolume(ctx context.Context, cfg *Config, nc *nats.Conn, volumeName string, readOnly bool) (types.EBSMountResponse, error) { + // A volume this node already exports must not get a second nbdkit, which + // is the double-writer hazard the provider boundary exists to prevent. + // The guard lives here rather than in one handler because the legacy + // ebs.mount subject is the route production actually takes: a retried + // attach racing a fresh one would otherwise start two real exports. + if mv, ok := findMountedVolume(cfg, volumeName); ok { + // Access mode is fixed when nbdkit starts, so a remount asking for the + // other mode cannot be answered with the running export. + if mv.ReadOnly != readOnly { + err := fmt.Errorf("volume %s is already mounted read_only=%t on this node", volumeName, mv.ReadOnly) + return types.EBSMountResponse{Error: err.Error()}, err + } + slog.InfoContext(ctx, "ebs.mount: already mounted, returning existing export", "volume", volumeName, "uri", mv.NBDURI) + return types.EBSMountResponse{URI: mv.NBDURI, Mounted: true}, nil + } + + // Clear any receipt left by a previous mount before anything else can + // return early, so a stale receipt can never survive into this mount. + clearStaleSealReceipt(cfg.BaseDir, volumeName) + + ctx, mountSpan := otel.Tracer(viperblockdTracerName).Start(ctx, "ebs.mount", + trace.WithAttributes(attribute.String("volume.id", volumeName))) + + var ebsResponse types.EBSMountResponse + ebsResponse.Mounted = false + defer func() { endSpanWithResponseError(mountSpan, ebsResponse.Error) }() + + vb, nbdCacheSize, lease, err := cfg.buildVB(ctx, volumeName) + if err != nil { + ebsResponse.Error = err.Error() + return ebsResponse, err + } + + // The lease belongs to the export from here on. Every failure below leaves + // without one, so it has to go back or the volume stays wedged until it + // expires. + mounted := false + defer func() { + if !mounted { + cfg.releaseVolumeLease(ctx, lease) + } + }() + + mountSpan.SetAttributes(attribute.Int64("volume.size_bytes", utils.SafeUint64ToInt64(vb.GetVolumeSize()))) + + useTCP := cfg.NBDTransport == types.NBDTransportTCP + + var nbdURI string + var nbdSocket string + var nbdPort int + + if useTCP { + // TCP transport - find a free port + portStr, err := viperblock.FindFreePort() + if err != nil { + ebsResponse.Error = err.Error() + return ebsResponse, err + } + + // Parse the port from the address + parts := strings.Split(portStr, ":") + nbdPort, err = strconv.Atoi(parts[len(parts)-1]) + if err != nil { + slog.ErrorContext(ctx, "Failed to convert port to int", "err", err) + ebsResponse.Error = fmt.Sprintf("failed to parse port: %v", err) + return ebsResponse, err + } + + nbdURI = utils.FormatNBDTCPURI("127.0.0.1", nbdPort) + slog.InfoContext(ctx, "Mounting volume (TCP)", "name", volumeName, "port", nbdPort, "uri", nbdURI) + } else { + // Unix socket transport (default) - generate unique socket path + nbdSocket, err = utils.GenerateUniqueSocketFile(volumeName) + if err != nil { + ebsResponse.Error = err.Error() + return ebsResponse, err + } + + nbdURI = utils.FormatNBDSocketURI(nbdSocket) + slog.InfoContext(ctx, "Mounting volume (socket)", "name", volumeName, "socket", nbdSocket, "uri", nbdURI) + } + + // Generate PID file for nbdkit process + nbdPidFile, err := utils.GeneratePidFile(fmt.Sprintf("nbdkit-vol-%s", volumeName)) + if err != nil { + slog.ErrorContext(ctx, "Failed to generate nbdkit pid file", "err", err) + ebsResponse.Error = fmt.Sprintf("failed to generate pid file: %v", err) + return ebsResponse, err + } + + nbdConfig := nbd.NBDKitConfig{ + Port: nbdPort, + Socket: nbdSocket, + UseTCP: useTCP, + PidFile: nbdPidFile, + PluginPath: cfg.PluginPath, + BaseDir: cfg.BaseDir, + Host: admin.DialTarget(cfg.S3Host), + Verbose: false, + Size: utils.SafeUint64ToInt64(vb.GetVolumeSize()), + Volume: volumeName, + Bucket: cfg.Bucket, + Region: cfg.Region, + AccessKey: cfg.AccessKey, + SecretKey: cfg.SecretKey, + CacheSize: nbdCacheSize, + ShardWAL: cfg.ShardWAL, + GCEnabled: cfg.GCEnabled, + EncryptionKeyFile: cfg.EncryptionKeyFile, + ReadOnly: readOnly, + } + + // Create a unique error channel for this specific mount request + processChan := make(chan int, 1) + exitChan := make(chan int, 1) + + // TODO: Improve, use a process manager to track the (multiple) nbdkit process + go func() { + slog.Debug("Executing nbdkit") + + cmd, err := nbdConfig.Execute() + if err != nil { + slog.Error("Failed to execute nbdkit", "err", err) + // Signal error (no PID) to parent goroutine + processChan <- 0 + return + } + + pid := cmd.Process.Pid + // Signal successful startup w/ PID + processChan <- pid + + err = cmd.Wait() + + if err != nil { + slog.Error("Failed to wait for nbdkit", "err", err) + exitChan <- 1 + return + } + + exitCode := cmd.ProcessState.ExitCode() + + exitChan <- exitCode + + slog.Error("NBDKit exited", "code", exitCode) + }() + + pid := <-processChan + + if pid == 0 { + ebsResponse.Error = "Failed to start nbdkit" + return ebsResponse, errors.New(ebsResponse.Error) + } + + ctx, readySpan := otel.Tracer(viperblockdTracerName).Start(ctx, "ebs.mount.nbdkit_ready") + readyDeadline := time.Now().Add(nbdkitReadyDeadline) + + network, address := "tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(nbdPort)) + if nbdSocket != "" { + network, address = "unix", nbdSocket + + // NBDKit creates the socket with its own umask (typically 0755). + // The daemon (different user, same group) needs write access to + // connect, and the file has to exist before it can be chmod'd. + if err := waitForSocketFile(ctx, nbdSocket, exitChan, readyDeadline); err != nil { + readySpan.End() + ebsResponse.Error = err.Error() + return ebsResponse, err + } + if err := os.Chmod(nbdSocket, 0770); err != nil { //nolint:gosec // socket needs group-write for cross-service access + slog.WarnContext(ctx, "Failed to chmod NBD socket", "socket", nbdSocket, "err", err) + } + } + + if err := waitForNBDDial(ctx, network, address, exitChan, readyDeadline); err != nil { + readySpan.End() + ebsResponse.Error = err.Error() + return ebsResponse, err + } + readySpan.End() + + slog.InfoContext(ctx, "NBDKit is accepting connections", "network", network, "address", address) + + ebsResponse.Mounted = true + ebsResponse.URI = nbdURI + + // Subscribe to volume-specific config-update topic so encrypted-volume + // metadata writes route to this node's live VB (the StateSeqNum owner). + configSub, err := nc.Subscribe(fmt.Sprintf("ebs.config.%s", volumeName), makeConfigUpdateHandler(vb, volumeName)) + if err != nil { + slog.ErrorContext(ctx, "Failed to subscribe to volume config topic", "volume", volumeName, "err", err) + } + + // Owner-routed subscriptions let snapshot/expand/describe requests reach + // this node directly while it holds the live engine, instead of the + // spinifex-workers queue group delivering them to a random node. + ownerSubs := subscribeOwnerSubjects(ctx, cfg, nc, volumeName) + + cfg.mu.Lock() + cfg.MountedVolumes = append(cfg.MountedVolumes, MountedVolume{ + Name: volumeName, + Port: nbdPort, + Socket: nbdSocket, + NBDURI: nbdURI, + PID: pid, + VB: vb, + ConfigSub: configSub, + OwnerSubs: ownerSubs, + ReadOnly: readOnly, + Lease: lease, + }) + cfg.mu.Unlock() + + mounted = true + return ebsResponse, nil +} + +// unmountVolume is launchService's ebs.unmount body, extracted for the same +// reason as mountVolume: the legacy ebs.unmount handler and the +// ebs.provider.v1.*.unmount handler must share the one seal-decision path +// rather than risk it diverging between two copies. +// +// A failed seal leaves volumeName in cfg.MountedVolumes (response.Error set, +// response.Mounted stays false but the entry is not removed) so a retry +// re-attempts the seal; a volume with no matching entry gets +// response.NotFound set instead of a bare error, so callers can tell "never +// mounted here" apart from "seal failed". +func unmountVolume(ctx context.Context, cfg *Config, volumeName string) (types.EBSUnMountResponse, error) { + ctx, unmountSpan := otel.Tracer(viperblockdTracerName).Start(ctx, "ebs.unmount", + trace.WithAttributes(attribute.String("volume.id", volumeName))) + + // Find the volume and extract references while holding the lock, + // then release before calling VB.Close() (which does heavy S3 I/O). + var ebsResponse types.EBSUnMountResponse + defer func() { endSpanWithResponseError(unmountSpan, ebsResponse.Error) }() + var matched MountedVolume + var matchIdx = -1 + cfg.mu.Lock() + for i, volume := range cfg.MountedVolumes { + if volume.Name == volumeName { + matched = volume + matchIdx = i + break + } + } + cfg.mu.Unlock() + + if matchIdx >= 0 { + ebsResponse = types.EBSUnMountResponse{ + Volume: matched.Name, + Mounted: false, + } + + // Unsubscribe from volume-specific config-update topic + if matched.ConfigSub != nil { + if err := matched.ConfigSub.Unsubscribe(); err != nil { + slog.ErrorContext(ctx, "Failed to unsubscribe config topic", "volume", volumeName, "err", err) + } + } + unsubscribeOwnerSubjects(matched.Name, matched.OwnerSubs) + + // Stop background goroutines on the state-tracking VB. + // Actual I/O is in the nbdkit plugin process; sealVolumeVB below + // opens a fresh VB and calls Close() for the proper seal. + if matched.VB != nil { + matched.VB.StopChunkUploader() + matched.VB.StopWALSyncer() + } + + if err := utils.KillProcess(matched.PID); err != nil { + slog.ErrorContext(ctx, "Failed to kill nbdkit process", "pid", matched.PID, "err", err) + } + + // nbdkit is now dead, so no process writes the shared BaseDir: seal + // the block map to predastore for volumes that hold local state to + // flush (see volumeNeedsSeal). + if isAuxVolume(matched.Name) { + // Auxiliary volumes carry no durable guest data, so there is + // nothing to seal even when local state is present. + } else if volumeNeedsSeal(matched.Name, cfg.BaseDir) { + // Local state survived: the plugin's seal either failed or was + // cut short, so this fallback is the real seal. + if err := cfg.seal(ctx, matched.Name); err != nil { + slog.ErrorContext(ctx, "ebs.unmount: failed to seal volume to predastore", "volume", matched.Name, "err", err) + ebsResponse.Error = fmt.Sprintf("seal volume: %v", err) + } else { + slog.InfoContext(ctx, "ebs.unmount: volume sealed to predastore", "volume", matched.Name) + } + } else if consumeSealReceipt(cfg.BaseDir, matched.Name) { + // Healthy path: the plugin sealed to predastore and removed + // its local state itself, leaving this receipt as proof. + slog.InfoContext(ctx, "ebs.unmount: volume already sealed by nbdkit plugin", "volume", matched.Name) + } else { + // A durable volume reached unmount with no local WAL and no + // seal receipt: this node never held its state, so there is + // nothing to seal. WARN since this can mask a durability gap + // the seal would otherwise close. + slog.WarnContext(ctx, "ebs.unmount: no local viperblock state for volume, skipping seal", "volume", matched.Name, "baseDir", cfg.BaseDir) + } + + // Remove the socket file if using socket transport + if matched.Socket != "" { + slog.InfoContext(ctx, "Removing socket file", "socket", matched.Socket) + if err := os.Remove(matched.Socket); err != nil && !os.IsNotExist(err) { + slog.ErrorContext(ctx, "Failed to delete nbd socket", "err", err, "socket", matched.Socket) + } + } + + // Only drop the volume from MountedVolumes once the seal actually + // succeeded (or none was needed): a failed seal must leave it + // mounted so a retry re-attempts sealVolumeVB, rather than a + // caller seeing "not found" and mistaking a failed seal for a + // completed one. + if ebsResponse.Error == "" { + cfg.mu.Lock() + for i, volume := range cfg.MountedVolumes { + if volume.Name == matched.Name { + cfg.MountedVolumes = append(cfg.MountedVolumes[:i], cfg.MountedVolumes[i+1:]...) + break + } + } + cfg.mu.Unlock() + + // The export is gone, so the volume may be opened elsewhere. A + // failed seal keeps the entry, and so keeps the lease, because the + // retry has to be the only writer too. + cfg.releaseVolumeLease(ctx, matched.Lease) + } + } + + if matchIdx < 0 { + ebsResponse = types.EBSUnMountResponse{ + Volume: volumeName, + Error: fmt.Sprintf("Volume %s not found", volumeName), + NotFound: true, + } + } + + if ebsResponse.Error != "" { + return ebsResponse, errors.New(ebsResponse.Error) + } + return ebsResponse, nil +} + +// handlePublishVolume serves ebs.provider.v1..mount, the provider-neutral +// front for the same nbdkit mount path ebs.mount uses. It is node-addressed +// (registerProviderSubjects only subscribes it when cfg.NodeName is set), so +// unlike the legacy handler there is no queue-group fallback to reason about. +func handlePublishVolume(ctx context.Context, cfg *Config, nc *nats.Conn, msg *nats.Msg) { + var req ebsprovider.PublishVolumeRequest + if err := json.Unmarshal(msg.Data, &req); err != nil { + slog.Error("ebs.provider.volume.publish: bad request", "err", err) + respondProvider(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)}) + return + } + if req.SchemaVersion != ebsprovider.SchemaVersion { + slog.Error("ebs.provider.volume.publish: unsupported schema version", "version", req.SchemaVersion) + respondProvider(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: unsupportedVersionError(req.SchemaVersion)}) + return + } + if !validVolumeName(req.VolumeID) { + respondProvider(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("invalid volume id %q", req.VolumeID)}) + return + } + // Idempotent republish: a volume this node already has mounted must not + // start a second nbdkit against it (the double-writer hazard this + // provider boundary exists to prevent). Match memory.go's behaviour. + if mv, ok := findMountedVolume(cfg, req.VolumeID); ok { + // Access mode is fixed when nbdkit starts, so a republish asking for + // the other mode cannot be answered with the running export. + if mv.ReadOnly != req.ReadOnly { + respondProvider(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: volumeInUseError("volume %s is already published read_only=%t on this node", req.VolumeID, mv.ReadOnly)}) + return + } + slog.Info("ebs.provider.volume.publish: already published, returning existing attachment", "volume", req.VolumeID) + respondProvider(ctx, msg, ebsprovider.PublishVolumeResponse{ + Versioned: ebsprovider.NewVersioned(), + Published: &ebsprovider.PublishedVolume{ + VolumeID: req.VolumeID, + NodeID: cfg.NodeName, + NBDURI: mv.NBDURI, + }, + }) + return + } + + mountResp, err := mountVolume(ctx, cfg, nc, req.VolumeID, req.ReadOnly) + if err != nil { + if errors.Is(err, viperblock.ErrStateNotFound) { + respondProvider(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: notFoundError("volume %s not found", req.VolumeID)}) + return + } + // Another node holds the volume's lease. This is the exclusion working, + // not a failure: reporting it as internal tells the caller something + // broke when the answer is that someone else has the volume. + if errors.Is(err, errVolumeLeaseHeld) { + respondProvider(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: volumeInUseError("volume %s is held by another node: %v", req.VolumeID, err)}) + return + } + slog.Error("ebs.provider.volume.publish: mount failed", "volume", req.VolumeID, "err", err) + respondProvider(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("mount volume: %v", err)}) + return + } + + slog.Info("ebs.provider.volume.publish: published", "volume", req.VolumeID, "node", cfg.NodeName) + respondProvider(ctx, msg, ebsprovider.PublishVolumeResponse{ + Versioned: ebsprovider.NewVersioned(), + Published: &ebsprovider.PublishedVolume{ + VolumeID: req.VolumeID, + NodeID: cfg.NodeName, + NBDURI: mountResp.URI, + }, + }) +} + +// handleUnpublishVolume serves ebs.provider.v1..unmount, fronting the +// same seal-decision path ebs.unmount uses. A failed seal is reported as an +// error while leaving the volume in cfg.MountedVolumes (see unmountVolume), +// never as success. +func handleUnpublishVolume(ctx context.Context, cfg *Config, msg *nats.Msg) { + var req ebsprovider.UnpublishVolumeRequest + if err := json.Unmarshal(msg.Data, &req); err != nil { + slog.Error("ebs.provider.volume.unpublish: bad request", "err", err) + respondProvider(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)}) + return + } + if req.SchemaVersion != ebsprovider.SchemaVersion { + slog.Error("ebs.provider.volume.unpublish: unsupported schema version", "version", req.SchemaVersion) + respondProvider(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: unsupportedVersionError(req.SchemaVersion)}) + return + } + if !validVolumeName(req.VolumeID) { + respondProvider(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("invalid volume id %q", req.VolumeID)}) + return + } + + unmountResp, err := unmountVolume(ctx, cfg, req.VolumeID) + if err != nil { + // Nothing mounted here is already the state unpublish asks for, so a + // retry after a request that completed server-side converges rather + // than erroring, matching the legacy path's unmountResponseError. + if unmountResp.NotFound { + slog.Info("ebs.provider.volume.unpublish: already unpublished", "volume", req.VolumeID, "node", cfg.NodeName) + respondProvider(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: ebsprovider.NewVersioned()}) + return + } + slog.Error("ebs.provider.volume.unpublish: unmount failed", "volume", req.VolumeID, "err", err) + respondProvider(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("unmount volume: %v", err)}) + return + } + + slog.Info("ebs.provider.volume.unpublish: unpublished", "volume", req.VolumeID, "node", cfg.NodeName) + respondProvider(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: ebsprovider.NewVersioned()}) +} diff --git a/spinifex/services/viperblockd/provider_handlers_invariants_test.go b/spinifex/services/viperblockd/provider_handlers_invariants_test.go new file mode 100644 index 000000000..67acd50e6 --- /dev/null +++ b/spinifex/services/viperblockd/provider_handlers_invariants_test.go @@ -0,0 +1,298 @@ +package viperblockd + +// Tests proving a mounted volume's snapshot handlers reuse the live VB via +// findMountedVolume rather than open a second engine, pinned on the same +// "viperblock volume opened" log record a production alarm would watch. + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "runtime" + "slices" + "strings" + "testing" + "time" + + "github.com/mulgadc/spinifex/spinifex/ebsprovider" + "github.com/nats-io/nats.go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// volumeOpenCount counts "viperblock volume opened" records in logs +// (captureLogs' output) naming volumeID via a "volume=" token, so a +// volume ID that is a prefix of another one never falsely matches. +func volumeOpenCount(logs, volumeID string) int { + count := 0 + for line := range strings.SplitSeq(logs, "\n") { + if !strings.Contains(line, `msg="viperblock volume opened"`) { + continue + } + if slices.Contains(strings.Fields(line), "volume="+volumeID) { + count++ + } + } + return count +} + +// fastFailingS3Host starts a local server answering every request with 400 +// Bad Request, for use as cfg.S3Host. A plain 4xx isn't retried by the AWS +// SDK's default retryer, so a forced engine construction fails immediately. +func fastFailingS3Host(t *testing.T) string { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadRequest) + })) + t.Cleanup(srv.Close) + return srv.URL +} + +// stackFrameCount counts live stack frames naming starter. A running +// background goroutine contributes a fixed number of them, so a before/after +// comparison detects one that was never stopped. +func stackFrameCount(t *testing.T, starter string) int { + t.Helper() + buf := make([]byte, 1<<20) + n := runtime.Stack(buf, true) + return strings.Count(string(buf[:n]), starter) +} + +func chunkUploaderFrames(t *testing.T) int { + t.Helper() + return stackFrameCount(t, "viperblock.(*VB).StartChunkUploader") +} + +func walSyncerFrames(t *testing.T) int { + t.Helper() + return stackFrameCount(t, "viperblock.(*VB).StartWALSyncer") +} + +// TestOpenVolumeVB_FailedOpenLeavesNoChunkUploader pins the release half of +// the same invariant: viperblock.New starts an uploader before the backend is +// touched, so an open that fails afterwards must not abandon it. +func TestOpenVolumeVB_FailedOpenLeavesNoChunkUploader(t *testing.T) { + _, natsURL := setupEmbeddedNATS(t) + cfg := setupTestConfig(t, natsURL) + cfg.S3Host = fastFailingS3Host(t) + + before := chunkUploaderFrames(t) + + vb, lease, err := openVolumeVB(t.Context(), cfg, "vol-invariantleak001") + require.Error(t, err, "the fast-failing backend must fail the open, or this test proves nothing") + require.Nil(t, vb) + require.Nil(t, lease) + + assert.Equal(t, before, chunkUploaderFrames(t), + "a failed open must stop the uploader viperblock.New started; the caller gets no handle to stop it with") +} + +// TestConstructMountedVB_FailedOpenLeavesNoBackgroundGoroutines is the same +// invariant for the mount/recovery constructor: it stops the uploader up +// front, so the syncer viperblock.New started is what a late failure abandons. +func TestConstructMountedVB_FailedOpenLeavesNoBackgroundGoroutines(t *testing.T) { + _, natsURL := setupEmbeddedNATS(t) + cfg := setupTestConfig(t, natsURL) + cfg.S3Host = fastFailingS3Host(t) + + beforeUploaders := chunkUploaderFrames(t) + beforeSyncers := walSyncerFrames(t) + + vb, _, err := constructMountedVB(context.Background(), cfg, "vol-invariantcvbleak") + require.Error(t, err, "the fast-failing backend must fail construction, or this test proves nothing") + require.Nil(t, vb) + + assert.Equal(t, beforeUploaders, chunkUploaderFrames(t), + "a failed construction must leave no chunk uploader running") + assert.Equal(t, beforeSyncers, walSyncerFrames(t), + "a failed construction must stop the WAL syncer viperblock.New started; the caller gets no handle to stop it with") +} + +// TestProviderHandlers_SnapshotPath_MountedVolumeDoesNotOpenEngine drives +// handleCreateSnapshot and handleCopySnapshot against a live mounted VB and +// asserts zero "viperblock volume opened" records name that volume. +func TestProviderHandlers_SnapshotPath_MountedVolumeDoesNotOpenEngine(t *testing.T) { + _, natsURL := setupEmbeddedNATS(t) + cfg := setupTestConfig(t, natsURL) + nc := startProviderSubjects(t, cfg, natsURL) + + const volumeName = "vol-invariantmount01" + const srcSnapshotID = "snap-invariantsrc001" + const dstSnapshotID = "snap-invariantdst001" + + vb := createTestVBWithState(t, volumeName) + t.Cleanup(func() { + vb.StopChunkUploader() + vb.StopWALSyncer() + }) + cfg.MountedVolumes = append(cfg.MountedVolumes, MountedVolume{Name: volumeName, VB: vb}) + + logs := captureLogs(t) + + wantCompletionSubject, err := ebsprovider.SnapshotCompletionSubject(srcSnapshotID) + require.NoError(t, err) + completionSub, err := nc.SubscribeSync(wantCompletionSubject) + require.NoError(t, err) + require.NoError(t, nc.Flush()) + + createBody := marshalRequest(t, map[string]any{ + "schema_version": ebsprovider.SchemaVersion, + "volume_id": volumeName, + "snapshot_id": srcSnapshotID, + }) + requestProvider(t, nc, ebsprovider.SnapshotCreateSubjectPrefix+volumeName, createBody) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + completionMsg, err := completionSub.NextMsgWithContext(ctx) + require.NoError(t, err) + var completed ebsprovider.CreateSnapshotResponse + require.NoError(t, json.Unmarshal(completionMsg.Data, &completed)) + require.Nil(t, completed.Error, "snapshot create on a mounted volume must succeed with no S3 backend involved") + + copyBody := marshalRequest(t, map[string]any{ + "schema_version": ebsprovider.SchemaVersion, + "source_snapshot_id": srcSnapshotID, + "destination_snapshot_id": dstSnapshotID, + "volume_id": volumeName, + }) + copyMsg := requestProvider(t, nc, ebsprovider.CopySnapshotSubject, copyBody) + var copyResp ebsprovider.CopySnapshotResponse + require.NoError(t, json.Unmarshal(copyMsg.Data, ©Resp)) + require.Nil(t, copyResp.Error, "snapshot copy on a mounted volume must succeed with no S3 backend involved") + + assert.Equal(t, 0, volumeOpenCount(logs.String(), volumeName), + "snapshot create+copy on an already-mounted volume must reuse the live VB, never open a second engine") +} + +// TestProviderHandlers_CreateSnapshot_UnmountedVolumeOpensEngine is the +// contrast case: with no mounted VB, handleCreateSnapshot must construct its +// own engine (one log record), even though the fast-failing backend then fails it. +func TestProviderHandlers_CreateSnapshot_UnmountedVolumeOpensEngine(t *testing.T) { + _, natsURL := setupEmbeddedNATS(t) + cfg := setupTestConfig(t, natsURL) + cfg.S3Host = fastFailingS3Host(t) + nc := startProviderSubjects(t, cfg, natsURL) + + const volumeName = "vol-invariantopen001" + const snapshotID = "snap-invariantopen01" + + logs := captureLogs(t) + + wantCompletionSubject, err := ebsprovider.SnapshotCompletionSubject(snapshotID) + require.NoError(t, err) + completionSub, err := nc.SubscribeSync(wantCompletionSubject) + require.NoError(t, err) + require.NoError(t, nc.Flush()) + + body := marshalRequest(t, map[string]any{ + "schema_version": ebsprovider.SchemaVersion, + "volume_id": volumeName, + "snapshot_id": snapshotID, + }) + requestProvider(t, nc, ebsprovider.SnapshotCreateSubjectPrefix+volumeName, body) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + completionMsg, err := completionSub.NextMsgWithContext(ctx) + require.NoError(t, err) + var completed ebsprovider.CreateSnapshotResponse + require.NoError(t, json.Unmarshal(completionMsg.Data, &completed)) + require.NotNil(t, completed.Error, + "the fast-failing backend must fail the snapshot, proving this really took the construct-a-new-engine path rather than silently reusing something") + + assert.Equal(t, 1, volumeOpenCount(logs.String(), volumeName), + "snapshot create on an unmounted volume must construct exactly one engine") +} + +// TestProviderHandlers_SnapshotPath_DifferentNodeConfigDoesNotOpenEngine puts +// the mount on cfgOwner, reachable only by owner subject, and makes cfgOther +// the queue group's sole member. Zero opens proves the owner subject won. +func TestProviderHandlers_SnapshotPath_DifferentNodeConfigDoesNotOpenEngine(t *testing.T) { + _, natsURL := setupEmbeddedNATS(t) + + cfgOther := setupTestConfig(t, natsURL) + cfgOther.S3Host = fastFailingS3Host(t) + startProviderSubjects(t, cfgOther, natsURL) + + cfgOwner := setupTestConfig(t, natsURL) + ncOwner, err := nats.Connect(natsURL) + require.NoError(t, err) + t.Cleanup(ncOwner.Close) + + const volumeName = "vol-diffnodeowner01" + const srcSnapshotID = "snap-diffnodesrc001" + const dstSnapshotID = "snap-diffnodedst001" + + vb := createTestVBWithState(t, volumeName) + t.Cleanup(func() { + vb.StopChunkUploader() + vb.StopWALSyncer() + }) + + ownerSubs := subscribeOwnerSubjects(context.Background(), cfgOwner, ncOwner, volumeName) + require.Len(t, ownerSubs, 4, "all four owner verbs must subscribe cleanly") + require.NoError(t, ncOwner.Flush()) + + cfgOwner.MountedVolumes = append(cfgOwner.MountedVolumes, MountedVolume{Name: volumeName, VB: vb, OwnerSubs: ownerSubs}) + + logs := captureLogs(t) + + // The client connects through cfgOther's connection: with no owner + // routing this would have no bearing on which node answers (NATS + // subjects are server-wide, not connection-scoped), so this alone does + // not bias the result toward the fix passing. + ncClient, err := nats.Connect(natsURL) + require.NoError(t, err) + t.Cleanup(ncClient.Close) + provider := ebsprovider.NewNATSProvider(ncClient, 5*time.Second) + + snap, err := provider.CreateSnapshot(context.Background(), ebsprovider.CreateSnapshotRequest{ + Versioned: ebsprovider.NewVersioned(), + SnapshotID: srcSnapshotID, + VolumeID: volumeName, + }) + require.NoError(t, err, "the mounting node's owner subject must serve this, not the other node's fast-failing queue handler") + require.NotNil(t, snap) + + _, err = provider.CopySnapshot(context.Background(), ebsprovider.CopySnapshotRequest{ + Versioned: ebsprovider.NewVersioned(), + SourceSnapshotID: srcSnapshotID, + DestinationSnapshotID: dstSnapshotID, + VolumeID: volumeName, + }) + require.NoError(t, err) + + assert.Equal(t, 0, volumeOpenCount(logs.String(), volumeName), + "a snapshot request for a volume mounted by a different node's config must never open a second engine") +} + +// TestProviderHandlers_CopySnapshot_UnmountedVolumeOpensEngine is +// handleCopySnapshot's counterpart: with no mounted VB, copySnapshotOnVolume +// falls through to openVolumeVB, which calls viperblock.New() exactly once. +func TestProviderHandlers_CopySnapshot_UnmountedVolumeOpensEngine(t *testing.T) { + _, natsURL := setupEmbeddedNATS(t) + cfg := setupTestConfig(t, natsURL) + cfg.S3Host = fastFailingS3Host(t) + nc := startProviderSubjects(t, cfg, natsURL) + + const volumeName = "vol-invariantcopy001" + + logs := captureLogs(t) + + body := marshalRequest(t, map[string]any{ + "schema_version": ebsprovider.SchemaVersion, + "source_snapshot_id": "snap-invariantcopysrc", + "destination_snapshot_id": "snap-invariantcopydst", + "volume_id": volumeName, + }) + msg := requestProvider(t, nc, ebsprovider.CopySnapshotSubject, body) + var resp ebsprovider.CopySnapshotResponse + require.NoError(t, json.Unmarshal(msg.Data, &resp)) + require.NotNil(t, resp.Error, + "the fast-failing backend must fail the copy, proving this really took the construct-a-new-engine path rather than silently reusing something") + + assert.Equal(t, 1, volumeOpenCount(logs.String(), volumeName), + "snapshot copy on an unmounted volume must construct exactly one engine") +} diff --git a/spinifex/services/viperblockd/provider_handlers_test.go b/spinifex/services/viperblockd/provider_handlers_test.go new file mode 100644 index 000000000..c6049be9a --- /dev/null +++ b/spinifex/services/viperblockd/provider_handlers_test.go @@ -0,0 +1,1046 @@ +package viperblockd + +// Tests for the ebs.provider.v1.* handlers registered by registerProviderSubjects. +// Engine-backed cases (CreateVolume idempotency, ExpandVolume in-use, CreateSnapshot) +// use a live mounted VB on the file backend (createTestVBWithState from +// viperblockd_handlers_test.go), matching this package's existing convention for +// tests that need a real viperblock engine but not real predastore. Success paths +// that require opening a DETACHED engine against S3 (CreateVolume of a fresh +// volume, GetVolume/ExpandVolume of an unmounted one) are out of scope here for +// the same reason handlers/ec2/volume's unit tests stop at validation: nothing +// past Backend.Init() can execute without a live backend, and this file must not +// add a network dependency to a unit test. + +import ( + "bytes" + "context" + "encoding/json" + "maps" + "os" + "path/filepath" + "testing" + "time" + + awssdk "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/mulgadc/spinifex/spinifex/ebsprovider" + "github.com/mulgadc/spinifex/spinifex/objectstore" + "github.com/nats-io/nats.go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// versionedErrorResponse decodes the common shape every ebs.provider.v1.* +// response carries: an embedded Versioned and an optional ProviderError. +// Every concrete response type marshals a superset of these fields, so +// decoding into this narrower struct works regardless of which handler +// answered. +type versionedErrorResponse struct { + ebsprovider.Versioned + + Error *ebsprovider.ProviderError `json:"error,omitempty"` +} + +// startProviderSubjects wires registerProviderSubjects onto a fresh NATS +// connection for cfg, without launching the rest of launchService (mount, +// unmount, sync, config, delete), which are unrelated to the ebs.provider.v1.* +// contract under test here. +func startProviderSubjects(t *testing.T, cfg *Config, natsURL string) *nats.Conn { + t.Helper() + nc, err := nats.Connect(natsURL) + require.NoError(t, err) + t.Cleanup(nc.Close) + require.NoError(t, registerProviderSubjects(cfg, nc)) + return nc +} + +func requestProvider(t *testing.T, nc *nats.Conn, subject string, payload []byte) *nats.Msg { + t.Helper() + msg, err := nc.Request(subject, payload, 5*time.Second) + require.NoError(t, err) + return msg +} + +func marshalRequest(t *testing.T, req map[string]any) []byte { + t.Helper() + data, err := json.Marshal(req) + require.NoError(t, err) + return data +} + +// providerSubjectCase is a subject plus a validly-shaped request body (minus +// schema_version) for the generic malformed-JSON / wrong-version table below. +type providerSubjectCase struct { + name string + subject string + body map[string]any +} + +func providerSubjectCases() []providerSubjectCase { + return []providerSubjectCase{ + {"capabilities", ebsprovider.CapabilitiesSubject, map[string]any{}}, + {"create_volume", ebsprovider.CreateVolumeSubject, map[string]any{ + "volume_id": "vol-testcreate0001", + "capacity_range": map[string]any{"required_bytes": 1073741824}, + }}, + {"get_volume", ebsprovider.GetVolumeSubject, map[string]any{"volume_id": "vol-testget00001"}}, + {"expand_volume", ebsprovider.ExpandVolumeSubject, map[string]any{ + "volume_id": "vol-testexpand001", + "capacity_range": map[string]any{"required_bytes": 1073741824}, + }}, + {"delete_volume", ebsprovider.DeleteVolumeSubject, map[string]any{"volume_id": "vol-testdelete001"}}, + {"delete_snapshot", ebsprovider.DeleteSnapshotSubject, map[string]any{"snapshot_id": "snap-testdelete01"}}, + {"copy_snapshot", ebsprovider.CopySnapshotSubject, map[string]any{ + "source_snapshot_id": "snap-testcopysrc01", + "destination_snapshot_id": "snap-testcopydst01", + "volume_id": "vol-testcopysnap01", + }}, + {"create_snapshot", ebsprovider.SnapshotCreateSubjectPrefix + "vol-testsnap0001", map[string]any{ + "volume_id": "vol-testsnap0001", + "snapshot_id": "snap-testsnap0001", + }}, + // publish_volume/unpublish_volume are node-addressed; the subject + // below must match setupTestConfig's NodeName ("test-node"). + {"publish_volume", "ebs.provider.v1.test-node.mount", map[string]any{ + "volume_id": "vol-testpublish001", + "node_id": "test-node", + }}, + {"unpublish_volume", "ebs.provider.v1.test-node.unmount", map[string]any{ + "volume_id": "vol-testunpublish01", + "node_id": "test-node", + }}, + } +} + +func TestProviderHandlers_UnsupportedSchemaVersion(t *testing.T) { + _, natsURL := setupEmbeddedNATS(t) + cfg := setupTestConfig(t, natsURL) + nc := startProviderSubjects(t, cfg, natsURL) + + for _, tc := range providerSubjectCases() { + t.Run(tc.name, func(t *testing.T) { + body := map[string]any{"schema_version": ebsprovider.SchemaVersion + 1} + maps.Copy(body, tc.body) + msg := requestProvider(t, nc, tc.subject, marshalRequest(t, body)) + + var resp versionedErrorResponse + require.NoError(t, json.Unmarshal(msg.Data, &resp)) + require.NotNil(t, resp.Error, "expected an error for unsupported schema version") + assert.Equal(t, ebsprovider.ErrorCodeUnsupportedVersion, resp.Error.Code) + // The zero-version guard: even an error response must carry a + // valid SchemaVersion, or NATSProvider's responseError would + // reject it before ever inspecting Error. + assert.Equal(t, ebsprovider.SchemaVersion, resp.SchemaVersion) + }) + } +} + +func TestProviderHandlers_MalformedJSON(t *testing.T) { + _, natsURL := setupEmbeddedNATS(t) + cfg := setupTestConfig(t, natsURL) + nc := startProviderSubjects(t, cfg, natsURL) + + for _, tc := range providerSubjectCases() { + t.Run(tc.name, func(t *testing.T) { + msg := requestProvider(t, nc, tc.subject, []byte("{not valid json")) + + var resp versionedErrorResponse + require.NoError(t, json.Unmarshal(msg.Data, &resp)) + require.NotNil(t, resp.Error, "expected an error for malformed JSON") + assert.Equal(t, ebsprovider.ErrorCodeInvalidArgument, resp.Error.Code) + assert.Equal(t, ebsprovider.SchemaVersion, resp.SchemaVersion) + }) + } +} + +// TestProviderHandlers_InvalidVolumeName covers the validVolumeName rejection +// path shared by every handler that takes a volume ID: empty, ".", "..", and +// path traversal must all come back invalid_argument, never reach the +// filesystem or a backend call. +func TestProviderHandlers_InvalidVolumeName(t *testing.T) { + _, natsURL := setupEmbeddedNATS(t) + cfg := setupTestConfig(t, natsURL) + nc := startProviderSubjects(t, cfg, natsURL) + + invalidNames := []string{"", ".", "..", "../..", "a/b", "/etc/passwd", "vol/../.."} + + for _, name := range invalidNames { + t.Run("delete_volume/"+name, func(t *testing.T) { + body := marshalRequest(t, map[string]any{"schema_version": ebsprovider.SchemaVersion, "volume_id": name}) + msg := requestProvider(t, nc, ebsprovider.DeleteVolumeSubject, body) + + var resp ebsprovider.DeleteVolumeResponse + require.NoError(t, json.Unmarshal(msg.Data, &resp)) + require.NotNil(t, resp.Error) + assert.Equal(t, ebsprovider.ErrorCodeInvalidArgument, resp.Error.Code) + }) + + t.Run("get_volume/"+name, func(t *testing.T) { + body := marshalRequest(t, map[string]any{"schema_version": ebsprovider.SchemaVersion, "volume_id": name}) + msg := requestProvider(t, nc, ebsprovider.GetVolumeSubject, body) + + var resp ebsprovider.GetVolumeResponse + require.NoError(t, json.Unmarshal(msg.Data, &resp)) + require.NotNil(t, resp.Error) + assert.Equal(t, ebsprovider.ErrorCodeInvalidArgument, resp.Error.Code) + }) + + t.Run("delete_snapshot/"+name, func(t *testing.T) { + body := marshalRequest(t, map[string]any{"schema_version": ebsprovider.SchemaVersion, "snapshot_id": name}) + msg := requestProvider(t, nc, ebsprovider.DeleteSnapshotSubject, body) + + var resp ebsprovider.DeleteSnapshotResponse + require.NoError(t, json.Unmarshal(msg.Data, &resp)) + require.NotNil(t, resp.Error) + assert.Equal(t, ebsprovider.ErrorCodeInvalidArgument, resp.Error.Code) + }) + + t.Run("copy_snapshot/"+name, func(t *testing.T) { + body := marshalRequest(t, map[string]any{ + "schema_version": ebsprovider.SchemaVersion, + "source_snapshot_id": name, + "destination_snapshot_id": "snap-testcopyvalid1", + "volume_id": "vol-testcopyvalid01", + }) + msg := requestProvider(t, nc, ebsprovider.CopySnapshotSubject, body) + + var resp ebsprovider.CopySnapshotResponse + require.NoError(t, json.Unmarshal(msg.Data, &resp)) + require.NotNil(t, resp.Error) + assert.Equal(t, ebsprovider.ErrorCodeInvalidArgument, resp.Error.Code) + }) + + t.Run("publish_volume/"+name, func(t *testing.T) { + publishSubject, err := ebsprovider.PublishSubject(cfg.NodeName) + require.NoError(t, err) + body := marshalRequest(t, map[string]any{"schema_version": ebsprovider.SchemaVersion, "volume_id": name, "node_id": cfg.NodeName}) + msg := requestProvider(t, nc, publishSubject, body) + + var resp ebsprovider.PublishVolumeResponse + require.NoError(t, json.Unmarshal(msg.Data, &resp)) + require.NotNil(t, resp.Error) + assert.Equal(t, ebsprovider.ErrorCodeInvalidArgument, resp.Error.Code) + }) + + t.Run("unpublish_volume/"+name, func(t *testing.T) { + unpublishSubject, err := ebsprovider.UnpublishSubject(cfg.NodeName) + require.NoError(t, err) + body := marshalRequest(t, map[string]any{"schema_version": ebsprovider.SchemaVersion, "volume_id": name, "node_id": cfg.NodeName}) + msg := requestProvider(t, nc, unpublishSubject, body) + + var resp ebsprovider.UnpublishVolumeResponse + require.NoError(t, json.Unmarshal(msg.Data, &resp)) + require.NotNil(t, resp.Error) + assert.Equal(t, ebsprovider.ErrorCodeInvalidArgument, resp.Error.Code) + }) + } +} + +func TestProviderHandlers_DeleteVolume_AbsentIsIdempotent(t *testing.T) { + _, natsURL := setupEmbeddedNATS(t) + cfg := setupTestConfig(t, natsURL) + nc := startProviderSubjects(t, cfg, natsURL) + + store := objectstore.NewMemoryObjectStore() + prev := providerObjectStoreFactory + providerObjectStoreFactory = func(*Config) objectstore.ObjectStore { return store } + t.Cleanup(func() { providerObjectStoreFactory = prev }) + + body := marshalRequest(t, map[string]any{"schema_version": ebsprovider.SchemaVersion, "volume_id": "vol-neverexisted1"}) + msg := requestProvider(t, nc, ebsprovider.DeleteVolumeSubject, body) + + var resp ebsprovider.DeleteVolumeResponse + require.NoError(t, json.Unmarshal(msg.Data, &resp)) + assert.Nil(t, resp.Error, "deleting an absent volume must succeed") +} + +// TestProviderHandlers_DeleteVolume_UnansweredOwnerProbeRefuses pins the +// difference between "nobody holds this volume" and "nobody answered". A +// subscriber that never replies is what a busy or partitioned owner looks +// like from here, and reading it as unmounted deletes a volume a guest may +// still be writing to. +func TestProviderHandlers_DeleteVolume_UnansweredOwnerProbeRefuses(t *testing.T) { + _, natsURL := setupEmbeddedNATS(t) + cfg := setupTestConfig(t, natsURL) + nc := startProviderSubjects(t, cfg, natsURL) + + store := objectstore.NewMemoryObjectStore() + prev := providerObjectStoreFactory + providerObjectStoreFactory = func(*Config) objectstore.ObjectStore { return store } + t.Cleanup(func() { providerObjectStoreFactory = prev }) + + const volumeName = "vol-silentowner01" + subject, err := ebsprovider.GetVolumeOwnerSubject(volumeName) + require.NoError(t, err) + + // Subscribed but silent. A responder exists, so the probe times out rather + // than coming back as ErrNoResponders. + sub, err := nc.Subscribe(subject, func(*nats.Msg) {}) + require.NoError(t, err) + require.NoError(t, nc.Flush()) + t.Cleanup(func() { _ = sub.Unsubscribe() }) + + body := marshalRequest(t, map[string]any{"schema_version": ebsprovider.SchemaVersion, "volume_id": volumeName}) + msg := requestProvider(t, nc, ebsprovider.DeleteVolumeSubject, body) + + var resp ebsprovider.DeleteVolumeResponse + require.NoError(t, json.Unmarshal(msg.Data, &resp)) + require.NotNil(t, resp.Error, "a delete that could not establish ownership must refuse, not proceed") + assert.Equal(t, ebsprovider.ErrorCodeUnavailable, resp.Error.Code) + assert.True(t, resp.Error.Code.Retryable(), + "an owner that did not answer may answer later, so this refusal must read as retryable; a permanent code strands the volume") +} + +func TestProviderHandlers_DeleteSnapshot_AbsentIsIdempotent(t *testing.T) { + _, natsURL := setupEmbeddedNATS(t) + cfg := setupTestConfig(t, natsURL) + nc := startProviderSubjects(t, cfg, natsURL) + + store := objectstore.NewMemoryObjectStore() + prev := providerObjectStoreFactory + providerObjectStoreFactory = func(*Config) objectstore.ObjectStore { return store } + t.Cleanup(func() { providerObjectStoreFactory = prev }) + + body := marshalRequest(t, map[string]any{"schema_version": ebsprovider.SchemaVersion, "snapshot_id": "snap-neverexisted1"}) + msg := requestProvider(t, nc, ebsprovider.DeleteSnapshotSubject, body) + + var resp ebsprovider.DeleteSnapshotResponse + require.NoError(t, json.Unmarshal(msg.Data, &resp)) + assert.Nil(t, resp.Error, "deleting an absent snapshot must succeed") +} + +// TestProviderHandlers_CreateVolume_IdempotentOnLiveVB exercises CreateVolume's +// idempotency check (describeVolumeEngine) against a live mounted VB, which +// needs no S3 backend at all: an identical repeat must return the existing +// volume, and a conflicting capacity must be rejected as already_exists. +func TestProviderHandlers_CreateVolume_IdempotentOnLiveVB(t *testing.T) { + _, natsURL := setupEmbeddedNATS(t) + cfg := setupTestConfig(t, natsURL) + nc := startProviderSubjects(t, cfg, natsURL) + + const volumeName = "vol-liveidempotent" + vb := createTestVBWithState(t, volumeName) + cfg.MountedVolumes = append(cfg.MountedVolumes, MountedVolume{Name: volumeName, VB: vb}) + size := int64(vb.GetVolumeSize()) + + t.Run("identical repeat returns existing volume", func(t *testing.T) { + body := marshalRequest(t, map[string]any{ + "schema_version": ebsprovider.SchemaVersion, + "volume_id": volumeName, + "capacity_range": map[string]any{"required_bytes": size}, + }) + msg := requestProvider(t, nc, ebsprovider.CreateVolumeSubject, body) + + var resp ebsprovider.CreateVolumeResponse + require.NoError(t, json.Unmarshal(msg.Data, &resp)) + require.Nil(t, resp.Error) + require.NotNil(t, resp.Volume) + assert.Equal(t, volumeName, resp.Volume.ID) + assert.Equal(t, size, resp.Volume.CapacityBytes) + }) + + t.Run("conflicting capacity is already_exists", func(t *testing.T) { + body := marshalRequest(t, map[string]any{ + "schema_version": ebsprovider.SchemaVersion, + "volume_id": volumeName, + "capacity_range": map[string]any{"required_bytes": size + bytesPerGiB}, + }) + msg := requestProvider(t, nc, ebsprovider.CreateVolumeSubject, body) + + var resp ebsprovider.CreateVolumeResponse + require.NoError(t, json.Unmarshal(msg.Data, &resp)) + require.NotNil(t, resp.Error) + assert.Equal(t, ebsprovider.ErrorCodeAlreadyExists, resp.Error.Code) + }) +} + +// TestProviderHandlers_ExpandVolume_MountedIsVolumeInUse confirms ExpandVolume +// refuses a volume that is mounted with a live VB, matching the capability +// advertised (OnlineExpansion: false). +func TestProviderHandlers_ExpandVolume_MountedIsVolumeInUse(t *testing.T) { + _, natsURL := setupEmbeddedNATS(t) + cfg := setupTestConfig(t, natsURL) + nc := startProviderSubjects(t, cfg, natsURL) + + const volumeName = "vol-expandmounted1" + vb := createTestVBWithState(t, volumeName) + cfg.MountedVolumes = append(cfg.MountedVolumes, MountedVolume{Name: volumeName, VB: vb}) + + body := marshalRequest(t, map[string]any{ + "schema_version": ebsprovider.SchemaVersion, + "volume_id": volumeName, + "capacity_range": map[string]any{"required_bytes": int64(vb.GetVolumeSize()) + bytesPerGiB}, + }) + msg := requestProvider(t, nc, ebsprovider.ExpandVolumeSubject, body) + + var resp ebsprovider.ExpandVolumeResponse + require.NoError(t, json.Unmarshal(msg.Data, &resp)) + require.NotNil(t, resp.Error) + assert.Equal(t, ebsprovider.ErrorCodeVolumeInUse, resp.Error.Code) +} + +// TestProviderHandlers_CreateSnapshot_AcceptThenPublish exercises the accept +// reply directly (non-empty OperationID, pending state, correct completion +// subject) against a live mounted VB, which lets completeCreateSnapshot's +// background work finish with no S3 backend involved. +func TestProviderHandlers_CreateSnapshot_AcceptThenPublish(t *testing.T) { + _, natsURL := setupEmbeddedNATS(t) + cfg := setupTestConfig(t, natsURL) + nc := startProviderSubjects(t, cfg, natsURL) + + const volumeName = "vol-snapaccept0001" + const snapshotID = "snap-accept00000001" + vb := createTestVBWithState(t, volumeName) + cfg.MountedVolumes = append(cfg.MountedVolumes, MountedVolume{Name: volumeName, VB: vb}) + + wantCompletionSubject, err := ebsprovider.SnapshotCompletionSubject(snapshotID) + require.NoError(t, err) + + completionSub, err := nc.SubscribeSync(wantCompletionSubject) + require.NoError(t, err) + require.NoError(t, nc.Flush()) + + body := marshalRequest(t, map[string]any{ + "schema_version": ebsprovider.SchemaVersion, + "volume_id": volumeName, + "snapshot_id": snapshotID, + }) + msg := requestProvider(t, nc, ebsprovider.SnapshotCreateSubjectPrefix+volumeName, body) + + var accepted ebsprovider.CreateSnapshotResponse + require.NoError(t, json.Unmarshal(msg.Data, &accepted)) + require.Nil(t, accepted.Error) + assert.NotEmpty(t, accepted.OperationID) + assert.Equal(t, wantCompletionSubject, accepted.CompletionSubject) + require.NotNil(t, accepted.Snapshot) + assert.Equal(t, ebsprovider.SnapshotStatePending, accepted.Snapshot.State) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + completionMsg, err := completionSub.NextMsgWithContext(ctx) + require.NoError(t, err) + + var completed ebsprovider.CreateSnapshotResponse + require.NoError(t, json.Unmarshal(completionMsg.Data, &completed)) + require.Nil(t, completed.Error) + assert.Equal(t, accepted.OperationID, completed.OperationID) + require.NotNil(t, completed.Snapshot) + assert.Equal(t, ebsprovider.SnapshotStateCompleted, completed.Snapshot.State) + assert.Equal(t, snapshotID, completed.Snapshot.ID) +} + +// TestProviderHandlers_CreateSnapshot_FullRoundTripViaNATSProvider drives the +// same flow through ebsprovider.NewNATSProvider, the real client this server +// must satisfy, rather than raw NATS requests. +func TestProviderHandlers_CreateSnapshot_FullRoundTripViaNATSProvider(t *testing.T) { + _, natsURL := setupEmbeddedNATS(t) + cfg := setupTestConfig(t, natsURL) + nc := startProviderSubjects(t, cfg, natsURL) + + const volumeName = "vol-snaproundtrip1" + vb := createTestVBWithState(t, volumeName) + cfg.MountedVolumes = append(cfg.MountedVolumes, MountedVolume{Name: volumeName, VB: vb}) + + provider := ebsprovider.NewNATSProvider(nc, 10*time.Second) + snap, err := provider.CreateSnapshot(context.Background(), ebsprovider.CreateSnapshotRequest{ + Versioned: ebsprovider.NewVersioned(), + SnapshotID: "snap-roundtrip00001", + VolumeID: volumeName, + }) + require.NoError(t, err) + require.NotNil(t, snap) + assert.Equal(t, ebsprovider.SnapshotStateCompleted, snap.State) + assert.Equal(t, "snap-roundtrip00001", snap.ID) + assert.Equal(t, volumeName, snap.SourceVolumeID) +} + +// TestProviderHandlers_CopySnapshot_FullRoundTripViaNATSProvider drives +// CopySnapshot through ebsprovider.NewNATSProvider against a live mounted VB +// on the file backend: CreateSnapshot seeds a real source snapshot, then +// CopySnapshot must produce a second, independently readable one. +func TestProviderHandlers_CopySnapshot_FullRoundTripViaNATSProvider(t *testing.T) { + _, natsURL := setupEmbeddedNATS(t) + cfg := setupTestConfig(t, natsURL) + nc := startProviderSubjects(t, cfg, natsURL) + + const volumeName = "vol-copysnaproundtr1" + vb := createTestVBWithState(t, volumeName) + cfg.MountedVolumes = append(cfg.MountedVolumes, MountedVolume{Name: volumeName, VB: vb}) + + provider := ebsprovider.NewNATSProvider(nc, 10*time.Second) + _, err := provider.CreateSnapshot(context.Background(), ebsprovider.CreateSnapshotRequest{ + Versioned: ebsprovider.NewVersioned(), + SnapshotID: "snap-copysrc0000001", + VolumeID: volumeName, + }) + require.NoError(t, err) + + copied, err := provider.CopySnapshot(context.Background(), ebsprovider.CopySnapshotRequest{ + Versioned: ebsprovider.NewVersioned(), + SourceSnapshotID: "snap-copysrc0000001", + DestinationSnapshotID: "snap-copydst0000001", + VolumeID: volumeName, + }) + require.NoError(t, err) + require.NotNil(t, copied) + assert.Equal(t, ebsprovider.SnapshotStateCompleted, copied.State) + assert.Equal(t, "snap-copydst0000001", copied.ID) + assert.Equal(t, volumeName, copied.SourceVolumeID) + assert.NotEmpty(t, copied.Handle) + + // The checkpoint really landed on the backend under the new ID: a second + // copy from the destination must itself succeed, proving snap-copydst is a + // real, independently readable snapshot rather than a synthesized answer. + _, err = provider.CopySnapshot(context.Background(), ebsprovider.CopySnapshotRequest{ + Versioned: ebsprovider.NewVersioned(), + SourceSnapshotID: "snap-copydst0000001", + DestinationSnapshotID: "snap-copydst0000002", + VolumeID: volumeName, + }) + require.NoError(t, err) +} + +// TestProviderHandlers_CopySnapshot_MissingSourceIsNotFound proves the +// missing-source guard: copying a snapshot that was never created on the +// volume must fail as not_found, not internal. +func TestProviderHandlers_CopySnapshot_MissingSourceIsNotFound(t *testing.T) { + _, natsURL := setupEmbeddedNATS(t) + cfg := setupTestConfig(t, natsURL) + nc := startProviderSubjects(t, cfg, natsURL) + + const volumeName = "vol-copysnapmissing1" + vb := createTestVBWithState(t, volumeName) + cfg.MountedVolumes = append(cfg.MountedVolumes, MountedVolume{Name: volumeName, VB: vb}) + + provider := ebsprovider.NewNATSProvider(nc, 10*time.Second) + _, err := provider.CopySnapshot(context.Background(), ebsprovider.CopySnapshotRequest{ + Versioned: ebsprovider.NewVersioned(), + SourceSnapshotID: "snap-nevercreated001", + DestinationSnapshotID: "snap-copydstmissing1", + VolumeID: volumeName, + }) + require.Error(t, err) + var providerErr *ebsprovider.ProviderError + require.ErrorAs(t, err, &providerErr) + assert.Equal(t, ebsprovider.ErrorCodeNotFound, providerErr.Code) +} + +// TestProviderHandlers_CopySnapshot_ExistingDestinationIsAlreadyExists proves +// the destination-exists guard: copying onto an ID that already has a +// snapshot must refuse as already_exists, matching viperblock.CopySnapshotMeta +// refusing to clobber a committed destination. +func TestProviderHandlers_CopySnapshot_ExistingDestinationIsAlreadyExists(t *testing.T) { + _, natsURL := setupEmbeddedNATS(t) + cfg := setupTestConfig(t, natsURL) + nc := startProviderSubjects(t, cfg, natsURL) + + const volumeName = "vol-copysnapexists01" + vb := createTestVBWithState(t, volumeName) + cfg.MountedVolumes = append(cfg.MountedVolumes, MountedVolume{Name: volumeName, VB: vb}) + + provider := ebsprovider.NewNATSProvider(nc, 10*time.Second) + _, err := provider.CreateSnapshot(context.Background(), ebsprovider.CreateSnapshotRequest{ + Versioned: ebsprovider.NewVersioned(), + SnapshotID: "snap-copyexists-a001", + VolumeID: volumeName, + }) + require.NoError(t, err) + _, err = provider.CreateSnapshot(context.Background(), ebsprovider.CreateSnapshotRequest{ + Versioned: ebsprovider.NewVersioned(), + SnapshotID: "snap-copyexists-b001", + VolumeID: volumeName, + }) + require.NoError(t, err) + + _, err = provider.CopySnapshot(context.Background(), ebsprovider.CopySnapshotRequest{ + Versioned: ebsprovider.NewVersioned(), + SourceSnapshotID: "snap-copyexists-a001", + DestinationSnapshotID: "snap-copyexists-b001", + VolumeID: volumeName, + }) + require.Error(t, err) + var providerErr *ebsprovider.ProviderError + require.ErrorAs(t, err, &providerErr) + assert.Equal(t, ebsprovider.ErrorCodeAlreadyExists, providerErr.Code) +} + +// TestProviderHandlers_CopySnapshot_SameIDIsInvalidArgument proves +// handleCopySnapshot's own source-equals-destination guard fires before any +// engine work happens (no volume needs to be mounted for this to fail). +func TestProviderHandlers_CopySnapshot_SameIDIsInvalidArgument(t *testing.T) { + _, natsURL := setupEmbeddedNATS(t) + cfg := setupTestConfig(t, natsURL) + nc := startProviderSubjects(t, cfg, natsURL) + + provider := ebsprovider.NewNATSProvider(nc, 10*time.Second) + _, err := provider.CopySnapshot(context.Background(), ebsprovider.CopySnapshotRequest{ + Versioned: ebsprovider.NewVersioned(), + SourceSnapshotID: "snap-sameid00000001", + DestinationSnapshotID: "snap-sameid00000001", + VolumeID: "vol-copysnapsameid01", + }) + require.Error(t, err) + var providerErr *ebsprovider.ProviderError + require.ErrorAs(t, err, &providerErr) + assert.Equal(t, ebsprovider.ErrorCodeInvalidArgument, providerErr.Code) +} + +// TestProviderHandlers_SnapshotCreateWildcard_DoesNotCaptureDelete guards the +// SnapshotCreateSubjectPrefix wildcard subscription against ever matching +// ebs.provider.v1.snapshot.delete: a DeleteSnapshot request must come back as +// a DeleteSnapshotResponse, never a CreateSnapshotResponse (which carries a +// non-empty operation_id). +func TestProviderHandlers_SnapshotCreateWildcard_DoesNotCaptureDelete(t *testing.T) { + _, natsURL := setupEmbeddedNATS(t) + cfg := setupTestConfig(t, natsURL) + nc := startProviderSubjects(t, cfg, natsURL) + + store := objectstore.NewMemoryObjectStore() + prev := providerObjectStoreFactory + providerObjectStoreFactory = func(*Config) objectstore.ObjectStore { return store } + t.Cleanup(func() { providerObjectStoreFactory = prev }) + + body := marshalRequest(t, map[string]any{"schema_version": ebsprovider.SchemaVersion, "snapshot_id": "snap-notcreate0001"}) + msg := requestProvider(t, nc, ebsprovider.DeleteSnapshotSubject, body) + + var raw map[string]any + require.NoError(t, json.Unmarshal(msg.Data, &raw)) + _, hasOperationID := raw["operation_id"] + assert.False(t, hasOperationID, "DeleteSnapshot response must not look like a CreateSnapshot accept") +} + +// TestProviderHandlers_DeleteVolume_RemovesExistingObjects covers +// deleteObjectPrefix's non-empty loop body (DeleteVolume's absent-volume test +// only exercises the immediate zero-objects return): objects under both the +// volume's main prefix and its -efi/ auxiliary prefix must be removed. +func TestProviderHandlers_DeleteVolume_RemovesExistingObjects(t *testing.T) { + _, natsURL := setupEmbeddedNATS(t) + cfg := setupTestConfig(t, natsURL) + nc := startProviderSubjects(t, cfg, natsURL) + + store := objectstore.NewMemoryObjectStore() + prev := providerObjectStoreFactory + providerObjectStoreFactory = func(*Config) objectstore.ObjectStore { return store } + t.Cleanup(func() { providerObjectStoreFactory = prev }) + + const volumeName = "vol-deleteobjects01" + putTestObject(t, store, cfg.Bucket, volumeName+"/config.json") + putTestObject(t, store, cfg.Bucket, volumeName+"/chunk-0000.bin") + putTestObject(t, store, cfg.Bucket, volumeName+"-efi/config.json") + + body := marshalRequest(t, map[string]any{"schema_version": ebsprovider.SchemaVersion, "volume_id": volumeName}) + msg := requestProvider(t, nc, ebsprovider.DeleteVolumeSubject, body) + + var resp ebsprovider.DeleteVolumeResponse + require.NoError(t, json.Unmarshal(msg.Data, &resp)) + assert.Nil(t, resp.Error) + assert.Equal(t, 0, store.Count(), "every object under both prefixes must be deleted") +} + +// TestProviderHandlers_DeleteSnapshot_RemovesExistingObjects is +// DeleteVolume's counterpart for DeleteSnapshot's deleteObjectPrefix call. +func TestProviderHandlers_DeleteSnapshot_RemovesExistingObjects(t *testing.T) { + _, natsURL := setupEmbeddedNATS(t) + cfg := setupTestConfig(t, natsURL) + nc := startProviderSubjects(t, cfg, natsURL) + + store := objectstore.NewMemoryObjectStore() + prev := providerObjectStoreFactory + providerObjectStoreFactory = func(*Config) objectstore.ObjectStore { return store } + t.Cleanup(func() { providerObjectStoreFactory = prev }) + + const snapshotName = "snap-deleteobjects01" + putTestObject(t, store, cfg.Bucket, snapshotName+"/metadata.json") + putTestObject(t, store, cfg.Bucket, snapshotName+"/chunk-0000.bin") + + body := marshalRequest(t, map[string]any{"schema_version": ebsprovider.SchemaVersion, "snapshot_id": snapshotName}) + msg := requestProvider(t, nc, ebsprovider.DeleteSnapshotSubject, body) + + var resp ebsprovider.DeleteSnapshotResponse + require.NoError(t, json.Unmarshal(msg.Data, &resp)) + assert.Nil(t, resp.Error) + assert.Equal(t, 0, store.Count(), "every object under the snapshot prefix must be deleted") +} + +func putTestObject(t *testing.T, store objectstore.ObjectStore, bucket, key string) { + t.Helper() + _, err := store.PutObject(context.Background(), &s3.PutObjectInput{ + Bucket: awssdk.String(bucket), Key: awssdk.String(key), Body: bytes.NewReader([]byte("data")), + }) + require.NoError(t, err) +} + +// TestProviderErrorConstructors covers the notFoundError and internalError +// helpers directly: every other *Error constructor in this file is already +// exercised via a handler that returns it, but nothing in this package's +// current test set reaches a not_found or internal response without a live +// S3/viperblock backend. +func TestProviderErrorConstructors(t *testing.T) { + notFound := notFoundError("volume %s not found", "vol-1") + assert.Equal(t, ebsprovider.ErrorCodeNotFound, notFound.Code) + assert.Equal(t, "volume vol-1 not found", notFound.Message) + + internal := internalError("backend init: %v", assert.AnError) + assert.Equal(t, ebsprovider.ErrorCodeInternal, internal.Code) + assert.Contains(t, internal.Message, assert.AnError.Error()) +} + +// TestBuildProviderVBConfig covers buildProviderVBConfig's field wiring: the +// GCEnabled/MasterKey passthrough this file's own comment calls out as +// control-plane-owned (TenantID, Tags, VolumeType, IOPS, Throughput, +// AvailabilityZone) staying out of VolumeMetadata, and the SnapshotID/ +// SourceVolumeName fields only being set when a source snapshot is given. +func TestBuildProviderVBConfig(t *testing.T) { + cfg := &Config{BaseDir: "/tmp/vb-base", GCEnabled: true} + + vbconfig := buildProviderVBConfig(cfg, "vol-1", 8<<30, "", "") + assert.Equal(t, "vol-1", vbconfig.VolumeName) + assert.Equal(t, uint64(8<<30), vbconfig.VolumeSize) + assert.Equal(t, "/tmp/vb-base", vbconfig.BaseDir) + assert.True(t, vbconfig.GCEnabled) + assert.Empty(t, vbconfig.VolumeConfig.VolumeMetadata.TenantID, "control-plane facts must not be set here") + assert.Empty(t, vbconfig.SnapshotID) + assert.Empty(t, vbconfig.SourceVolumeName) + + fromSnapshot := buildProviderVBConfig(cfg, "vol-2", 8<<30, "snap-1", "vol-src") + assert.Equal(t, "snap-1", fromSnapshot.SnapshotID) + assert.Equal(t, "vol-src", fromSnapshot.SourceVolumeName) +} + +// TestProviderHandlers_DeleteVolume_MountedVolumeRefused covers +// handleDeleteVolume's mounted-volume path: a published volume is undeletable, +// so the request must be refused with volume_in_use and the mount left exactly +// as it was. Deleting under a live export leaves the guest writing into +// storage whose metadata has gone. +func TestProviderHandlers_DeleteVolume_MountedVolumeRefused(t *testing.T) { + _, natsURL := setupEmbeddedNATS(t) + cfg := setupTestConfig(t, natsURL) + nc := startProviderSubjects(t, cfg, natsURL) + + store := objectstore.NewMemoryObjectStore() + prev := providerObjectStoreFactory + providerObjectStoreFactory = func(*Config) objectstore.ObjectStore { return store } + t.Cleanup(func() { providerObjectStoreFactory = prev }) + + const volumeName = "vol-deletemounted01" + vb := createTestVBWithState(t, volumeName) + socketPath := filepath.Join(t.TempDir(), volumeName+".sock") + require.NoError(t, os.WriteFile(socketPath, []byte("fake-socket"), 0600)) + + cfg.MountedVolumes = append(cfg.MountedVolumes, MountedVolume{ + Name: volumeName, VB: vb, Socket: socketPath, + // A PID that (almost certainly) does not exist: KillProcess must fail + // gracefully (logged, not fatal) rather than block or panic. + PID: 999999, + }) + + body := marshalRequest(t, map[string]any{"schema_version": ebsprovider.SchemaVersion, "volume_id": volumeName}) + msg := requestProvider(t, nc, ebsprovider.DeleteVolumeSubject, body) + + var resp ebsprovider.DeleteVolumeResponse + require.NoError(t, json.Unmarshal(msg.Data, &resp)) + require.NotNil(t, resp.Error, "deleting a published volume must be refused") + assert.Equal(t, ebsprovider.ErrorCodeVolumeInUse, resp.Error.Code) + + cfg.mu.Lock() + remaining := cfg.MountedVolumes + cfg.mu.Unlock() + var stillMounted bool + for _, v := range remaining { + if v.Name == volumeName { + stillMounted = true + } + } + assert.True(t, stillMounted, "a refused delete must leave the mount in place") + + _, statErr := os.Stat(socketPath) + assert.NoError(t, statErr, "a refused delete must leave the NBD socket alone") +} + +// TestProviderHandlers_PublishVolume_IdempotentRepublish covers the double- +// writer hazard this provider boundary exists to prevent: republishing a +// volume this node already has mounted must return the existing attachment +// unchanged rather than mounting it a second time. The mounted-volume count +// and PID (not just the returned URI) prove no second nbdkit was started. +func TestProviderHandlers_PublishVolume_IdempotentRepublish(t *testing.T) { + _, natsURL := setupEmbeddedNATS(t) + cfg := setupTestConfig(t, natsURL) + nc := startProviderSubjects(t, cfg, natsURL) + + const volumeName = "vol-pubidempotent1" + const existingURI = "nbd:unix:/fake/vol-pubidempotent1.sock" + vb := createTestVBWithState(t, volumeName) + cfg.MountedVolumes = append(cfg.MountedVolumes, MountedVolume{ + Name: volumeName, + VB: vb, + PID: 424242, + NBDURI: existingURI, + }) + + publishSubject, err := ebsprovider.PublishSubject(cfg.NodeName) + require.NoError(t, err) + body := marshalRequest(t, map[string]any{ + "schema_version": ebsprovider.SchemaVersion, + "volume_id": volumeName, + "node_id": cfg.NodeName, + }) + msg := requestProvider(t, nc, publishSubject, body) + + var resp ebsprovider.PublishVolumeResponse + require.NoError(t, json.Unmarshal(msg.Data, &resp)) + require.Nil(t, resp.Error) + require.NotNil(t, resp.Published) + assert.Equal(t, volumeName, resp.Published.VolumeID) + assert.Equal(t, cfg.NodeName, resp.Published.NodeID) + assert.Equal(t, existingURI, resp.Published.NBDURI) + + cfg.mu.Lock() + count := len(cfg.MountedVolumes) + pid := cfg.MountedVolumes[0].PID + cfg.mu.Unlock() + assert.Equal(t, 1, count, "republish of an already-mounted volume must not add a second entry") + assert.Equal(t, 424242, pid, "republish must not spawn a second nbdkit (PID must be unchanged)") +} + +// TestProviderHandlers_PublishVolume_ReadOnlyModeMismatchRefused covers the +// half of read-only publishing that is not visible to an NBD client: the +// access mode is fixed when nbdkit starts, so a republish asking for the other +// mode must be refused rather than answered with the running export. +func TestProviderHandlers_PublishVolume_ReadOnlyModeMismatchRefused(t *testing.T) { + _, natsURL := setupEmbeddedNATS(t) + cfg := setupTestConfig(t, natsURL) + nc := startProviderSubjects(t, cfg, natsURL) + + const volumeName = "vol-readonlypub001" + vb := createTestVBWithState(t, volumeName) + cfg.MountedVolumes = append(cfg.MountedVolumes, MountedVolume{ + Name: volumeName, + VB: vb, + PID: 424243, + NBDURI: "nbd:unix:/fake/vol-readonlypub001.sock", + ReadOnly: false, + }) + + publishSubject, err := ebsprovider.PublishSubject(cfg.NodeName) + require.NoError(t, err) + body := marshalRequest(t, map[string]any{ + "schema_version": ebsprovider.SchemaVersion, + "volume_id": volumeName, + "node_id": cfg.NodeName, + "read_only": true, + }) + msg := requestProvider(t, nc, publishSubject, body) + + var resp ebsprovider.PublishVolumeResponse + require.NoError(t, json.Unmarshal(msg.Data, &resp)) + require.NotNil(t, resp.Error, "a read-only republish of a writable export must not succeed") + assert.Equal(t, ebsprovider.ErrorCodeVolumeInUse, resp.Error.Code) + + cfg.mu.Lock() + count := len(cfg.MountedVolumes) + cfg.mu.Unlock() + assert.Equal(t, 1, count, "a refused republish must not add a second entry") +} + +// TestProviderHandlers_UnpublishVolume_RemovesMountedVolume mirrors +// TestIntegration_EBSUnmountRequest (the legacy ebs.unmount handler's happy +// path) but drives it through the provider subject instead, confirming the +// shared unmountVolume extraction behaves the same way from either front. +func TestProviderHandlers_UnpublishVolume_RemovesMountedVolume(t *testing.T) { + _, natsURL := setupEmbeddedNATS(t) + cfg := setupTestConfig(t, natsURL) + nc := startProviderSubjects(t, cfg, natsURL) + + const volumeName = "vol-unpubremoved01" + cfg.MountedVolumes = append(cfg.MountedVolumes, MountedVolume{ + Name: volumeName, + PID: 999999, // Fake PID that doesn't exist; KillProcess must fail gracefully. + }) + + unpublishSubject, err := ebsprovider.UnpublishSubject(cfg.NodeName) + require.NoError(t, err) + body := marshalRequest(t, map[string]any{ + "schema_version": ebsprovider.SchemaVersion, + "volume_id": volumeName, + "node_id": cfg.NodeName, + }) + msg := requestProvider(t, nc, unpublishSubject, body) + + var resp ebsprovider.UnpublishVolumeResponse + require.NoError(t, json.Unmarshal(msg.Data, &resp)) + assert.Nil(t, resp.Error) + + cfg.mu.Lock() + remaining := cfg.MountedVolumes + cfg.mu.Unlock() + for _, v := range remaining { + assert.NotEqual(t, volumeName, v.Name, "unpublished volume must be removed from MountedVolumes") + } +} + +// TestProviderHandlers_UnpublishVolume_AbsentIsIdempotent covers a volume this +// node never mounted. The legacy path treats that as a successful seal (see +// unmountResponseError), and so must this one: a retry after a request that +// timed out client-side but completed server-side has to converge. +func TestProviderHandlers_UnpublishVolume_AbsentIsIdempotent(t *testing.T) { + _, natsURL := setupEmbeddedNATS(t) + cfg := setupTestConfig(t, natsURL) + nc := startProviderSubjects(t, cfg, natsURL) + + unpublishSubject, err := ebsprovider.UnpublishSubject(cfg.NodeName) + require.NoError(t, err) + body := marshalRequest(t, map[string]any{ + "schema_version": ebsprovider.SchemaVersion, + "volume_id": "vol-neverpublished1", + "node_id": cfg.NodeName, + }) + msg := requestProvider(t, nc, unpublishSubject, body) + + var resp ebsprovider.UnpublishVolumeResponse + require.NoError(t, json.Unmarshal(msg.Data, &resp)) + assert.Nil(t, resp.Error, "unpublishing a volume this node never mounted must succeed") +} + +// TestProviderHandlers_PublishUnpublish_RequireNodeName covers +// registerProviderSubjects' NodeName gate: PublishVolume/UnpublishVolume must +// not be registered when cfg.NodeName is empty (PublishSubject has no node to +// address), while every other ebs.provider.v1.* subject still is. +func TestProviderHandlers_PublishUnpublish_RequireNodeName(t *testing.T) { + _, natsURL := setupEmbeddedNATS(t) + cfgNoNode := setupTestConfig(t, natsURL) + cfgNoNode.NodeName = "" + ncNoNode := startProviderSubjects(t, cfgNoNode, natsURL) + + _, natsURL2 := setupEmbeddedNATS(t) + cfgNamed := setupTestConfig(t, natsURL2) + ncNamed := startProviderSubjects(t, cfgNamed, natsURL2) + + assert.Equal(t, ncNamed.NumSubscriptions()-2, ncNoNode.NumSubscriptions(), + "an empty NodeName must skip exactly the publish and unpublish subscriptions") + + // The other provider subjects must still work without a node name. + body := marshalRequest(t, map[string]any{"schema_version": ebsprovider.SchemaVersion}) + msg := requestProvider(t, ncNoNode, ebsprovider.CapabilitiesSubject, body) + var resp ebsprovider.GetCapabilitiesResponse + require.NoError(t, json.Unmarshal(msg.Data, &resp)) + assert.Nil(t, resp.Error) +} + +// TestProviderHandlers_CreateVolume_SeedValidation covers the guards that must +// reject a seed before any engine is built. The bound exists because a []byte +// is base64-encoded into the JSON request and has to clear NATS max_payload. +func TestProviderHandlers_CreateVolume_SeedValidation(t *testing.T) { + _, natsURL := setupEmbeddedNATS(t) + cfg := setupTestConfig(t, natsURL) + nc := startProviderSubjects(t, cfg, natsURL) + + requestError := func(t *testing.T, body []byte) *ebsprovider.ProviderError { + t.Helper() + msg := requestProvider(t, nc, ebsprovider.CreateVolumeSubject, body) + var resp ebsprovider.CreateVolumeResponse + require.NoError(t, json.Unmarshal(msg.Data, &resp)) + require.NotNil(t, resp.Error) + require.Nil(t, resp.Volume) + return resp.Error + } + + t.Run("seed above MaxSeedBytes is invalid_argument", func(t *testing.T) { + err := requestError(t, marshalRequest(t, map[string]any{ + "schema_version": ebsprovider.SchemaVersion, + "volume_id": "vol-seedtoobig0001", + "capacity_range": map[string]any{"required_bytes": int64(bytesPerGiB)}, + "seed_data": make([]byte, ebsprovider.MaxSeedBytes+1), + })) + assert.Equal(t, ebsprovider.ErrorCodeInvalidArgument, err.Code) + }) + + t.Run("seed larger than capacity is invalid_argument", func(t *testing.T) { + err := requestError(t, marshalRequest(t, map[string]any{ + "schema_version": ebsprovider.SchemaVersion, + "volume_id": "vol-seedovercap001", + "capacity_range": map[string]any{"required_bytes": 512}, + "seed_data": make([]byte, 4096), + })) + assert.Equal(t, ebsprovider.ErrorCodeInvalidArgument, err.Code) + }) + + // A snapshot clone already fills the volume, so seeding on top of one would + // overwrite the cloned image rather than initialise a blank volume. + t.Run("seed with a source snapshot is invalid_argument", func(t *testing.T) { + err := requestError(t, marshalRequest(t, map[string]any{ + "schema_version": ebsprovider.SchemaVersion, + "volume_id": "vol-seedwithsnap01", + "capacity_range": map[string]any{"required_bytes": int64(bytesPerGiB)}, + "source_snapshot_id": "snap-0000000000001", + "source_snapshot_volume_id": "vol-source00000001", + "seed_data": make([]byte, 16), + })) + assert.Equal(t, ebsprovider.ErrorCodeInvalidArgument, err.Code) + }) +} + +// TestProviderHandlers_OwnerSubscription_UnmountRemovesItAndFallsBack proves +// the owner subscription's lifecycle: live while a volume is mounted (a raw +// request to its owner subject gets a responder), gone once unmountVolume +// runs (the same request comes back nats.ErrNoResponders), at which point +// NATSProvider.GetVolume falls back to the queue-group subject and takes the +// fresh-engine path (the fast-failing backend then fails it, proving this +// really happened rather than silently reusing the removed VB). +func TestProviderHandlers_OwnerSubscription_UnmountRemovesItAndFallsBack(t *testing.T) { + _, natsURL := setupEmbeddedNATS(t) + cfg := setupTestConfig(t, natsURL) + nc := startProviderSubjects(t, cfg, natsURL) + + const volumeName = "vol-ownerunmount001" + vb := createTestVBWithState(t, volumeName) + + ownerSubs := subscribeOwnerSubjects(context.Background(), cfg, nc, volumeName) + require.Len(t, ownerSubs, 4, "all four owner verbs must subscribe cleanly") + require.NoError(t, nc.Flush()) + + cfg.MountedVolumes = append(cfg.MountedVolumes, MountedVolume{ + Name: volumeName, VB: vb, OwnerSubs: ownerSubs, + // A PID that (almost certainly) does not exist: KillProcess must fail + // gracefully during unmountVolume rather than block or panic. + PID: 999999, + }) + + ownerSubject, err := ebsprovider.GetVolumeOwnerSubject(volumeName) + require.NoError(t, err) + getVolumeBody := marshalRequest(t, map[string]any{ + "schema_version": ebsprovider.SchemaVersion, "volume_id": volumeName, + }) + + _, err = nc.Request(ownerSubject, getVolumeBody, 2*time.Second) + require.NoError(t, err, "owner subject must have a live responder while the volume is mounted") + + _, err = unmountVolume(context.Background(), cfg, volumeName) + require.NoError(t, err) + + _, err = nc.Request(ownerSubject, getVolumeBody, 2*time.Second) + require.ErrorIs(t, err, nats.ErrNoResponders, "unmount must drop the owner subscription") + + cfg.S3Host = fastFailingS3Host(t) + provider := ebsprovider.NewNATSProvider(nc, 5*time.Second) + _, err = provider.GetVolume(context.Background(), ebsprovider.GetVolumeRequest{ + Versioned: ebsprovider.NewVersioned(), VolumeID: volumeName, + }) + require.Error(t, err, "with no owner and a broken S3 backend, the queue fallback's fresh engine must fail") +} + +// The EFI variable store depends on CreateVolume writing a seed, so the daemon +// must advertise that capability; a caller branching on it would otherwise fall +// back to building its own engine. +func TestProviderHandlers_Capabilities_AdvertisesVolumeSeeding(t *testing.T) { + _, natsURL := setupEmbeddedNATS(t) + cfg := setupTestConfig(t, natsURL) + nc := startProviderSubjects(t, cfg, natsURL) + + msg := requestProvider(t, nc, ebsprovider.CapabilitiesSubject, marshalRequest(t, map[string]any{ + "schema_version": ebsprovider.SchemaVersion, + })) + var resp ebsprovider.GetCapabilitiesResponse + require.NoError(t, json.Unmarshal(msg.Data, &resp)) + require.Nil(t, resp.Error) + assert.True(t, resp.Capabilities.VolumeSeeding) +} diff --git a/spinifex/services/viperblockd/provider_tracing_test.go b/spinifex/services/viperblockd/provider_tracing_test.go new file mode 100644 index 000000000..72534b482 --- /dev/null +++ b/spinifex/services/viperblockd/provider_tracing_test.go @@ -0,0 +1,76 @@ +package viperblockd + +import ( + "encoding/json" + "testing" + "time" + + "github.com/mulgadc/spinifex/spinifex/ebsprovider" + "github.com/mulgadc/spinifex/spinifex/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/codes" +) + +// TestProviderTracing_ServerSpanJoinsTheCallersTrace covers the provider that +// actually runs in production. viperblockd serves the contract with its own +// handlers rather than natsserve, so its propagation is a separate claim. +func TestProviderTracing_ServerSpanJoinsTheCallersTrace(t *testing.T) { + recorder := testutil.RecordSpans(t) + _, natsURL := setupEmbeddedNATS(t) + cfg := setupTestConfig(t, natsURL) + nc := startProviderSubjects(t, cfg, natsURL) + + client := ebsprovider.NewNATSProvider(nc, 5*time.Second) + _, err := client.GetCapabilities(t.Context(), ebsprovider.GetCapabilitiesRequest{Versioned: ebsprovider.NewVersioned()}) + require.NoError(t, err) + + clientSpan, serverSpan := testutil.SpansByKind(t, recorder) + require.NotNil(t, clientSpan, "no client span recorded") + require.NotNil(t, serverSpan, "no server span recorded") + assert.Equal(t, "ebs.capabilities", clientSpan.Name()) + assert.Equal(t, "ebs.capabilities", serverSpan.Name()) + assert.Equal(t, clientSpan.SpanContext().TraceID(), serverSpan.SpanContext().TraceID()) + assert.Equal(t, clientSpan.SpanContext().SpanID(), serverSpan.Parent().SpanID()) +} + +// TestProviderTracing_SpanNamesDropTheVolumeID pins the owner-routed path, +// where the subject carries a volume ID that must not reach the span name. +// The request is version-skewed so it is rejected before any backend work, +// which also exercises the error tagging on a reply built by the handler. +func TestProviderTracing_SpanNamesDropTheVolumeID(t *testing.T) { + const volumeID = "vol-testtracing0001" + recorder := testutil.RecordSpans(t) + _, natsURL := setupEmbeddedNATS(t) + cfg := setupTestConfig(t, natsURL) + nc := startProviderSubjects(t, cfg, natsURL) + + ownerSubject, err := ebsprovider.GetVolumeOwnerSubject(volumeID) + require.NoError(t, err) + subs := subscribeOwnerSubjects(t.Context(), cfg, nc, volumeID) + t.Cleanup(func() { + for _, sub := range subs { + _ = sub.Unsubscribe() + } + }) + require.NoError(t, nc.Flush()) + + payload := marshalRequest(t, map[string]any{ + "schema_version": ebsprovider.SchemaVersion + 1, + "volume_id": volumeID, + }) + msg := requestProvider(t, nc, ownerSubject, payload) + var response versionedErrorResponse + require.NoError(t, json.Unmarshal(msg.Data, &response)) + require.NotNil(t, response.Error) + + _, serverSpan := testutil.SpansByKind(t, recorder) + require.NotNil(t, serverSpan, "no server span recorded") + assert.Equal(t, "ebs.volume.describe", serverSpan.Name()) + assert.NotContains(t, serverSpan.Name(), volumeID) + assert.Equal(t, ownerSubject, testutil.SpanAttribute(t, serverSpan, "ebs.provider.subject").AsString()) + assert.Equal(t, volumeID, testutil.SpanAttribute(t, serverSpan, "ebs.provider.volume_id").AsString()) + assert.Equal(t, codes.Error, serverSpan.Status().Code) + assert.Equal(t, string(ebsprovider.ErrorCodeUnsupportedVersion), + testutil.SpanAttribute(t, serverSpan, "ebs.provider.error_code").AsString()) +} diff --git a/spinifex/services/viperblockd/viperblockd.go b/spinifex/services/viperblockd/viperblockd.go index 025eba3b9..1f373db11 100644 --- a/spinifex/services/viperblockd/viperblockd.go +++ b/spinifex/services/viperblockd/viperblockd.go @@ -1,15 +1,16 @@ package viperblockd import ( + "context" "encoding/json" "errors" "fmt" "log/slog" + "net/http" "os" "os/exec" "os/signal" "path/filepath" - "strconv" "strings" "sync" "syscall" @@ -17,15 +18,12 @@ import ( "github.com/mulgadc/bluebottle/pkg/masterkey" "github.com/mulgadc/spinifex/spinifex/admin" - "github.com/mulgadc/spinifex/spinifex/nbd" "github.com/mulgadc/spinifex/spinifex/types" "github.com/mulgadc/spinifex/spinifex/utils" "github.com/mulgadc/viperblock/viperblock" "github.com/mulgadc/viperblock/viperblock/backends/s3" "github.com/nats-io/nats.go" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/trace" ) @@ -76,10 +74,13 @@ func retryLoadState(volume string, attempts int, baseDelay time.Duration, sleep return fmt.Errorf("LoadState exhausted %d retries: %w", attempts, err) } -// loadStateWithRetry calls vb.LoadState with the production retry budget -// (loadStateRetryAttempts / loadStateRetryBaseDelay). -func loadStateWithRetry(vb *viperblock.VB, volume string) error { - return retryLoadState(volume, loadStateRetryAttempts, loadStateRetryBaseDelay, time.Sleep, vb.LoadState) +// loadStateWithRetry calls vb.LoadStateCtx with the production retry budget +// (loadStateRetryAttempts / loadStateRetryBaseDelay). The context matters +// beyond cancellation: the S3 backend only emits a span per request when the +// request carries one, so the no-context variant makes this time invisible. +func loadStateWithRetry(ctx context.Context, vb *viperblock.VB, volume string) error { + load := func() error { return vb.LoadStateCtx(ctx) } + return retryLoadState(volume, loadStateRetryAttempts, loadStateRetryBaseDelay, time.Sleep, load) } var serviceName = "viperblock" @@ -92,6 +93,22 @@ type MountedVolume struct { PID int VB *viperblock.VB // Reference to viperblock instance for state sync/flush ConfigSub *nats.Subscription // Per-volume config-update subscription (ebs.config.{volumeID}) + + // OwnerSubs are the plain (non-queue-group) ebs.provider.v1.owner.{Name}.* + // subscriptions registered while this volume is mounted, so + // snapshot/expand/describe requests reach this node directly instead of a + // random spinifex-workers member. Registered in mountVolume, dropped + // alongside ConfigSub whenever this entry leaves cfg.MountedVolumes. + OwnerSubs []*nats.Subscription + + // ReadOnly records the access mode nbdkit was started with. It is fixed + // for the life of the export, so a republish asking for the other mode + // cannot reuse this one. + ReadOnly bool + + // Lease is the cluster-wide claim on this volume, held for as long as the + // export is up and given back when this entry leaves cfg.MountedVolumes. + Lease *volumeLease } type Config struct { ConfigPath string @@ -132,24 +149,64 @@ type Config struct { masterKey *masterkey.Key + // s3HTTPClient is shared by every viperblock backend this service builds. + // One client per volume would be one connection pool per volume, so every + // engine open and state read would pay its own TLS handshake. + s3HTTPClientOnce sync.Once + s3HTTPClient *http.Client + // sealVolume overrides how a detached volume is sealed to predastore. // Nil means sealVolumeVB, the real seal. Tests that need a seal to FAIL // inject here rather than pointing S3Host at an unreachable endpoint: the // real failure only arrives once the S3 client exhausts its jittered // backoff, which takes anywhere from one to five seconds and so decides the // test on where the dice land relative to the caller's deadline. - sealVolume func(volumeName string) error + sealVolume func(ctx context.Context, volumeName string) error + + // constructVB overrides constructMountedVB. Nil means the real + // construction. Tests inject a fake (e.g. file-backed) VB here to avoid + // standing up a real S3 backend. + constructVB func(ctx context.Context, volumeName string) (*viperblock.VB, int, error) + + // leases excludes a second viperblock engine on a volume this node has + // open. Nil means exclusion cannot be established, and every engine open + // refuses rather than proceeding blind. + leases *volumeLeases mu sync.Mutex } // seal persists volumeName's block map to predastore, honouring a test's // injected seal if there is one. -func (cfg *Config) seal(volumeName string) error { +func (cfg *Config) seal(ctx context.Context, volumeName string) error { if cfg.sealVolume != nil { - return cfg.sealVolume(volumeName) + return cfg.sealVolume(ctx, volumeName) } - return sealVolumeVB(cfg, volumeName) + return sealVolumeVB(ctx, cfg, volumeName) +} + +// buildVB constructs volumeName's daemon-side VB, honouring a test's +// injected constructVB if there is one. The returned lease is the caller's to +// release once the engine is closed. +func (cfg *Config) buildVB(ctx context.Context, volumeName string) (*viperblock.VB, int, *volumeLease, error) { + lease, err := cfg.acquireVolumeLease(ctx, volumeName) + if err != nil { + return nil, 0, nil, err + } + + construct := func(ctx context.Context, volumeName string) (*viperblock.VB, int, error) { + return constructMountedVB(ctx, cfg, volumeName) + } + if cfg.constructVB != nil { + construct = cfg.constructVB + } + + vb, cacheSize, err := construct(ctx, volumeName) + if err != nil { + cfg.releaseVolumeLease(ctx, lease) + return nil, 0, nil, err + } + return vb, cacheSize, lease, nil } type Service struct { @@ -175,7 +232,7 @@ func New(config any) (svc *Service, err error) { // AES-GCM tag under the volume's current StateSeqNum, so the caller MUST own the // volume exclusively (live mounted VB, or a freshly opened detached one) to keep // the GCM nonce unique. -func applyConfigUpdate(vb *viperblock.VB, req types.EBSConfigUpdateRequest) error { +func applyConfigUpdate(ctx context.Context, vb *viperblock.VB, req types.EBSConfigUpdateRequest) error { var vc viperblock.VolumeConfig if err := json.Unmarshal(req.VolumeConfig, &vc); err != nil { return fmt.Errorf("unmarshal VolumeConfig: %w", err) @@ -185,7 +242,7 @@ func applyConfigUpdate(vb *viperblock.VB, req types.EBSConfigUpdateRequest) erro if sz := vc.VolumeMetadata.SizeGiB * 1024 * 1024 * 1024; sz > vb.VolumeSize { vb.VolumeSize = sz } - return vb.SaveState() + return vb.SaveStateCtx(ctx) } // makeConfigUpdateHandler returns a NATS handler for volume-specific config @@ -193,14 +250,17 @@ func applyConfigUpdate(vb *viperblock.VB, req types.EBSConfigUpdateRequest) erro // the single writer that owns the volume's StateSeqNum. func makeConfigUpdateHandler(vb *viperblock.VB, volumeName string) nats.MsgHandler { return func(msg *nats.Msg) { + ctx, span := utils.StartConsumerSpan(msg) + defer span.End() + var req types.EBSConfigUpdateRequest if err := json.Unmarshal(msg.Data, &req); err != nil { - slog.Error("Failed to unmarshal ebs.config message", "volume", volumeName, "err", err) + slog.ErrorContext(ctx, "Failed to unmarshal ebs.config message", "volume", volumeName, "err", err) respondJSON(msg, types.EBSConfigUpdateResponse{Volume: volumeName, Error: fmt.Sprintf("bad request: %v", err)}) return } - if err := applyConfigUpdate(vb, req); err != nil { - slog.Error("ebs.config: live VB update failed", "volume", volumeName, "err", err) + if err := applyConfigUpdate(ctx, vb, req); err != nil { + slog.ErrorContext(ctx, "ebs.config: live VB update failed", "volume", volumeName, "err", err) respondJSON(msg, types.EBSConfigUpdateResponse{Volume: volumeName, Error: err.Error()}) return } @@ -209,22 +269,27 @@ func makeConfigUpdateHandler(vb *viperblock.VB, volumeName string) nats.MsgHandl } } -// openVolumeVB constructs and opens an existing viperblock volume with its -// config state loaded (LoadState) but NOT its block map. Construction mirrors -// the ebs.mount path so encrypted volumes open with the master key and matching -// encryption state. Callers that Close() the VB MUST go through -// openLoadedVolumeVB instead, so the block map is restored before Close() -// flushes it back to predastore. -func openVolumeVB(cfg *Config, volumeName string) (*viperblock.VB, error) { - s3cfg := s3.S3Config{ +// volumeS3Config builds the backend config for one volume. Every viperblock +// backend this service constructs goes through here so they all share one +// connection pool. +func (cfg *Config) volumeS3Config(volumeName string) s3.S3Config { + cfg.s3HTTPClientOnce.Do(func() { cfg.s3HTTPClient = s3.NewHTTPClient() }) + return s3.S3Config{ VolumeName: volumeName, Bucket: cfg.Bucket, Region: cfg.Region, AccessKey: cfg.AccessKey, SecretKey: cfg.SecretKey, Host: admin.DialTarget(cfg.S3Host), + HTTPClient: cfg.s3HTTPClient, } - vbconfig := viperblock.VB{ +} + +// volumeVBConfig builds the engine and backend config for one volume. Shared +// by the read-write open and the read-only state read so the two cannot drift +// on encryption, bucket or base directory. +func volumeVBConfig(cfg *Config, volumeName string) (viperblock.VB, s3.S3Config) { + return viperblock.VB{ VolumeName: volumeName, VolumeSize: 1, // Recalculated on LoadState. BaseDir: cfg.BaseDir, @@ -232,18 +297,79 @@ func openVolumeVB(cfg *Config, volumeName string) (*viperblock.VB, error) { MasterKey: cfg.masterKey, EncryptionEnabled: cfg.masterKey != nil, GCEnabled: cfg.GCEnabled, + }, cfg.volumeS3Config(volumeName) +} + +// readVolumeState reads and verifies a volume's persisted state without +// opening the volume: no background goroutines, no reachability probe and no +// SeqNum window claimed. A describe is then one GetObject that writes nothing, +// which matters because it runs on whichever worker took the request rather +// than on the node that owns the volume. +func readVolumeState(ctx context.Context, cfg *Config, volumeName string) (viperblock.VBState, error) { + vbconfig, s3cfg := volumeVBConfig(cfg, volumeName) + vb, err := viperblock.NewStateReader(&vbconfig, "s3", s3cfg) + if err != nil { + return viperblock.VBState{}, fmt.Errorf("new state reader: %w", err) } + if err := vb.InitBackendForRead(ctx); err != nil { + return viperblock.VBState{}, fmt.Errorf("backend init: %w", err) + } + + var state viperblock.VBState + read := func() error { + var rerr error + state, rerr = vb.ReadStateCtx(ctx) + return rerr + } + if err := retryLoadState(volumeName, loadStateRetryAttempts, loadStateRetryBaseDelay, time.Sleep, read); err != nil { + return viperblock.VBState{}, fmt.Errorf("read state: %w", err) + } + return state, nil +} + +// openVolumeVB constructs and opens an existing viperblock volume with its +// config state loaded (LoadState) but NOT its block map. Construction mirrors +// the ebs.mount path so encrypted volumes open with the master key and matching +// encryption state. Callers that Close() the VB MUST go through +// openLoadedVolumeVB instead, so the block map is restored before Close() +// flushes it back to predastore. +// +// The returned lease is the volume's cluster-wide claim. It is the caller's to +// release, and must outlive the VB: releasing it while the engine is still +// open readmits a second writer. +func openVolumeVB(ctx context.Context, cfg *Config, volumeName string) (*viperblock.VB, *volumeLease, error) { + lease, err := cfg.acquireVolumeLease(ctx, volumeName) + if err != nil { + return nil, nil, err + } + + vbconfig, s3cfg := volumeVBConfig(cfg, volumeName) vb, err := viperblock.New(&vbconfig, "s3", s3cfg) if err != nil { - return nil, fmt.Errorf("new viperblock: %w", err) + cfg.releaseVolumeLease(ctx, lease) + return nil, nil, fmt.Errorf("new viperblock: %w", err) } - if err := vb.Backend.Init(); err != nil { - return nil, fmt.Errorf("backend init: %w", err) + + // New starts the chunk uploader and WAL syncer, so a failure below must + // release them: the caller gets no handle to stop them with. + opened := false + defer func() { + if !opened { + vb.StopChunkUploader() + vb.StopWALSyncer() + cfg.releaseVolumeLease(ctx, lease) + } + }() + + if err := vb.Backend.InitCtx(ctx); err != nil { + return nil, nil, fmt.Errorf("backend init: %w", err) } - if err := loadStateWithRetry(vb, volumeName); err != nil { - return nil, fmt.Errorf("load state: %w", err) + if err := loadStateWithRetry(ctx, vb, volumeName); err != nil { + return nil, nil, fmt.Errorf("load state: %w", err) } - return vb, nil + + opened = true + return vb, lease, nil } // isAuxVolume reports whether a volume is an -efi auxiliary volume. Auxiliary @@ -371,22 +497,24 @@ func clearStaleSealReceipt(baseDir, volume string) { // MUST Close the returned VB; on error the WAL syncer is stopped and no VB is // returned. The caller MUST ensure no nbdkit process is writing the shared // BaseDir first (post-KillProcess, or volume detached). -func openLoadedVolumeVB(cfg *Config, volumeName string) (*viperblock.VB, error) { - vb, err := openVolumeVB(cfg, volumeName) +func openLoadedVolumeVB(ctx context.Context, cfg *Config, volumeName string) (*viperblock.VB, *volumeLease, error) { + vb, lease, err := openVolumeVB(ctx, cfg, volumeName) if err != nil { - return nil, err + return nil, nil, err } - if err := vb.LoadBlockState(); err != nil { + if err := vb.LoadBlockStateCtx(ctx); err != nil { vb.StopWALSyncer() - return nil, fmt.Errorf("load block state: %w", err) + cfg.releaseVolumeLease(ctx, lease) + return nil, nil, fmt.Errorf("load block state: %w", err) } // RecoverLocalWALs is fail-closed on integrity errors and persists recovered // state itself; on failure retain the local WAL (no Close) for retry. if err := vb.RecoverLocalWALs(); err != nil { vb.StopWALSyncer() - return nil, fmt.Errorf("recover local WALs: %w", err) + cfg.releaseVolumeLease(ctx, lease) + return nil, nil, fmt.Errorf("recover local WALs: %w", err) } - return vb, nil + return vb, lease, nil } // sealVolumeVB persists a detached volume's block->object map to predastore. @@ -395,14 +523,16 @@ func openLoadedVolumeVB(cfg *Config, volumeName string) (*viperblock.VB, error) // node lacking the local WAL finds no checkpoint (bad superblock). It mirrors // the plugin's recover sequence (LoadBlockState + RecoverLocalWALs replay // un-sealed chunk WALs) then Close()s to flush the map. -func sealVolumeVB(cfg *Config, volumeName string) error { - vb, err := openLoadedVolumeVB(cfg, volumeName) +func sealVolumeVB(ctx context.Context, cfg *Config, volumeName string) error { + vb, lease, err := openLoadedVolumeVB(ctx, cfg, volumeName) if err != nil { return err } + defer cfg.releaseVolumeLease(ctx, lease) + // Close removes local files only after the predastore writes succeed, so a // failed seal leaves the WAL intact rather than losing data. - if err := vb.Close(); err != nil { + if err := vb.CloseCtx(ctx); err != nil { return fmt.Errorf("seal close: %w", err) } return nil @@ -489,6 +619,20 @@ func launchService(cfg *Config) (err error) { slog.Info("Viperblock config", "shardwal", cfg.ShardWAL, "gc_enabled", cfg.GCEnabled) + // Bound before recovery, which opens engines: without the store every + // engine open refuses, and the daemon would come up unable to adopt the + // exports that outlived it. + leases, err := newVolumeLeases(context.Background(), nc, cfg.leaseOwner()) + if err != nil { + return fmt.Errorf("volume leases: %w", err) + } + cfg.leases = leases + + // Rebuild MountedVolumes from any nbdkit processes that survived a + // restart before the daemon accepts a single request, so a handler can + // never race recovery and open a second engine against a live volume. + recoverMountedVolumes(context.Background(), cfg, nc, "/proc") + if cfg.NodeName != "" { slog.Info("Waiting for EBS events", "node", cfg.NodeName) } else { @@ -559,6 +703,8 @@ func launchService(cfg *Config) (err error) { } } + cfg.releaseVolumeLease(ctx, matched.Lease) + slog.InfoContext(ctx, "ebs.delete: cleaned up mounted volume", "volume", ebsRequest.Volume, "pid", matched.PID) } else { // Volume not mounted is expected for "available" volumes @@ -608,110 +754,7 @@ func launchService(cfg *Config) (err error) { return } - _, unmountSpan := otel.Tracer(viperblockdTracerName).Start(ctx, "ebs.unmount", - trace.WithAttributes(attribute.String("volume.id", ebsRequest.Name))) - - // Find the volume and extract references while holding the lock, - // then release before calling VB.Close() (which does heavy S3 I/O). - var ebsResponse types.EBSUnMountResponse - defer func() { endSpanWithResponseError(unmountSpan, ebsResponse.Error) }() - var matched MountedVolume - var matchIdx = -1 - cfg.mu.Lock() - for i, volume := range cfg.MountedVolumes { - if volume.Name == ebsRequest.Name { - matched = volume - matchIdx = i - break - } - } - cfg.mu.Unlock() - - if matchIdx >= 0 { - ebsResponse = types.EBSUnMountResponse{ - Volume: matched.Name, - Mounted: false, - } - - // Unsubscribe from volume-specific config-update topic - if matched.ConfigSub != nil { - if err := matched.ConfigSub.Unsubscribe(); err != nil { - slog.ErrorContext(ctx, "Failed to unsubscribe config topic", "volume", ebsRequest.Name, "err", err) - } - } - - // Stop background goroutines on the state-tracking VB. - // Actual I/O is in the nbdkit plugin process; sealVolumeVB below - // opens a fresh VB and calls Close() for the proper seal. - if matched.VB != nil { - matched.VB.StopChunkUploader() - matched.VB.StopWALSyncer() - } - - if err := utils.KillProcess(matched.PID); err != nil { - slog.ErrorContext(ctx, "Failed to kill nbdkit process", "pid", matched.PID, "err", err) - } - - // nbdkit is now dead, so no process writes the shared BaseDir: seal - // the block map to predastore for volumes that hold local state to - // flush (see volumeNeedsSeal). - if isAuxVolume(matched.Name) { - // Auxiliary volumes carry no durable guest data, so there is - // nothing to seal even when local state is present. - } else if volumeNeedsSeal(matched.Name, cfg.BaseDir) { - // Local state survived: the plugin's seal either failed or was - // cut short, so this fallback is the real seal. - if err := cfg.seal(matched.Name); err != nil { - slog.ErrorContext(ctx, "ebs.unmount: failed to seal volume to predastore", "volume", matched.Name, "err", err) - ebsResponse.Error = fmt.Sprintf("seal volume: %v", err) - } else { - slog.InfoContext(ctx, "ebs.unmount: volume sealed to predastore", "volume", matched.Name) - } - } else if consumeSealReceipt(cfg.BaseDir, matched.Name) { - // Healthy path: the plugin sealed to predastore and removed - // its local state itself, leaving this receipt as proof. - slog.InfoContext(ctx, "ebs.unmount: volume already sealed by nbdkit plugin", "volume", matched.Name) - } else { - // A durable volume reached unmount with no local WAL and no - // seal receipt: this node never held its state, so there is - // nothing to seal. WARN since this can mask a durability gap - // the seal would otherwise close. - slog.WarnContext(ctx, "ebs.unmount: no local viperblock state for volume, skipping seal", "volume", matched.Name, "baseDir", cfg.BaseDir) - } - - // Remove the socket file if using socket transport - if matched.Socket != "" { - slog.InfoContext(ctx, "Removing socket file", "socket", matched.Socket) - if err := os.Remove(matched.Socket); err != nil && !os.IsNotExist(err) { - slog.ErrorContext(ctx, "Failed to delete nbd socket", "err", err, "socket", matched.Socket) - } - } - - // Only drop the volume from MountedVolumes once the seal actually - // succeeded (or none was needed): a failed seal must leave it - // mounted so a retry re-attempts sealVolumeVB, rather than a - // caller seeing "not found" and mistaking a failed seal for a - // completed one. - if ebsResponse.Error == "" { - cfg.mu.Lock() - for i, volume := range cfg.MountedVolumes { - if volume.Name == matched.Name { - cfg.MountedVolumes = append(cfg.MountedVolumes[:i], cfg.MountedVolumes[i+1:]...) - break - } - } - cfg.mu.Unlock() - } - } - - if matchIdx < 0 { - ebsResponse = types.EBSUnMountResponse{ - Volume: ebsRequest.Name, - Error: fmt.Sprintf("Volume %s not found", ebsRequest.Name), - NotFound: true, - } - } - + ebsResponse, _ := unmountVolume(ctx, cfg, ebsRequest.Name) respondAndPublish(msg, nc, "ebs.unmount.response", ebsResponse) }); err != nil { return fmt.Errorf("failed to subscribe to %s: %w", unmountTopic, err) @@ -803,7 +846,7 @@ func launchService(cfg *Config) (err error) { cfg.mu.Unlock() if live != nil { - if err := applyConfigUpdate(live, req); err != nil { + if err := applyConfigUpdate(ctx, live, req); err != nil { slog.ErrorContext(ctx, "ebs.config: live VB update failed", "volume", req.Volume, "err", err) utils.MarkSpanError(span, err) respondJSON(msg, types.EBSConfigUpdateResponse{Volume: req.Volume, Error: err.Error()}) @@ -814,15 +857,17 @@ func launchService(cfg *Config) (err error) { return } - vb, err := openLoadedVolumeVB(cfg, req.Volume) + vb, lease, err := openLoadedVolumeVB(ctx, cfg, req.Volume) if err != nil { slog.ErrorContext(ctx, "ebs.config: failed to open detached volume", "volume", req.Volume, "err", err) utils.MarkSpanError(span, err) respondJSON(msg, types.EBSConfigUpdateResponse{Volume: req.Volume, Error: fmt.Sprintf("open volume: %v", err)}) return } - applyErr := applyConfigUpdate(vb, req) - if closeErr := vb.Close(); closeErr != nil { + defer cfg.releaseVolumeLease(ctx, lease) + + applyErr := applyConfigUpdate(ctx, vb, req) + if closeErr := vb.CloseCtx(ctx); closeErr != nil { slog.ErrorContext(ctx, "ebs.config: VB close failed", "volume", req.Volume, "err", closeErr) } if applyErr != nil { @@ -879,272 +924,19 @@ func launchService(cfg *Config) (err error) { return } - // Clear any receipt left by a previous mount before anything else can - // return early, so a stale receipt can never survive into this mount. - clearStaleSealReceipt(cfg.BaseDir, ebsRequest.Name) - - _, mountSpan := otel.Tracer(viperblockdTracerName).Start(ctx, "ebs.mount", - trace.WithAttributes(attribute.String("volume.id", ebsRequest.Name))) - - var ebsResponse types.EBSMountResponse - ebsResponse.Mounted = false - defer func() { endSpanWithResponseError(mountSpan, ebsResponse.Error) }() - - s3cfg := s3.S3Config{ - VolumeName: ebsRequest.Name, - Bucket: cfg.Bucket, - Region: cfg.Region, - AccessKey: cfg.AccessKey, - SecretKey: cfg.SecretKey, - Host: admin.DialTarget(cfg.S3Host), - } - - // TODO: Improve based on system availability. Default 128MB cache - defaultCache := (128 * 1024 * 1024) / int(viperblock.DefaultBlockSize) - - vbconfig := viperblock.VB{ - VolumeName: ebsRequest.Name, - VolumeSize: 1, // Workaround, calculated on LoadState() - BaseDir: cfg.BaseDir, - Cache: viperblock.Cache{ - Config: viperblock.CacheConfig{ - // TODO: Improve, based on system memory - Size: defaultCache, - }, - }, - VolumeConfig: viperblock.VolumeConfig{}, - MasterKey: cfg.masterKey, - EncryptionEnabled: cfg.masterKey != nil, - } - - vb, err := viperblock.New(&vbconfig, "s3", s3cfg) - - // Enable 128MB cache for main volumes, disable for efi (small, rarely read) - // This cacheSize is passed to nbdkit plugin (separate viperblock instance) - var nbdCacheSize int - if strings.HasSuffix(ebsRequest.Name, "-efi") { - slog.InfoContext(ctx, "Disabling cache for auxiliary volume", "volume", ebsRequest.Name) - if err := vb.SetCacheSize(0, 0); err != nil { - slog.ErrorContext(ctx, "Failed to set cache size", "err", err) - } - nbdCacheSize = 0 - } else { - slog.InfoContext(ctx, "Enabling 128MB cache for main volume", "volume", ebsRequest.Name, "blocks", defaultCache) - if err := vb.SetCacheSize(defaultCache, 0); err != nil { - slog.ErrorContext(ctx, "Failed to set cache size", "err", err) - } - nbdCacheSize = defaultCache - } - - if err != nil { - ebsResponse.Error = fmt.Sprintf("Failed to connect to Viperblock store: %v", err) - respondAndPublish(msg, nc, "ebs.mount.response", ebsResponse) - return - } - - // This daemon-side VB tracks state only; the nbdkit plugin process owns - // the data path and its own uploader. Stop this VB's background uploader - // so it cannot overwrite the live checkpoint every 30s (AEAD corruption). - vb.StopChunkUploader() - - if cfg.Debug { - vb.SetDebug(true) - } - - // Initialize the backend - err = vb.Backend.Init() - - if err != nil { - ebsResponse.Error = err.Error() - respondAndPublish(msg, nc, "ebs.mount.response", ebsResponse) - return - } - - // Retry on transient backend errors so daemon recovery doesn't tip a healthy volume into cleanup. - err = loadStateWithRetry(vb, ebsRequest.Name) - - if err != nil { - ebsResponse.Error = err.Error() - respondAndPublish(msg, nc, "ebs.mount.response", ebsResponse) - return - } - mountSpan.SetAttributes(attribute.Int64("volume.size_bytes", utils.SafeUint64ToInt64(vb.GetVolumeSize()))) - - useTCP := cfg.NBDTransport == types.NBDTransportTCP - - var nbdURI string - var nbdSocket string - var nbdPort int - - if useTCP { - // TCP transport - find a free port - portStr, err := viperblock.FindFreePort() - if err != nil { - ebsResponse.Error = err.Error() - respondAndPublish(msg, nc, "ebs.mount.response", ebsResponse) - return - } - - // Parse the port from the address - parts := strings.Split(portStr, ":") - nbdPort, err = strconv.Atoi(parts[len(parts)-1]) - if err != nil { - slog.ErrorContext(ctx, "Failed to convert port to int", "err", err) - ebsResponse.Error = fmt.Sprintf("failed to parse port: %v", err) - respondAndPublish(msg, nc, "ebs.mount.response", ebsResponse) - return - } - - nbdURI = utils.FormatNBDTCPURI("127.0.0.1", nbdPort) - slog.InfoContext(ctx, "Mounting volume (TCP)", "name", ebsRequest.Name, "port", nbdPort, "uri", nbdURI) - } else { - // Unix socket transport (default) - generate unique socket path - nbdSocket, err = utils.GenerateUniqueSocketFile(ebsRequest.Name) - if err != nil { - ebsResponse.Error = err.Error() - respondAndPublish(msg, nc, "ebs.mount.response", ebsResponse) - return - } - - nbdURI = utils.FormatNBDSocketURI(nbdSocket) - slog.InfoContext(ctx, "Mounting volume (socket)", "name", ebsRequest.Name, "socket", nbdSocket, "uri", nbdURI) - } - - // Generate PID file for nbdkit process - nbdPidFile, err := utils.GeneratePidFile(fmt.Sprintf("nbdkit-vol-%s", ebsRequest.Name)) - if err != nil { - slog.ErrorContext(ctx, "Failed to generate nbdkit pid file", "err", err) - ebsResponse.Error = fmt.Sprintf("failed to generate pid file: %v", err) - respondAndPublish(msg, nc, "ebs.mount.response", ebsResponse) - return - } - - nbdConfig := nbd.NBDKitConfig{ - Port: nbdPort, - Socket: nbdSocket, - UseTCP: useTCP, - PidFile: nbdPidFile, - PluginPath: cfg.PluginPath, - BaseDir: cfg.BaseDir, - Host: admin.DialTarget(cfg.S3Host), - Verbose: false, - Size: utils.SafeUint64ToInt64(vb.GetVolumeSize()), - Volume: ebsRequest.Name, - Bucket: cfg.Bucket, - Region: cfg.Region, - AccessKey: cfg.AccessKey, - SecretKey: cfg.SecretKey, - CacheSize: nbdCacheSize, - ShardWAL: cfg.ShardWAL, - GCEnabled: cfg.GCEnabled, - EncryptionKeyFile: cfg.EncryptionKeyFile, - } - - // Create a unique error channel for this specific mount request - processChan := make(chan int, 1) - exitChan := make(chan int, 1) - - // TODO: Improve, use a process manager to track the (multiple) nbdkit process - go func() { - slog.Debug("Executing nbdkit") - - cmd, err := nbdConfig.Execute() - if err != nil { - slog.Error("Failed to execute nbdkit", "err", err) - // Signal error (no PID) to parent goroutine - processChan <- 0 - return - } - - pid := cmd.Process.Pid - // Signal successful startup w/ PID - processChan <- pid - - err = cmd.Wait() - - if err != nil { - slog.Error("Failed to wait for nbdkit", "err", err) - exitChan <- 1 - return - } - - exitCode := cmd.ProcessState.ExitCode() - - exitChan <- exitCode - - slog.Error("NBDKit exited", "code", exitCode) - }() - - pid := <-processChan - - if pid == 0 { - ebsResponse.Error = "Failed to start nbdkit" - respondAndPublish(msg, nc, "ebs.mount.response", ebsResponse) - return - } - - // Poll for up to 1 second to confirm nbdkit is running, checking every - // 50ms so a fast failure is detected quickly instead of always - // costing the full second. Any exit within that window means NBDKit - // failed to stay up. - const nbdkitConfirmPollInterval = 50 * time.Millisecond - const nbdkitConfirmDeadline = 1 * time.Second - confirmDeadline := time.Now().Add(nbdkitConfirmDeadline) - for { - select { - case exitErr := <-exitChan: - ebsResponse.Error = fmt.Sprintf("nbdkit exited unexpectedly (code=%d)", exitErr) - respondAndPublish(msg, nc, "ebs.mount.response", ebsResponse) - return - default: - } - - if time.Now().After(confirmDeadline) { - break - } - - time.Sleep(nbdkitConfirmPollInterval) - } - - // nbdkit is still running after the poll window, which means it started successfully - slog.InfoContext(ctx, "NBDKit started successfully and is running") - - // NBDKit creates the socket with its own umask (typically 0755). - // The daemon (different user, same group) needs write access to connect. - if nbdSocket != "" { - if err := os.Chmod(nbdSocket, 0770); err != nil { //nolint:gosec // socket needs group-write for cross-service access - slog.WarnContext(ctx, "Failed to chmod NBD socket", "socket", nbdSocket, "err", err) - } - } - - ebsResponse.Mounted = true - ebsResponse.URI = nbdURI - - // Subscribe to volume-specific config-update topic so encrypted-volume - // metadata writes route to this node's live VB (the StateSeqNum owner). - configSub, err := nc.Subscribe(fmt.Sprintf("ebs.config.%s", ebsRequest.Name), makeConfigUpdateHandler(vb, ebsRequest.Name)) - if err != nil { - slog.ErrorContext(ctx, "Failed to subscribe to volume config topic", "volume", ebsRequest.Name, "err", err) - } - - cfg.mu.Lock() - cfg.MountedVolumes = append(cfg.MountedVolumes, MountedVolume{ - Name: ebsRequest.Name, - Port: nbdPort, - Socket: nbdSocket, - NBDURI: nbdURI, - PID: pid, - VB: vb, - ConfigSub: configSub, - }) - cfg.mu.Unlock() - + ebsResponse, _ := mountVolume(ctx, cfg, nc, ebsRequest.Name, false) respondAndPublish(msg, nc, "ebs.mount.response", ebsResponse) - slog.Debug("Sent ebs.mount response") + if ebsResponse.Mounted { + slog.Debug("Sent ebs.mount response") + } }); err != nil { return fmt.Errorf("failed to subscribe to %s: %w", mountTopic, err) } + if err := registerProviderSubjects(cfg, nc); err != nil { + return err + } + // Create a channel to receive shutdown signals sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) diff --git a/spinifex/services/viperblockd/viperblockd_integration_test.go b/spinifex/services/viperblockd/viperblockd_integration_test.go index b002931dd..6b3ad3cf3 100644 --- a/spinifex/services/viperblockd/viperblockd_integration_test.go +++ b/spinifex/services/viperblockd/viperblockd_integration_test.go @@ -6,6 +6,7 @@ package viperblockd import ( "bytes" + "context" "encoding/json" "fmt" "log/slog" @@ -24,21 +25,40 @@ import ( "github.com/stretchr/testify/require" ) -// setupEmbeddedNATS starts an embedded NATS server for testing. +// setupEmbeddedNATS starts an embedded NATS server for testing. JetStream is +// on because volume leases live in a KV bucket, and a daemon that cannot take +// one refuses to open an engine. func setupEmbeddedNATS(t *testing.T) (*server.Server, string) { opts := &server.Options{ - Host: "127.0.0.1", - Port: -1, // Random available port + Host: "127.0.0.1", + Port: -1, // Random available port + JetStream: true, + StoreDir: t.TempDir(), } ns := natstest.RunServer(opts) if ns == nil { t.Fatal("Failed to start embedded NATS server") } + t.Cleanup(ns.Shutdown) return ns, ns.ClientURL() } +// installTestVolumeLeases gives cfg a lease store backed by natsURL's +// JetStream, which every engine-open path needs before it will proceed. +func installTestVolumeLeases(t *testing.T, cfg *Config, natsURL string) { + t.Helper() + + nc, err := nats.Connect(natsURL) + require.NoError(t, err) + t.Cleanup(nc.Close) + + leases, err := newVolumeLeases(t.Context(), nc, cfg.leaseOwner()) + require.NoError(t, err) + cfg.leases = leases +} + // setupTestConfig creates a test configuration with proper paths. func setupTestConfig(t *testing.T, natsURL string) *Config { testDir := t.TempDir() @@ -56,6 +76,7 @@ func setupTestConfig(t *testing.T, natsURL string) *Config { MountedVolumes: []MountedVolume{}, NodeName: "test-node", } + installTestVolumeLeases(t, cfg, natsURL) return cfg } @@ -422,7 +443,7 @@ func TestIntegration_EBSUnmountSealFailureKeepsVolumeMounted(t *testing.T) { // is what put this test's outcome inside the tail of the 5s deadline below. createMockVolumeState(t, cfg.BaseDir, "vol-seal-fail") var sealCalls atomic.Int32 - cfg.sealVolume = func(volumeName string) error { + cfg.sealVolume = func(_ context.Context, volumeName string) error { sealCalls.Add(1) return fmt.Errorf("seal %s: injected backend failure", volumeName) } @@ -475,7 +496,7 @@ func TestIntegration_EBSUnmountReceiptSkipsFallbackSeal(t *testing.T) { cfg := setupTestConfig(t, natsURL) writeSealReceipt(t, cfg.BaseDir, "vol-clean-receipt") var sealCalls atomic.Int32 - cfg.sealVolume = func(volumeName string) error { + cfg.sealVolume = func(_ context.Context, volumeName string) error { sealCalls.Add(1) return nil } @@ -527,7 +548,7 @@ func TestIntegration_EBSUnmountStateDirSealsRegardlessOfReceipt(t *testing.T) { createMockVolumeState(t, cfg.BaseDir, "vol-state-and-receipt") writeSealReceipt(t, cfg.BaseDir, "vol-state-and-receipt") var sealCalls atomic.Int32 - cfg.sealVolume = func(volumeName string) error { + cfg.sealVolume = func(_ context.Context, volumeName string) error { sealCalls.Add(1) return nil } @@ -570,7 +591,7 @@ func TestIntegration_EBSUnmountAuxVolumeNeverSeals(t *testing.T) { cfg := setupTestConfig(t, natsURL) createMockVolumeState(t, cfg.BaseDir, "vol-aux-efi") var sealCalls atomic.Int32 - cfg.sealVolume = func(volumeName string) error { + cfg.sealVolume = func(_ context.Context, volumeName string) error { sealCalls.Add(1) return nil } @@ -613,7 +634,7 @@ func TestIntegration_EBSUnmountNoStateNoReceiptWarns(t *testing.T) { cfg := setupTestConfig(t, natsURL) var sealCalls atomic.Int32 - cfg.sealVolume = func(volumeName string) error { + cfg.sealVolume = func(_ context.Context, volumeName string) error { sealCalls.Add(1) return nil } diff --git a/spinifex/services/viperblockd/volume_lease.go b/spinifex/services/viperblockd/volume_lease.go new file mode 100644 index 000000000..3ca6c60a4 --- /dev/null +++ b/spinifex/services/viperblockd/volume_lease.go @@ -0,0 +1,339 @@ +package viperblockd + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "regexp" + "sync" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +// volumeLeaseBucket holds one entry per volume that some node has a viperblock +// engine open on. JetStream serialises the create, so the entry is the +// exclusion rather than a record of it. +const volumeLeaseBucket = "VIPERBLOCK_VOLUME_LEASES" + +// volumeLeaseTTL is how long a lease outlives the node holding it. A node that +// dies mid-mount leaves its entry behind and nothing else may open the volume +// until the entry ages out. +const volumeLeaseTTL = 45 * time.Second + +// volumeLeaseRenewInterval keeps a live holder's entry young. Well inside the +// TTL so one missed renewal does not surrender the lease. +const volumeLeaseRenewInterval = 15 * time.Second + +// errVolumeLeaseHeld reports that another claimant owns the volume. It is the +// one lease failure a caller can do something about, so it is distinguishable. +var errVolumeLeaseHeld = errors.New("volume is leased by another owner") + +// volumeLeaseKeyPattern is what JetStream KV accepts as a key. Volume names +// reach here from the wire, and a name carrying "." or ">" would address +// somebody else's key. +var volumeLeaseKeyPattern = regexp.MustCompile(`^[a-zA-Z0-9_-]+$`) + +// volumeLeaseRecord is what a holder publishes about itself. Generation is the +// revision that won the race: monotonic across the bucket, so a later opener +// always carries a higher one than the writer it displaced. +type volumeLeaseRecord struct { + Owner string `json:"owner"` + Generation uint64 `json:"generation"` + AcquiredAt time.Time `json:"acquired_at"` +} + +// volumeLeases hands out cluster-wide volume leases from a JetStream KV +// bucket, and remembers which ones this node holds. +type volumeLeases struct { + kv jetstream.KeyValue + owner string + + mu sync.Mutex + held map[string]*volumeLease +} + +// newVolumeLeases binds the lease bucket, creating it if this is the first +// node up. owner identifies this node in the entries it writes. +func newVolumeLeases(ctx context.Context, nc *nats.Conn, owner string) (*volumeLeases, error) { + js, err := jetstream.New(nc) + if err != nil { + return nil, fmt.Errorf("jetstream: %w", err) + } + kv, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{ + Bucket: volumeLeaseBucket, + Description: "one entry per volume with a viperblock engine open on it", + TTL: volumeLeaseTTL, + History: 1, + }) + if err != nil { + return nil, fmt.Errorf("volume lease bucket: %w", err) + } + return &volumeLeases{kv: kv, owner: owner, held: make(map[string]*volumeLease)}, nil +} + +// volumeLease is one held lease and the goroutine keeping it alive. +type volumeLease struct { + leases *volumeLeases + key string + volume string + generation uint64 + + stop context.CancelFunc + done chan struct{} + + mu sync.Mutex + // revision is the entry version this holder last wrote, and is what every + // subsequent write is conditioned on. + revision uint64 + // lost records that the entry moved out from under this holder, which + // means somebody else may now be writing the volume. + lost bool + // refs counts opens on this node sharing the lease. The lease is released + // when the last one lets go. + refs int +} + +// acquire claims volumeName for this node, or reports who has it. Repeat +// acquisitions on this node share one lease: cross-node exclusion is what the +// lease is for, and the per-node case is already held by the volume flock. +func (l *volumeLeases) acquire(ctx context.Context, volumeName string) (*volumeLease, error) { + if !volumeLeaseKeyPattern.MatchString(volumeName) { + return nil, fmt.Errorf("volume name %q cannot be a lease key", volumeName) + } + + l.mu.Lock() + if lease, ok := l.held[volumeName]; ok { + lease.mu.Lock() + lease.refs++ + lease.mu.Unlock() + l.mu.Unlock() + return lease, nil + } + l.mu.Unlock() + + record := volumeLeaseRecord{Owner: l.owner, AcquiredAt: time.Now().UTC()} + payload, err := json.Marshal(record) + if err != nil { + return nil, fmt.Errorf("marshal lease: %w", err) + } + + revision, err := l.kv.Create(ctx, volumeName, payload) + switch { + case errors.Is(err, jetstream.ErrKeyExists): + if revision, err = l.takeOver(ctx, volumeName, payload); err != nil { + return nil, err + } + case err != nil: + return nil, fmt.Errorf("claim lease for %s: %w", volumeName, err) + } + + lease := &volumeLease{ + leases: l, + key: volumeName, + volume: volumeName, + generation: revision, + revision: revision, + done: make(chan struct{}), + refs: 1, + } + + // The generation is the revision the create returned, so it can only be + // published afterwards. A failure here costs observability, not exclusion. + record.Generation = revision + if published, perr := json.Marshal(record); perr == nil { + if rev, uerr := l.kv.Update(ctx, volumeName, published, revision); uerr == nil { + lease.revision = rev + } else { + slog.Warn("volume lease: could not publish generation", "volume", volumeName, "err", uerr) + } + } + + l.mu.Lock() + l.held[volumeName] = lease + l.mu.Unlock() + + renewCtx, cancel := context.WithCancel(context.WithoutCancel(ctx)) + lease.stop = cancel + go lease.renewLoop(renewCtx) + + slog.Info("volume lease acquired", "volume", volumeName, "owner", l.owner, "generation", revision) + return lease, nil +} + +// takeOver reclaims an entry this node left behind. A daemon restart abandons +// the leases of exports that outlived it, and those exports are still running: +// waiting out the TTL would leave them untracked, and the entry cannot belong +// to a live remote holder because it names this node. +// +// The write is conditioned on the revision just read, so two claimants racing +// to reclaim the same entry cannot both win. +func (l *volumeLeases) takeOver(ctx context.Context, volumeName string, payload []byte) (uint64, error) { + entry, err := l.kv.Get(ctx, volumeName) + if err != nil { + return 0, fmt.Errorf("%w: %s", errVolumeLeaseHeld, l.describeHolder(ctx, volumeName)) + } + + var record volumeLeaseRecord + if err := json.Unmarshal(entry.Value(), &record); err != nil || record.Owner != l.owner { + return 0, fmt.Errorf("%w: %s", errVolumeLeaseHeld, l.describeHolder(ctx, volumeName)) + } + + revision, err := l.kv.Update(ctx, volumeName, payload, entry.Revision()) + if err != nil { + return 0, fmt.Errorf("%w: %s", errVolumeLeaseHeld, l.describeHolder(ctx, volumeName)) + } + slog.Info("volume lease reclaimed from this node's previous run", "volume", volumeName, "owner", l.owner) + return revision, nil +} + +// describeHolder reports who owns volumeName, for the error the loser gets. +// Best effort: the point of the message is triage, not control flow. +func (l *volumeLeases) describeHolder(ctx context.Context, volumeName string) string { + entry, err := l.kv.Get(ctx, volumeName) + if err != nil { + return "holder unknown" + } + var record volumeLeaseRecord + if err := json.Unmarshal(entry.Value(), &record); err != nil { + return "holder unreadable" + } + return fmt.Sprintf("held by %s since %s", record.Owner, record.AcquiredAt.Format(time.RFC3339)) +} + +// release drops one reference, and gives the lease up once the last one goes. +func (l *volumeLeases) release(ctx context.Context, lease *volumeLease) { + if lease == nil { + return + } + + lease.mu.Lock() + lease.refs-- + remaining := lease.refs + lease.mu.Unlock() + if remaining > 0 { + return + } + + l.mu.Lock() + delete(l.held, lease.key) + l.mu.Unlock() + + lease.stop() + <-lease.done + + // A lost lease belongs to somebody else now. Deleting the key would evict + // the live holder and hand the volume to a third opener. + lease.mu.Lock() + lost, revision := lease.lost, lease.revision + lease.mu.Unlock() + if lost { + slog.Warn("volume lease released without deleting: entry no longer ours", "volume", lease.volume) + return + } + + if err := l.kv.Delete(ctx, lease.key, jetstream.LastRevision(revision)); err != nil { + slog.Warn("volume lease: delete failed, entry will expire", "volume", lease.volume, "ttl", volumeLeaseTTL, "err", err) + return + } + slog.Info("volume lease released", "volume", lease.volume, "generation", lease.generation) +} + +// renewLoop refreshes the entry so the bucket TTL does not expire a live +// holder, and notices when the lease has been taken away. +func (lease *volumeLease) renewLoop(ctx context.Context) { + defer close(lease.done) + + ticker := time.NewTicker(volumeLeaseRenewInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if !lease.renew(ctx) { + return + } + } + } +} + +// renew rewrites the entry conditioned on the revision this holder last saw, +// and reports whether the lease is still ours. +// +// Losing it is logged rather than enforced: until the object store can refuse +// a stale writer's PUT, failing live guest I/O here would add a failure mode +// without stopping anything. +func (lease *volumeLease) renew(ctx context.Context) bool { + record := volumeLeaseRecord{Owner: lease.leases.owner, Generation: lease.generation, AcquiredAt: time.Now().UTC()} + payload, err := json.Marshal(record) + if err != nil { + slog.Error("volume lease: marshal renewal", "volume", lease.volume, "err", err) + return true + } + + lease.mu.Lock() + revision := lease.revision + lease.mu.Unlock() + + renewed, err := lease.leases.kv.Update(ctx, lease.key, payload, revision) + switch { + case err == nil: + lease.mu.Lock() + lease.revision = renewed + lease.mu.Unlock() + return true + case errors.Is(err, context.Canceled): + return false + case errors.Is(err, jetstream.ErrKeyExists), errors.Is(err, jetstream.ErrKeyNotFound), isWrongLastSequence(err): + lease.mu.Lock() + lease.lost = true + lease.mu.Unlock() + slog.Error("volume lease lost: another opener may hold this volume", "volume", lease.volume, "generation", lease.generation, "err", err) + return false + default: + // A transient JetStream error is not a lost lease. Keep renewing; the + // TTL is several intervals wide, so there is room to recover. + slog.Warn("volume lease: renewal failed", "volume", lease.volume, "err", err) + return true + } +} + +// isWrongLastSequence reports whether err is JetStream refusing a write whose +// expected revision no longer matches, which is how a lost lease presents. +func isWrongLastSequence(err error) bool { + var apiErr *jetstream.APIError + return errors.As(err, &apiErr) && apiErr.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequence +} + +// leaseOwner names this node in the lease entries it writes. A daemon with no +// NodeName is single-node by construction, but its entries still have to be +// attributable, so it says so rather than writing an empty owner. +func (cfg *Config) leaseOwner() string { + if cfg.NodeName != "" { + return cfg.NodeName + } + return "unnamed-node" +} + +// acquireVolumeLease claims volumeName before an engine is opened on it. A +// daemon with no lease store cannot establish that it is the only opener, so +// it refuses rather than opening blind. +func (cfg *Config) acquireVolumeLease(ctx context.Context, volumeName string) (*volumeLease, error) { + if cfg.leases == nil { + return nil, fmt.Errorf("no volume lease store: cannot establish exclusive access to %s", volumeName) + } + return cfg.leases.acquire(ctx, volumeName) +} + +// releaseVolumeLease gives up a lease taken by acquireVolumeLease. +func (cfg *Config) releaseVolumeLease(ctx context.Context, lease *volumeLease) { + if cfg.leases == nil || lease == nil { + return + } + cfg.leases.release(ctx, lease) +} diff --git a/spinifex/services/viperblockd/volume_lease_test.go b/spinifex/services/viperblockd/volume_lease_test.go new file mode 100644 index 000000000..d8d2ce6a4 --- /dev/null +++ b/spinifex/services/viperblockd/volume_lease_test.go @@ -0,0 +1,131 @@ +package viperblockd + +import ( + "testing" + + "github.com/nats-io/nats.go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// newTestLeases binds a lease store for owner against natsURL's JetStream. +func newTestLeases(t *testing.T, natsURL, owner string) *volumeLeases { + t.Helper() + + nc, err := nats.Connect(natsURL) + require.NoError(t, err) + t.Cleanup(nc.Close) + + leases, err := newVolumeLeases(t.Context(), nc, owner) + require.NoError(t, err) + return leases +} + +// TestVolumeLease_SecondNodeIsRefused is the exclusion itself. Two viperblock +// engines on one encrypted volume issue overlapping AES-GCM nonces, so the +// second claimant losing is the whole point of the lease. +func TestVolumeLease_SecondNodeIsRefused(t *testing.T) { + _, natsURL := setupEmbeddedNATS(t) + + first := newTestLeases(t, natsURL, "node-a") + second := newTestLeases(t, natsURL, "node-b") + + const volumeName = "vol-leaseexclusion1" + held, err := first.acquire(t.Context(), volumeName) + require.NoError(t, err) + require.NotNil(t, held) + + _, err = second.acquire(t.Context(), volumeName) + require.ErrorIs(t, err, errVolumeLeaseHeld, "a second node must not get an engine on a volume this one holds") + assert.Contains(t, err.Error(), "node-a", "the loser needs to know who to chase") +} + +// TestVolumeLease_ReleasedVolumeIsClaimableAgain pins that exclusion is not a +// one-way door: a volume detached from one node has to be attachable on the +// next, which is the ordinary EBS lifecycle. +func TestVolumeLease_ReleasedVolumeIsClaimableAgain(t *testing.T) { + _, natsURL := setupEmbeddedNATS(t) + + first := newTestLeases(t, natsURL, "node-a") + second := newTestLeases(t, natsURL, "node-b") + + const volumeName = "vol-leasehandover1" + held, err := first.acquire(t.Context(), volumeName) + require.NoError(t, err) + first.release(t.Context(), held) + + taken, err := second.acquire(t.Context(), volumeName) + require.NoError(t, err, "a released volume must be claimable by another node") + require.Greater(t, taken.generation, held.generation, "generations must advance, or a stale writer is indistinguishable from a live one") +} + +// TestVolumeLease_RepeatAcquireOnOneNodeShares covers the unmount seal, which +// opens a detached engine while the mount that is being torn down still holds +// the lease. Refusing itself there would fail every seal. +func TestVolumeLease_RepeatAcquireOnOneNodeShares(t *testing.T) { + _, natsURL := setupEmbeddedNATS(t) + + leases := newTestLeases(t, natsURL, "node-a") + other := newTestLeases(t, natsURL, "node-b") + + const volumeName = "vol-leaserefcount1" + mount, err := leases.acquire(t.Context(), volumeName) + require.NoError(t, err) + seal, err := leases.acquire(t.Context(), volumeName) + require.NoError(t, err, "the same node must be able to open a second handle on a volume it already holds") + require.Same(t, mount, seal, "a repeat acquisition must share the lease, not allocate a second one") + + // The seal finishing must not surrender the mount's claim. + leases.release(t.Context(), seal) + _, err = other.acquire(t.Context(), volumeName) + require.ErrorIs(t, err, errVolumeLeaseHeld, "releasing one handle must not hand the volume to another node while the mount holds it") + + leases.release(t.Context(), mount) + _, err = other.acquire(t.Context(), volumeName) + require.NoError(t, err, "the last release must actually give the volume up") +} + +// TestVolumeLease_NodeReclaimsItsOwnStaleEntry covers a daemon restart. The +// nbdkit exports outlive the daemon, so recovery has to re-adopt them; waiting +// out the TTL would leave live exports untracked. +func TestVolumeLease_NodeReclaimsItsOwnStaleEntry(t *testing.T) { + _, natsURL := setupEmbeddedNATS(t) + + const volumeName = "vol-leaserestart01" + before := newTestLeases(t, natsURL, "node-a") + stale, err := before.acquire(t.Context(), volumeName) + require.NoError(t, err) + + // A restart, not a release: the entry stays behind with nothing renewing it. + stale.stop() + <-stale.done + + after := newTestLeases(t, natsURL, "node-a") + reclaimed, err := after.acquire(t.Context(), volumeName) + require.NoError(t, err, "a restarted daemon must reclaim the leases of the exports that outlived it") + require.Greater(t, reclaimed.generation, stale.generation) +} + +// TestVolumeLease_OpenRefusesWithoutALeaseStore pins the fail-closed default. +// A daemon that cannot reach JetStream cannot establish that it is the only +// opener, and opening blind is what the lease exists to prevent. +func TestVolumeLease_OpenRefusesWithoutALeaseStore(t *testing.T) { + cfg := &Config{NodeName: "test-node"} + + lease, err := cfg.acquireVolumeLease(t.Context(), "vol-nostore00000001") + require.Error(t, err, "no lease store must refuse the open, not wave it through") + assert.Nil(t, lease) +} + +// TestVolumeLease_RejectsUnsafeKeys pins that a volume name off the wire +// cannot address another volume's entry: "." and ">" are JetStream subject +// tokens, and a name carrying them would claim or read the wrong key. +func TestVolumeLease_RejectsUnsafeKeys(t *testing.T) { + _, natsURL := setupEmbeddedNATS(t) + leases := newTestLeases(t, natsURL, "node-a") + + for _, name := range []string{"vol.other", "vol>", "", "vol/../other", "vol *"} { + _, err := leases.acquire(t.Context(), name) + require.Errorf(t, err, "volume name %q must not become a lease key", name) + } +} diff --git a/spinifex/testutil/ebsfake/ebsfake.go b/spinifex/testutil/ebsfake/ebsfake.go new file mode 100644 index 000000000..55e096b40 --- /dev/null +++ b/spinifex/testutil/ebsfake/ebsfake.go @@ -0,0 +1,51 @@ +package ebsfake + +import ( + "context" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/mulgadc/spinifex/spinifex/ebsprovider" + "github.com/mulgadc/spinifex/spinifex/objectstore" +) + +// Provider is ebsprovider.MemoryProvider plus the object-prefix delete +// viperblockd performs on ebs.provider.v1.volume.delete. Without it a deleted +// volume's legacy config.json survives in the test store and the ebsmetadata +// legacy fallback resurrects the volume. +type Provider struct { + ebsprovider.EBSProvider + + store objectstore.ObjectStore + bucket string +} + +var _ ebsprovider.EBSProvider = (*Provider)(nil) + +// New returns an in-memory EBS provider backed by store. +func New(store objectstore.ObjectStore, bucket string) *Provider { + return &Provider{ + EBSProvider: ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{}), + store: store, bucket: bucket, + } +} + +func (p *Provider) DeleteVolume(ctx context.Context, req ebsprovider.DeleteVolumeRequest) error { + if err := p.EBSProvider.DeleteVolume(ctx, req); err != nil { + return err + } + for _, prefix := range []string{req.VolumeID + "-efi/", req.VolumeID + "/"} { + listed, err := p.store.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ + Bucket: aws.String(p.bucket), Prefix: aws.String(prefix), + }) + if err != nil { + return err + } + for _, obj := range listed.Contents { + if _, err := p.store.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(p.bucket), Key: obj.Key}); err != nil { + return err + } + } + } + return nil +} diff --git a/spinifex/testutil/nats.go b/spinifex/testutil/nats.go index 36e4961e0..4f74cf0ce 100644 --- a/spinifex/testutil/nats.go +++ b/spinifex/testutil/nats.go @@ -12,8 +12,9 @@ import ( ) // StartTestNATS starts an embedded NATS server (no JetStream) and returns -// the server and a connected client. Both are cleaned up via t.Cleanup. -func StartTestNATS(t *testing.T) (*server.Server, *nats.Conn) { +// the server and a connected client. Both are cleaned up via t.Cleanup. It +// takes a testing.TB so benchmarks can stand up the same transport tests do. +func StartTestNATS(t testing.TB) (*server.Server, *nats.Conn) { t.Helper() opts := &server.Options{ Host: "127.0.0.1", diff --git a/spinifex/testutil/tracing.go b/spinifex/testutil/tracing.go new file mode 100644 index 000000000..cddabafdb --- /dev/null +++ b/spinifex/testutil/tracing.go @@ -0,0 +1,59 @@ +package testutil + +import ( + "testing" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/propagation" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + "go.opentelemetry.io/otel/trace" +) + +// RecordSpans installs a recording tracer provider and the W3C trace-context +// propagator for the duration of the test, restoring both afterwards. The +// defaults are no-ops, so without this a span assertion sees nothing. +// +// Both are process globals, so a test using this cannot run in parallel. +func RecordSpans(t *testing.T) *tracetest.SpanRecorder { + t.Helper() + recorder := tracetest.NewSpanRecorder() + previousTracer := otel.GetTracerProvider() + previousPropagator := otel.GetTextMapPropagator() + otel.SetTracerProvider(sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder))) + otel.SetTextMapPropagator(propagation.TraceContext{}) + t.Cleanup(func() { + otel.SetTracerProvider(previousTracer) + otel.SetTextMapPropagator(previousPropagator) + }) + return recorder +} + +// SpanAttribute returns span's value for key, failing the test when absent. +func SpanAttribute(t *testing.T, span sdktrace.ReadOnlySpan, key string) attribute.Value { + t.Helper() + for _, attr := range span.Attributes() { + if string(attr.Key) == key { + return attr.Value + } + } + t.Fatalf("span %q has no attribute %q", span.Name(), key) + return attribute.Value{} +} + +// SpansByKind returns the last recorded span of each kind. Owner-first +// routing can produce more than one client span for a single call, and the +// leg that answered is the last one. +func SpansByKind(t *testing.T, recorder *tracetest.SpanRecorder) (client, server sdktrace.ReadOnlySpan) { + t.Helper() + for _, span := range recorder.Ended() { + switch span.SpanKind() { + case trace.SpanKindClient: + client = span + case trace.SpanKindServer: + server = span + } + } + return client, server +} diff --git a/spinifex/types/ebs.go b/spinifex/types/ebs.go index 30c14404e..842f23d8a 100644 --- a/spinifex/types/ebs.go +++ b/spinifex/types/ebs.go @@ -24,6 +24,22 @@ type EBSRequest struct { HotplugPort int `json:"HotplugPort,omitempty"` } +// VolumeTypeGP3 is the only EBS volume type this platform serves. +const VolumeTypeGP3 = "gp3" + +const ( + // gp3 IOPS envelope (AWS): 3000 baseline on any size, up to 500 IOPS/GiB, + // capped at 16000. + DefaultGP3IOPS = 3000 + MaxGP3IOPS = 16000 + GP3IOPSPerGiB = 500 + + // gp3 Throughput envelope (AWS): 125 MiB/s baseline, 1000 MiB/s ceiling, + // flat range independent of volume size. + DefaultGP3Throughput = 125 + MaxGP3Throughput = 1000 +) + // NBDTransport defines the transport type for NBD connections. type NBDTransport string diff --git a/spinifex/utils/nbd.go b/spinifex/utils/nbd.go index 859ecb112..6d26250e9 100644 --- a/spinifex/utils/nbd.go +++ b/spinifex/utils/nbd.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "net" + "net/url" "path/filepath" "strconv" "strings" @@ -56,9 +57,11 @@ func IsSocketURI(nbdURI string) bool { return strings.HasSuffix(nbdURI, ".sock") || strings.Contains(nbdURI, "unix:") } -// FormatNBDSocketURI formats a socket path as an NBD URI (nbd:unix:/path/to/socket.sock). +// FormatNBDSocketURI formats a socket path as an NBD URI +// (nbd+unix:///?socket=/path/to/socket.sock), the form libnbd, qemu and +// nbdkit all accept. QEMU's older nbd:unix: syntax is not a URI. func FormatNBDSocketURI(socketPath string) string { - return fmt.Sprintf("nbd:unix:%s", socketPath) + return fmt.Sprintf("nbd+unix:///?socket=%s", socketPath) } // FormatNBDTCPURI formats a host:port as an NBD TCP URI (nbd://host:port). @@ -98,14 +101,33 @@ func waitForTCPListener(addr string, timeout time.Duration) error { } } -// ParseNBDURI parses an NBD URI into components: nbd:unix:/path → ("unix", path); nbd://host:port → ("inet", host, port). -func ParseNBDURI(nbdURI string) (serverType, path, host string, port int, err error) { +// cutNBDUnixSocket extracts the socket path from either the nbd+unix:// URI +// form or QEMU's legacy nbd:unix: filename form, reporting whether the URI +// named a Unix socket at all. +func cutNBDUnixSocket(nbdURI string) (string, bool) { if after, ok := strings.CutPrefix(nbdURI, "nbd:unix:"); ok { - path = after - if path == "" { + return after, true + } + if !strings.HasPrefix(nbdURI, "nbd+unix:") { + return "", false + } + + parsed, err := url.Parse(nbdURI) + if err != nil { + return "", true + } + return parsed.Query().Get("socket"), true +} + +// ParseNBDURI parses an NBD URI into components: nbd+unix:///?socket=/path → ("unix", path); +// nbd://host:port → ("inet", host, port). The legacy nbd:unix:/path form is +// still accepted for state written by older daemons. +func ParseNBDURI(nbdURI string) (serverType, path, host string, port int, err error) { + if socket, ok := cutNBDUnixSocket(nbdURI); ok { + if socket == "" { return "", "", "", 0, fmt.Errorf("empty socket path in NBD URI: %s", nbdURI) } - return "unix", path, "", 0, nil + return "unix", socket, "", 0, nil } if after, ok := strings.CutPrefix(nbdURI, "nbd://"); ok { diff --git a/spinifex/utils/utils_test.go b/spinifex/utils/utils_test.go index ad836a7e0..fcd86c0b8 100644 --- a/spinifex/utils/utils_test.go +++ b/spinifex/utils/utils_test.go @@ -429,7 +429,19 @@ func TestParseNBDURI(t *testing.T) { wantErr bool }{ { - name: "Unix socket", + name: "Unix socket URI", + uri: "nbd+unix:///?socket=/run/user/1000/nbd-vol-123.sock", + wantType: "unix", + wantPath: "/run/user/1000/nbd-vol-123.sock", + }, + { + name: "Unix socket URI without the empty export path", + uri: "nbd+unix://?socket=/run/nbd-vol.sock", + wantType: "unix", + wantPath: "/run/nbd-vol.sock", + }, + { + name: "legacy QEMU nbd:unix: filename form", uri: "nbd:unix:/run/user/1000/nbd-vol-123.sock", wantType: "unix", wantPath: "/run/user/1000/nbd-vol-123.sock", @@ -453,6 +465,11 @@ func TestParseNBDURI(t *testing.T) { uri: "nbd:unix:", wantErr: true, }, + { + name: "Unix socket URI with no socket parameter", + uri: "nbd+unix:///", + wantErr: true, + }, { name: "Missing port in TCP", uri: "nbd://127.0.0.1", @@ -804,8 +821,18 @@ func TestIsSocketURI(t *testing.T) { } func TestFormatNBDSocketURI(t *testing.T) { - assert.Equal(t, "nbd:unix:/run/nbd-vol.sock", FormatNBDSocketURI("/run/nbd-vol.sock")) - assert.Equal(t, "nbd:unix:/tmp/test.sock", FormatNBDSocketURI("/tmp/test.sock")) + assert.Equal(t, "nbd+unix:///?socket=/run/nbd-vol.sock", FormatNBDSocketURI("/run/nbd-vol.sock")) + assert.Equal(t, "nbd+unix:///?socket=/tmp/test.sock", FormatNBDSocketURI("/tmp/test.sock")) +} + +// TestFormatNBDSocketURIRoundTrips guards the pairing that a third-party NBD +// client depends on: what we publish must be what we can parse back. +func TestFormatNBDSocketURIRoundTrips(t *testing.T) { + const path = "/run/spinifex/nbd/nbd-vol-abc123.sock" + serverType, parsed, _, _, err := ParseNBDURI(FormatNBDSocketURI(path)) + require.NoError(t, err) + assert.Equal(t, "unix", serverType) + assert.Equal(t, path, parsed) } func TestFormatNBDTCPURI(t *testing.T) { diff --git a/spinifex/vm/lifecycle.go b/spinifex/vm/lifecycle.go index 9db724acc..4b17be350 100644 --- a/spinifex/vm/lifecycle.go +++ b/spinifex/vm/lifecycle.go @@ -21,7 +21,6 @@ import ( "github.com/mulgadc/spinifex/spinifex/qmp" "github.com/mulgadc/spinifex/spinifex/types" "github.com/mulgadc/spinifex/spinifex/utils" - "github.com/mulgadc/viperblock/viperblock" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" @@ -381,7 +380,7 @@ func (m *Manager) startQEMU(instance *VM) error { m.appendDevHostfwdNIC(instance) } } else { - sshDebugAddr, err := viperblock.FindFreePort() + sshDebugAddr, err := findFreePort() if err != nil { slog.Error("Failed to find free port", "err", err) return err @@ -609,9 +608,21 @@ func (m *Manager) startQEMU(instance *VM) error { return nil } -// findFreePort is a var so tests can swap in a stub to reach appendDevHostfwdNIC's -// failure paths (viperblock.FindFreePort always succeeds in production). -var findFreePort = viperblock.FindFreePort +// findFreePort is a var so tests can swap in a stub to reach callers' failure +// paths without depending on the OS allocator. +var findFreePort = allocateFreePort + +// allocateFreePort asks the OS for an available loopback TCP port. The listener +// is closed before returning, so callers must still handle a subsequent bind +// race in the usual way. +func allocateFreePort() (string, error) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return "", err + } + defer listener.Close() + return listener.Addr().String(), nil +} // appendDevHostfwdNIC adds a user-mode NIC with SSH hostfwd for dev access. func (m *Manager) appendDevHostfwdNIC(instance *VM) { diff --git a/tests/bench/ebsctl/cmd/ebsctl-compare/main.go b/tests/bench/ebsctl/cmd/ebsctl-compare/main.go new file mode 100644 index 000000000..f77272b4d --- /dev/null +++ b/tests/bench/ebsctl/cmd/ebsctl-compare/main.go @@ -0,0 +1,52 @@ +// Command ebsctl-compare reads two ebsctl-bench result JSON files and emits +// a markdown comparison table (medians, p95s, deltas, sample counts, error +// rates, and a bootstrap-CI significance signal per operation). No cluster +// or build tag required — it only consumes the JSON schema, so it builds +// and runs anywhere `go build` does. +package main + +import ( + "flag" + "fmt" + "os" + "time" + + "github.com/mulgadc/spinifex/tests/bench/ebsctl" +) + +func usage() { + fmt.Fprintf(os.Stderr, "usage: %s [-out path] \n", os.Args[0]) + flag.PrintDefaults() +} + +func main() { + out := flag.String("out", "", "write markdown to this path instead of stdout") + flag.Parse() + + if flag.NArg() != 2 { + usage() + os.Exit(2) + } + + a, err := ebsctl.LoadResult(flag.Arg(0)) + if err != nil { + fmt.Fprintf(os.Stderr, "ebsctl-compare: %v\n", err) + os.Exit(1) + } + b, err := ebsctl.LoadResult(flag.Arg(1)) + if err != nil { + fmt.Fprintf(os.Stderr, "ebsctl-compare: %v\n", err) + os.Exit(1) + } + + md := ebsctl.CompareRuns(a, b, time.Now().UnixNano()) + + if *out == "" { + fmt.Print(md) + return + } + if err := os.WriteFile(*out, []byte(md), 0o644); err != nil { + fmt.Fprintf(os.Stderr, "ebsctl-compare: write %s: %v\n", *out, err) + os.Exit(1) + } +} diff --git a/tests/bench/ebsctl/collector.go b/tests/bench/ebsctl/collector.go new file mode 100644 index 000000000..14194cd32 --- /dev/null +++ b/tests/bench/ebsctl/collector.go @@ -0,0 +1,158 @@ +package ebsctl + +import ( + "errors" + "sync" + "time" + + "github.com/aws/aws-sdk-go/aws/awserr" +) + +// msFloat converts a duration to fractional milliseconds, the unit every +// SeriesStats field is reported in. +func msFloat(d time.Duration) float64 { + return float64(d) / float64(time.Millisecond) +} + +// errCodeMessage extracts an AWS error code/message pair from err. Non-AWS +// errors (e.g. a transport failure) get a synthetic "non-aws-error" code so +// they still show up in the tally instead of vanishing into "unknown". +func errCodeMessage(err error) (code, message string) { + if err == nil { + return "", "" + } + var aerr awserr.Error + if errors.As(err, &aerr) { + return aerr.Code(), aerr.Message() + } + return "non-aws-error", err.Error() +} + +// collector accumulates one operation's samples/errors across concurrent +// workers. Safe for concurrent use. +type collector struct { + mu sync.Mutex + api []float64 + settle []float64 + settleTimeouts int + tally *ErrorTally + attempts int +} + +func newCollector() *collector { + return &collector{tally: newErrorTally()} +} + +// record logs one attempt. Exactly one of (err != nil) or +// (api recorded, optionally settle/settleTimedOut) applies: +// - err != nil: the API call itself failed; settle/settleTimedOut are +// ignored since there is nothing to settle. +// - err == nil, settleTimedOut: the call succeeded but the resource never +// reached the target state within budget. +// - err == nil, settle != nil: the call succeeded and settle records how +// long the resource took to reach the target state. +// - err == nil, settle == nil, !settleTimedOut: the call succeeded and +// settle time isn't tracked for this operation (e.g. DescribeVolumes). +func (c *collector) record(api time.Duration, settle *time.Duration, settleTimedOut bool, err error) { + c.mu.Lock() + defer c.mu.Unlock() + c.attempts++ + if err != nil { + code, msg := errCodeMessage(err) + c.tally.add(code, msg) + return + } + c.api = append(c.api, msFloat(api)) + switch { + case settleTimedOut: + c.settleTimeouts++ + case settle != nil: + c.settle = append(c.settle, msFloat(*settle)) + } +} + +// result finalises the collected samples into an OpResult named name. +func (c *collector) result(name string) *OpResult { + c.mu.Lock() + defer c.mu.Unlock() + res := &OpResult{ + Operation: name, + Attempts: c.attempts, + APILatency: computeSeries(c.api), + SettleTimeouts: c.settleTimeouts, + Errors: *c.tally, + ErrorRate: c.tally.rate(c.attempts), + } + if len(c.settle) > 0 { + s := computeSeries(c.settle) + res.SettleTime = &s + } + return res +} + +// parallelFor runs fn across `concurrency` workers (>=1). Each worker first +// runs `warmup` discarded iterations, then `iterations` measured ones. Warm-up +// is per-worker, not globally barriered — for the default concurrency=1 this +// is exact; at higher concurrency it's an approximation (workers' warm-up +// windows can overlap in wall-clock time), which is an acceptable trade for +// not needing a synchronization barrier no one asked for. +func parallelFor(concurrency, warmup, iterations int, fn func(workerID, iter int, warm bool)) { + if concurrency < 1 { + concurrency = 1 + } + var wg sync.WaitGroup + for w := range concurrency { + wg.Go(func() { + for i := range warmup { + fn(w, i, true) + } + for i := range iterations { + fn(w, i, false) + } + }) + } + wg.Wait() +} + +// drainDelete concurrently drains ids across `concurrency` workers, calling +// del for each and recording the result into the returned collector. The +// first warmupCount completions (in dequeue order, which is not necessarily +// creation order once concurrency>1) are discarded — a work-queue analogue +// of parallelFor's per-worker warm-up, needed because a delete pass doesn't +// have a natural "iterations per worker" shape: it must delete exactly the +// set of ids a paired create pass produced, however many that turned out to +// be if some creates failed. +func drainDelete(ids []string, concurrency, warmupCount int, del func(id string) (api time.Duration, settle *time.Duration, settleTimedOut bool, err error)) *collector { + c := newCollector() + if concurrency < 1 { + concurrency = 1 + } + idCh := make(chan string, len(ids)) + for _, id := range ids { + idCh <- id + } + close(idCh) + + var mu sync.Mutex + warmLeft := warmupCount + var wg sync.WaitGroup + for range concurrency { + wg.Go(func() { + for id := range idCh { + mu.Lock() + warm := warmLeft > 0 + if warm { + warmLeft-- + } + mu.Unlock() + + api, settle, timedOut, err := del(id) + if !warm { + c.record(api, settle, timedOut, err) + } + } + }) + } + wg.Wait() + return c +} diff --git a/tests/bench/ebsctl/collector_test.go b/tests/bench/ebsctl/collector_test.go new file mode 100644 index 000000000..3e5cabe59 --- /dev/null +++ b/tests/bench/ebsctl/collector_test.go @@ -0,0 +1,175 @@ +package ebsctl + +import ( + "errors" + "fmt" + "sync" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws/awserr" +) + +func TestCollectorRecordSuccessWithSettle(t *testing.T) { + c := newCollector() + settle := 5 * time.Millisecond + c.record(10*time.Millisecond, &settle, false, nil) + c.record(20*time.Millisecond, &settle, false, nil) + + res := c.result("Op") + if res.Attempts != 2 { + t.Errorf("Attempts = %d, want 2", res.Attempts) + } + if res.APILatency.Count != 2 { + t.Errorf("APILatency.Count = %d, want 2", res.APILatency.Count) + } + if res.SettleTime == nil || res.SettleTime.Count != 2 { + t.Errorf("SettleTime = %+v, want Count=2", res.SettleTime) + } + if res.Errors.Total != 0 { + t.Errorf("Errors.Total = %d, want 0", res.Errors.Total) + } +} + +func TestCollectorRecordSettleTimeout(t *testing.T) { + c := newCollector() + c.record(10*time.Millisecond, nil, true, nil) + + res := c.result("Op") + if res.SettleTimeouts != 1 { + t.Errorf("SettleTimeouts = %d, want 1", res.SettleTimeouts) + } + if res.SettleTime != nil { + t.Errorf("SettleTime = %+v, want nil (no settle samples recorded)", res.SettleTime) + } + if res.APILatency.Count != 1 { + t.Errorf("APILatency.Count = %d, want 1 (the API call itself still succeeded)", res.APILatency.Count) + } +} + +func TestCollectorRecordNoSettleTracked(t *testing.T) { + c := newCollector() + c.record(10*time.Millisecond, nil, false, nil) + + res := c.result("Op") + if res.SettleTime != nil { + t.Errorf("SettleTime = %+v, want nil", res.SettleTime) + } + if res.SettleTimeouts != 0 { + t.Errorf("SettleTimeouts = %d, want 0", res.SettleTimeouts) + } +} + +type fakeAWSError struct{ code, msg string } + +func (e fakeAWSError) Error() string { return e.code + ": " + e.msg } +func (e fakeAWSError) Code() string { return e.code } +func (e fakeAWSError) Message() string { return e.msg } +func (e fakeAWSError) OrigErr() error { return nil } + +var _ awserr.Error = fakeAWSError{} + +func TestCollectorRecordErrorTally(t *testing.T) { + c := newCollector() + c.record(0, nil, false, fakeAWSError{code: "VolumeInUse", msg: "boom"}) + c.record(0, nil, false, fakeAWSError{code: "VolumeInUse", msg: "boom"}) + c.record(0, nil, false, errors.New("plain error")) + + res := c.result("Op") + if res.Errors.Total != 3 { + t.Errorf("Errors.Total = %d, want 3", res.Errors.Total) + } + if res.Errors.ByCode["VolumeInUse"] != 2 { + t.Errorf("ByCode[VolumeInUse] = %d, want 2", res.Errors.ByCode["VolumeInUse"]) + } + if res.Errors.ByCode["non-aws-error"] != 1 { + t.Errorf("ByCode[non-aws-error] = %d, want 1", res.Errors.ByCode["non-aws-error"]) + } + if res.Attempts != 3 { + t.Errorf("Attempts = %d, want 3", res.Attempts) + } + if res.APILatency.Count != 0 { + t.Errorf("APILatency.Count = %d, want 0 (all attempts errored)", res.APILatency.Count) + } + wantRate := 1.0 + if res.ErrorRate != wantRate { + t.Errorf("ErrorRate = %v, want %v", res.ErrorRate, wantRate) + } +} + +func TestParallelForCounts(t *testing.T) { + var mu sync.Mutex + warmCount, measuredCount := 0, 0 + parallelFor(4, 3, 5, func(_, _ int, warm bool) { + mu.Lock() + defer mu.Unlock() + if warm { + warmCount++ + } else { + measuredCount++ + } + }) + if warmCount != 4*3 { + t.Errorf("warmCount = %d, want %d", warmCount, 4*3) + } + if measuredCount != 4*5 { + t.Errorf("measuredCount = %d, want %d", measuredCount, 4*5) + } +} + +func TestParallelForDefaultsConcurrencyToOne(t *testing.T) { + var count int + var mu sync.Mutex + parallelFor(0, 0, 3, func(_, _ int, _ bool) { + mu.Lock() + count++ + mu.Unlock() + }) + if count != 3 { + t.Errorf("count = %d, want 3 (concurrency<1 should default to 1)", count) + } +} + +func TestDrainDeleteDistributesAllIDs(t *testing.T) { + ids := make([]string, 10) + for i := range ids { + ids[i] = fmt.Sprintf("id-%d", i) + } + var mu sync.Mutex + seen := map[string]int{} + col := drainDelete(ids, 3, 2, func(id string) (time.Duration, *time.Duration, bool, error) { + mu.Lock() + seen[id]++ + mu.Unlock() + return time.Millisecond, nil, false, nil + }) + if len(seen) != 10 { + t.Errorf("saw %d distinct ids, want 10", len(seen)) + } + for id, n := range seen { + if n != 1 { + t.Errorf("id %s processed %d times, want 1", id, n) + } + } + res := col.result("DeleteX") + if res.Attempts != 8 { + t.Errorf("Attempts = %d, want 8 (10 ids - 2 discarded warm-up)", res.Attempts) + } +} + +func TestDrainDeletePropagatesErrors(t *testing.T) { + ids := []string{"a", "b", "c"} + col := drainDelete(ids, 1, 0, func(id string) (time.Duration, *time.Duration, bool, error) { + if id == "b" { + return 0, nil, false, fakeAWSError{code: "NotFound", msg: "gone"} + } + return time.Millisecond, nil, false, nil + }) + res := col.result("DeleteX") + if res.Errors.Total != 1 { + t.Errorf("Errors.Total = %d, want 1", res.Errors.Total) + } + if res.APILatency.Count != 2 { + t.Errorf("APILatency.Count = %d, want 2", res.APILatency.Count) + } +} diff --git a/tests/bench/ebsctl/compare.go b/tests/bench/ebsctl/compare.go new file mode 100644 index 000000000..0bbbc3841 --- /dev/null +++ b/tests/bench/ebsctl/compare.go @@ -0,0 +1,203 @@ +package ebsctl + +import ( + "encoding/json" + "fmt" + "math/rand/v2" + "os" + "sort" + "strconv" + "strings" + "time" +) + +// LoadResult reads and parses a RunResult JSON file written by the +// benchmark. +func LoadResult(path string) (*RunResult, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read %s: %w", path, err) + } + var r RunResult + if err := json.Unmarshal(data, &r); err != nil { + return nil, fmt.Errorf("parse %s: %w", path, err) + } + return &r, nil +} + +// bootstrapIters is the resample count for the percentile-bootstrap CI on +// the median difference. 2000 is a conventional floor for a stable 95% CI +// without being slow even for the largest sample vectors this tool sees. +const bootstrapIters = 2000 + +// resample draws len(s) values from s with replacement. +func resample(s []float64, rng *rand.Rand) []float64 { + out := make([]float64, len(s)) + for i := range out { + out[i] = s[rng.IntN(len(s))] + } + return out +} + +// bootstrapMedianDiffCI returns a 95% percentile-bootstrap confidence +// interval for median(b) - median(a). Returns ok=false when either sample is +// empty (no meaningful CI to compute). +func bootstrapMedianDiffCI(a, b []float64, rng *rand.Rand) (lo, hi float64, ok bool) { + if len(a) == 0 || len(b) == 0 { + return 0, 0, false + } + diffs := make([]float64, bootstrapIters) + for i := range bootstrapIters { + aSorted := resample(a, rng) + bSorted := resample(b, rng) + sort.Float64s(aSorted) + sort.Float64s(bSorted) + diffs[i] = median(bSorted) - median(aSorted) + } + sort.Float64s(diffs) + loIdx := int(0.025 * float64(bootstrapIters)) + hiIdx := int(0.975 * float64(bootstrapIters)) + if hiIdx >= bootstrapIters { + hiIdx = bootstrapIters - 1 + } + return diffs[loIdx], diffs[hiIdx], true +} + +// unionOps returns the sorted union of operation names present in a and/or +// b, so the comparison table covers every operation either run measured +// (e.g. one run with attach/detach included, one without). +func unionOps(a, b *RunResult) []string { + seen := map[string]struct{}{} + for name := range a.Operations { + seen[name] = struct{}{} + } + for name := range b.Operations { + seen[name] = struct{}{} + } + out := make([]string, 0, len(seen)) + for name := range seen { + out = append(out, name) + } + sort.Strings(out) + return out +} + +// seriesFor picks the api or settle series from op, or nil if op is nil or +// the requested series doesn't exist (settle time isn't tracked for every +// operation, e.g. DescribeVolumes). +func seriesFor(op *OpResult, metric string) *SeriesStats { + if op == nil { + return nil + } + if metric == "settle" { + return op.SettleTime + } + return &op.APILatency +} + +// CompareRuns renders a markdown comparison of a vs b: one table row per +// (operation, metric) pair present in either run, with medians, p95s, +// absolute/percentage delta, sample counts, error rates, and a bootstrap CI +// on the median difference as a crude, honest significance signal. seed +// makes the bootstrap resampling reproducible for tests; pass +// time.Now().UnixNano() for real use. +func CompareRuns(a, b *RunResult, seed int64) string { + rng := rand.New(rand.NewPCG(0, uint64(seed))) //nolint:gosec // deterministic bootstrap resampling, not security-sensitive + var sb strings.Builder + + fmt.Fprintf(&sb, "# ebsctl-bench comparison\n\n") + fmt.Fprintf(&sb, "- **A**: provider=%s source=%s iterations/worker=%d concurrency=%d timestamp=%s sha=%s\n", + a.Meta.Provider, a.Meta.ProviderSource, a.Meta.IterationsPerWorker, a.Meta.Concurrency, + a.Meta.Timestamp.Format(time.RFC3339), a.Meta.GitSHA) + fmt.Fprintf(&sb, "- **B**: provider=%s source=%s iterations/worker=%d concurrency=%d timestamp=%s sha=%s\n\n", + b.Meta.Provider, b.Meta.ProviderSource, b.Meta.IterationsPerWorker, b.Meta.Concurrency, + b.Meta.Timestamp.Format(time.RFC3339), b.Meta.GitSHA) + + fmt.Fprintf(&sb, "Deltas are indicative only unless the 95%% bootstrap CI on the median "+ + "difference excludes zero (Signal column). With modest sample counts in a shared, "+ + "noisy environment, a small median difference is not necessarily real — treat any row "+ + "whose CI straddles zero as noise until re-run with more samples.\n\n") + + fmt.Fprintf(&sb, "| Operation | Metric | A median (ms) | B median (ms) | Δ median (ms) | Δ%% | A p95 (ms) | B p95 (ms) | A n | B n | A err%% | B err%% | 95%% CI on Δ median (ms) | Signal |\n") + fmt.Fprintf(&sb, "|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---|---|\n") + + for _, name := range unionOps(a, b) { + opA, okA := a.Operations[name] + opB, okB := b.Operations[name] + writeRow(&sb, rng, name, "api", opA, opB, okA, okB) + if (okA && opA.SettleTime != nil) || (okB && opB.SettleTime != nil) { + writeRow(&sb, rng, name, "settle", opA, opB, okA, okB) + } + } + + return sb.String() +} + +func writeRow(sb *strings.Builder, rng *rand.Rand, op, metric string, opA, opB *OpResult, okA, okB bool) { + sa := seriesFor(opA, metric) + sb2 := seriesFor(opB, metric) + + if sa == nil || sb2 == nil || sa.Count == 0 || sb2.Count == 0 { + fmt.Fprintf(sb, "| %s | %s | %s | %s | n/a | n/a | %s | %s | %s | %s | %s | %s | n/a | missing on one side |\n", + op, metric, + naf(sa, "median"), naf(sb2, "median"), + naf(sa, "p95"), naf(sb2, "p95"), + nai(sa), nai(sb2), + errRate(opA, okA), errRate(opB, okB)) + return + } + + delta := sb2.Median - sa.Median + pct := 0.0 + if sa.Median != 0 { + pct = delta / sa.Median * 100 + } + + loCI, hiCI, ok := bootstrapMedianDiffCI(sa.Samples, sb2.Samples, rng) + ciStr := "n/a" + signal := "insufficient data" + if ok { + ciStr = fmt.Sprintf("[%.2f, %.2f]", loCI, hiCI) + if loCI > 0 || hiCI < 0 { + signal = "likely real (CI excludes 0)" + } else { + signal = "indicative only (CI includes 0)" + } + } + + p95Note := "" + if !sa.P95Reliable || !sb2.P95Reliable { + p95Note = "*" + } + + fmt.Fprintf(sb, "| %s | %s | %.2f | %.2f | %+.2f | %+.1f%% | %.2f%s | %.2f%s | %d | %d | %s | %s | %s | %s |\n", + op, metric, sa.Median, sb2.Median, delta, pct, + sa.P95, p95Note, sb2.P95, p95Note, + sa.Count, sb2.Count, + errRate(opA, okA), errRate(opB, okB), + ciStr, signal) +} + +func naf(s *SeriesStats, field string) string { + if s == nil || s.Count == 0 { + return "n/a" + } + if field == "p95" { + return fmt.Sprintf("%.2f", s.P95) + } + return fmt.Sprintf("%.2f", s.Median) +} + +func nai(s *SeriesStats) string { + if s == nil { + return "0" + } + return strconv.Itoa(s.Count) +} + +func errRate(op *OpResult, present bool) string { + if !present || op == nil { + return "n/a" + } + return fmt.Sprintf("%.1f%%", op.ErrorRate*100) +} diff --git a/tests/bench/ebsctl/compare_test.go b/tests/bench/ebsctl/compare_test.go new file mode 100644 index 000000000..504bd5671 --- /dev/null +++ b/tests/bench/ebsctl/compare_test.go @@ -0,0 +1,101 @@ +package ebsctl + +import ( + "math/rand/v2" + "strings" + "testing" + "time" +) + +func mkRun(provider string, samples map[string][]float64) *RunResult { + ops := map[string]*OpResult{} + for name, s := range samples { + series := computeSeries(s) + ops[name] = &OpResult{ + Operation: name, + Attempts: len(s), + APILatency: series, + Errors: ErrorTally{ByCode: map[string]int{}, ByMessage: map[string]int{}}, + } + } + return &RunResult{ + Meta: RunMeta{ + Timestamp: time.Now(), + Provider: provider, + ProviderSource: "flag", + IterationsPerWorker: len(samples), + Concurrency: 1, + }, + Operations: ops, + } +} + +func TestBootstrapMedianDiffCIIdenticalSamples(t *testing.T) { + rng := rand.New(rand.NewPCG(0, 1)) + s := []float64{10, 11, 9, 10, 12, 8, 10, 11, 9, 10} + lo, hi, ok := bootstrapMedianDiffCI(s, s, rng) + if !ok { + t.Fatal("bootstrapMedianDiffCI: ok = false, want true") + } + // Identical samples: the diff distribution should be tightly centred on 0. + if lo > 1 || hi < -1 { + t.Errorf("CI [%v, %v] for identical samples looks implausibly wide", lo, hi) + } +} + +func TestBootstrapMedianDiffCIClearShift(t *testing.T) { + rng := rand.New(rand.NewPCG(0, 1)) + a := []float64{10, 11, 9, 10, 12, 8, 10, 11, 9, 10} + b := make([]float64, len(a)) + for i, v := range a { + b[i] = v + 100 // unmistakable shift + } + lo, hi, ok := bootstrapMedianDiffCI(a, b, rng) + if !ok { + t.Fatal("bootstrapMedianDiffCI: ok = false, want true") + } + if lo <= 0 { + t.Errorf("CI [%v, %v] should exclude 0 for a 100ms shift", lo, hi) + } +} + +func TestBootstrapMedianDiffCIEmptySamples(t *testing.T) { + rng := rand.New(rand.NewPCG(0, 1)) + if _, _, ok := bootstrapMedianDiffCI(nil, []float64{1, 2}, rng); ok { + t.Error("ok = true for empty sample, want false") + } +} + +func TestCompareRunsRendersTableAndHandlesMissingOps(t *testing.T) { + a := mkRun("embedded", map[string][]float64{ + "CreateVolume": {10, 12, 11, 9, 13, 10, 11, 12, 9, 10}, + }) + b := mkRun("viperblockd", map[string][]float64{ + "CreateVolume": {20, 22, 21, 19, 23, 20, 21, 22, 19, 20}, + "AttachVolume": {5, 6, 5, 7, 5}, // present only in b + }) + + md := CompareRuns(a, b, 1) + + if !strings.Contains(md, "CreateVolume") { + t.Error("comparison missing CreateVolume row") + } + if !strings.Contains(md, "AttachVolume") { + t.Error("comparison missing AttachVolume row for the op only present in B") + } + if !strings.Contains(md, "missing on one side") { + t.Error("comparison should flag AttachVolume as missing on one side (A has no data)") + } + if !strings.Contains(md, "embedded") || !strings.Contains(md, "viperblockd") { + t.Error("comparison header should name both providers") + } +} + +func TestCompareRunsNoP99Mentioned(t *testing.T) { + a := mkRun("embedded", map[string][]float64{"CreateVolume": {1, 2, 3}}) + b := mkRun("viperblockd", map[string][]float64{"CreateVolume": {4, 5, 6}}) + md := CompareRuns(a, b, 1) + if strings.Contains(strings.ToLower(md), "p99") { + t.Error("comparison output must never mention p99 (see schema.go: not statistically supported at these sample sizes)") + } +} diff --git a/tests/bench/ebsctl/flags.go b/tests/bench/ebsctl/flags.go new file mode 100644 index 000000000..e257f9c5f --- /dev/null +++ b/tests/bench/ebsctl/flags.go @@ -0,0 +1,28 @@ +//go:build e2e && bench + +package ebsctl + +import "flag" + +// buildGitSHA is overridable at build time via +// -ldflags "-X 'github.com/mulgadc/spinifex/tests/bench/ebsctl.buildGitSHA='". +// gitSHA() (main_test.go) prefers a runtime `git rev-parse HEAD` (works when +// run from a repo checkout, the common case) and falls back to this when +// that fails. +var buildGitSHA = "unknown" + +// Flags are declared in a regular (non-_test.go) file, not main_test.go, +// so that provider.go and other non-test files in this package can reference +// them directly: a non-test file may not reference a symbol declared only in +// a _test.go file (Go excludes _test.go from a plain `go build`/`go vet` of +// the package, only including it when building the test binary via +// `go test`). +var ( + flagOut = flag.String("out", "", "output JSON path (default: $ARTIFACT_DIR/ebsctl-bench-.json)") + flagIterations = flag.Int("iterations", 50, "measured iterations per operation, per worker") + flagWarmup = flag.Int("warmup", 5, "discarded warm-up iterations per operation, per worker") + flagConcurrency = flag.Int("concurrency", 1, "concurrent workers per operation") + flagProvider = flag.String("provider", "", "expected/fallback [ebs] provider (viperblockd); cross-checked against cluster detection when detection succeeds, required when it doesn't") + flagAttachDetach = flag.Bool("attach-detach", true, "include the attach/detach phase (boots and terminates one instance)") + flagVolumeGiB = flag.Int64("volume-size-gib", 1, "size in GiB of every volume this benchmark creates") +) diff --git a/tests/bench/ebsctl/main_test.go b/tests/bench/ebsctl/main_test.go new file mode 100644 index 000000000..28d46cadc --- /dev/null +++ b/tests/bench/ebsctl/main_test.go @@ -0,0 +1,179 @@ +//go:build e2e && bench + +package ebsctl + +import ( + "encoding/json" + "flag" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mulgadc/spinifex/tests/e2e/harness" +) + +func TestMain(m *testing.M) { + flag.Parse() + os.Exit(m.Run()) +} + +// TestEBSControlPlaneLatency is the benchmark's single entry point, compiled +// to a standalone binary via `go test -tags "e2e bench" -c`. It measures +// per-operation API latency and settle time for CreateVolume, DescribeVolumes +// (single + list), CreateSnapshot, DeleteSnapshot, DeleteVolume, and +// (unless -attach-detach=false) AttachVolume/DetachVolume, then writes a +// RunResult JSON document. +func TestEBSControlPlaneLatency(t *testing.T) { + env := harness.LoadEnv(t) // skips the test if SPINIFEX_E2E is unset + if *flagIterations < 1 { + t.Fatalf("-iterations must be >= 1, got %d", *flagIterations) + } + if *flagWarmup < 0 { + t.Fatalf("-warmup must be >= 0, got %d", *flagWarmup) + } + if *flagConcurrency < 1 { + t.Fatalf("-concurrency must be >= 1, got %d", *flagConcurrency) + } + + awsCli := harness.NewAWSClient(t, env) + fix := harness.NewFixture(t, awsCli) + + provider, source := DetectEBSProvider(t, env) + t.Logf("ebs provider in effect: %s (source=%s)", provider, source) + + az := harness.DiscoverDefaultAZ(t, fix) + runID := fmt.Sprintf("%d-%d", time.Now().Unix(), os.Getpid()) + tags := []*ec2.Tag{ + {Key: aws.String("Name"), Value: aws.String("ebsctl-bench")}, + {Key: aws.String("ebsctl:run"), Value: aws.String(runID)}, + } + + result := &RunResult{ + Meta: RunMeta{ + Timestamp: time.Now().UTC(), + GitSHA: gitSHA(), + Provider: provider, + ProviderSource: source, + NodeCount: len(env.NodeIPs), + IterationsPerWorker: *flagIterations, + WarmupPerWorker: *flagWarmup, + Concurrency: *flagConcurrency, + AttachDetachIncluded: *flagAttachDetach, + VolumeSizeGiB: *flagVolumeGiB, + }, + Operations: map[string]*OpResult{}, + } + + t.Log("phase: CreateVolume / DeleteVolume") + createRes, volIDs := runCreateVolume(t, awsCli, tags, az, *flagVolumeGiB, *flagWarmup, *flagIterations, *flagConcurrency) + result.Operations[createRes.Operation] = createRes + if len(volIDs) > 0 { + deleteRes := runDeleteVolume(awsCli, volIDs, *flagWarmup, *flagConcurrency) + result.Operations[deleteRes.Operation] = deleteRes + } else { + t.Log("CreateVolume produced no volumes to delete; skipping DeleteVolume phase") + } + + t.Log("phase: DescribeVolumes (single + list)") + describePoolSize := min(*flagIterations, 5) + describePool := createDescribePool(t, awsCli, tags, az, *flagVolumeGiB, describePoolSize) + if len(describePool) > 0 { + single, list := runDescribeVolumes(awsCli, describePool, *flagWarmup, *flagIterations, *flagConcurrency) + result.Operations[single.Operation] = single + result.Operations[list.Operation] = list + } else { + t.Log("no describe-pool volumes available; skipping DescribeVolumes phase") + } + + t.Log("phase: CreateSnapshot / DeleteSnapshot") + snapCreate, snapDelete, snapErr := runSnapshotPhase(t, awsCli, tags, az, *flagVolumeGiB, *flagWarmup, *flagIterations, *flagConcurrency) + if snapErr != nil { + t.Errorf("snapshot phase: %v", snapErr) + } else { + result.Operations[snapCreate.Operation] = snapCreate + result.Operations[snapDelete.Operation] = snapDelete + } + + if *flagAttachDetach { + t.Log("phase: AttachVolume / DetachVolume") + attachRes, detachRes, err := runAttachDetachPhase(t, fix, awsCli, tags, az, *flagVolumeGiB, *flagWarmup, *flagIterations, *flagConcurrency) + if err != nil { + t.Errorf("attach/detach phase: %v", err) + } else { + result.Operations[attachRes.Operation] = attachRes + result.Operations[detachRes.Operation] = detachRes + } + } else { + t.Log("attach/detach phase skipped (-attach-detach=false)") + } + + writeResult(t, env, result) +} + +// createDescribePool creates n durable volumes for the DescribeVolumes +// phase, tolerant of partial failure (returns however many succeeded). +func createDescribePool(t *testing.T, c *harness.AWSClient, tags []*ec2.Tag, az string, sizeGiB int64, n int) []string { + t.Helper() + ids := make([]string, 0, n) + for i := range n { + out, err := c.EC2.CreateVolume(&ec2.CreateVolumeInput{ + AvailabilityZone: aws.String(az), + Size: aws.Int64(sizeGiB), + }) + if err != nil { + t.Logf("describe-pool volume %d/%d: %v", i+1, n, err) + continue + } + id := aws.StringValue(out.VolumeId) + harness.RegisterVolumeTeardown(t, c, id) + tagResource(c, id, tags) + if _, timedOut := pollVolumeState(c, id, ec2.VolumeStateAvailable, volumeSettleTimeout); timedOut { + t.Logf("describe-pool volume %s never became available", id) + continue + } + ids = append(ids, id) + } + return ids +} + +// gitSHA prefers the runtime repo's HEAD (accurate when run from a checkout, +// the expected common case) and falls back to buildGitSHA. +func gitSHA() string { + if out, err := exec.Command("git", "rev-parse", "HEAD").Output(); err == nil { + if sha := strings.TrimSpace(string(out)); sha != "" { + return sha + } + } + return buildGitSHA +} + +// writeResult marshals result to JSON and writes it to -out, or +// $ARTIFACT_DIR/ebsctl-bench-.json when -out is unset. +func writeResult(t *testing.T, env *harness.Env, result *RunResult) { + t.Helper() + path := *flagOut + if path == "" { + dir := env.ArtifactDir + if dir == "" { + dir = "." + } + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("mkdir %s: %v", dir, err) + } + path = filepath.Join(dir, fmt.Sprintf("ebsctl-bench-%s.json", result.Meta.Timestamp.Format("20060102T150405Z"))) + } + data, err := json.MarshalIndent(result, "", " ") + if err != nil { + t.Fatalf("marshal result: %v", err) + } + if err := os.WriteFile(path, data, 0o644); err != nil { + t.Fatalf("write result %s: %v", path, err) + } + t.Logf("wrote benchmark result to %s", path) +} diff --git a/tests/bench/ebsctl/ops_attach.go b/tests/bench/ebsctl/ops_attach.go new file mode 100644 index 000000000..45c693985 --- /dev/null +++ b/tests/bench/ebsctl/ops_attach.go @@ -0,0 +1,132 @@ +//go:build e2e && bench + +package ebsctl + +import ( + "fmt" + "sync/atomic" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mulgadc/spinifex/tests/e2e/harness" +) + +// attachDeviceLetters gives each worker its own guest device letter so +// concurrent attaches to the shared boot instance don't collide. Indexing a +// string (not a byte(int) conversion) sidesteps any overflow-conversion +// concern for the workerID -> device-letter mapping. +const attachDeviceLetters = "fghijklmnopqrstuvwxyz" + +// maxAttachDetachConcurrency bounds concurrency for the attach/detach phase: +// one distinct guest device letter is available per worker. +const maxAttachDetachConcurrency = len(attachDeviceLetters) + +// runAttachDetachPhase boots one nano instance, pre-creates +// concurrency*(warmup+iterations) volumes (so CreateVolume latency doesn't +// pollute the attach/detach measurement), then times AttachVolume/DetachVolume +// against them. This is the newest server code (PublishVolume/UnpublishVolume) +// and the part of the provider boundary most likely to regress, so it is +// included by default — see -attach-detach to skip it. +func runAttachDetachPhase(t *testing.T, fix *harness.Fixture, c *harness.AWSClient, tags []*ec2.Tag, az string, sizeGiB int64, warmup, iterations, concurrency int) (attachRes, detachRes *OpResult, err error) { + t.Helper() + if concurrency > maxAttachDetachConcurrency { + return nil, nil, fmt.Errorf("attach/detach phase: concurrency=%d exceeds the %d distinct guest "+ + "device letters (/dev/sdf../dev/sdz) this phase assigns one-per-worker", concurrency, maxAttachDetachConcurrency) + } + + instanceType, arch := harness.DiscoverNanoInstanceType(t, fix) + ami := harness.DiscoverUbuntuAMI(t, fix, arch) + vpc := harness.EnsureDefaultVPC(t, fix) + keyName, _ := harness.EnsureKeyPair(t, fix, t.TempDir()) + + instanceID := harness.EnsureInstance(t, fix, harness.InstanceSpec{ + AMIID: ami, + InstanceType: instanceType, + KeyName: keyName, + SubnetID: vpc.SubnetID, + SGID: vpc.SGID, + }) + // EnsureInstance already registers terminate-on-cleanup. + harness.WaitForInstanceState(t, c, instanceID, ec2.InstanceStateNameRunning) + + total := concurrency * (warmup + iterations) + volIDs := make([]string, 0, total) + for i := range total { + out, cerr := c.EC2.CreateVolume(&ec2.CreateVolumeInput{ + AvailabilityZone: aws.String(az), + Size: aws.Int64(sizeGiB), + }) + if cerr != nil { + return nil, nil, fmt.Errorf("attach/detach phase: pre-create volume %d/%d: %w", i+1, total, cerr) + } + id := aws.StringValue(out.VolumeId) + harness.RegisterVolumeTeardown(t, c, id) + tagResource(c, id, tags) + if _, timedOut := pollVolumeState(c, id, ec2.VolumeStateAvailable, volumeSettleTimeout); timedOut { + return nil, nil, fmt.Errorf("attach/detach phase: volume %s never became available", id) + } + volIDs = append(volIDs, id) + } + + colAttach := newCollector() + colDetach := newCollector() + var next atomic.Int64 + + parallelFor(concurrency, warmup, iterations, func(workerID, _ int, warm bool) { + i := int(next.Add(1) - 1) + if i >= len(volIDs) { + return // pre-create failed to reach `total`; nothing left to attach. + } + volID := volIDs[i] + device := "/dev/sd" + string(attachDeviceLetters[workerID]) + + start := time.Now() + _, err := c.EC2.AttachVolume(&ec2.AttachVolumeInput{ + VolumeId: aws.String(volID), + InstanceId: aws.String(instanceID), + Device: aws.String(device), + }) + api := time.Since(start) + if err != nil { + if !warm { + colAttach.record(api, nil, false, err) + } + return + } + settle, timedOut := pollVolumeState(c, volID, ec2.VolumeStateInUse, attachSettleTimeout) + if !warm { + var s *time.Duration + if !timedOut { + s = &settle + } + colAttach.record(api, s, timedOut, nil) + } + // A volume stuck attached (settle timeout) can't usefully be + // detach-timed; the DeleteVolume teardown still force-detaches it. + if timedOut { + return + } + + dstart := time.Now() + _, derr := c.EC2.DetachVolume(&ec2.DetachVolumeInput{VolumeId: aws.String(volID)}) + dapi := time.Since(dstart) + if derr != nil { + if !warm { + colDetach.record(dapi, nil, false, derr) + } + return + } + dsettle, dtimedOut := pollVolumeState(c, volID, ec2.VolumeStateAvailable, attachSettleTimeout) + if !warm { + var ds *time.Duration + if !dtimedOut { + ds = &dsettle + } + colDetach.record(dapi, ds, dtimedOut, nil) + } + }) + + return colAttach.result("AttachVolume"), colDetach.result("DetachVolume"), nil +} diff --git a/tests/bench/ebsctl/ops_snapshot.go b/tests/bench/ebsctl/ops_snapshot.go new file mode 100644 index 000000000..a1832e8e6 --- /dev/null +++ b/tests/bench/ebsctl/ops_snapshot.go @@ -0,0 +1,97 @@ +//go:build e2e && bench + +package ebsctl + +import ( + "fmt" + "sync" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mulgadc/spinifex/tests/e2e/harness" +) + +// registerSnapshotTeardown deletes snapID at test cleanup. Mirrors +// harness.RegisterVolumeTeardown's already-gone-is-fine contract; harness has +// no snapshot equivalent, so this is the local one. +func registerSnapshotTeardown(t *testing.T, c *harness.AWSClient, snapID string) { + t.Helper() + t.Cleanup(func() { + _, err := c.EC2.DeleteSnapshot(&ec2.DeleteSnapshotInput{SnapshotId: aws.String(snapID)}) + if err != nil && !harness.ErrorCodeIs(err, "InvalidSnapshot.NotFound") { + t.Errorf("cleanup: LEAKED snapshot %s: %v", snapID, err) + } + }) +} + +// runSnapshotPhase creates one source volume, then times CreateSnapshot and +// DeleteSnapshot against it warmup+iterations times per worker. The source +// volume and every created snapshot get a t.Cleanup teardown registered +// immediately on creation, so a failure partway through still reclaims them. +func runSnapshotPhase(t *testing.T, c *harness.AWSClient, tags []*ec2.Tag, az string, sizeGiB int64, warmup, iterations, concurrency int) (createRes, deleteRes *OpResult, err error) { + t.Helper() + + out, cerr := c.EC2.CreateVolume(&ec2.CreateVolumeInput{ + AvailabilityZone: aws.String(az), + Size: aws.Int64(sizeGiB), + }) + if cerr != nil { + return nil, nil, fmt.Errorf("snapshot phase: create source volume: %w", cerr) + } + volID := aws.StringValue(out.VolumeId) + harness.RegisterVolumeTeardown(t, c, volID) + tagResource(c, volID, tags) + + if _, timedOut := pollVolumeState(c, volID, ec2.VolumeStateAvailable, volumeSettleTimeout); timedOut { + return nil, nil, fmt.Errorf("snapshot phase: source volume %s never became available", volID) + } + + colCreate := newCollector() + var mu sync.Mutex + var snapIDs []string + parallelFor(concurrency, warmup, iterations, func(_, _ int, warm bool) { + start := time.Now() + out, err := c.EC2.CreateSnapshot(&ec2.CreateSnapshotInput{VolumeId: aws.String(volID)}) + api := time.Since(start) + if err != nil { + if !warm { + colCreate.record(api, nil, false, err) + } + return + } + + snapID := aws.StringValue(out.SnapshotId) + registerSnapshotTeardown(t, c, snapID) + tagResource(c, snapID, tags) + mu.Lock() + snapIDs = append(snapIDs, snapID) + mu.Unlock() + + settle, timedOut := pollSnapshotState(c, snapID, ec2.SnapshotStateCompleted, snapshotSettleTimeout) + if !warm { + var s *time.Duration + if !timedOut { + s = &settle + } + colCreate.record(api, s, timedOut, nil) + } + }) + + colDelete := drainDelete(snapIDs, concurrency, warmup*concurrency, func(id string) (time.Duration, *time.Duration, bool, error) { + start := time.Now() + _, err := c.EC2.DeleteSnapshot(&ec2.DeleteSnapshotInput{SnapshotId: aws.String(id)}) + api := time.Since(start) + if err != nil { + return api, nil, false, err + } + settle, timedOut := pollSnapshotGone(c, id, snapshotSettleTimeout) + if timedOut { + return api, nil, true, nil + } + return api, &settle, false, nil + }) + + return colCreate.result("CreateSnapshot"), colDelete.result("DeleteSnapshot"), nil +} diff --git a/tests/bench/ebsctl/ops_volume.go b/tests/bench/ebsctl/ops_volume.go new file mode 100644 index 000000000..5ed8f312b --- /dev/null +++ b/tests/bench/ebsctl/ops_volume.go @@ -0,0 +1,117 @@ +//go:build e2e && bench + +package ebsctl + +import ( + "sync" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mulgadc/spinifex/tests/e2e/harness" +) + +// runCreateVolume creates concurrency*(warmup+iterations) volumes of sizeGiB +// in az, timing CreateVolume's own latency and the settle time to +// "available" separately. Every created volume (including warm-up ones) gets +// a t.Cleanup teardown registered immediately via harness.RegisterVolumeTeardown, +// so a panic or Fatal partway through still reclaims whatever was created — +// and also gets its ID returned so a paired runDeleteVolume can measure +// deleting exactly this set instead of leaving cleanup to the test-end sweep. +func runCreateVolume(t *testing.T, c *harness.AWSClient, tags []*ec2.Tag, az string, sizeGiB int64, warmup, iterations, concurrency int) (*OpResult, []string) { + t.Helper() + col := newCollector() + var mu sync.Mutex + var ids []string + + parallelFor(concurrency, warmup, iterations, func(_, _ int, warm bool) { + start := time.Now() + out, err := c.EC2.CreateVolume(&ec2.CreateVolumeInput{ + AvailabilityZone: aws.String(az), + Size: aws.Int64(sizeGiB), + }) + api := time.Since(start) + if err != nil { + if !warm { + col.record(api, nil, false, err) + } + return + } + + id := aws.StringValue(out.VolumeId) + harness.RegisterVolumeTeardown(t, c, id) + tagResource(c, id, tags) + mu.Lock() + ids = append(ids, id) + mu.Unlock() + + settle, timedOut := pollVolumeState(c, id, ec2.VolumeStateAvailable, volumeSettleTimeout) + if !warm { + var s *time.Duration + if !timedOut { + s = &settle + } + col.record(api, s, timedOut, nil) + } + }) + + return col.result("CreateVolume"), ids +} + +// runDeleteVolume deletes every id (already-tagged, already-teardown-registered +// by runCreateVolume — a duplicate delete here just races the t.Cleanup one +// harmlessly since both tolerate InvalidVolume.NotFound), timing DeleteVolume's +// own latency and the settle time to gone. The first warmup*concurrency +// completions are discarded as warm-up. +func runDeleteVolume(c *harness.AWSClient, ids []string, warmup, concurrency int) *OpResult { + col := drainDelete(ids, concurrency, warmup*concurrency, func(id string) (time.Duration, *time.Duration, bool, error) { + start := time.Now() + _, err := c.EC2.DeleteVolume(&ec2.DeleteVolumeInput{VolumeId: aws.String(id)}) + api := time.Since(start) + if err != nil { + return api, nil, false, err + } + settle, timedOut := pollVolumeGone(c, id, volumeSettleTimeout) + if timedOut { + return api, nil, true, nil + } + return api, &settle, false, nil + }) + return col.result("DeleteVolume") +} + +// runDescribeVolumes times two DescribeVolumes shapes against poolIDs: a +// single-volume-by-id lookup (round-robin over the pool) and a full +// unfiltered list. The provider path enumerates these differently, so they +// are measured — and reported — as separate operations, never combined. +func runDescribeVolumes(c *harness.AWSClient, poolIDs []string, warmup, iterations, concurrency int) (single, list *OpResult) { + colSingle := newCollector() + var idx int + var idxMu sync.Mutex + parallelFor(concurrency, warmup, iterations, func(_, _ int, warm bool) { + idxMu.Lock() + id := poolIDs[idx%len(poolIDs)] + idx++ + idxMu.Unlock() + + start := time.Now() + _, err := c.EC2.DescribeVolumes(&ec2.DescribeVolumesInput{VolumeIds: []*string{aws.String(id)}}) + api := time.Since(start) + if !warm { + colSingle.record(api, nil, false, err) + } + }) + + colList := newCollector() + parallelFor(concurrency, warmup, iterations, func(_, _ int, warm bool) { + start := time.Now() + _, err := c.EC2.DescribeVolumes(&ec2.DescribeVolumesInput{}) + api := time.Since(start) + if !warm { + colList.record(api, nil, false, err) + } + }) + + return colSingle.result("DescribeVolumes.Single"), colList.result("DescribeVolumes.List") +} diff --git a/tests/bench/ebsctl/poll.go b/tests/bench/ebsctl/poll.go new file mode 100644 index 000000000..1c6c8d542 --- /dev/null +++ b/tests/bench/ebsctl/poll.go @@ -0,0 +1,108 @@ +//go:build e2e && bench + +package ebsctl + +import ( + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mulgadc/spinifex/tests/e2e/harness" +) + +// Settle-poll budgets. Bounded well under harness's own fatal Wait* defaults +// (5-10min) because a stuck settle here must not stall the whole benchmark — +// it becomes a SettleTimeouts count instead, which is itself a measurement. +const ( + pollInterval = 500 * time.Millisecond + volumeSettleTimeout = 2 * time.Minute + snapshotSettleTimeout = 3 * time.Minute + attachSettleTimeout = 2 * time.Minute +) + +// pollVolumeState polls DescribeVolumes for id until State == target or +// timeout elapses. Non-fatal by design — unlike harness.WaitForVolumeState +// (which calls t.Fatalf), a slow settle here is a data point, not a harness +// bug, and must not abort the run. +func pollVolumeState(c *harness.AWSClient, id, target string, timeout time.Duration) (elapsed time.Duration, timedOut bool) { + start := time.Now() + deadline := start.Add(timeout) + for { + out, err := c.EC2.DescribeVolumes(&ec2.DescribeVolumesInput{VolumeIds: []*string{aws.String(id)}}) + if err == nil && len(out.Volumes) > 0 && aws.StringValue(out.Volumes[0].State) == target { + return time.Since(start), false + } + if time.Now().After(deadline) { + return time.Since(start), true + } + time.Sleep(pollInterval) + } +} + +// pollVolumeGone polls until DescribeVolumes reports id absent +// (InvalidVolume.NotFound) or state "deleted", or timeout elapses. +func pollVolumeGone(c *harness.AWSClient, id string, timeout time.Duration) (elapsed time.Duration, timedOut bool) { + start := time.Now() + deadline := start.Add(timeout) + for { + out, err := c.EC2.DescribeVolumes(&ec2.DescribeVolumesInput{VolumeIds: []*string{aws.String(id)}}) + if err != nil && harness.ErrorCodeIs(err, "InvalidVolume.NotFound") { + return time.Since(start), false + } + if err == nil && (len(out.Volumes) == 0 || aws.StringValue(out.Volumes[0].State) == "deleted") { + return time.Since(start), false + } + if time.Now().After(deadline) { + return time.Since(start), true + } + time.Sleep(pollInterval) + } +} + +// pollSnapshotState polls DescribeSnapshots for id until State == target +// (typically "completed") or timeout. +func pollSnapshotState(c *harness.AWSClient, id, target string, timeout time.Duration) (elapsed time.Duration, timedOut bool) { + start := time.Now() + deadline := start.Add(timeout) + for { + out, err := c.EC2.DescribeSnapshots(&ec2.DescribeSnapshotsInput{SnapshotIds: []*string{aws.String(id)}}) + if err == nil && len(out.Snapshots) > 0 && aws.StringValue(out.Snapshots[0].State) == target { + return time.Since(start), false + } + if time.Now().After(deadline) { + return time.Since(start), true + } + time.Sleep(pollInterval) + } +} + +// pollSnapshotGone polls until DescribeSnapshots reports id absent +// (InvalidSnapshot.NotFound), or timeout. +func pollSnapshotGone(c *harness.AWSClient, id string, timeout time.Duration) (elapsed time.Duration, timedOut bool) { + start := time.Now() + deadline := start.Add(timeout) + for { + _, err := c.EC2.DescribeSnapshots(&ec2.DescribeSnapshotsInput{SnapshotIds: []*string{aws.String(id)}}) + if err != nil && harness.ErrorCodeIs(err, "InvalidSnapshot.NotFound") { + return time.Since(start), false + } + if time.Now().After(deadline) { + return time.Since(start), true + } + time.Sleep(pollInterval) + } +} + +// tagResource best-effort tags id (volume, snapshot, or instance) so orphans +// left by a killed run can be found and swept later. Not timed as part of +// any operation's latency and not fatal on failure — the resource still +// functions untagged, it just won't show up in a tag-based sweep. +func tagResource(c *harness.AWSClient, id string, tags []*ec2.Tag) { + if len(tags) == 0 { + return + } + _, _ = c.EC2.CreateTags(&ec2.CreateTagsInput{ + Resources: []*string{aws.String(id)}, + Tags: tags, + }) +} diff --git a/tests/bench/ebsctl/provider.go b/tests/bench/ebsctl/provider.go new file mode 100644 index 000000000..0ca84d35b --- /dev/null +++ b/tests/bench/ebsctl/provider.go @@ -0,0 +1,173 @@ +//go:build e2e && bench + +package ebsctl + +import ( + "bufio" + "context" + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + "time" + + "github.com/mulgadc/spinifex/spinifex/config" + "github.com/mulgadc/spinifex/tests/e2e/harness" +) + +// A benchmark that mislabels which [ebs] provider produced its numbers is +// worse than no benchmark, so DetectEBSProvider reads the value off the +// cluster itself rather than trusting an operator-supplied flag by default. +// +// Single-node mode: the test binary runs on the same host as the config +// (mirrors harness's own detectPoolMode), so the file is read locally. +// Multinode/baremetal: the binary typically runs off-cluster, so every node +// in SPINIFEX_NODE_IPS is read over SSH (SPINIFEX_SSH_USER/SPINIFEX_SSH_KEY) +// and the values must agree — a split cluster is a genuine configuration +// problem, not something to paper over with the first answer found. +// +// -provider is the fallback: required and validated when detection fails, +// and cross-checked against a successful detection rather than silently +// overriding it. +func DetectEBSProvider(t *testing.T, env *harness.Env) (provider, source string) { + t.Helper() + + var detected string + var detErr error + switch env.Mode { + case harness.ModeSingle: + detected, detErr = detectProviderLocal(env) + source = "cluster-local-config" + default: + detected, detErr = detectProviderSSH(env) + source = "cluster-ssh" + } + + if detErr != nil { + p := requireProviderFlag(t, detErr.Error()) + return p, "flag" + } + if *flagProvider != "" && *flagProvider != detected { + t.Fatalf("-provider=%q conflicts with the cluster-detected value %q (source=%s); "+ + "fix the flag or omit it to trust detection", *flagProvider, detected, source) + } + return detected, source +} + +func requireProviderFlag(t *testing.T, reason string) string { + t.Helper() + if *flagProvider == "" { + t.Fatalf("cannot determine [ebs] provider from the cluster (%s); "+ + "pass -provider=%s explicitly", reason, config.EBSProviderViperblockd) + } + if *flagProvider != config.EBSProviderViperblockd { + t.Fatalf("-provider=%q is not a recognised value (want %q)", + *flagProvider, config.EBSProviderViperblockd) + } + t.Logf("WARNING: could not detect [ebs] provider from the cluster (%s); trusting -provider=%s", + reason, *flagProvider) + return *flagProvider +} + +// detectProviderLocal reads spinifex.toml from the local filesystem, trying +// the same candidate paths as harness.LoadEnv/detectPoolMode. +func detectProviderLocal(env *harness.Env) (string, error) { + candidates := []string{} + if env.ConfigDir != "" { + candidates = append(candidates, filepath.Join(env.ConfigDir, "spinifex.toml")) + } + candidates = append(candidates, + "/etc/spinifex/spinifex.toml", + os.ExpandEnv("$HOME/spinifex/config/spinifex.toml"), + ) + + var lastErr error + for _, path := range candidates { + data, err := os.ReadFile(path) + if err != nil { + lastErr = err + continue + } + return resolvedProviderFromTOML(string(data)) + } + return "", fmt.Errorf("no readable spinifex.toml among %v: %w", candidates, lastErr) +} + +// detectProviderSSH reads spinifex.toml from every node in env.NodeIPs over +// SSH and requires unanimous agreement on the resolved provider value. +func detectProviderSSH(env *harness.Env) (string, error) { + if len(env.NodeIPs) == 0 { + return "", errors.New("no SPINIFEX_NODE_IPS to query") + } + + peer := harness.NewPeerSSH() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + seen := map[string][]string{} // provider -> node IPs reporting it + var details []string + for _, ip := range env.NodeIPs { + out, err := peer.Run(ctx, ip, "cat /etc/spinifex/spinifex.toml 2>/dev/null || cat $HOME/spinifex/config/spinifex.toml 2>/dev/null") + if err != nil || strings.TrimSpace(string(out)) == "" { + details = append(details, fmt.Sprintf("%s: unreachable or empty config (%v)", ip, err)) + continue + } + p, perr := resolvedProviderFromTOML(string(out)) + if perr != nil { + details = append(details, fmt.Sprintf("%s: %v", ip, perr)) + continue + } + seen[p] = append(seen[p], ip) + details = append(details, fmt.Sprintf("%s: %s", ip, p)) + } + + if len(seen) == 1 { + for p := range seen { + return p, nil + } + } + return "", fmt.Errorf("%d/%d node(s) queried disagree or were unreachable: %s", + len(details), len(env.NodeIPs), strings.Join(details, "; ")) +} + +var ( + tomlTopNodeLine = regexp.MustCompile(`(?i)^\s*node\s*=\s*"([^"]+)"`) + tomlProviderLine = regexp.MustCompile(`(?i)^\s*provider\s*=\s*"([^"]*)"`) +) + +// resolvedProviderFromTOML extracts [nodes..ebs] provider from a +// spinifex.toml's contents, where is the file's own top-level `node` +// key (the node's name for itself). An unset provider resolves to +// config.EBSProviderViperblockd via config.EBSConfig.ResolvedProvider, the same +// default the daemon itself applies — so detection never invents its own +// notion of "default". +func resolvedProviderFromTOML(content string) (string, error) { + var node, provider string + inEBS := false + sc := bufio.NewScanner(strings.NewReader(content)) + for sc.Scan() { + line := strings.TrimSpace(sc.Text()) + if strings.HasPrefix(line, "[") { + inEBS = node != "" && line == "[nodes."+node+".ebs]" + continue + } + if node == "" { + if m := tomlTopNodeLine.FindStringSubmatch(line); m != nil { + node = m[1] + } + continue + } + if inEBS { + if m := tomlProviderLine.FindStringSubmatch(line); m != nil { + provider = m[1] + } + } + } + if node == "" { + return "", errors.New("no top-level node key found in spinifex.toml") + } + return config.EBSConfig{Provider: provider}.ResolvedProvider(), nil +} diff --git a/tests/bench/ebsctl/schema.go b/tests/bench/ebsctl/schema.go new file mode 100644 index 000000000..3f8c89bd1 --- /dev/null +++ b/tests/bench/ebsctl/schema.go @@ -0,0 +1,122 @@ +// Package ebsctl benchmarks EC2 control-plane volume-operation latency +// against a live Spinifex cluster under both [ebs] provider settings +// (embedded vs viperblockd), isolating API round-trip latency from +// time-to-desired-state settle time and tallying the error surface each +// provider produces. +// +// This file and stats.go/collector.go/compare.go carry no build tag: they +// hold the JSON schema and pure statistics/comparison math, so they compile +// and unit-test without a cluster. The cluster-driving code lives in files +// tagged `e2e bench` (both required, since it imports the e2e harness +// package, which is itself gated on `e2e`). +package ebsctl + +import "time" + +// RunMeta is the run-level metadata recorded alongside the per-operation +// results, so a result file is self-describing when read back later without +// the invocation that produced it. +type RunMeta struct { + Timestamp time.Time `json:"timestamp"` + // GitSHA is the build/runtime git commit, best-effort (see gitSHA in + // main_test.go). "unknown" when neither the runtime repo lookup nor an + // -ldflags override was available. + GitSHA string `json:"git_sha"` + // Provider is the [ebs] provider value determined to be in effect on the + // cluster for this run: "viperblockd". + Provider string `json:"ebs_provider"` + // ProviderSource records how Provider was determined: "cluster-local-config", + // "cluster-ssh", or "flag" (operator-supplied, cluster detection failed). + ProviderSource string `json:"ebs_provider_source"` + NodeCount int `json:"node_count"` + IterationsPerWorker int `json:"iterations_per_worker"` + WarmupPerWorker int `json:"warmup_per_worker"` + Concurrency int `json:"concurrency"` + AttachDetachIncluded bool `json:"attach_detach_included"` + VolumeSizeGiB int64 `json:"volume_size_gib"` +} + +// MinSamplesForP95 is the sample-count heuristic below which a reported p95 +// is a near-single-point estimate rather than a meaningful percentile (at +// n=20, p95 sits at rank ~19, i.e. only the top sample or two beyond it). +// It is not a hard statistical threshold — SeriesStats.P95Reliable just +// makes the honesty call visible in the output rather than silent. +const MinSamplesForP95 = 20 + +// SeriesStats summarises one latency sample series in milliseconds, and +// carries the raw samples so a later comparison can be redone without +// re-running the benchmark. p99 is deliberately not reported anywhere in +// this schema: at the default sample sizes here (tens of samples) p99 is +// one data point dressed up as a percentile. +type SeriesStats struct { + Count int `json:"count"` + // Median/P95/Mean/Min/Max are all in milliseconds. + Median float64 `json:"median_ms"` + P95 float64 `json:"p95_ms"` + // P95Reliable is false when Count < MinSamplesForP95 — a signal to the + // reader that P95 above is indicative only. + P95Reliable bool `json:"p95_reliable"` + Mean float64 `json:"mean_ms"` + Min float64 `json:"min_ms"` + Max float64 `json:"max_ms"` + Samples []float64 `json:"samples_ms"` +} + +// ErrorTally counts operation errors by AWS error code and by message, for +// the same operation over the same run. +type ErrorTally struct { + Total int `json:"total"` + ByCode map[string]int `json:"by_code,omitempty"` + ByMessage map[string]int `json:"by_message,omitempty"` +} + +func newErrorTally() *ErrorTally { + return &ErrorTally{ByCode: map[string]int{}, ByMessage: map[string]int{}} +} + +// add records one error occurrence. Empty code/message are folded into +// "unknown" so the tally maps never grow an empty-string key. +func (e *ErrorTally) add(code, message string) { + e.Total++ + if code == "" { + code = "unknown" + } + if message == "" { + message = "unknown" + } + e.ByCode[code]++ + e.ByMessage[message]++ +} + +// rate returns Total as a fraction of attempts, or 0 when attempts is 0. +func (e *ErrorTally) rate(attempts int) float64 { + if attempts == 0 { + return 0 + } + return float64(e.Total) / float64(attempts) +} + +// OpResult is the full measured result for one operation kind (e.g. +// "CreateVolume", "DescribeVolumes.Single"). +type OpResult struct { + Operation string `json:"operation"` + // Attempts is every call made (successes + errors), across every worker, + // excluding discarded warm-up iterations. + Attempts int `json:"attempts"` + APILatency SeriesStats `json:"api_latency"` + SettleTime *SeriesStats `json:"settle_time,omitempty"` + // SettleTimeouts counts successful API calls whose settle-state poll + // exceeded its budget — distinct from Errors, which is only API-call + // failures. A settle timeout means the call succeeded but the resource + // never (within budget) reached the expected state. + SettleTimeouts int `json:"settle_timeouts"` + Errors ErrorTally `json:"errors"` + ErrorRate float64 `json:"error_rate"` +} + +// RunResult is the top-level JSON document the benchmark writes and the +// comparison tool reads. +type RunResult struct { + Meta RunMeta `json:"meta"` + Operations map[string]*OpResult `json:"operations"` +} diff --git a/tests/bench/ebsctl/stats.go b/tests/bench/ebsctl/stats.go new file mode 100644 index 000000000..a0be1007e --- /dev/null +++ b/tests/bench/ebsctl/stats.go @@ -0,0 +1,61 @@ +package ebsctl + +import ( + "math" + "sort" +) + +// percentile returns the p-th percentile (0<=p<=1) of sorted using linear +// interpolation between closest ranks (the common "R-7" convention, same as +// numpy's default). sorted must already be ascending. +func percentile(sorted []float64, p float64) float64 { + n := len(sorted) + if n == 0 { + return 0 + } + if n == 1 { + return sorted[0] + } + rank := p * float64(n-1) + lo := int(math.Floor(rank)) + hi := int(math.Ceil(rank)) + if lo == hi { + return sorted[lo] + } + frac := rank - float64(lo) + return sorted[lo] + (sorted[hi]-sorted[lo])*frac +} + +// median is percentile(sorted, 0.5). +func median(sorted []float64) float64 { + return percentile(sorted, 0.5) +} + +// computeSeries reduces samples (in the order they were recorded) to a +// SeriesStats: the raw Samples field preserves recording order (not sorted) +// so a reader can see arrival-order trends; every statistic is computed off +// a sorted copy. +func computeSeries(samples []float64) SeriesStats { + n := len(samples) + if n == 0 { + return SeriesStats{Count: 0, Samples: []float64{}} + } + sorted := append([]float64(nil), samples...) + sort.Float64s(sorted) + + sum := 0.0 + for _, v := range sorted { + sum += v + } + + return SeriesStats{ + Count: n, + Median: median(sorted), + P95: percentile(sorted, 0.95), + P95Reliable: n >= MinSamplesForP95, + Mean: sum / float64(n), + Min: sorted[0], + Max: sorted[n-1], + Samples: samples, + } +} diff --git a/tests/bench/ebsctl/stats_test.go b/tests/bench/ebsctl/stats_test.go new file mode 100644 index 000000000..9f8af3fe5 --- /dev/null +++ b/tests/bench/ebsctl/stats_test.go @@ -0,0 +1,98 @@ +package ebsctl + +import ( + "math" + "testing" +) + +func almostEqual(a, b float64) bool { + return math.Abs(a-b) < 1e-9 +} + +func TestPercentile(t *testing.T) { + sorted := []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10} + cases := []struct { + p float64 + want float64 + }{ + {0, 1}, + {1, 10}, + {0.5, 5.5}, + } + for _, c := range cases { + got := percentile(sorted, c.p) + if !almostEqual(got, c.want) { + t.Errorf("percentile(%v, %v) = %v, want %v", sorted, c.p, got, c.want) + } + } +} + +func TestPercentileSingleValue(t *testing.T) { + if got := percentile([]float64{42}, 0.95); got != 42 { + t.Errorf("percentile single value = %v, want 42", got) + } +} + +func TestPercentileEmpty(t *testing.T) { + if got := percentile(nil, 0.95); got != 0 { + t.Errorf("percentile empty = %v, want 0", got) + } +} + +func TestMedianOddEven(t *testing.T) { + odd := []float64{1, 2, 3, 4, 5} + if got := median(odd); !almostEqual(got, 3) { + t.Errorf("median(odd) = %v, want 3", got) + } + even := []float64{1, 2, 3, 4} + if got := median(even); !almostEqual(got, 2.5) { + t.Errorf("median(even) = %v, want 2.5", got) + } +} + +func TestComputeSeriesBasic(t *testing.T) { + samples := []float64{10, 20, 30, 40, 50} + s := computeSeries(samples) + if s.Count != 5 { + t.Errorf("Count = %d, want 5", s.Count) + } + if !almostEqual(s.Median, 30) { + t.Errorf("Median = %v, want 30", s.Median) + } + if !almostEqual(s.Mean, 30) { + t.Errorf("Mean = %v, want 30", s.Mean) + } + if s.Min != 10 || s.Max != 50 { + t.Errorf("Min/Max = %v/%v, want 10/50", s.Min, s.Max) + } + if s.P95Reliable { + t.Errorf("P95Reliable = true for n=5, want false (< MinSamplesForP95)") + } + // Samples must preserve recording (input) order, not be sorted. + if s.Samples[0] != 10 || s.Samples[4] != 50 { + t.Errorf("Samples order not preserved: %v", s.Samples) + } +} + +func TestComputeSeriesEmpty(t *testing.T) { + s := computeSeries(nil) + if s.Count != 0 { + t.Errorf("Count = %d, want 0", s.Count) + } +} + +func TestComputeSeriesP95ReliableThreshold(t *testing.T) { + samples := make([]float64, MinSamplesForP95) + for i := range samples { + samples[i] = float64(i) + } + s := computeSeries(samples) + if !s.P95Reliable { + t.Errorf("P95Reliable = false for n=%d, want true (== MinSamplesForP95)", MinSamplesForP95) + } + + s2 := computeSeries(samples[:MinSamplesForP95-1]) + if s2.P95Reliable { + t.Errorf("P95Reliable = true for n=%d, want false (< MinSamplesForP95)", MinSamplesForP95-1) + } +} diff --git a/tests/e2e/run-bm-multinode-e2e.sh b/tests/e2e/run-bm-multinode-e2e.sh index 1ded8b12a..a0a56ce11 100755 --- a/tests/e2e/run-bm-multinode-e2e.sh +++ b/tests/e2e/run-bm-multinode-e2e.sh @@ -1585,9 +1585,10 @@ else SPREAD_NODES=() for inst_id in "${NATGW_PRIV_IDS[@]}"; do - # spx get vms output is pipe-delimited: INSTANCE | STATUS | TYPE | VCPU | MEM | NODE | IP | AGE - # NODE is field 6 - NODE_NAME=$(echo "$SPX_VMS" | grep "$inst_id" | awk -F'|' '{gsub(/^[ \t]+|[ \t]+$/, "", $6); print $6}' || echo "unknown") + # spx get vms is pipe-delimited, with ANSI colour codes around each + # cell: INSTANCE | STATUS | HEALTH | CRASHES | TYPE | VCPU | MEM | + # NODE | IP | AGE. NODE is field 8; reading field 6 yields VCPU. + NODE_NAME=$(echo "$SPX_VMS" | grep "$inst_id" | sed -e 's/\x1b\[[0-9;]*m//g' | awk -F'|' '{gsub(/^[ \t]+|[ \t]+$/, "", $8); print $8}' || echo "unknown") if [ -z "$NODE_NAME" ] || [ "$NODE_NAME" = "unknown" ]; then NODE_NAME=$(find_instance_node "$inst_id" || echo "unknown") fi diff --git a/tests/fixtures/predastore/predastore.go b/tests/fixtures/predastore/predastore.go index 9d22d36cc..7bf718fdb 100644 --- a/tests/fixtures/predastore/predastore.go +++ b/tests/fixtures/predastore/predastore.go @@ -166,11 +166,15 @@ func Start(t *testing.T) *Fixture { // SSL_CERT_FILE injects the cert into the OS trust store for clients that // verify it rather than skipping verification (objectstore's S3 client, - // for one). t.Setenv reverts when this test ends, while the cluster lives - // on for the whole binary, so the pool is loaded here and now: crypto/x509 - // caches it behind a sync.Once, and this call is what fixes our cert in it - // for every later test in the process. - t.Setenv("SSL_CERT_FILE", certPath) + // for one). Set for the life of the process, not the calling test: the + // daemon outlives that test, and child processes started by later tests + // (nbdkit, whose plugin talks to this daemon) read the environment when + // they are spawned, long after a t.Setenv would have reverted it. + if err := os.Setenv("SSL_CERT_FILE", certPath); err != nil { //nolint:usetesting // must outlive the calling test, like the daemon + t.Fatalf("predastore fixture: set SSL_CERT_FILE: %v", err) + } + // crypto/x509 caches the pool behind a sync.Once, so this call is what + // fixes our cert in it for every in-process client that follows. if _, err := x509.SystemCertPool(); err != nil { t.Fatalf("predastore fixture: load system cert pool: %v", err) } diff --git a/tests/integration/volume_daemon_lite.go b/tests/integration/volume_daemon_lite.go index 52ec0e118..0099d00f2 100644 --- a/tests/integration/volume_daemon_lite.go +++ b/tests/integration/volume_daemon_lite.go @@ -5,10 +5,13 @@ package integration import ( "context" "testing" + "time" "github.com/mulgadc/spinifex/spinifex/config" + "github.com/mulgadc/spinifex/spinifex/ebsprovider" handlers_ec2_volume "github.com/mulgadc/spinifex/spinifex/handlers/ec2/volume" "github.com/mulgadc/spinifex/spinifex/objectstore" + "github.com/mulgadc/spinifex/spinifex/services/viperblockd" testpredastore "github.com/mulgadc/spinifex/tests/fixtures/predastore" "github.com/nats-io/nats.go" "github.com/stretchr/testify/require" @@ -21,6 +24,15 @@ import ( // dials it for real, so chunk uploads and config.json persistence are real. const testVolumeBucket = "integration-test-volumes" +// testProviderNodeName names the node the in-process provider answers as. Only +// publish and unpublish are node-addressed, and nothing here publishes, but an +// empty name makes the provider log a warning about attachments it cannot serve. +const testProviderNodeName = "integration-node" + +// providerRequestTimeout bounds a control-plane call to the provider. Generous +// because the first CreateVolume pays the real predastore's cold start. +const providerRequestTimeout = 60 * time.Second + // StartVolumeDaemonLite subscribes a real handlers_ec2_volume.VolumeServiceImpl // — the same production code a live daemon runs (daemon/daemon_handlers_volume.go) // — to the ec2.CreateVolume/DeleteVolume/DescribeVolumes subjects, backed by a @@ -63,6 +75,20 @@ func StartVolumeDaemonLite(t *testing.T, gw *Gateway) *handlers_ec2_volume.Volum nc := gw.NATSConn svc := handlers_ec2_volume.NewVolumeServiceImplWithStore(cfg, store, nc) + // The control plane no longer builds volumes itself; it asks a provider. + // Serve the provider contract from viperblockd against the same fixture, + // so the blocks are still written by real viperblock to real predastore. + require.NoError(t, viperblockd.RegisterProviderSubjects(&viperblockd.Config{ + S3Host: "https://" + fixture.Host, + Bucket: testVolumeBucket, + Region: fixture.Region, + AccessKey: fixture.AccessKey, + SecretKey: fixture.SecretKey, + BaseDir: cfg.WalDir, + NodeName: testProviderNodeName, + }, nc), "register provider subjects") + svc.SetEBSProvider(ebsprovider.NewNATSProvider(nc, providerRequestTimeout)) + sub(t, nc, "ec2.CreateVolume", func(m *nats.Msg) { dispatch(m, svc.CreateVolume) }) sub(t, nc, "ec2.DeleteVolume", func(m *nats.Msg) { dispatch(m, svc.DeleteVolume) }) sub(t, nc, "ec2.DescribeVolumes", func(m *nats.Msg) { dispatch(m, svc.DescribeVolumes) }) diff --git a/tests/integration/volume_test.go b/tests/integration/volume_test.go index 1aebe6713..2c0fa2646 100644 --- a/tests/integration/volume_test.go +++ b/tests/integration/volume_test.go @@ -9,9 +9,8 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" "github.com/mulgadc/spinifex/spinifex/objectstore" - testpredastore "github.com/mulgadc/spinifex/tests/fixtures/predastore" "github.com/mulgadc/spinifex/spinifex/testutil/vbscan" - "github.com/mulgadc/spinifex/spinifex/utils" + testpredastore "github.com/mulgadc/spinifex/tests/fixtures/predastore" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -75,9 +74,16 @@ func TestCreateVolume_PersistsToRealPredastore(t *testing.T) { assert.Equal(t, aws.StringValue(out.VolumeId), meta.VolumeID, "persisted VolumeID") assert.Equal(t, uint64(8), meta.SizeGiB, "persisted VolumeMetadata.SizeGiB") assert.Equal(t, "available", meta.State, "persisted VolumeMetadata.State") - assert.Equal(t, "gp3", meta.VolumeType, "persisted VolumeMetadata.VolumeType") - assert.Equal(t, testAZ, meta.AvailabilityZone, "persisted VolumeMetadata.AvailabilityZone") - assert.Equal(t, utils.GlobalAccountID, meta.TenantID, "persisted VolumeMetadata.TenantID") + + // Tenancy, volume type and placement are control-plane facts, held in the + // ebsmetadata document rather than storage state: the provider writes only + // what it owns. DescribeVolumes above is what checks them, and its answer + // is account-scoped, so returning this volume at all proves the tenant was + // recorded. + assert.Empty(t, meta.TenantID, "tenancy belongs to ebsmetadata, not viperblock state") + assert.Empty(t, meta.VolumeType, "volume type belongs to ebsmetadata, not viperblock state") + assert.Empty(t, meta.AvailabilityZone, "placement belongs to ebsmetadata, not viperblock state") + assert.Equal(t, testAZ, aws.StringValue(desc.Volumes[0].AvailabilityZone), "reported AvailabilityZone") // The on-disk VolumeSize is bytes, derived independently of // VolumeMetadata.SizeGiB (the field DescribeVolumes actually reports) —