mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-23 20:57:02 +00:00
Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| ab93d1fdcb | |||
| 3a79c1b866 | |||
| 8eefbc405f |
@@ -65,7 +65,7 @@ An incompatible change to the subject, fields or vocabulary needs a new major.
|
||||
|
||||
`SealAccountSettingsSubject` (`ec2.v1.SealAccountSettings`) is a request/reply route on which a daemon in the `spinifex-workers` queue group irreversibly seals one account's EC2 settings record.
|
||||
EC2 owns it: `domains/ec2/account` serves it through the daemon.
|
||||
`accountteardown`'s StageAccount seal reaper is the consumer: it calls this subject only after the seal-capability probe below reports every configured node compatible, and completes only once it holds a successful sealed reply.
|
||||
`accountteardown` is the intended consumer from a later release; no teardown calls it yet.
|
||||
|
||||
The request names the account in its body (`account_id`) and may carry an opaque `correlation_ref` that is echoed for logs.
|
||||
The daemon ignores `X-Account-ID` for this subject: the account sealed is the one named in the body.
|
||||
@@ -82,20 +82,3 @@ The subject name carries the contract major; `account_settings_seal_test.go` pre
|
||||
Producer and consumer are in the coordinated Spinifex release, so the window is N and N-1.
|
||||
A daemon that predates the subject has no subscription, so the request gets no responder.
|
||||
An incompatible change to the subject or fields needs a new major.
|
||||
|
||||
## Seal-account-settings capability
|
||||
|
||||
`AccountSettingsSealCapabilitySubject(node)` (`ec2.v1.AccountSettingsSealCapability.<node>`) is a per-node, read-only request/reply route on which `accountteardown` asks one configured daemon whether it supports the account-settings seal, immediately before every seal attempt.
|
||||
EC2 owns it: the daemon serves it with no queue group, so a request addressed to one node reaches only subscribers configured as that node, and two daemons sharing a node ID both answer.
|
||||
`accountteardown` is the consumer: its StageAccount seal reaper probes every configured node and sends no seal request unless every one answers compatible.
|
||||
|
||||
`AccountSettingsSealCapabilityVersion` is a claim about the whole binary's writers, not about the responder: version 1 means every EC2 account-settings writer in the binary is revision-guarded and refuses a sealed record, and the binary serves the seal subject.
|
||||
A daemon binary without this responder — predating the capability, or carrying only the seal subject without it — has no subscriber, so the probe for that node reads as incomplete (`no-reply`), the same as a stopped or partitioned node.
|
||||
The reply proves nothing beyond the moment it was sent: a daemon downgraded or replaced after a successful probe is outside the guarantee.
|
||||
|
||||
### Compatibility boundary
|
||||
|
||||
The subject name carries the contract major; `account_settings_seal_capability_test.go` preserves the version-1 JSON shape and the subject builder.
|
||||
Producer and consumer are in the coordinated Spinifex release, so the window is N and N-1.
|
||||
A daemon that predates the capability has no subscription, so the probe gets no responder for that node.
|
||||
An incompatible change to the subject or fields needs a new major.
|
||||
|
||||
@@ -1,39 +0,0 @@
|
||||
package ec2v1
|
||||
|
||||
import "strings"
|
||||
|
||||
// AccountSettingsSealCapabilityVersion is the version this binary reports.
|
||||
// Version 1: every EC2 account-settings writer in the binary is
|
||||
// revision-guarded and refuses a sealed record, and the seal subject is served.
|
||||
const AccountSettingsSealCapabilityVersion = 1
|
||||
|
||||
// AccountSettingsSealCapabilitySubject is one daemon's read-only
|
||||
// seal-capability route, addressed per node so a probe reaches exactly the
|
||||
// node it asks about.
|
||||
func AccountSettingsSealCapabilitySubject(node string) string {
|
||||
return "ec2.v1.AccountSettingsSealCapability." + node
|
||||
}
|
||||
|
||||
// ValidSealCapabilityNode reports whether node is exactly one NATS subject
|
||||
// token. Responder and probe both check it before building a subject, so
|
||||
// neither can address a wildcard or another node's route.
|
||||
func ValidSealCapabilityNode(node string) bool {
|
||||
return node != "" && !strings.ContainsAny(node, ".*>\t\n\r ")
|
||||
}
|
||||
|
||||
// AccountSettingsSealCapabilityRequest asks node whether it supports the
|
||||
// account-settings seal. CorrelationRef is an opaque caller reference echoed
|
||||
// for logs; it grants nothing and is never a fence or authority to write.
|
||||
type AccountSettingsSealCapabilityRequest struct {
|
||||
Node string `json:"node"`
|
||||
CorrelationRef string `json:"correlation_ref,omitempty"`
|
||||
}
|
||||
|
||||
// AccountSettingsSealCapabilityReply reports the responding node's capability
|
||||
// version. It carries no account data and proves nothing beyond the moment it
|
||||
// was sent.
|
||||
type AccountSettingsSealCapabilityReply struct {
|
||||
Node string `json:"node"`
|
||||
CapabilityVersion int `json:"capability_version"`
|
||||
CorrelationRef string `json:"correlation_ref,omitempty"`
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
package ec2v1
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestAccountSettingsSealCapabilitySubject(t *testing.T) {
|
||||
require.Equal(t, "ec2.v1.AccountSettingsSealCapability.node-1", AccountSettingsSealCapabilitySubject("node-1"))
|
||||
require.Equal(t, "ec2.v1.AccountSettingsSealCapability.node-2", AccountSettingsSealCapabilitySubject("node-2"))
|
||||
}
|
||||
|
||||
func TestAccountSettingsSealCapabilityRequestJSONContract(t *testing.T) {
|
||||
data, err := json.Marshal(AccountSettingsSealCapabilityRequest{Node: "node-1", CorrelationRef: "req-1"})
|
||||
require.NoError(t, err)
|
||||
require.JSONEq(t, `{"node":"node-1","correlation_ref":"req-1"}`, string(data))
|
||||
|
||||
data, err = json.Marshal(AccountSettingsSealCapabilityRequest{Node: "node-1"})
|
||||
require.NoError(t, err)
|
||||
require.JSONEq(t, `{"node":"node-1"}`, string(data))
|
||||
}
|
||||
|
||||
func TestAccountSettingsSealCapabilityReplyJSONContract(t *testing.T) {
|
||||
reply := AccountSettingsSealCapabilityReply{Node: "node-1", CapabilityVersion: 1, CorrelationRef: "req-1"}
|
||||
data, err := json.Marshal(reply)
|
||||
require.NoError(t, err)
|
||||
require.JSONEq(t, `{"node":"node-1","capability_version":1,"correlation_ref":"req-1"}`, string(data))
|
||||
|
||||
data, err = json.Marshal(AccountSettingsSealCapabilityReply{Node: "node-1", CapabilityVersion: 1})
|
||||
require.NoError(t, err)
|
||||
require.JSONEq(t, `{"node":"node-1","capability_version":1}`, string(data))
|
||||
}
|
||||
|
||||
func TestValidSealCapabilityNode(t *testing.T) {
|
||||
for _, node := range []string{"node-1", "node_2", "n3"} {
|
||||
require.True(t, ValidSealCapabilityNode(node), node)
|
||||
}
|
||||
for _, node := range []string{"", "a.b", "*", ">", "node-*", "node >", "node\t1", "node\n1"} {
|
||||
require.False(t, ValidSealCapabilityNode(node), "%q", node)
|
||||
}
|
||||
}
|
||||
@@ -48,6 +48,11 @@ type EBSRequest struct {
|
||||
NBDURI string `json:"NBDURI"`
|
||||
DeviceName string `json:"DeviceName"`
|
||||
HotplugPort int `json:"HotplugPort,omitempty"`
|
||||
// BlockDevice and BlockSizeBytes describe a host block device export, the
|
||||
// alternative to NBDURI; the guest sees BlockSizeBytes of it. Exactly one
|
||||
// of NBDURI and BlockDevice may be set.
|
||||
BlockDevice string `json:"BlockDevice,omitempty"`
|
||||
BlockSizeBytes uint64 `json:"BlockSizeBytes,omitempty"`
|
||||
}
|
||||
|
||||
// EBSMountResponse is the response to an EBSRequest on MountSubject.
|
||||
|
||||
@@ -35,6 +35,18 @@ func TestEBSRequestJSONContract(t *testing.T) {
|
||||
"NBDURI":"nbd:unix:/run/spinifex/vol-123.sock",
|
||||
"DeviceName":"/dev/sdf", "HotplugPort":3
|
||||
}`, string(data))
|
||||
|
||||
block, err := json.Marshal(EBSRequest{
|
||||
Name: "vol-456",
|
||||
BlockDevice: "/dev/sdx",
|
||||
BlockSizeBytes: 10 << 30,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.JSONEq(t, `{
|
||||
"Name":"vol-456", "VolType":"", "Boot":false, "EFI":false,
|
||||
"DeleteOnTermination":false, "NBDURI":"", "DeviceName":"",
|
||||
"BlockDevice":"/dev/sdx", "BlockSizeBytes":10737418240
|
||||
}`, string(block))
|
||||
}
|
||||
|
||||
func TestVolumeResponseJSONContracts(t *testing.T) {
|
||||
|
||||
@@ -446,7 +446,7 @@ These residuals were recorded by individual slices and must be resolved once the
|
||||
- `providers/ebs/viperblock` → `runtime/compute/nbd` and `runtime/host/process` (dependency, moved unchanged from `services/viperblockd`): the adapter formats NBD endpoints and manages nbdkit processes through runtime packages. ADR-0001 does not list providers → runtime as forbidden and the layering test does not enforce it (same category as `providers/ebs/conformance` → `runtime/compute/nbd`), but a provider should not depend on node runtime. Clears when NBD endpoint naming has a provider-facing home (a candidate for `contracts/ebs/v1`) and process supervision is injected by the role. The adapter also imports `providers/objectstore`, a provider-to-provider edge that is allowed.
|
||||
- `gateway/ec2.go` (registration): the EC2 dispatch map stays a `GatewayConfig` method in `gateway` and imports every `domains/ec2/awsapi/*` package; `gateway/ec2_idempotency.go` also imports `domains/ec2/awsapi/idem` (`KVBucket`, `TTL`, `TokenAndParams`) for the dispatch-time idempotency wrapper. ADR-0001 makes the domain own its action registration; moving the map waits on the generic ingress registration seam and must not be done as a package move.
|
||||
- `gateway/requestaudit.go`: the outermost request-audit middleware records AWS fields and is updated from four places in the gateway's authentication flow through the gateway's own context key. Moving it would export about seven names used only by that flow, so it stays in `gateway` until the generic ingress registration seam defines how ingress middleware receives authentication results.
|
||||
- `accountteardown` (dependency): its EC2 reapers call `domains/ec2/awsapi/*` action adapters directly, as they did under `gateway/ec2`. The ADR's `accountteardown` current-to-target map row replaces this with domain-owned cleanup capabilities before the package moves. It also holds a read-only consumer of the EC2 platform-inventory contract (`contracts/ec2/v1`, `ec2.v1.PlatformInventory`), served by `domains/ec2/accountinventory`; only the explicit dry run (`Engine.DryRun`) calls it, to log a shadow comparison, and the key-pair, placement-group and launch-template reapers still use the `awsapi` adapters. A domain-owned cleanup capability for the proven-safe subset of those three kinds (key pairs and empty placement groups no instance, spot request or live launch-template version names, and launch-template residue; never live launch templates) exists as `domains/ec2/accountcleanup` (a Go interface with injected owners and reference readers, no NATS subject); nothing calls it yet. It also holds the StageAccount settings-seal reaper, a consumer of `contracts/ec2/v1`'s `ec2.v1.SealAccountSettings` and per-node `ec2.v1.AccountSettingsSealCapability.<node>` subjects, served by `domains/ec2/account` through the daemon; the reaper reads no KV and imports no domain package, reaching both subjects only through NATS and the configured node set `NewClusterEngine` already receives.
|
||||
- `accountteardown` (dependency): its EC2 reapers call `domains/ec2/awsapi/*` action adapters directly, as they did under `gateway/ec2`. The ADR's `accountteardown` current-to-target map row replaces this with domain-owned cleanup capabilities before the package moves. It also holds a read-only consumer of the EC2 platform-inventory contract (`contracts/ec2/v1`, `ec2.v1.PlatformInventory`), served by `domains/ec2/accountinventory`; only the explicit dry run (`Engine.DryRun`) calls it, to log a shadow comparison, and the key-pair, placement-group and launch-template reapers still use the `awsapi` adapters. A domain-owned cleanup capability for the proven-safe subset of those three kinds (key pairs and empty placement groups no instance, spot request or live launch-template version names, and launch-template residue; never live launch templates) exists as `domains/ec2/accountcleanup` (a Go interface with injected owners and reference readers, no NATS subject); nothing calls it yet.
|
||||
- `domains/admission/quota` (dependency, moved from `handlers/quota` unchanged): `live.go` imports `gateway/elbv2` and `handlers/rds` for its live ELBv2 and RDS counters; `records.go` imports `runtime/compute/vm` for the instance-record vCPU sweep; `live.go`, `vcpu.go` and `storage.go` call `domains/ec2/awsapi/{vpc,eip,volume}` describe adapters to count VPCs, subnets, addresses and volumes, as they did under `gateway/ec2`. The ADR's `handlers/quota` current-to-target map row replaces these with reader capabilities once their final locations exist.
|
||||
- `domains/ec2/awsapi/instance` (dependency): imports `handlers/iam` (`IAMService` in `DescribeInstances.go`, `RunInstances.go` and `IamInstanceProfileAssociation.go`; `InstanceProfile` in the latter two), `runtime/compute/cache` (`NodeState`, `NodeUnknown`, `NodeLive`) and `runtime/compute/vm` (`VM`), both in `status_synthesis.go`. The IAM edge needs an IAM capability; the runtime edges are domain-to-runtime and need the instance record and node liveness exposed through an EC2-owned projection.
|
||||
- `domains/ec2/awsapi/spotinstance` (dependency): imports `handlers/iam.IAMService` and `domains/admission/quota.Service` in `RequestSpotInstances.go`.
|
||||
|
||||
@@ -70,11 +70,6 @@ func NewClusterEngine(
|
||||
reapers = append(reapers, S3Reapers(buckets)...)
|
||||
reapers = append(reapers, ACMReapers(nc)...)
|
||||
reapers = append(reapers, IAMReapers(svc)...)
|
||||
|
||||
// Before the account counters and reservations: the seal must still be
|
||||
// able to find the account IAM holds as TERMINATING, which the account
|
||||
// record reaper does not remove.
|
||||
reapers = append(reapers, EC2AccountSettingsReapers(nc, nodeIDs)...)
|
||||
reapers = append(reapers, AccountReapers(svc, names, usage)...)
|
||||
|
||||
engine := NewEngine(NewIAMAccounts(svc), reapers...)
|
||||
|
||||
@@ -41,27 +41,6 @@ func TestNewClusterEngineCoversEveryStage(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The EC2 settings seal reaper must be registered in StageAccount on every
|
||||
// surface that builds a cluster engine, or teardown would finalise accounts
|
||||
// without ever sealing their settings.
|
||||
func TestNewClusterEngineRegistersTheEC2SettingsSealReaper(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
|
||||
masterKey, err := handlers_iam.GenerateMasterKey()
|
||||
require.NoError(t, err)
|
||||
svc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey)
|
||||
require.NoError(t, err)
|
||||
|
||||
engine, err := NewClusterEngine(t.Context(), nc, 1, []string{"node-1"}, svc, newFakeBuckets())
|
||||
require.NoError(t, err)
|
||||
|
||||
var kinds []string
|
||||
for _, reaper := range engine.reapersFor(StageAccount) {
|
||||
kinds = append(kinds, reaper.Kind())
|
||||
}
|
||||
assert.Contains(t, kinds, "ec2-account-settings")
|
||||
}
|
||||
|
||||
// A teardown must refuse the accounts no credential may delete before it reads
|
||||
// anything, and refuse a name confirmation that does not match the record.
|
||||
func TestClusterEnginePrechecks(t *testing.T) {
|
||||
|
||||
@@ -1,125 +0,0 @@
|
||||
package accountteardown
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
|
||||
natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// ec2AccountSettingsSealTimeout bounds one seal request; an earlier caller
|
||||
// deadline wins.
|
||||
const ec2AccountSettingsSealTimeout = 30 * time.Second
|
||||
|
||||
// ec2AccountSettingsSealer is EC2's seal of one account's settings record.
|
||||
type ec2AccountSettingsSealer interface {
|
||||
SealAccountSettings(ctx context.Context, accountID string) (revision uint64, err error)
|
||||
}
|
||||
|
||||
// ec2AccountSettingsSealClient reaches the EC2 settings-seal contract over
|
||||
// NATS. Only a reply for this account, disposition sealed and a non-zero
|
||||
// revision is a seal; anything else, including no responder, is an error.
|
||||
type ec2AccountSettingsSealClient struct {
|
||||
nc *nats.Conn
|
||||
timeout time.Duration
|
||||
}
|
||||
|
||||
var _ ec2AccountSettingsSealer = (*ec2AccountSettingsSealClient)(nil)
|
||||
|
||||
func newEC2AccountSettingsSealClient(nc *nats.Conn) *ec2AccountSettingsSealClient {
|
||||
return &ec2AccountSettingsSealClient{nc: nc, timeout: ec2AccountSettingsSealTimeout}
|
||||
}
|
||||
|
||||
func (c *ec2AccountSettingsSealClient) SealAccountSettings(ctx context.Context, accountID string) (uint64, error) {
|
||||
timeout := c.timeout
|
||||
if deadline, ok := ctx.Deadline(); ok {
|
||||
timeout = min(timeout, time.Until(deadline))
|
||||
}
|
||||
if ctx.Err() != nil || timeout <= 0 {
|
||||
return 0, fmt.Errorf("seal account settings: %w", context.DeadlineExceeded)
|
||||
}
|
||||
|
||||
reply, err := natsmsg.NATSRequest[ec2v1.SealAccountSettingsReply](ctx, c.nc, ec2v1.SealAccountSettingsSubject,
|
||||
&ec2v1.SealAccountSettingsRequest{AccountID: accountID}, timeout, accountID)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("seal account settings: %w", err)
|
||||
}
|
||||
if reply.AccountID != accountID {
|
||||
return 0, fmt.Errorf("seal account settings: reply named account %q, not %q", reply.AccountID, accountID)
|
||||
}
|
||||
if reply.Disposition != ec2v1.AccountSettingsDispositionSealed {
|
||||
return 0, fmt.Errorf("seal account settings: unexpected disposition %q", reply.Disposition)
|
||||
}
|
||||
if reply.Revision == 0 {
|
||||
return 0, errors.New("seal account settings: reply carried no revision")
|
||||
}
|
||||
return reply.Revision, nil
|
||||
}
|
||||
|
||||
// accountSettingsSealReaper seals EC2's settings record in StageAccount. List
|
||||
// does no I/O and stays pending until this instance has sealed, so DryRun never
|
||||
// seals; state is per engine, so every fresh teardown run seals again.
|
||||
type accountSettingsSealReaper struct {
|
||||
sealer ec2AccountSettingsSealer
|
||||
capability sealCapabilityChecker
|
||||
|
||||
mu sync.Mutex
|
||||
sealed map[string]uint64 // account ID -> leader revision, this run only
|
||||
}
|
||||
|
||||
func (r *accountSettingsSealReaper) Kind() string { return "ec2-account-settings" }
|
||||
func (r *accountSettingsSealReaper) Stage() Stage { return StageAccount }
|
||||
|
||||
func (r *accountSettingsSealReaper) alreadySealed(accountID string) bool {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
_, ok := r.sealed[accountID]
|
||||
return ok
|
||||
}
|
||||
|
||||
func (r *accountSettingsSealReaper) List(_ context.Context, accountID string) ([]Resource, error) {
|
||||
if r.alreadySealed(accountID) {
|
||||
return nil, nil
|
||||
}
|
||||
return []Resource{{Kind: r.Kind(), ID: accountID, Detail: "seal"}}, nil
|
||||
}
|
||||
|
||||
// Delete probes every configured daemon immediately before each seal attempt,
|
||||
// never reusing an earlier probe. Incomplete capability sends no seal and is
|
||||
// retryable; force is ignored, since nothing may force past the probe.
|
||||
func (r *accountSettingsSealReaper) Delete(ctx context.Context, accountID string, _ Resource, _ bool) error {
|
||||
report := r.capability.CheckSealCapability(ctx, accountID)
|
||||
if !report.Complete {
|
||||
return errSealCapabilityIncomplete(report)
|
||||
}
|
||||
|
||||
revision, err := r.sealer.SealAccountSettings(ctx, accountID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
r.mu.Lock()
|
||||
if r.sealed == nil {
|
||||
r.sealed = map[string]uint64{}
|
||||
}
|
||||
r.sealed[accountID] = revision
|
||||
r.mu.Unlock()
|
||||
slog.InfoContext(ctx, "Account teardown sealed EC2 account settings", "accountID", accountID, "revision", revision)
|
||||
return nil
|
||||
}
|
||||
|
||||
// EC2AccountSettingsReapers returns the StageAccount reaper that seals EC2's
|
||||
// settings record. nodeIDs is the configured node set the capability probe
|
||||
// treats as "every relevant writer"; an empty set is unknown, never complete.
|
||||
func EC2AccountSettingsReapers(nc *nats.Conn, nodeIDs []string) []Reaper {
|
||||
return []Reaper{&accountSettingsSealReaper{
|
||||
sealer: newEC2AccountSettingsSealClient(nc),
|
||||
capability: newSealCapabilityClient(nc, nodeIDs),
|
||||
}}
|
||||
}
|
||||
@@ -1,190 +0,0 @@
|
||||
package accountteardown
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
|
||||
awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// sealCapabilityWait bounds the whole probe; shortened by the caller's
|
||||
// deadline. Every configured node is probed in parallel against this one
|
||||
// wait, so N nodes cost one wait, not N.
|
||||
const sealCapabilityWait = 3 * time.Second
|
||||
|
||||
// sealCapabilityGrace is how long a node's probe keeps listening after its
|
||||
// one expected responder has answered, long enough for a genuine second
|
||||
// subscriber on the same node to be caught as a duplicate.
|
||||
const sealCapabilityGrace = 150 * time.Millisecond
|
||||
|
||||
// Seal capability reasons, one per failing node. Each names a category, never
|
||||
// wire error text.
|
||||
const (
|
||||
sealCapabilityReasonNoReply = "no-reply"
|
||||
sealCapabilityReasonDuplicate = "duplicate"
|
||||
sealCapabilityReasonWrongIdentity = "wrong-identity"
|
||||
sealCapabilityReasonErrorReply = "error-reply"
|
||||
sealCapabilityReasonMalformed = "malformed"
|
||||
sealCapabilityReasonOldVersion = "old-version"
|
||||
sealCapabilityReasonTransport = "transport"
|
||||
sealCapabilityReasonInvalidNodeID = "invalid-node-id"
|
||||
sealCapabilityReasonSetUnknown = "configured-node-set-unknown"
|
||||
)
|
||||
|
||||
// sealCapabilityChecker reports whether every configured daemon supports the
|
||||
// seal.
|
||||
type sealCapabilityChecker interface {
|
||||
CheckSealCapability(ctx context.Context, correlationRef string) sealCapabilityReport
|
||||
}
|
||||
|
||||
// sealCapabilityReport is one probe's result, one entry per configured node.
|
||||
type sealCapabilityReport struct {
|
||||
Complete bool
|
||||
Nodes []nodeSealCapability // one per configured node, sorted by node
|
||||
}
|
||||
|
||||
// nodeSealCapability is one configured node's compatibility. Reason is ""
|
||||
// when the node is compatible.
|
||||
type nodeSealCapability struct {
|
||||
Node string
|
||||
Version int // 0 unless a valid reply arrived
|
||||
Reason string // "" when compatible
|
||||
}
|
||||
|
||||
// sealCapabilityClient probes every configured node's seal-capability subject
|
||||
// in parallel. "Configured" is the same node set NewClusterEngine already
|
||||
// receives; there is no separate liveness or membership source.
|
||||
type sealCapabilityClient struct {
|
||||
nc *nats.Conn
|
||||
nodeIDs []string
|
||||
wait time.Duration // 3 s, shortened by the caller's deadline
|
||||
grace time.Duration // 150 ms ResponderGrace
|
||||
}
|
||||
|
||||
var _ sealCapabilityChecker = (*sealCapabilityClient)(nil)
|
||||
|
||||
func newSealCapabilityClient(nc *nats.Conn, nodeIDs []string) *sealCapabilityClient {
|
||||
return &sealCapabilityClient{nc: nc, nodeIDs: nodeIDs, wait: sealCapabilityWait, grace: sealCapabilityGrace}
|
||||
}
|
||||
|
||||
// CheckSealCapability probes every configured node in parallel, bounded by
|
||||
// one shared deadline, run immediately before each seal attempt. A cached or
|
||||
// earlier result is never reused.
|
||||
func (c *sealCapabilityClient) CheckSealCapability(ctx context.Context, correlationRef string) sealCapabilityReport {
|
||||
if len(c.nodeIDs) == 0 {
|
||||
return sealCapabilityReport{Complete: false}
|
||||
}
|
||||
|
||||
wait := c.wait
|
||||
if deadline, ok := ctx.Deadline(); ok {
|
||||
wait = min(wait, time.Until(deadline))
|
||||
}
|
||||
|
||||
nodes := append([]string(nil), c.nodeIDs...)
|
||||
sort.Strings(nodes)
|
||||
|
||||
results := make([]nodeSealCapability, len(nodes))
|
||||
if ctx.Err() != nil || wait <= 0 {
|
||||
for i, node := range nodes {
|
||||
results[i] = nodeSealCapability{Node: node, Reason: sealCapabilityReasonNoReply}
|
||||
}
|
||||
return sealCapabilityReport{Complete: false, Nodes: results}
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for i, node := range nodes {
|
||||
wg.Add(1)
|
||||
go func(i int, node string) {
|
||||
defer wg.Done()
|
||||
results[i] = c.checkNode(ctx, node, correlationRef, wait)
|
||||
}(i, node)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
complete := true
|
||||
for _, r := range results {
|
||||
if r.Reason != "" {
|
||||
complete = false
|
||||
break
|
||||
}
|
||||
}
|
||||
return sealCapabilityReport{Complete: complete, Nodes: results}
|
||||
}
|
||||
|
||||
// checkNode probes one node and classifies the result. A node is compatible
|
||||
// only when exactly one identified, decodable, matching, version >= 1 frame
|
||||
// arrived, with no duplicate, conflict or cap hit.
|
||||
func (c *sealCapabilityClient) checkNode(ctx context.Context, node, correlationRef string, wait time.Duration) nodeSealCapability {
|
||||
if !ec2v1.ValidSealCapabilityNode(node) {
|
||||
return nodeSealCapability{Node: node, Reason: sealCapabilityReasonInvalidNodeID}
|
||||
}
|
||||
|
||||
data, err := json.Marshal(ec2v1.AccountSettingsSealCapabilityRequest{Node: node, CorrelationRef: correlationRef})
|
||||
if err != nil {
|
||||
return nodeSealCapability{Node: node, Reason: sealCapabilityReasonMalformed}
|
||||
}
|
||||
|
||||
frames, sum, err := natsmsg.Gather(ctx, c.nc, ec2v1.AccountSettingsSealCapabilitySubject(node), data, natsmsg.GatherOpts{
|
||||
Timeout: wait,
|
||||
ExpectedResponders: 1,
|
||||
Mode: natsmsg.CollectUntilDeadline,
|
||||
ResponderGrace: c.grace,
|
||||
})
|
||||
if err != nil {
|
||||
return nodeSealCapability{Node: node, Reason: sealCapabilityReasonTransport}
|
||||
}
|
||||
|
||||
switch {
|
||||
case sum.CapHit, len(sum.ConflictNodes) > 0, sum.DuplicateFrames > 0:
|
||||
return nodeSealCapability{Node: node, Reason: sealCapabilityReasonDuplicate}
|
||||
case sum.Unidentified > 0:
|
||||
return nodeSealCapability{Node: node, Reason: sealCapabilityReasonWrongIdentity}
|
||||
case len(sum.ErrorResponders) > 0:
|
||||
return nodeSealCapability{Node: node, Reason: sealCapabilityReasonErrorReply}
|
||||
case sum.Received == 0, len(frames) == 0:
|
||||
return nodeSealCapability{Node: node, Reason: sealCapabilityReasonNoReply}
|
||||
}
|
||||
|
||||
frame := frames[0]
|
||||
if frame.NodeID != node {
|
||||
return nodeSealCapability{Node: node, Reason: sealCapabilityReasonWrongIdentity}
|
||||
}
|
||||
|
||||
var reply ec2v1.AccountSettingsSealCapabilityReply
|
||||
if json.Unmarshal(frame.Data, &reply) != nil {
|
||||
return nodeSealCapability{Node: node, Reason: sealCapabilityReasonMalformed}
|
||||
}
|
||||
if reply.Node != node {
|
||||
return nodeSealCapability{Node: node, Reason: sealCapabilityReasonWrongIdentity}
|
||||
}
|
||||
if reply.CapabilityVersion < 1 {
|
||||
return nodeSealCapability{Node: node, Version: reply.CapabilityVersion, Reason: sealCapabilityReasonOldVersion}
|
||||
}
|
||||
return nodeSealCapability{Node: node, Version: reply.CapabilityVersion}
|
||||
}
|
||||
|
||||
// errSealCapabilityIncomplete builds the retryable refusal from report. It
|
||||
// names node IDs and reason categories only, never wire error text.
|
||||
func errSealCapabilityIncomplete(report sealCapabilityReport) error {
|
||||
detail := sealCapabilityReasonSetUnknown
|
||||
var parts []string
|
||||
for _, n := range report.Nodes {
|
||||
if n.Reason != "" {
|
||||
parts = append(parts, n.Node+"="+n.Reason)
|
||||
}
|
||||
}
|
||||
if len(parts) > 0 {
|
||||
detail = strings.Join(parts, ", ")
|
||||
}
|
||||
return awserrors.WithRetryAfter(
|
||||
awserrors.Errorf(awserrors.ErrorServiceUnavailable, "EC2 account-settings seal capability incomplete: %s", detail),
|
||||
5*time.Second,
|
||||
)
|
||||
}
|
||||
@@ -1,311 +0,0 @@
|
||||
package accountteardown
|
||||
|
||||
//test:in-package — the probe client, its report types and reason constants
|
||||
// are unexported, and what they guarantee (incomplete never seals, complete
|
||||
// does) is only observable from inside the package.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
|
||||
"github.com/mulgadc/spinifex/internal/testkit"
|
||||
awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// testCapabilityClient builds a probe client with test-sized timeouts so the
|
||||
// incomplete cases below do not each pay the production 3 s wait.
|
||||
func testCapabilityClient(nc *nats.Conn, nodeIDs []string) *sealCapabilityClient {
|
||||
c := newSealCapabilityClient(nc, nodeIDs)
|
||||
c.wait = 200 * time.Millisecond
|
||||
c.grace = 30 * time.Millisecond
|
||||
return c
|
||||
}
|
||||
|
||||
// serveCapability subscribes reply on subject with no queue group, which is
|
||||
// how the real responder is addressed: per node, so two subscribers on the
|
||||
// same node's subject both answer.
|
||||
func serveCapability(t *testing.T, nc *nats.Conn, subject string, reply nats.MsgHandler) *nats.Subscription {
|
||||
t.Helper()
|
||||
sub, err := nc.Subscribe(subject, reply)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
require.NoError(t, nc.Flush())
|
||||
return sub
|
||||
}
|
||||
|
||||
// capabilityReply answers with a well-formed reply naming replyNode (which
|
||||
// may deliberately differ from the subject's node) and headerNode as the
|
||||
// X-Node-ID ("" omits the header).
|
||||
func capabilityReply(replyNode, headerNode string, version int) nats.MsgHandler {
|
||||
return func(msg *nats.Msg) {
|
||||
data, _ := json.Marshal(ec2v1.AccountSettingsSealCapabilityReply{Node: replyNode, CapabilityVersion: version})
|
||||
reply := nats.NewMsg(msg.Reply)
|
||||
reply.Data = data
|
||||
if headerNode != "" {
|
||||
reply.Header.Set(natsmsg.NodeIDHeader, headerNode)
|
||||
}
|
||||
_ = msg.RespondMsg(reply)
|
||||
}
|
||||
}
|
||||
|
||||
func nodeResult(t *testing.T, report sealCapabilityReport, node string) nodeSealCapability {
|
||||
t.Helper()
|
||||
for _, n := range report.Nodes {
|
||||
if n.Node == node {
|
||||
return n
|
||||
}
|
||||
}
|
||||
t.Fatalf("no result for node %q in %+v", node, report)
|
||||
return nodeSealCapability{}
|
||||
}
|
||||
|
||||
// Each sub-case names exactly one way a node fails to prove capability
|
||||
// version 1, and that failure alone makes the whole probe incomplete.
|
||||
func TestSealCapabilityIncomplete(t *testing.T) {
|
||||
t.Run("missing", func(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
report := testCapabilityClient(nc, []string{"node-1"}).CheckSealCapability(testCtx(t), "req")
|
||||
assert.False(t, report.Complete)
|
||||
assert.Equal(t, sealCapabilityReasonNoReply, nodeResult(t, report, "node-1").Reason)
|
||||
})
|
||||
|
||||
t.Run("duplicate", func(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
serveCapability(t, nc, ec2v1.AccountSettingsSealCapabilitySubject("node-1"), capabilityReply("node-1", "node-1", 1))
|
||||
serveCapability(t, nc, ec2v1.AccountSettingsSealCapabilitySubject("node-1"), capabilityReply("node-1", "node-1", 1))
|
||||
report := testCapabilityClient(nc, []string{"node-1"}).CheckSealCapability(testCtx(t), "req")
|
||||
assert.False(t, report.Complete)
|
||||
assert.Equal(t, sealCapabilityReasonDuplicate, nodeResult(t, report, "node-1").Reason)
|
||||
})
|
||||
|
||||
t.Run("unreachable timeout", func(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
serveCapability(t, nc, ec2v1.AccountSettingsSealCapabilitySubject("node-1"), func(msg *nats.Msg) {
|
||||
time.Sleep(time.Second)
|
||||
_ = msg.Respond(nil)
|
||||
})
|
||||
client := testCapabilityClient(nc, []string{"node-1"})
|
||||
report := client.CheckSealCapability(testCtx(t), "req")
|
||||
assert.False(t, report.Complete)
|
||||
assert.Equal(t, sealCapabilityReasonNoReply, nodeResult(t, report, "node-1").Reason)
|
||||
})
|
||||
|
||||
t.Run("transport closed connection", func(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
nc.Close()
|
||||
report := testCapabilityClient(nc, []string{"node-1"}).CheckSealCapability(testCtx(t), "req")
|
||||
assert.False(t, report.Complete)
|
||||
assert.Equal(t, sealCapabilityReasonTransport, nodeResult(t, report, "node-1").Reason)
|
||||
})
|
||||
|
||||
t.Run("malformed body", func(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
serveCapability(t, nc, ec2v1.AccountSettingsSealCapabilitySubject("node-1"), func(msg *nats.Msg) {
|
||||
reply := nats.NewMsg(msg.Reply)
|
||||
reply.Data = []byte("not json")
|
||||
reply.Header.Set(natsmsg.NodeIDHeader, "node-1")
|
||||
_ = msg.RespondMsg(reply)
|
||||
})
|
||||
report := testCapabilityClient(nc, []string{"node-1"}).CheckSealCapability(testCtx(t), "req")
|
||||
assert.False(t, report.Complete)
|
||||
assert.Equal(t, sealCapabilityReasonMalformed, nodeResult(t, report, "node-1").Reason)
|
||||
})
|
||||
|
||||
t.Run("error reply", func(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
serveCapability(t, nc, ec2v1.AccountSettingsSealCapabilitySubject("node-1"), func(msg *nats.Msg) {
|
||||
reply := nats.NewMsg(msg.Reply)
|
||||
reply.Data = awserrors.GenerateErrorPayload(awserrors.ErrorServerInternal)
|
||||
reply.Header.Set(natsmsg.NodeIDHeader, "node-1")
|
||||
_ = msg.RespondMsg(reply)
|
||||
})
|
||||
report := testCapabilityClient(nc, []string{"node-1"}).CheckSealCapability(testCtx(t), "req")
|
||||
assert.False(t, report.Complete)
|
||||
assert.Equal(t, sealCapabilityReasonErrorReply, nodeResult(t, report, "node-1").Reason)
|
||||
})
|
||||
|
||||
t.Run("wrong identity no header", func(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
serveCapability(t, nc, ec2v1.AccountSettingsSealCapabilitySubject("node-1"), capabilityReply("node-1", "", 1))
|
||||
report := testCapabilityClient(nc, []string{"node-1"}).CheckSealCapability(testCtx(t), "req")
|
||||
assert.False(t, report.Complete)
|
||||
assert.Equal(t, sealCapabilityReasonWrongIdentity, nodeResult(t, report, "node-1").Reason)
|
||||
})
|
||||
|
||||
t.Run("wrong identity another node header", func(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
serveCapability(t, nc, ec2v1.AccountSettingsSealCapabilitySubject("node-1"), capabilityReply("node-1", "node-9", 1))
|
||||
report := testCapabilityClient(nc, []string{"node-1"}).CheckSealCapability(testCtx(t), "req")
|
||||
assert.False(t, report.Complete)
|
||||
assert.Equal(t, sealCapabilityReasonWrongIdentity, nodeResult(t, report, "node-1").Reason)
|
||||
})
|
||||
|
||||
t.Run("wrong identity body node differs", func(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
serveCapability(t, nc, ec2v1.AccountSettingsSealCapabilitySubject("node-1"), capabilityReply("node-9", "node-1", 1))
|
||||
report := testCapabilityClient(nc, []string{"node-1"}).CheckSealCapability(testCtx(t), "req")
|
||||
assert.False(t, report.Complete)
|
||||
assert.Equal(t, sealCapabilityReasonWrongIdentity, nodeResult(t, report, "node-1").Reason)
|
||||
})
|
||||
|
||||
t.Run("lower version", func(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
serveCapability(t, nc, ec2v1.AccountSettingsSealCapabilitySubject("node-1"), capabilityReply("node-1", "node-1", 0))
|
||||
report := testCapabilityClient(nc, []string{"node-1"}).CheckSealCapability(testCtx(t), "req")
|
||||
assert.False(t, report.Complete)
|
||||
assert.Equal(t, sealCapabilityReasonOldVersion, nodeResult(t, report, "node-1").Reason)
|
||||
})
|
||||
|
||||
t.Run("configured set empty", func(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
report := testCapabilityClient(nc, nil).CheckSealCapability(testCtx(t), "req")
|
||||
assert.False(t, report.Complete)
|
||||
assert.Empty(t, report.Nodes)
|
||||
})
|
||||
|
||||
t.Run("invalid node id", func(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
report := testCapabilityClient(nc, []string{"bad.node"}).CheckSealCapability(testCtx(t), "req")
|
||||
assert.False(t, report.Complete)
|
||||
assert.Equal(t, sealCapabilityReasonInvalidNodeID, nodeResult(t, report, "bad.node").Reason)
|
||||
})
|
||||
}
|
||||
|
||||
// Every configured node answering version 1 or later makes the probe
|
||||
// complete, and the reaper sends exactly one seal request.
|
||||
func TestSealCapabilityCompleteAllowsSeal(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
serveCapability(t, nc, ec2v1.AccountSettingsSealCapabilitySubject("node-1"), capabilityReply("node-1", "node-1", 1))
|
||||
serveCapability(t, nc, ec2v1.AccountSettingsSealCapabilitySubject("node-2"), capabilityReply("node-2", "node-2", 2))
|
||||
|
||||
client := testCapabilityClient(nc, []string{"node-1", "node-2"})
|
||||
report := client.CheckSealCapability(testCtx(t), "req")
|
||||
require.True(t, report.Complete)
|
||||
assert.Equal(t, 1, nodeResult(t, report, "node-1").Version)
|
||||
assert.Equal(t, 2, nodeResult(t, report, "node-2").Version)
|
||||
|
||||
sealer := &fakeSealer{}
|
||||
reaper := &accountSettingsSealReaper{sealer: sealer, capability: client}
|
||||
require.NoError(t, reaper.Delete(testCtx(t), testAccountID, Resource{Kind: reaper.Kind(), ID: testAccountID}, false))
|
||||
assert.Equal(t, []string{testAccountID}, sealer.calls)
|
||||
}
|
||||
|
||||
// An incomplete probe keeps the account TERMINATING through the full engine,
|
||||
// names the silent node and its reason, sends no seal request, and leaves the
|
||||
// settings record untouched (the fake sealer records no call).
|
||||
func TestSealCapabilityIncompleteKeepsAccountTerminating(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
serveCapability(t, nc, ec2v1.AccountSettingsSealCapabilitySubject("node-1"), capabilityReply("node-1", "node-1", 1))
|
||||
// node-2 has no subscriber.
|
||||
|
||||
sealer := &fakeSealer{}
|
||||
reaper := &accountSettingsSealReaper{sealer: sealer, capability: testCapabilityClient(nc, []string{"node-1", "node-2"})}
|
||||
accounts := newFakeAccounts()
|
||||
engine := quietEngine(accounts, reaper)
|
||||
|
||||
result, err := engine.Teardown(testCtx(t), request())
|
||||
require.ErrorIs(t, err, ErrResourcesStuck)
|
||||
assert.False(t, result.AccountDeleted)
|
||||
assert.Equal(t, AccountStatusTerminating, accounts.account.Status)
|
||||
assert.Empty(t, sealer.calls)
|
||||
|
||||
var reason string
|
||||
for _, stage := range result.Stages {
|
||||
for _, stuck := range stage.Stuck {
|
||||
if stuck.Resource.Kind == "ec2-account-settings" {
|
||||
reason = stuck.Reason
|
||||
}
|
||||
}
|
||||
}
|
||||
assert.Contains(t, reason, "node-2="+sealCapabilityReasonNoReply)
|
||||
}
|
||||
|
||||
// The silent node starting to answer during the drain makes the very next
|
||||
// Delete re-probe and succeed, which proves the result is retryable and the
|
||||
// probe is never cached across attempts.
|
||||
func TestSealCapabilityRetriesThenSeals(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
serveCapability(t, nc, ec2v1.AccountSettingsSealCapabilitySubject("node-1"), capabilityReply("node-1", "node-1", 1))
|
||||
|
||||
var node2Up atomic.Bool
|
||||
sub, err := nc.Subscribe(ec2v1.AccountSettingsSealCapabilitySubject("node-2"), func(msg *nats.Msg) {
|
||||
if !node2Up.Load() {
|
||||
return
|
||||
}
|
||||
capabilityReply("node-2", "node-2", 1)(msg)
|
||||
})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
require.NoError(t, nc.Flush())
|
||||
|
||||
sealer := &fakeSealer{}
|
||||
reaper := &accountSettingsSealReaper{sealer: sealer, capability: testCapabilityClient(nc, []string{"node-1", "node-2"})}
|
||||
accounts := newFakeAccounts()
|
||||
engine := quietEngine(accounts, reaper)
|
||||
engine.Timeouts = Timeouts{StageDrain: time.Second, DrainPoll: 5 * time.Millisecond}
|
||||
|
||||
go func() {
|
||||
time.Sleep(80 * time.Millisecond)
|
||||
node2Up.Store(true)
|
||||
}()
|
||||
|
||||
result, err := engine.Teardown(testCtx(t), request())
|
||||
require.NoError(t, err)
|
||||
assert.True(t, result.AccountDeleted)
|
||||
assert.Equal(t, []string{testAccountID}, sealer.calls)
|
||||
}
|
||||
|
||||
// Two Delete calls on one reaper probe twice: the probe runs in Delete
|
||||
// itself, never cached from a previous attempt.
|
||||
func TestSealCapabilityProbedBeforeEverySeal(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
var probes int
|
||||
serveCapability(t, nc, ec2v1.AccountSettingsSealCapabilitySubject("node-1"), func(msg *nats.Msg) {
|
||||
probes++
|
||||
capabilityReply("node-1", "node-1", 1)(msg)
|
||||
})
|
||||
|
||||
sealer := &fakeSealer{}
|
||||
client := testCapabilityClient(nc, []string{"node-1"})
|
||||
reaper := &accountSettingsSealReaper{sealer: sealer, capability: client}
|
||||
|
||||
require.NoError(t, reaper.Delete(testCtx(t), testAccountID, Resource{Kind: reaper.Kind(), ID: testAccountID}, false))
|
||||
// alreadySealed now makes List empty, but Delete itself does not consult
|
||||
// it, so a second direct call still probes and seals again.
|
||||
require.NoError(t, reaper.Delete(testCtx(t), testAccountID, Resource{Kind: reaper.Kind(), ID: testAccountID}, false))
|
||||
assert.Equal(t, 2, probes)
|
||||
assert.Len(t, sealer.calls, 2)
|
||||
}
|
||||
|
||||
// Five silent nodes still return within one deadline, not five: every node
|
||||
// is probed in parallel against the same wait.
|
||||
func TestSealCapabilityBoundedByOneDeadline(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
client := testCapabilityClient(nc, []string{"node-1", "node-2", "node-3", "node-4", "node-5"})
|
||||
|
||||
started := time.Now()
|
||||
report := client.CheckSealCapability(testCtx(t), "req")
|
||||
elapsed := time.Since(started)
|
||||
|
||||
assert.False(t, report.Complete)
|
||||
assert.Less(t, elapsed, client.wait+time.Second, "five silent nodes must not cost five deadlines")
|
||||
}
|
||||
|
||||
// The stuck reason is logged and stored as one line naming each failing node;
|
||||
// the retry hint and the ServiceUnavailable code travel with it.
|
||||
func TestSealCapabilityIncompleteErrorIsOneLine(t *testing.T) {
|
||||
err := errSealCapabilityIncomplete(sealCapabilityReport{Nodes: []nodeSealCapability{{Node: "node-1", Reason: sealCapabilityReasonDuplicate}}})
|
||||
|
||||
assert.Equal(t, "EC2 account-settings seal capability incomplete: node-1=duplicate: ServiceUnavailable", err.Error())
|
||||
d, ok := awserrors.ResolveRetryAfter(err)
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, 5*time.Second, d)
|
||||
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorServiceUnavailable))
|
||||
}
|
||||
@@ -1,273 +0,0 @@
|
||||
package accountteardown
|
||||
|
||||
//test:in-package — the reaper, sealer and capability checker are unexported;
|
||||
// no seal without a complete probe, completion only after a sealed reply and
|
||||
// idempotent fresh runs are only observable from inside the package.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
|
||||
"github.com/mulgadc/spinifex/internal/testkit"
|
||||
ec2account "github.com/mulgadc/spinifex/spinifex/domains/ec2/account"
|
||||
awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// fakeSealer is a recording ec2AccountSettingsSealer. Each call beyond
|
||||
// failFor succeeds with a fresh, strictly increasing revision, unless err is
|
||||
// set, in which case every call after failFor fails with it.
|
||||
type fakeSealer struct {
|
||||
mu sync.Mutex
|
||||
calls []string
|
||||
failFor int
|
||||
err error
|
||||
revision uint64
|
||||
}
|
||||
|
||||
func (f *fakeSealer) SealAccountSettings(_ context.Context, accountID string) (uint64, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.calls = append(f.calls, accountID)
|
||||
if len(f.calls) <= f.failFor {
|
||||
return 0, errors.New("account settings owner temporarily unavailable")
|
||||
}
|
||||
if f.err != nil {
|
||||
return 0, f.err
|
||||
}
|
||||
f.revision++
|
||||
return f.revision, nil
|
||||
}
|
||||
|
||||
// fakeSealCapability is a recording sealCapabilityChecker returning a fixed report.
|
||||
type fakeSealCapability struct {
|
||||
calls int
|
||||
report sealCapabilityReport
|
||||
}
|
||||
|
||||
func (f *fakeSealCapability) CheckSealCapability(context.Context, string) sealCapabilityReport {
|
||||
f.calls++
|
||||
return f.report
|
||||
}
|
||||
|
||||
// completeReport builds a sealCapabilityReport where every named node answers
|
||||
// compatible at version 1.
|
||||
func completeReport(nodeIDs ...string) sealCapabilityReport {
|
||||
var nodes []nodeSealCapability
|
||||
for _, node := range nodeIDs {
|
||||
nodes = append(nodes, nodeSealCapability{Node: node, Version: 1})
|
||||
}
|
||||
return sealCapabilityReport{Complete: true, Nodes: nodes}
|
||||
}
|
||||
|
||||
// alwaysTerminatingChecker stands in for IAM in the end-to-end test: the
|
||||
// account under test is always TERMINATING.
|
||||
type alwaysTerminatingChecker struct{}
|
||||
|
||||
func (alwaysTerminatingChecker) AccountTerminating(context.Context, string) (bool, error) {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// No subscriber on either subject makes the capability probe incomplete, so
|
||||
// no seal request is ever sent and the account stays TERMINATING with one
|
||||
// stuck ec2-account-settings item.
|
||||
func TestSealReaperNoResponderKeepsAccountTerminating(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
|
||||
capability := newSealCapabilityClient(nc, []string{"node-1"})
|
||||
capability.wait = 50 * time.Millisecond
|
||||
capability.grace = 10 * time.Millisecond
|
||||
sealer := newEC2AccountSettingsSealClient(nc)
|
||||
sealer.timeout = 50 * time.Millisecond
|
||||
|
||||
reaper := &accountSettingsSealReaper{sealer: sealer, capability: capability}
|
||||
accounts := newFakeAccounts()
|
||||
engine := quietEngine(accounts, reaper)
|
||||
|
||||
result, err := engine.Teardown(testCtx(t), request())
|
||||
require.ErrorIs(t, err, ErrResourcesStuck)
|
||||
assert.False(t, result.AccountDeleted)
|
||||
assert.Equal(t, AccountStatusTerminating, accounts.account.Status)
|
||||
|
||||
var stuck []Stuck
|
||||
for _, stage := range result.Stages {
|
||||
stuck = append(stuck, stage.Stuck...)
|
||||
}
|
||||
require.Len(t, stuck, 1)
|
||||
assert.Equal(t, "ec2-account-settings", stuck[0].Resource.Kind)
|
||||
}
|
||||
|
||||
// Whatever the owner refuses with, the account stays TERMINATING: an owner
|
||||
// refusal, a timeout, and each reply shape the client rejects (wrong account,
|
||||
// wrong disposition, no revision).
|
||||
func TestSealReaperOwnerFailureKeepsAccountTerminating(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
err error
|
||||
}{
|
||||
{"incorrect-state", errors.New(awserrors.ErrorIncorrectState)},
|
||||
{"internal-error", errors.New(awserrors.ErrorInternalError)},
|
||||
{"timeout", context.DeadlineExceeded},
|
||||
{"wrong-account", errors.New(`seal account settings: reply named account "999999999999", not "000000000042"`)},
|
||||
{"wrong-disposition", errors.New(`seal account settings: unexpected disposition "pending"`)},
|
||||
{"zero-revision", errors.New("seal account settings: reply carried no revision")},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
sealer := &fakeSealer{err: tc.err}
|
||||
reaper := &accountSettingsSealReaper{sealer: sealer, capability: &fakeSealCapability{report: completeReport("node-1")}}
|
||||
accounts := newFakeAccounts()
|
||||
engine := quietEngine(accounts, reaper)
|
||||
|
||||
result, err := engine.Teardown(testCtx(t), request())
|
||||
require.ErrorIs(t, err, ErrResourcesStuck)
|
||||
assert.False(t, result.AccountDeleted)
|
||||
assert.Equal(t, AccountStatusTerminating, accounts.account.Status)
|
||||
assert.NotEmpty(t, sealer.calls)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The reaper retries through the engine's own drain mechanics: it fails
|
||||
// twice, then succeeds, and only the success lets the account be deleted.
|
||||
// List is non-empty before the success and empty only after it.
|
||||
func TestSealReaperCompletesOnlyAfterSealed(t *testing.T) {
|
||||
sealer := &fakeSealer{failFor: 2}
|
||||
reaper := &accountSettingsSealReaper{sealer: sealer, capability: &fakeSealCapability{report: completeReport("node-1")}}
|
||||
accounts := newFakeAccounts()
|
||||
engine := quietEngine(accounts, reaper)
|
||||
|
||||
before, err := reaper.List(testCtx(t), testAccountID)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, before, 1)
|
||||
|
||||
result, err := engine.Teardown(testCtx(t), request())
|
||||
require.NoError(t, err)
|
||||
assert.True(t, result.AccountDeleted)
|
||||
assert.GreaterOrEqual(t, len(sealer.calls), 3)
|
||||
|
||||
after, err := reaper.List(testCtx(t), testAccountID)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, after)
|
||||
}
|
||||
|
||||
// Each teardown builds a fresh engine and a fresh reaper, so a retried or
|
||||
// taken-over run seals again: two engines against the same owner both
|
||||
// succeed, each with a higher revision than the last.
|
||||
func TestSealReaperFreshRunSealsAgain(t *testing.T) {
|
||||
sealer := &fakeSealer{}
|
||||
capability := &fakeSealCapability{report: completeReport("node-1")}
|
||||
|
||||
firstEngine := quietEngine(newFakeAccounts(), &accountSettingsSealReaper{sealer: sealer, capability: capability})
|
||||
firstResult, err := firstEngine.Teardown(testCtx(t), request())
|
||||
require.NoError(t, err)
|
||||
assert.True(t, firstResult.AccountDeleted)
|
||||
|
||||
secondEngine := quietEngine(newFakeAccounts(), &accountSettingsSealReaper{sealer: sealer, capability: capability})
|
||||
secondResult, err := secondEngine.Teardown(testCtx(t), request())
|
||||
require.NoError(t, err)
|
||||
assert.True(t, secondResult.AccountDeleted)
|
||||
|
||||
require.Len(t, sealer.calls, 2)
|
||||
assert.Equal(t, uint64(2), sealer.revision)
|
||||
}
|
||||
|
||||
// Inventory and the explicit dry run, and a Teardown request marked dry run,
|
||||
// all list the pending seal and send no seal request: only a real teardown
|
||||
// may seal.
|
||||
func TestSealReaperInventoryDoesNotSeal(t *testing.T) {
|
||||
sealer := &fakeSealer{}
|
||||
capability := &fakeSealCapability{report: completeReport("node-1")}
|
||||
reaper := &accountSettingsSealReaper{sealer: sealer, capability: capability}
|
||||
accounts := newFakeAccounts()
|
||||
engine := quietEngine(accounts, reaper)
|
||||
|
||||
inventory, err := engine.Inventory(testCtx(t), testAccountID)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, sealer.calls)
|
||||
assert.Zero(t, capability.calls)
|
||||
found := false
|
||||
for _, stage := range inventory.Stages {
|
||||
for _, resource := range stage.Deleted {
|
||||
if resource.Kind == "ec2-account-settings" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
}
|
||||
assert.True(t, found, "inventory must still list the pending seal")
|
||||
|
||||
dryRun := request()
|
||||
dryRun.DryRun = true
|
||||
_, err = engine.Teardown(testCtx(t), dryRun)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, sealer.calls)
|
||||
assert.Zero(t, capability.calls)
|
||||
}
|
||||
|
||||
// A real settings service subscribed on the real seal subject, behind a real
|
||||
// engine: teardown seals the record, replacing the characterisation test that
|
||||
// showed it used to survive.
|
||||
func TestTeardownSealsAccountSettings(t *testing.T) {
|
||||
_, nc, js := testutil.StartTestJetStream(t)
|
||||
ctx := testCtx(t)
|
||||
|
||||
settings, err := ec2account.NewAccountSettingsServiceImplWithNATS(ctx, nil, nc)
|
||||
require.NoError(t, err)
|
||||
settings.SetAccountTerminationChecker(alwaysTerminatingChecker{})
|
||||
_, err = settings.EnableEbsEncryptionByDefault(ctx, &ec2.EnableEbsEncryptionByDefaultInput{}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
sealSub, err := nc.QueueSubscribe(ec2v1.SealAccountSettingsSubject, "spinifex-workers", func(msg *nats.Msg) {
|
||||
natsmsg.ServeNATSRequestCtx(msg, func(ctx context.Context, req *ec2v1.SealAccountSettingsRequest) (*ec2v1.SealAccountSettingsReply, error) {
|
||||
seal, err := settings.SealAccount(ctx, req.AccountID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &ec2v1.SealAccountSettingsReply{
|
||||
AccountID: req.AccountID, Disposition: ec2v1.AccountSettingsDispositionSealed, Revision: seal.Revision,
|
||||
}, nil
|
||||
})
|
||||
})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = sealSub.Unsubscribe() })
|
||||
|
||||
capSub, err := nc.Subscribe(ec2v1.AccountSettingsSealCapabilitySubject("node-1"), func(msg *nats.Msg) {
|
||||
reply, err := json.Marshal(ec2v1.AccountSettingsSealCapabilityReply{Node: "node-1", CapabilityVersion: 1})
|
||||
require.NoError(t, err)
|
||||
out := nats.NewMsg("")
|
||||
out.Data = reply
|
||||
out.Header.Set(natsmsg.NodeIDHeader, "node-1")
|
||||
require.NoError(t, msg.RespondMsg(out))
|
||||
})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = capSub.Unsubscribe() })
|
||||
require.NoError(t, nc.Flush())
|
||||
|
||||
accounts := newFakeAccounts()
|
||||
engine := quietEngine(accounts, EC2AccountSettingsReapers(nc, []string{"node-1"})...)
|
||||
// The real capability probe holds CollectUntilDeadline's full ResponderGrace
|
||||
// (150 ms) even when the one expected node answers instantly, so the quiet
|
||||
// engine's default 200 ms StageDrain leaves no room for the sweep after it.
|
||||
engine.Timeouts = Timeouts{StageDrain: 2 * time.Second, DrainPoll: 10 * time.Millisecond}
|
||||
|
||||
result, err := engine.Teardown(ctx, request())
|
||||
require.NoError(t, err)
|
||||
assert.True(t, result.AccountDeleted)
|
||||
|
||||
bucket, err := js.KeyValue(ctx, ec2account.KVBucketAccountSettings)
|
||||
require.NoError(t, err)
|
||||
entry, err := bucket.Get(ctx, testAccountID)
|
||||
require.NoError(t, err)
|
||||
var rec ec2account.AccountSettingsRecord
|
||||
require.NoError(t, json.Unmarshal(entry.Value(), &rec))
|
||||
assert.Equal(t, ec2account.AccountSettingsStateSealed, rec.State)
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
package accountteardown
|
||||
|
||||
//test:in-package — drives the real final-stage reapers against the real
|
||||
// EC2 account-settings service to pin that teardown leaves the bucket alone.
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/mulgadc/spinifex/internal/testkit"
|
||||
ec2account "github.com/mulgadc/spinifex/spinifex/domains/ec2/account"
|
||||
handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// No reaper in this package references the EC2 account-settings bucket or
|
||||
// calls a settings delete. Running the real engine and the real StageAccount
|
||||
// reapers against an account with settings enabled shows the consequence:
|
||||
// the settings record outlives the account record it belongs to.
|
||||
func TestCurrentBehaviour_AccountTermination_EC2AccountSettingsSurviveTeardown(t *testing.T) {
|
||||
_, nc, js := testutil.StartTestJetStream(t)
|
||||
ctx := testCtx(t)
|
||||
|
||||
settings, err := ec2account.NewAccountSettingsServiceImplWithNATS(ctx, nil, nc)
|
||||
require.NoError(t, err)
|
||||
_, err = settings.EnableEbsEncryptionByDefault(ctx, &ec2.EnableEbsEncryptionByDefaultInput{}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
_, err = settings.EnableSerialConsoleAccess(ctx, &ec2.EnableSerialConsoleAccessInput{}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
accounts := newFakeAccounts()
|
||||
iam := &accountIAM{account: &handlers_iam.Account{AccountID: testAccountID, AccountName: testAccountName}}
|
||||
engine := quietEngine(accounts, AccountReapers(iam, nil, nil)...)
|
||||
|
||||
result, err := engine.Teardown(ctx, request())
|
||||
require.NoError(t, err)
|
||||
assert.True(t, result.AccountDeleted)
|
||||
assert.True(t, accounts.deleted)
|
||||
assert.Zero(t, result.StuckCount())
|
||||
|
||||
bucket, err := js.KeyValue(ctx, ec2account.KVBucketAccountSettings)
|
||||
require.NoError(t, err)
|
||||
_, err = bucket.Get(ctx, testAccountID)
|
||||
require.NoError(t, err, "the settings key must still be present after the account is torn down")
|
||||
|
||||
out, err := settings.GetEbsEncryptionByDefault(ctx, &ec2.GetEbsEncryptionByDefaultInput{}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, *out.EbsEncryptionByDefault)
|
||||
}
|
||||
@@ -1,89 +0,0 @@
|
||||
package accountteardown
|
||||
|
||||
//test:in-package — pins ADR-0003:S4 and ADR-0001 Context item 2 for the EC2
|
||||
// settings seal: the legacy engine must reach the owner only through the
|
||||
// versioned contract and NATS, never by reading the settings bucket itself.
|
||||
|
||||
import (
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"io/fs"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// forbiddenSettingsImports are imports that would let this package reach the
|
||||
// EC2 settings owner or the configured node set by a path other than the
|
||||
// versioned contract and NewClusterEngine's parameters.
|
||||
var forbiddenSettingsImports = []string{
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/ec2/account",
|
||||
"github.com/mulgadc/spinifex/spinifex/bootstrap/config",
|
||||
}
|
||||
|
||||
// forbiddenSettingsIdentifiers would let this package name the settings
|
||||
// bucket or its key directly instead of reaching it only through
|
||||
// contracts/ec2/v1 and NATS.
|
||||
var forbiddenSettingsIdentifiers = []string{"spinifex-ec2-account-settings", "KVBucketAccountSettings"}
|
||||
|
||||
// Teardown reaches EC2 settings only over contracts/ec2/v1: no import of the
|
||||
// settings owner or bootstrap/config (nodes arrive via NewClusterEngine), and
|
||||
// no literal naming the settings bucket or its constant.
|
||||
func TestAccountTeardownHasNoSettingsKVAccess(t *testing.T) {
|
||||
_, file, _, ok := runtime.Caller(0)
|
||||
if !ok {
|
||||
t.Fatal("cannot locate test source")
|
||||
}
|
||||
root := filepath.Dir(file)
|
||||
|
||||
fset := token.NewFileSet()
|
||||
scanned := 0
|
||||
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil || d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {
|
||||
return err
|
||||
}
|
||||
f, perr := parser.ParseFile(fset, path, nil, parser.ParseComments)
|
||||
if perr != nil {
|
||||
return perr
|
||||
}
|
||||
scanned++
|
||||
|
||||
for _, imp := range f.Imports {
|
||||
p, _ := strconv.Unquote(imp.Path.Value)
|
||||
for _, bad := range forbiddenSettingsImports {
|
||||
if p == bad {
|
||||
t.Errorf("%s: imports %s; the legacy engine must reach the EC2 settings owner only through contracts/ec2/v1 and the configured node set NewClusterEngine already receives",
|
||||
fset.Position(imp.Pos()), p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ast.Inspect(f, func(n ast.Node) bool {
|
||||
lit, ok := n.(*ast.BasicLit)
|
||||
if !ok || lit.Kind != token.STRING {
|
||||
return true
|
||||
}
|
||||
value, uerr := strconv.Unquote(lit.Value)
|
||||
if uerr != nil {
|
||||
return true
|
||||
}
|
||||
for _, bad := range forbiddenSettingsIdentifiers {
|
||||
if value == bad {
|
||||
t.Errorf("%s: names %q; the settings bucket and its key are the owner's own, never this package's",
|
||||
fset.Position(lit.Pos()), value)
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if scanned == 0 {
|
||||
t.Fatalf("scanned no Go files under %s; the invariant would pass vacuously", root)
|
||||
}
|
||||
}
|
||||
@@ -1289,15 +1289,6 @@ func (d *Daemon) subscribeAll() error {
|
||||
natsSub{viperblocklegacyv1.VolumeFencedSubject(d.node), d.handleVolumeFenced, ""},
|
||||
)
|
||||
|
||||
// Per-node, no queue group: a second daemon claiming this node ID also
|
||||
// answers, so the teardown probe sees a duplicate and refuses to seal. An
|
||||
// invalid node ID serves nothing, so this daemon reads as incompatible.
|
||||
if ec2v1.ValidSealCapabilityNode(d.node) {
|
||||
subs = append(subs, natsSub{ec2v1.AccountSettingsSealCapabilitySubject(d.node), handleNATSRequest(d.node, d.accountSettingsSealCapability), ""})
|
||||
} else {
|
||||
slog.Error("Not serving account-settings seal capability: node ID is not a single NATS subject token", "node", d.node)
|
||||
}
|
||||
|
||||
return d.registerNatsSubs(subs)
|
||||
}
|
||||
|
||||
|
||||
@@ -1,23 +0,0 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
|
||||
awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
)
|
||||
|
||||
// accountSettingsSealCapability serves this daemon's read-only seal capability.
|
||||
// It reads no KV and takes no account; a request naming another node is a
|
||||
// caller mistake, since the subject is already addressed to this node.
|
||||
func (d *Daemon) accountSettingsSealCapability(_ context.Context, req *ec2v1.AccountSettingsSealCapabilityRequest, _ string) (*ec2v1.AccountSettingsSealCapabilityReply, error) {
|
||||
if req.Node != d.node {
|
||||
return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue,
|
||||
"seal capability request for node %q answered by node %q", req.Node, d.node)
|
||||
}
|
||||
return &ec2v1.AccountSettingsSealCapabilityReply{
|
||||
Node: d.node,
|
||||
CapabilityVersion: ec2v1.AccountSettingsSealCapabilityVersion,
|
||||
CorrelationRef: req.CorrelationRef,
|
||||
}, nil
|
||||
}
|
||||
@@ -1,115 +0,0 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
|
||||
"github.com/mulgadc/spinifex/internal/testkit"
|
||||
awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// newCapabilityTestDaemon builds a daemon serving its real subscription table
|
||||
// with node as its configured node ID.
|
||||
func newCapabilityTestDaemon(t *testing.T, node string) (*nats.Conn, *nats.Conn) {
|
||||
t.Helper()
|
||||
ns, nc, _ := testutil.StartTestJetStream(t)
|
||||
|
||||
d := newSubscribeTestDaemon(t, nc, "", "")
|
||||
d.node = node
|
||||
require.NoError(t, d.subscribeAll())
|
||||
require.NoError(t, nc.Flush())
|
||||
|
||||
caller, err := nats.Connect(ns.ClientURL())
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(caller.Close)
|
||||
return nc, caller
|
||||
}
|
||||
|
||||
func capabilityRequest(t *testing.T, nc *nats.Conn, node string) *nats.Msg {
|
||||
t.Helper()
|
||||
data, err := json.Marshal(ec2v1.AccountSettingsSealCapabilityRequest{Node: node, CorrelationRef: "req-1"})
|
||||
require.NoError(t, err)
|
||||
reply, err := nc.Request(ec2v1.AccountSettingsSealCapabilitySubject(node), data, 5*time.Second)
|
||||
require.NoError(t, err)
|
||||
return reply
|
||||
}
|
||||
|
||||
// The responder answers its own node with version 1, the configured node and
|
||||
// no KV access (no settings service is wired into the test daemon at all).
|
||||
func TestAccountSettingsSealCapabilityAnswersOwnNode(t *testing.T) {
|
||||
_, caller := newCapabilityTestDaemon(t, "node-1")
|
||||
|
||||
raw := capabilityRequest(t, caller, "node-1")
|
||||
assert.Equal(t, "node-1", raw.Header.Get("X-Node-ID"))
|
||||
|
||||
var reply ec2v1.AccountSettingsSealCapabilityReply
|
||||
require.NoError(t, json.Unmarshal(raw.Data, &reply))
|
||||
assert.Equal(t, "node-1", reply.Node)
|
||||
assert.Equal(t, ec2v1.AccountSettingsSealCapabilityVersion, reply.CapabilityVersion)
|
||||
assert.Equal(t, "req-1", reply.CorrelationRef)
|
||||
}
|
||||
|
||||
// A request naming a node other than this daemon's own is refused, not
|
||||
// answered as if it were this node.
|
||||
func TestAccountSettingsSealCapabilityRefusesOtherNode(t *testing.T) {
|
||||
_, caller := newCapabilityTestDaemon(t, "node-1")
|
||||
|
||||
data, err := json.Marshal(ec2v1.AccountSettingsSealCapabilityRequest{Node: "node-2"})
|
||||
require.NoError(t, err)
|
||||
raw, err := caller.Request(ec2v1.AccountSettingsSealCapabilitySubject("node-1"), data, 5*time.Second)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, awserrors.ErrorInvalidParameterValue, replyErrCode(t, raw.Data))
|
||||
}
|
||||
|
||||
// The subject is per node with no queue group, so two daemons configured
|
||||
// with the same node ID both answer a request addressed to that node.
|
||||
func TestAccountSettingsSealCapabilityHasNoQueueGroup(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
|
||||
first := newSubscribeTestDaemon(t, nc, "", "")
|
||||
first.node = "node-1"
|
||||
require.NoError(t, first.subscribeAll())
|
||||
|
||||
second := newSubscribeTestDaemon(t, nc, "", "")
|
||||
second.node = "node-1"
|
||||
require.NoError(t, second.subscribeAll())
|
||||
require.NoError(t, nc.Flush())
|
||||
|
||||
sub, err := nc.SubscribeSync(nats.NewInbox())
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
|
||||
data, err := json.Marshal(ec2v1.AccountSettingsSealCapabilityRequest{Node: "node-1"})
|
||||
require.NoError(t, err)
|
||||
msg := nats.NewMsg(ec2v1.AccountSettingsSealCapabilitySubject("node-1"))
|
||||
msg.Data = data
|
||||
msg.Reply = sub.Subject
|
||||
require.NoError(t, nc.PublishMsg(msg))
|
||||
require.NoError(t, nc.Flush())
|
||||
|
||||
seen := 0
|
||||
for {
|
||||
_, err := sub.NextMsg(300 * time.Millisecond)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
seen++
|
||||
}
|
||||
assert.Equal(t, 2, seen)
|
||||
}
|
||||
|
||||
// A wildcard node ID must not become a wildcard subscription: built unchecked,
|
||||
// "node-*" would answer probes addressed to node-1 on node-1's behalf.
|
||||
func TestAccountSettingsSealCapabilityNotServedForInvalidNodeID(t *testing.T) {
|
||||
_, caller := newCapabilityTestDaemon(t, "node-*")
|
||||
|
||||
data, err := json.Marshal(ec2v1.AccountSettingsSealCapabilityRequest{Node: "node-1"})
|
||||
require.NoError(t, err)
|
||||
_, err = caller.Request(ec2v1.AccountSettingsSealCapabilitySubject("node-1"), data, time.Second)
|
||||
require.ErrorIs(t, err, nats.ErrNoResponders)
|
||||
}
|
||||
@@ -1,68 +0,0 @@
|
||||
package account
|
||||
|
||||
//test:in-package — pins what AccountSettingsSealCapabilityVersion 1 asserts:
|
||||
// nothing in this package writes the settings KV except through the
|
||||
// revision-guarded helpers.
|
||||
|
||||
import (
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"io/fs"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func accountPackageDir(t *testing.T) string {
|
||||
t.Helper()
|
||||
_, file, _, ok := runtime.Caller(0)
|
||||
if !ok {
|
||||
t.Fatal("cannot locate test source")
|
||||
}
|
||||
return filepath.Dir(file)
|
||||
}
|
||||
|
||||
// The capability daemons report as version 1 is a claim about every writer in
|
||||
// the binary, not just the responder: no non-test file in this package may
|
||||
// call Put or PutString, which bypass the revision-guarded mutate/seal paths.
|
||||
func TestAccountSettingsWritesAreRevisionGuarded(t *testing.T) {
|
||||
root := accountPackageDir(t)
|
||||
fset := token.NewFileSet()
|
||||
scanned := 0
|
||||
|
||||
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil || d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {
|
||||
return err
|
||||
}
|
||||
f, perr := parser.ParseFile(fset, path, nil, 0)
|
||||
if perr != nil {
|
||||
return perr
|
||||
}
|
||||
scanned++
|
||||
|
||||
ast.Inspect(f, func(n ast.Node) bool {
|
||||
call, ok := n.(*ast.CallExpr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
sel, ok := call.Fun.(*ast.SelectorExpr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
if sel.Sel.Name == "Put" || sel.Sel.Name == "PutString" {
|
||||
t.Errorf("%s: unconditional %s call bypasses the revision-guarded settings write; use mutateSettings or SealAccount's CAS loop instead",
|
||||
fset.Position(call.Pos()), sel.Sel.Name)
|
||||
}
|
||||
return true
|
||||
})
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if scanned == 0 {
|
||||
t.Fatalf("scanned no Go files under %s; the invariant would pass vacuously", root)
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package instance
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
|
||||
"log/slog"
|
||||
@@ -100,10 +101,10 @@ func DescribeInstanceTypes(ctx context.Context, input *ec2.DescribeInstanceTypes
|
||||
|
||||
if missing := missingInstanceTypes(finalInstanceTypes, input.InstanceTypes); len(missing) > 0 {
|
||||
if !proveAbsence || !everyNodeAnswered(nodeIDs, answered, sum) {
|
||||
return nil, awserrors.WithRetryAfter(
|
||||
return nil, errors.Join(
|
||||
awserrors.Errorf(awserrors.ErrorServiceUnavailable,
|
||||
"cannot confirm instance type(s) %s: describe fan-out did not complete", strings.Join(missing, ", ")),
|
||||
describeIncompleteRetryAfter,
|
||||
awserrors.RetryAfter(awserrors.ErrorServiceUnavailable, describeIncompleteRetryAfter),
|
||||
)
|
||||
}
|
||||
return nil, awserrors.Errorf(awserrors.ErrorInvalidInstanceType,
|
||||
|
||||
@@ -126,10 +126,10 @@ func DescribeInstancesChecked(ctx context.Context, input *ec2.DescribeInstancesI
|
||||
// An incomplete sweep cannot prove absence, so a missing id is
|
||||
// unresolved, not gone. NotFound would be a false negative and an
|
||||
// empty success would look identical to a real one, so return 503.
|
||||
return nil, awserrors.WithRetryAfter(
|
||||
return nil, errors.Join(
|
||||
awserrors.Errorf(awserrors.ErrorServiceUnavailable,
|
||||
"cannot confirm status of instance(s) %s: describe fan-out did not complete", strings.Join(missing, ", ")),
|
||||
describeIncompleteRetryAfter,
|
||||
awserrors.RetryAfter(awserrors.ErrorServiceUnavailable, describeIncompleteRetryAfter),
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -198,10 +198,10 @@ func DescribeAccountInstancesComplete(ctx context.Context, natsConn *nats.Conn,
|
||||
}
|
||||
|
||||
func incompleteListingError() error {
|
||||
return awserrors.WithRetryAfter(
|
||||
return errors.Join(
|
||||
awserrors.Errorf(awserrors.ErrorServiceUnavailable,
|
||||
"cannot list instances: not every node and instance store answered"),
|
||||
describeIncompleteRetryAfter,
|
||||
awserrors.RetryAfter(awserrors.ErrorServiceUnavailable, describeIncompleteRetryAfter),
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -658,22 +658,13 @@ func HasDependencies(kind, id string) error {
|
||||
// retryableError wraps a registered code with a suggested Retry-After
|
||||
// duration, for a 503 that a client can expect to clear on its own within a
|
||||
// bounded window (e.g. a warm-up race) rather than an open-ended outage.
|
||||
// Error() returns only the code, or the wrapped error's text, so code
|
||||
// resolution is unaffected.
|
||||
// Error() returns only the code, so code resolution is unaffected.
|
||||
type retryableError struct {
|
||||
code string
|
||||
err error
|
||||
retryAfter time.Duration
|
||||
}
|
||||
|
||||
func (e *retryableError) Error() string {
|
||||
if e.err != nil {
|
||||
return e.err.Error()
|
||||
}
|
||||
return e.code
|
||||
}
|
||||
|
||||
func (e *retryableError) Unwrap() error { return e.err }
|
||||
func (e *retryableError) Error() string { return e.code }
|
||||
|
||||
// RetryAfter wraps code (a registered ErrorLookup key) with a suggested
|
||||
// Retry-After duration for ErrorHandler to surface as a response header.
|
||||
@@ -681,13 +672,6 @@ func RetryAfter(code string, d time.Duration) error {
|
||||
return &retryableError{code: code, retryAfter: d}
|
||||
}
|
||||
|
||||
// WithRetryAfter attaches a suggested Retry-After duration to err without
|
||||
// changing its text, unlike joining err with RetryAfter, which appends the
|
||||
// bare code on a second line.
|
||||
func WithRetryAfter(err error, d time.Duration) error {
|
||||
return &retryableError{err: err, retryAfter: d}
|
||||
}
|
||||
|
||||
// ResolveRetryAfter returns the Retry-After duration attached to err via
|
||||
// RetryAfter, if any, walking the same unwrap tree ResolveErrorDetail does.
|
||||
func ResolveRetryAfter(err error) (time.Duration, bool) {
|
||||
|
||||
@@ -176,24 +176,6 @@ func TestResolveRetryAfter_WrappedAndResolvable(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// WithRetryAfter keeps the coded error's text and message while still
|
||||
// carrying the hint: a joined RetryAfter would log the code twice.
|
||||
func TestWithRetryAfter_KeepsTextAndResolves(t *testing.T) {
|
||||
inner := Errorf(ErrorServiceUnavailable, "probe incomplete")
|
||||
err := WithRetryAfter(inner, 5*time.Second)
|
||||
|
||||
if err.Error() != inner.Error() {
|
||||
t.Fatalf("Error() = %q, want %q", err.Error(), inner.Error())
|
||||
}
|
||||
if d, ok := ResolveRetryAfter(fmt.Errorf("wrapped: %w", err)); !ok || d != 5*time.Second {
|
||||
t.Fatalf("ResolveRetryAfter() = (%v, %v), want (5s, true)", d, ok)
|
||||
}
|
||||
code, msg, ok := ResolveErrorDetail(err)
|
||||
if !ok || code != ErrorServiceUnavailable || msg != "probe incomplete" {
|
||||
t.Fatalf("ResolveErrorDetail() = (%q, %q, %v), want (%q, %q, true)", code, msg, ok, ErrorServiceUnavailable, "probe incomplete")
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolveRetryAfter_PlainErrorNotFound is the compatibility case: an
|
||||
// ordinary errors.New(code) -- the overwhelming majority of call sites --
|
||||
// carries no Retry-After hint.
|
||||
|
||||
@@ -12,7 +12,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
|
||||
"github.com/mulgadc/spinifex/internal/testkit"
|
||||
"github.com/mulgadc/spinifex/spinifex/accountteardown"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
@@ -32,10 +31,6 @@ const deleteClientToken = "8b9c1d2e3f4a5b6c7d8e9f0a1b2c3d4e"
|
||||
// to keep step with the reapers NewClusterEngine wires.
|
||||
var teardownSubjects = []string{"ec2.>", "ecs.>", "eks.>", "rds.>", "elbv2.>", "acm.>", "bedrock.>"}
|
||||
|
||||
// sealCapabilitySubjectPrefix matches every per-node seal-capability subject,
|
||||
// which the generic ec2.> stub below must not also answer.
|
||||
const sealCapabilitySubjectPrefix = "ec2.v1.AccountSettingsSealCapability."
|
||||
|
||||
// newDeleteAccountGateway wires a gateway against a real IAM service and
|
||||
// embedded JetStream, with a stub cluster answering the subjects the reapers
|
||||
// drive. The tenant has no resources, so every stage drains empty and the test
|
||||
@@ -51,12 +46,8 @@ func newDeleteAccountGateway(t *testing.T) *GatewayConfig {
|
||||
|
||||
for _, subject := range teardownSubjects {
|
||||
// Replies carry a node ID: the instance inventory only counts a frame
|
||||
// it can attribute to a configured node. The seal subjects have their
|
||||
// own responders below, which a second reply here would race.
|
||||
// it can attribute to a configured node.
|
||||
sub, err := nc.Subscribe(subject, func(msg *nats.Msg) {
|
||||
if msg.Subject == ec2v1.SealAccountSettingsSubject || strings.HasPrefix(msg.Subject, sealCapabilitySubjectPrefix) {
|
||||
return
|
||||
}
|
||||
reply := nats.NewMsg(msg.Reply)
|
||||
reply.Data = []byte(`{}`)
|
||||
reply.Header.Set(natsmsg.NodeIDHeader, "node-1")
|
||||
@@ -65,38 +56,6 @@ func newDeleteAccountGateway(t *testing.T) *GatewayConfig {
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
}
|
||||
|
||||
// The seal reaper needs a real sealed disposition and revision, and every
|
||||
// configured node answering capability version 1, or it never finalises:
|
||||
// the generic `{}` stub above is neither.
|
||||
sealSub, err := nc.Subscribe(ec2v1.SealAccountSettingsSubject, func(msg *nats.Msg) {
|
||||
var req ec2v1.SealAccountSettingsRequest
|
||||
if json.Unmarshal(msg.Data, &req) != nil {
|
||||
return
|
||||
}
|
||||
data, _ := json.Marshal(ec2v1.SealAccountSettingsReply{
|
||||
AccountID: req.AccountID, CorrelationRef: req.CorrelationRef,
|
||||
Disposition: ec2v1.AccountSettingsDispositionSealed, Revision: 1,
|
||||
})
|
||||
reply := nats.NewMsg(msg.Reply)
|
||||
reply.Data = data
|
||||
reply.Header.Set(natsmsg.NodeIDHeader, "node-1")
|
||||
_ = msg.RespondMsg(reply)
|
||||
})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = sealSub.Unsubscribe() })
|
||||
|
||||
for _, node := range []string{"node-1"} {
|
||||
capSub, err := nc.Subscribe(ec2v1.AccountSettingsSealCapabilitySubject(node), func(msg *nats.Msg) {
|
||||
data, _ := json.Marshal(ec2v1.AccountSettingsSealCapabilityReply{Node: node, CapabilityVersion: ec2v1.AccountSettingsSealCapabilityVersion})
|
||||
reply := nats.NewMsg(msg.Reply)
|
||||
reply.Data = data
|
||||
reply.Header.Set(natsmsg.NodeIDHeader, node)
|
||||
_ = msg.RespondMsg(reply)
|
||||
})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = capSub.Unsubscribe() })
|
||||
}
|
||||
require.NoError(t, nc.Flush())
|
||||
|
||||
return &GatewayConfig{
|
||||
|
||||
@@ -71,7 +71,12 @@ func ImportImage(ctx context.Context, provider ebsprovider.EBSProvider, opts Imp
|
||||
return fmt.Errorf("import: publish volume %s: %w", opts.VolumeID, err)
|
||||
}
|
||||
|
||||
writeErr := writeImage(ctx, opts.SourcePath, published.NBDURI, opts.Progress)
|
||||
var writeErr error
|
||||
if published.Export.Kind == ebsprovider.ExportKindNBD && published.Export.NBD != nil {
|
||||
writeErr = writeImage(ctx, opts.SourcePath, published.Export.NBD.NBDURI, opts.Progress)
|
||||
} else {
|
||||
writeErr = fmt.Errorf("volume is published as a %q export; image import writes only through nbd", published.Export.Kind)
|
||||
}
|
||||
|
||||
// Unpublish regardless: a volume left published cannot be snapshotted or
|
||||
// attached, and on the failure path it would strand the export too.
|
||||
|
||||
@@ -22,6 +22,10 @@ type recordingProvider struct {
|
||||
publishes []ebsprovider.PublishVolumeRequest
|
||||
snapshots []ebsprovider.CreateSnapshotRequest
|
||||
publishURI string
|
||||
|
||||
// publishExport, when its Kind is set, replaces the nbd export built
|
||||
// from publishURI.
|
||||
publishExport ebsprovider.VolumeExport
|
||||
}
|
||||
|
||||
func newRecordingProvider() *recordingProvider {
|
||||
@@ -39,7 +43,11 @@ func (p *recordingProvider) PublishVolume(ctx context.Context, req ebsprovider.P
|
||||
if _, err := p.MemoryProvider.PublishVolume(ctx, req); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &ebsprovider.PublishedVolume{VolumeID: req.VolumeID, NodeID: req.NodeID, NBDURI: p.publishURI}, nil
|
||||
export := ebsprovider.NewNBDExport(p.publishURI)
|
||||
if p.publishExport.Kind != "" {
|
||||
export = p.publishExport
|
||||
}
|
||||
return &ebsprovider.PublishedVolume{VolumeID: req.VolumeID, NodeID: req.NodeID, Export: export}, nil
|
||||
}
|
||||
|
||||
func (p *recordingProvider) UnpublishVolume(ctx context.Context, req ebsprovider.UnpublishVolumeRequest) error {
|
||||
@@ -117,6 +125,32 @@ func TestImportImage_WriteFailureStillUnpublishes(t *testing.T) {
|
||||
"a failed copy must not be snapshotted, and must still be unpublished")
|
||||
}
|
||||
|
||||
// TestImportImage_BlockExportIsRefused covers a provider that publishes a host
|
||||
// block device: the import writes only through NBD, so it must fail without
|
||||
// copying and still release the export.
|
||||
func TestImportImage_BlockExportIsRefused(t *testing.T) {
|
||||
var wroteTo string
|
||||
stubWriteImage(t, nil, &wroteTo)
|
||||
provider := newRecordingProvider()
|
||||
provider.publishExport = ebsprovider.VolumeExport{
|
||||
Kind: ebsprovider.ExportKindBlock,
|
||||
Block: &ebsprovider.BlockExport{DevicePath: "/dev/sdx", SizeBytes: 1 << 30},
|
||||
}
|
||||
|
||||
err := ImportImage(context.Background(), provider, ImportOpts{
|
||||
VolumeID: "ami-import005",
|
||||
NodeID: "node-a",
|
||||
SizeBytes: 1 << 30,
|
||||
SourcePath: "/dev/null",
|
||||
Snapshot: true,
|
||||
})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), `"block" export`)
|
||||
assert.Empty(t, wroteTo, "a block export must not reach the NBD writer")
|
||||
assert.Equal(t, []string{"create", "publish", "unpublish"}, provider.calls,
|
||||
"a refused export must not be snapshotted, and must still be unpublished")
|
||||
}
|
||||
|
||||
// TestImportImage_NoSnapshotWhenNotAsked covers the weights-style import that
|
||||
// wants the volume but not an AMI snapshot.
|
||||
func TestImportImage_NoSnapshotWhenNotAsked(t *testing.T) {
|
||||
|
||||
@@ -791,7 +791,10 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
|
||||
require.NotNil(t, pub)
|
||||
assert.Equal(t, cfg.id("vol-pub-ok"), pub.VolumeID)
|
||||
assert.Equal(t, cfg.node(), pub.NodeID)
|
||||
assertNBDURI(t, pub.NBDURI)
|
||||
require.NoError(t, pub.Export.Validate(), "published export is not attachable")
|
||||
if pub.Export.Kind == ebsprovider.ExportKindNBD {
|
||||
assertNBDURI(t, pub.Export.NBD.NBDURI)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("idempotent republish to the same node", func(t *testing.T) {
|
||||
|
||||
@@ -167,9 +167,9 @@ func RunDataPathSuite(t *testing.T, newProvider func(t *testing.T) ebsprovider.E
|
||||
provider := newProvider(t)
|
||||
volumeID := cfg.VolumePrefix + "datapath"
|
||||
|
||||
pub := publishForNBD(t, provider, cfg, volumeID, false)
|
||||
assertNBDURI(t, pub.NBDURI)
|
||||
uri := dialURI(t, pub.NBDURI)
|
||||
nbdURI := publishForNBD(t, provider, cfg, volumeID, false)
|
||||
assertNBDURI(t, nbdURI)
|
||||
uri := dialURI(t, nbdURI)
|
||||
|
||||
export := nbdInfo(t, uri)
|
||||
require.Equalf(t, cfg.VolumeBytes, export.Size, "export size does not match the volume that was created")
|
||||
|
||||
@@ -177,9 +177,9 @@ func nbdReachable(uri string) bool {
|
||||
return exec.CommandContext(context.Background(), "nbdinfo", "--size", uri).Run() == nil
|
||||
}
|
||||
|
||||
// publishForNBD creates and publishes a volume, registering cleanup so a run
|
||||
// against a real cluster does not leave volumes behind.
|
||||
func publishForNBD(t *testing.T, provider ebsprovider.EBSProvider, cfg NBDClientConfig, volumeID string, readOnly bool) *ebsprovider.PublishedVolume {
|
||||
// publishForNBD creates and publishes a volume and returns its NBD URI,
|
||||
// registering cleanup so a run against a real cluster leaves nothing behind.
|
||||
func publishForNBD(t *testing.T, provider ebsprovider.EBSProvider, cfg NBDClientConfig, volumeID string, readOnly bool) string {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{
|
||||
@@ -198,7 +198,17 @@ func publishForNBD(t *testing.T, provider ebsprovider.EBSProvider, cfg NBDClient
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, pub)
|
||||
return pub
|
||||
return publishedNBDURI(t, pub)
|
||||
}
|
||||
|
||||
// publishedNBDURI fails the test outright when a provider publishes anything
|
||||
// but an nbd export: the NBD-dialling suites have nothing to dial otherwise.
|
||||
func publishedNBDURI(t *testing.T, pub *ebsprovider.PublishedVolume) string {
|
||||
t.Helper()
|
||||
require.NoErrorf(t, pub.Export.Validate(), "published export for %s is invalid", pub.VolumeID)
|
||||
require.Equalf(t, ebsprovider.ExportKindNBD, pub.Export.Kind,
|
||||
"published export for %s is %q; this suite drives only nbd exports", pub.VolumeID, pub.Export.Kind)
|
||||
return pub.Export.NBD.NBDURI
|
||||
}
|
||||
|
||||
// cleanupVolume unpublishes and deletes best-effort: cleanup runs after the
|
||||
@@ -234,21 +244,21 @@ func RunNBDClientSuiteWithConfig(t *testing.T, newProvider func(t *testing.T) eb
|
||||
|
||||
t.Run("published URI is a standard NBD URI", func(t *testing.T) {
|
||||
provider := newProvider(t)
|
||||
pub := publishForNBD(t, provider, cfg, cfg.VolumePrefix+"-uri", false)
|
||||
assertNBDURI(t, pub.NBDURI)
|
||||
nbdURI := publishForNBD(t, provider, cfg, cfg.VolumePrefix+"-uri", false)
|
||||
assertNBDURI(t, nbdURI)
|
||||
})
|
||||
|
||||
t.Run("export is connectable by a standard NBD client", func(t *testing.T) {
|
||||
provider := newProvider(t)
|
||||
pub := publishForNBD(t, provider, cfg, cfg.VolumePrefix+"-connect", false)
|
||||
export := nbdInfo(t, dialURI(t, pub.NBDURI))
|
||||
nbdURI := publishForNBD(t, provider, cfg, cfg.VolumePrefix+"-connect", false)
|
||||
export := nbdInfo(t, dialURI(t, nbdURI))
|
||||
assert.Equal(t, cfg.VolumeBytes, export.Size)
|
||||
})
|
||||
|
||||
t.Run("export advertises usable block size constraints", func(t *testing.T) {
|
||||
provider := newProvider(t)
|
||||
pub := publishForNBD(t, provider, cfg, cfg.VolumePrefix+"-blocksize", false)
|
||||
export := nbdInfo(t, dialURI(t, pub.NBDURI))
|
||||
nbdURI := publishForNBD(t, provider, cfg, cfg.VolumePrefix+"-blocksize", false)
|
||||
export := nbdInfo(t, dialURI(t, nbdURI))
|
||||
|
||||
require.Positivef(t, export.BlockSizeMinimum,
|
||||
"export advertises no minimum block size, so a client cannot know how to align its requests")
|
||||
@@ -261,8 +271,8 @@ func RunNBDClientSuiteWithConfig(t *testing.T, newProvider func(t *testing.T) eb
|
||||
// export. Publication is what holds the volume, not any one connection.
|
||||
t.Run("export accepts a client reconnecting after the first disconnects", func(t *testing.T) {
|
||||
provider := newProvider(t)
|
||||
pub := publishForNBD(t, provider, cfg, cfg.VolumePrefix+"-reconnect", false)
|
||||
uri := dialURI(t, pub.NBDURI)
|
||||
nbdURI := publishForNBD(t, provider, cfg, cfg.VolumePrefix+"-reconnect", false)
|
||||
uri := dialURI(t, nbdURI)
|
||||
|
||||
first := nbdInfo(t, uri)
|
||||
|
||||
@@ -275,7 +285,7 @@ func RunNBDClientSuiteWithConfig(t *testing.T, newProvider func(t *testing.T) eb
|
||||
|
||||
t.Run("data written by an external client reads back byte for byte", func(t *testing.T) {
|
||||
provider := newProvider(t)
|
||||
pub := publishForNBD(t, provider, cfg, cfg.VolumePrefix+"-roundtrip", false)
|
||||
nbdURI := publishForNBD(t, provider, cfg, cfg.VolumePrefix+"-roundtrip", false)
|
||||
|
||||
dir := t.TempDir()
|
||||
in := filepath.Join(dir, "in.bin")
|
||||
@@ -283,7 +293,7 @@ func RunNBDClientSuiteWithConfig(t *testing.T, newProvider func(t *testing.T) eb
|
||||
want := patternBytes(int(cfg.VolumeBytes))
|
||||
require.NoError(t, os.WriteFile(in, want, 0o600))
|
||||
|
||||
uri := dialURI(t, pub.NBDURI)
|
||||
uri := dialURI(t, nbdURI)
|
||||
runNBDCopy(t, in, uri)
|
||||
runNBDCopy(t, uri, out)
|
||||
|
||||
@@ -301,9 +311,9 @@ func RunNBDClientSuiteWithConfig(t *testing.T, newProvider func(t *testing.T) eb
|
||||
t.Run("sparse extent reporting matches the advertised capability", func(t *testing.T) {
|
||||
provider := newProvider(t)
|
||||
capabilities := capabilitiesOf(t, provider)
|
||||
pub := publishForNBD(t, provider, cfg, cfg.VolumePrefix+"-extents", false)
|
||||
nbdURI := publishForNBD(t, provider, cfg, cfg.VolumePrefix+"-extents", false)
|
||||
|
||||
holes := slices.ContainsFunc(nbdMap(t, dialURI(t, pub.NBDURI)), func(extent nbdExtent) bool {
|
||||
holes := slices.ContainsFunc(nbdMap(t, dialURI(t, nbdURI)), func(extent nbdExtent) bool {
|
||||
return strings.Contains(extent.Description, "hole")
|
||||
})
|
||||
assert.Equalf(t, capabilities.SparseExtentReporting, holes,
|
||||
@@ -334,16 +344,16 @@ func RunNBDClientSuiteWithConfig(t *testing.T, newProvider func(t *testing.T) eb
|
||||
return
|
||||
}
|
||||
|
||||
pub := publishForNBD(t, provider, cfg, cfg.VolumePrefix+"-readonly", true)
|
||||
export := nbdInfo(t, dialURI(t, pub.NBDURI))
|
||||
nbdURI := publishForNBD(t, provider, cfg, cfg.VolumePrefix+"-readonly", true)
|
||||
export := nbdInfo(t, dialURI(t, nbdURI))
|
||||
assert.True(t, export.ReadOnly, "a volume published ReadOnly must set the NBD read-only transmission flag")
|
||||
})
|
||||
|
||||
t.Run("unpublish tears the export down", func(t *testing.T) {
|
||||
provider := newProvider(t)
|
||||
volumeID := cfg.VolumePrefix + "-unpublish"
|
||||
pub := publishForNBD(t, provider, cfg, volumeID, false)
|
||||
uri := dialURI(t, pub.NBDURI)
|
||||
nbdURI := publishForNBD(t, provider, cfg, volumeID, false)
|
||||
uri := dialURI(t, nbdURI)
|
||||
require.True(t, nbdReachable(uri), "export should be reachable before unpublish")
|
||||
|
||||
require.NoError(t, provider.UnpublishVolume(context.Background(), ebsprovider.UnpublishVolumeRequest{
|
||||
|
||||
@@ -74,13 +74,24 @@ func corruptPublished(published *ebsprovider.PublishedVolume, flavour string) *e
|
||||
corrupted := *published
|
||||
switch flavour {
|
||||
case "handle", "capacity":
|
||||
corrupted.NBDURI = "nbd+unix:///?socket=/hostile/wrong.sock"
|
||||
corrupted.Export = corruptExport(published.Export)
|
||||
case "state":
|
||||
corrupted.NodeID = "hostile-wrong-node"
|
||||
}
|
||||
return &corrupted
|
||||
}
|
||||
|
||||
// corruptExport points a still-valid export at the wrong place, building new
|
||||
// variants so the inner provider's own export is never mutated.
|
||||
func corruptExport(export ebsprovider.VolumeExport) ebsprovider.VolumeExport {
|
||||
if export.Kind == ebsprovider.ExportKindBlock && export.Block != nil {
|
||||
block := *export.Block
|
||||
block.DevicePath = "/dev/hostile-wrong"
|
||||
return ebsprovider.VolumeExport{Kind: ebsprovider.ExportKindBlock, Block: &block}
|
||||
}
|
||||
return ebsprovider.NewNBDExport("nbd+unix:///?socket=/hostile/wrong.sock")
|
||||
}
|
||||
|
||||
func (p *Provider) GetCapabilities(ctx context.Context, req ebsprovider.GetCapabilitiesRequest) (*ebsprovider.GetCapabilitiesResponse, error) {
|
||||
injection := p.draw(verbCapabilities, "")
|
||||
if stop, err := p.apply(ctx, injection); stop || err != nil {
|
||||
|
||||
@@ -375,7 +375,7 @@ func (m *MemoryProvider) PublishVolume(_ context.Context, req PublishVolumeReque
|
||||
volume.published = &PublishedVolume{
|
||||
VolumeID: req.VolumeID,
|
||||
NodeID: req.NodeID,
|
||||
NBDURI: "nbd+unix:///?socket=/memory/" + req.VolumeID + ".sock",
|
||||
Export: NewNBDExport("nbd+unix:///?socket=/memory/" + req.VolumeID + ".sock"),
|
||||
}
|
||||
volume.volume.State = VolumeStateInUse
|
||||
return clonePublished(volume.published), nil
|
||||
@@ -421,5 +421,13 @@ func clonePublished(published *PublishedVolume) *PublishedVolume {
|
||||
return nil
|
||||
}
|
||||
clone := *published
|
||||
if published.Export.NBD != nil {
|
||||
nbd := *published.Export.NBD
|
||||
clone.Export.NBD = &nbd
|
||||
}
|
||||
if published.Export.Block != nil {
|
||||
block := *published.Export.Block
|
||||
clone.Export.Block = &block
|
||||
}
|
||||
return &clone
|
||||
}
|
||||
|
||||
@@ -40,7 +40,7 @@ func TestMemoryProviderLifecycleAndIdempotency(t *testing.T) {
|
||||
Versioned: NewVersioned(), VolumeID: volume.ID, Handle: volume.Handle, NodeID: "node-1",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "nbd+unix:///?socket=/memory/vol-1.sock", published.NBDURI)
|
||||
assert.Equal(t, NewNBDExport("nbd+unix:///?socket=/memory/vol-1.sock"), published.Export)
|
||||
|
||||
repeatedPublish, err := provider.PublishVolume(ctx, PublishVolumeRequest{
|
||||
Versioned: NewVersioned(), VolumeID: volume.ID, Handle: volume.Handle, NodeID: "node-1",
|
||||
|
||||
@@ -452,6 +452,9 @@ func (p *NATSProvider) PublishVolume(ctx context.Context, req PublishVolumeReque
|
||||
if response.Published == nil {
|
||||
return nil, fmt.Errorf("%s returned no published volume", subject)
|
||||
}
|
||||
if err := response.Published.Export.Validate(); err != nil {
|
||||
return nil, fmt.Errorf("%s returned an invalid export: %w", subject, err)
|
||||
}
|
||||
return response.Published, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -432,6 +432,50 @@ func TestNATSProviderRejectsEmptySuccessResponse(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
// TestNATSProviderRejectsInvalidExport covers the client-side export check: a
|
||||
// reply the server marks successful but whose export cannot be attached must
|
||||
// fail here, not later when QEMU is handed it.
|
||||
func TestNATSProviderRejectsInvalidExport(t *testing.T) {
|
||||
_, conn := testutil.StartTestNATS(t)
|
||||
provider := NewNATSProvider(conn, time.Second)
|
||||
|
||||
tests := []struct {
|
||||
node string
|
||||
export VolumeExport
|
||||
wantErr string
|
||||
}{
|
||||
{node: "node-nokind", export: VolumeExport{NBD: &NBDExport{NBDURI: "nbd://10.0.0.1:10809"}}, wantErr: `unknown export kind ""`},
|
||||
{node: "node-noblock", export: VolumeExport{Kind: ExportKindBlock}, wantErr: "block export has no block details"},
|
||||
{node: "node-emptyuri", export: VolumeExport{Kind: ExportKindNBD, NBD: &NBDExport{}}, wantErr: "nbd export has an empty NBD URI"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.node, func(t *testing.T) {
|
||||
subject, err := PublishSubject(tt.node)
|
||||
require.NoError(t, err)
|
||||
respondWith(t, conn, subject, PublishVolumeResponse{
|
||||
Versioned: NewVersioned(),
|
||||
Published: &PublishedVolume{VolumeID: "vol-1", NodeID: tt.node, Export: tt.export},
|
||||
})
|
||||
published, err := provider.PublishVolume(t.Context(), PublishVolumeRequest{Versioned: NewVersioned(), VolumeID: "vol-1", NodeID: tt.node})
|
||||
require.EqualError(t, err, subject+" returned an invalid export: "+tt.wantErr)
|
||||
assert.Nil(t, published)
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("valid block export passes", func(t *testing.T) {
|
||||
subject, err := PublishSubject("node-block")
|
||||
require.NoError(t, err)
|
||||
export := VolumeExport{Kind: ExportKindBlock, Block: &BlockExport{DevicePath: "/dev/sdb", SizeBytes: 10 << 30}}
|
||||
respondWith(t, conn, subject, PublishVolumeResponse{
|
||||
Versioned: NewVersioned(),
|
||||
Published: &PublishedVolume{VolumeID: "vol-1", NodeID: "node-block", Export: export},
|
||||
})
|
||||
published, err := provider.PublishVolume(t.Context(), PublishVolumeRequest{Versioned: NewVersioned(), VolumeID: "vol-1", NodeID: "node-block"})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, export, published.Export)
|
||||
})
|
||||
}
|
||||
|
||||
// TestNATSProviderPropagatesProviderErrors covers responseError's populated
|
||||
// branch on the enumeration and unpublish verbs, whose errors otherwise reach
|
||||
// the caller as an empty page or a silent success.
|
||||
|
||||
@@ -276,7 +276,7 @@ func TestServedProviderRoundTrip(t *testing.T) {
|
||||
Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-roundtrip", NodeID: "node-1",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "nbd+unix:///?socket=/memory/vol-roundtrip.sock", published.NBDURI)
|
||||
assert.Equal(t, ebsprovider.NewNBDExport("nbd+unix:///?socket=/memory/vol-roundtrip.sock"), published.Export)
|
||||
|
||||
inUse, err := client.GetVolume(ctx, ebsprovider.GetVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-roundtrip"})
|
||||
require.NoError(t, err)
|
||||
@@ -432,6 +432,42 @@ func (p *countingProvider) PublishVolume(ctx context.Context, req ebsprovider.Pu
|
||||
return p.EBSProvider.PublishVolume(ctx, req)
|
||||
}
|
||||
|
||||
// exportProvider publishes a fixed export, so the server's own export check
|
||||
// can be driven with a provider that returns one it must not relay.
|
||||
type exportProvider struct {
|
||||
ebsprovider.EBSProvider
|
||||
|
||||
export ebsprovider.VolumeExport
|
||||
}
|
||||
|
||||
func (p *exportProvider) PublishVolume(_ context.Context, req ebsprovider.PublishVolumeRequest) (*ebsprovider.PublishedVolume, error) {
|
||||
return &ebsprovider.PublishedVolume{VolumeID: req.VolumeID, NodeID: req.NodeID, Export: p.export}, nil
|
||||
}
|
||||
|
||||
// TestPublishVolumeRejectsInvalidExport covers the server-side export check:
|
||||
// an unattachable export is the provider's fault and goes out as internal.
|
||||
func TestPublishVolumeRejectsInvalidExport(t *testing.T) {
|
||||
provider := &exportProvider{
|
||||
EBSProvider: ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{}),
|
||||
export: ebsprovider.VolumeExport{Kind: ebsprovider.ExportKindBlock, Block: &ebsprovider.BlockExport{DevicePath: "/dev/sdb"}},
|
||||
}
|
||||
conn := serveProvider(t, provider, natsserve.Options{NoQueueGroup: true})
|
||||
subject, err := ebsprovider.PublishSubject("node-1")
|
||||
require.NoError(t, err)
|
||||
|
||||
payload, err := json.Marshal(ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-1", NodeID: "node-1"})
|
||||
require.NoError(t, err)
|
||||
msg, err := conn.Request(subject, payload, requestTimeout)
|
||||
require.NoError(t, err)
|
||||
|
||||
var resp ebsprovider.PublishVolumeResponse
|
||||
require.NoError(t, json.Unmarshal(msg.Data, &resp))
|
||||
assert.Nil(t, resp.Published, "an invalid export must not be relayed")
|
||||
require.NotNil(t, resp.Error)
|
||||
assert.Equal(t, ebsprovider.ErrorCodeInternal, resp.Error.Code)
|
||||
assert.Equal(t, "provider returned an invalid export for vol-1: block export has a zero size", resp.Error.Message)
|
||||
}
|
||||
|
||||
// handlerSettle is how long a handler count is watched after it reaches the
|
||||
// expected value, to catch a delivery that should not have happened arriving
|
||||
// late. Generous, because a false pass here is a silently broken queue group.
|
||||
@@ -543,7 +579,7 @@ func TestServeRejectsUnroutableNodeID(t *testing.T) {
|
||||
assert.Contains(t, err.Error(), "build publish subject")
|
||||
assert.Nil(t, stop)
|
||||
|
||||
_, err = conn.Request(ebsprovider.CapabilitiesSubject, []byte(`{"schema_version":1}`), 200*time.Millisecond)
|
||||
_, err = conn.Request(ebsprovider.CapabilitiesSubject, []byte(fmt.Sprintf(`{"schema_version":%d}`, ebsprovider.SchemaVersion)), 200*time.Millisecond)
|
||||
require.ErrorIs(t, err, nats.ErrNoResponders, "a failed Serve must leave no subscription behind")
|
||||
}
|
||||
|
||||
@@ -571,6 +607,6 @@ func TestServeStopIsIdempotent(t *testing.T) {
|
||||
stop()
|
||||
require.NoError(t, conn.Flush())
|
||||
|
||||
_, err = conn.Request(ebsprovider.CapabilitiesSubject, []byte(`{"schema_version":1}`), 200*time.Millisecond)
|
||||
_, err = conn.Request(ebsprovider.CapabilitiesSubject, []byte(fmt.Sprintf(`{"schema_version":%d}`, ebsprovider.SchemaVersion)), 200*time.Millisecond)
|
||||
require.ErrorIs(t, err, nats.ErrNoResponders, "stop must unsubscribe every subject Serve registered")
|
||||
}
|
||||
|
||||
@@ -501,6 +501,16 @@ func handlePublishVolume(provider ebsprovider.EBSProvider) msgHandler {
|
||||
respond(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: classifyError(err)})
|
||||
return
|
||||
}
|
||||
if pub == nil {
|
||||
respond(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("provider returned no published volume for %s", req.VolumeID)})
|
||||
return
|
||||
}
|
||||
// An unattachable export is the provider's fault, so it is reported as
|
||||
// internal rather than relayed for the client to reject.
|
||||
if err := pub.Export.Validate(); err != nil {
|
||||
respond(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("provider returned an invalid export for %s: %v", req.VolumeID, err)})
|
||||
return
|
||||
}
|
||||
respond(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Published: pub})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,14 +6,16 @@ package ebsprovider
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"time"
|
||||
)
|
||||
|
||||
// SchemaVersion is the only wire-contract version understood by this build.
|
||||
// Every request and response carries it so version skew fails at the boundary
|
||||
// instead of silently decoding a changed payload.
|
||||
const SchemaVersion uint16 = 1
|
||||
const SchemaVersion uint16 = 2
|
||||
|
||||
// Versioned is embedded in every request and response sent over NATS.
|
||||
type Versioned struct {
|
||||
@@ -443,12 +445,80 @@ type PublishVolumeRequest struct {
|
||||
ReadOnly bool `json:"read_only,omitempty"`
|
||||
}
|
||||
|
||||
// PublishedVolume is where a published volume is served: the NBD URI on NodeID
|
||||
// that QEMU connects to.
|
||||
// ExportKind names how a published volume is reached on its node, and so
|
||||
// which VolumeExport variant carries the details.
|
||||
type ExportKind string
|
||||
|
||||
const (
|
||||
ExportKindNBD ExportKind = "nbd"
|
||||
ExportKindBlock ExportKind = "block"
|
||||
)
|
||||
|
||||
// VolumeExport is how a published volume is reached; exactly one of NBD and
|
||||
// Block is set, and it is the one Kind names.
|
||||
type VolumeExport struct {
|
||||
Kind ExportKind `json:"kind"`
|
||||
NBD *NBDExport `json:"nbd,omitempty"`
|
||||
Block *BlockExport `json:"block,omitempty"`
|
||||
}
|
||||
|
||||
// NBDExport is a volume served over NBD: the URI on the node that QEMU
|
||||
// connects to.
|
||||
type NBDExport struct {
|
||||
NBDURI string `json:"nbd_uri"`
|
||||
}
|
||||
|
||||
// BlockExport is a volume exposed as a host block device. SizeBytes is the
|
||||
// size the guest sees, which may be smaller than the device itself.
|
||||
type BlockExport struct {
|
||||
DevicePath string `json:"device_path"`
|
||||
SizeBytes uint64 `json:"size_bytes"`
|
||||
}
|
||||
|
||||
// Validate rejects an export a consumer could not attach: an unknown kind, a
|
||||
// missing or extra variant, or a variant with an unusable field.
|
||||
func (e VolumeExport) Validate() error {
|
||||
switch e.Kind {
|
||||
case ExportKindNBD:
|
||||
if e.NBD == nil {
|
||||
return errors.New("nbd export has no nbd details")
|
||||
}
|
||||
if e.Block != nil {
|
||||
return errors.New("nbd export also sets block details")
|
||||
}
|
||||
if e.NBD.NBDURI == "" {
|
||||
return errors.New("nbd export has an empty NBD URI")
|
||||
}
|
||||
case ExportKindBlock:
|
||||
if e.Block == nil {
|
||||
return errors.New("block export has no block details")
|
||||
}
|
||||
if e.NBD != nil {
|
||||
return errors.New("block export also sets nbd details")
|
||||
}
|
||||
if !filepath.IsAbs(e.Block.DevicePath) {
|
||||
return fmt.Errorf("block export device path %q is not absolute", e.Block.DevicePath)
|
||||
}
|
||||
if e.Block.SizeBytes == 0 {
|
||||
return errors.New("block export has a zero size")
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("unknown export kind %q", e.Kind)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// PublishedVolume is where a published volume is served: the export on NodeID
|
||||
// that QEMU attaches.
|
||||
type PublishedVolume struct {
|
||||
VolumeID string `json:"volume_id"`
|
||||
NodeID string `json:"node_id"`
|
||||
NBDURI string `json:"nbd_uri"`
|
||||
VolumeID string `json:"volume_id"`
|
||||
NodeID string `json:"node_id"`
|
||||
Export VolumeExport `json:"export"`
|
||||
}
|
||||
|
||||
// NewNBDExport returns an nbd export for uri.
|
||||
func NewNBDExport(uri string) VolumeExport {
|
||||
return VolumeExport{Kind: ExportKindNBD, NBD: &NBDExport{NBDURI: uri}}
|
||||
}
|
||||
|
||||
// PublishVolumeResponse is the reply to PublishVolumeRequest: the mount, or
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
package ebsprovider
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestVolumeExportValidate(t *testing.T) {
|
||||
nbd := &NBDExport{NBDURI: "nbd+unix:///?socket=/run/vol-1.sock"}
|
||||
block := &BlockExport{DevicePath: "/dev/sdb", SizeBytes: 10 << 30}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
export VolumeExport
|
||||
wantErr string
|
||||
}{
|
||||
{name: "valid nbd", export: VolumeExport{Kind: ExportKindNBD, NBD: nbd}},
|
||||
{name: "valid block", export: VolumeExport{Kind: ExportKindBlock, Block: block}},
|
||||
{name: "empty kind", export: VolumeExport{NBD: nbd}, wantErr: `unknown export kind ""`},
|
||||
{name: "unknown kind", export: VolumeExport{Kind: "iscsi", NBD: nbd}, wantErr: `unknown export kind "iscsi"`},
|
||||
{name: "nbd kind without nbd variant", export: VolumeExport{Kind: ExportKindNBD}, wantErr: "nbd export has no nbd details"},
|
||||
{name: "nbd kind carrying only block variant", export: VolumeExport{Kind: ExportKindNBD, Block: block}, wantErr: "nbd export has no nbd details"},
|
||||
{name: "nbd kind with block variant", export: VolumeExport{Kind: ExportKindNBD, NBD: nbd, Block: block}, wantErr: "nbd export also sets block details"},
|
||||
{name: "empty nbd uri", export: VolumeExport{Kind: ExportKindNBD, NBD: &NBDExport{}}, wantErr: "nbd export has an empty NBD URI"},
|
||||
{name: "block kind without block variant", export: VolumeExport{Kind: ExportKindBlock}, wantErr: "block export has no block details"},
|
||||
{name: "block kind with nbd variant", export: VolumeExport{Kind: ExportKindBlock, NBD: nbd, Block: block}, wantErr: "block export also sets nbd details"},
|
||||
{name: "relative device path", export: VolumeExport{Kind: ExportKindBlock, Block: &BlockExport{DevicePath: "dev/sdb", SizeBytes: 1}}, wantErr: `block export device path "dev/sdb" is not absolute`},
|
||||
{name: "empty device path", export: VolumeExport{Kind: ExportKindBlock, Block: &BlockExport{SizeBytes: 1}}, wantErr: `block export device path "" is not absolute`},
|
||||
{name: "zero size", export: VolumeExport{Kind: ExportKindBlock, Block: &BlockExport{DevicePath: "/dev/sdb"}}, wantErr: "block export has a zero size"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := tt.export.Validate()
|
||||
if tt.wantErr == "" {
|
||||
require.NoError(t, err)
|
||||
return
|
||||
}
|
||||
require.EqualError(t, err, tt.wantErr)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestPublishedVolumeWireShape pins the JSON the contract carries: a renamed
|
||||
// tag would decode as an empty variant on a peer at the same schema version.
|
||||
func TestPublishedVolumeWireShape(t *testing.T) {
|
||||
wires := map[string]string{
|
||||
"nbd": `{"volume_id":"vol-1","node_id":"node-1","export":{"kind":"nbd","nbd":{"nbd_uri":"nbd://10.0.0.1:10809"}}}`,
|
||||
"block": `{"volume_id":"vol-1","node_id":"node-1","export":{"kind":"block","block":{"device_path":"/dev/sdb","size_bytes":10737418240}}}`,
|
||||
}
|
||||
for name, wire := range wires {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
var published PublishedVolume
|
||||
require.NoError(t, json.Unmarshal([]byte(wire), &published))
|
||||
require.NoError(t, published.Export.Validate())
|
||||
encoded, err := json.Marshal(published)
|
||||
require.NoError(t, err)
|
||||
assert.JSONEq(t, wire, string(encoded))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1765,7 +1765,7 @@ func handlePublishVolume(ctx context.Context, cfg *Config, nc *nats.Conn, msg *n
|
||||
Published: &ebsprovider.PublishedVolume{
|
||||
VolumeID: req.VolumeID,
|
||||
NodeID: cfg.NodeName,
|
||||
NBDURI: mv.NBDURI,
|
||||
Export: ebsprovider.NewNBDExport(mv.NBDURI),
|
||||
},
|
||||
})
|
||||
return
|
||||
@@ -1795,7 +1795,7 @@ func handlePublishVolume(ctx context.Context, cfg *Config, nc *nats.Conn, msg *n
|
||||
Published: &ebsprovider.PublishedVolume{
|
||||
VolumeID: req.VolumeID,
|
||||
NodeID: cfg.NodeName,
|
||||
NBDURI: mountResp.URI,
|
||||
Export: ebsprovider.NewNBDExport(mountResp.URI),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
@@ -837,7 +837,7 @@ func TestProviderHandlers_PublishVolume_IdempotentRepublish(t *testing.T) {
|
||||
require.NotNil(t, resp.Published)
|
||||
assert.Equal(t, volumeName, resp.Published.VolumeID)
|
||||
assert.Equal(t, cfg.NodeName, resp.Published.NodeID)
|
||||
assert.Equal(t, existingURI, resp.Published.NBDURI)
|
||||
assert.Equal(t, ebsprovider.NewNBDExport(existingURI), resp.Published.Export)
|
||||
|
||||
cfg.mu.Lock()
|
||||
count := len(cfg.MountedVolumes)
|
||||
|
||||
@@ -355,3 +355,35 @@ func VolumeBlockdev(nodeName string, server NBDServerOpts) Blockdev {
|
||||
opts = append(opts, "export=", fmt.Sprintf("reconnect-delay=%d", NBDReconnectDelaySeconds))
|
||||
return Blockdev{Value: strings.Join(opts, ",")}
|
||||
}
|
||||
|
||||
// HostDeviceOpts is a host block device export. QEMU opens Path through the
|
||||
// host_device protocol driver with O_DIRECT and native AIO, and a raw format
|
||||
// node over it exposes only the first SizeBytes to the guest.
|
||||
type HostDeviceOpts struct {
|
||||
Path string
|
||||
SizeBytes uint64
|
||||
}
|
||||
|
||||
// QMPArgs renders the blockdev-add arguments for the raw node and its nested
|
||||
// host_device file node.
|
||||
func (o HostDeviceOpts) QMPArgs(nodeName string) map[string]any {
|
||||
return map[string]any{
|
||||
"node-name": nodeName,
|
||||
"driver": "raw",
|
||||
"size": o.SizeBytes,
|
||||
"file": map[string]any{
|
||||
"driver": "host_device",
|
||||
"filename": o.Path,
|
||||
"aio": "native",
|
||||
"cache": map[string]any{"direct": true},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// VolumeHostDeviceBlockdev renders the -blockdev equivalent of QMPArgs for a
|
||||
// data volume backed by a host block device.
|
||||
func VolumeHostDeviceBlockdev(nodeName string, dev HostDeviceOpts) Blockdev {
|
||||
return Blockdev{Value: fmt.Sprintf(
|
||||
"driver=raw,node-name=%s,size=%d,file.driver=host_device,file.filename=%s,file.aio=native,file.cache.direct=on",
|
||||
nodeName, dev.SizeBytes, dev.Path)}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
package vm
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
|
||||
@@ -20,3 +24,45 @@ type EBSRequests struct {
|
||||
// VM's persisted attachment list. The alias keeps the VM state tied to the
|
||||
// deployed wire shape without making the old catch-all types package an owner.
|
||||
type EBSRequest = viperblocklegacyv1.EBSRequest
|
||||
|
||||
// volumeExportKind is how a volume's backing store reaches QEMU.
|
||||
type volumeExportKind int
|
||||
|
||||
const (
|
||||
volumeExportInvalid volumeExportKind = iota
|
||||
volumeExportNBD
|
||||
volumeExportBlock
|
||||
)
|
||||
|
||||
// classifyVolumeExport reports which export a request carries. A block device
|
||||
// must be absolute, comma-free (QEMU key=value options would split it) and
|
||||
// sized; both exports set, neither, or a stray size is invalid.
|
||||
func classifyVolumeExport(req EBSRequest) (volumeExportKind, error) {
|
||||
switch {
|
||||
case req.NBDURI != "" && req.BlockDevice != "":
|
||||
return volumeExportInvalid, fmt.Errorf("volume %s has both NBDURI and BlockDevice set", req.Name)
|
||||
case req.BlockSizeBytes != 0 && req.BlockDevice == "":
|
||||
return volumeExportInvalid, fmt.Errorf("volume %s has BlockSizeBytes set without BlockDevice", req.Name)
|
||||
case req.NBDURI != "":
|
||||
return volumeExportNBD, nil
|
||||
case req.BlockDevice != "":
|
||||
if !filepath.IsAbs(req.BlockDevice) {
|
||||
return volumeExportInvalid, fmt.Errorf("volume %s BlockDevice %q is not an absolute path", req.Name, req.BlockDevice)
|
||||
}
|
||||
if strings.Contains(req.BlockDevice, ",") {
|
||||
return volumeExportInvalid, fmt.Errorf("volume %s BlockDevice %q contains a comma", req.Name, req.BlockDevice)
|
||||
}
|
||||
if req.BlockSizeBytes == 0 {
|
||||
return volumeExportInvalid, fmt.Errorf("volume %s BlockDevice %q has zero BlockSizeBytes", req.Name, req.BlockDevice)
|
||||
}
|
||||
return volumeExportBlock, nil
|
||||
default:
|
||||
return volumeExportInvalid, fmt.Errorf("NBDURI not set for volume %s - was volume mounted?", req.Name)
|
||||
}
|
||||
}
|
||||
|
||||
// checkBlockDevice reports whether a block export's device node is present.
|
||||
func checkBlockDevice(path string) error {
|
||||
_, err := os.Stat(path)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -607,11 +607,18 @@ func (m *Manager) startQEMU(instance *VM) error {
|
||||
refreshTelemetryMeta(instance)
|
||||
}
|
||||
|
||||
// An NBD export is waited on until it accepts connections; a block export
|
||||
// is local to the host and only has to exist. An invalid request has no
|
||||
// endpoint to probe (buildDrives rejects it on the command-line path).
|
||||
instance.EBSRequests.Mu.Lock()
|
||||
nbdEndpoints := make([]struct{ name, uri string }, 0, len(instance.EBSRequests.Requests))
|
||||
var blockDevices []struct{ name, path string }
|
||||
for _, req := range instance.EBSRequests.Requests {
|
||||
if req.NBDURI != "" {
|
||||
switch kind, _ := classifyVolumeExport(req); kind {
|
||||
case volumeExportNBD:
|
||||
nbdEndpoints = append(nbdEndpoints, struct{ name, uri string }{req.Name, req.NBDURI})
|
||||
case volumeExportBlock:
|
||||
blockDevices = append(blockDevices, struct{ name, path string }{req.Name, req.BlockDevice})
|
||||
}
|
||||
}
|
||||
instance.EBSRequests.Mu.Unlock()
|
||||
@@ -620,6 +627,11 @@ func (m *Manager) startQEMU(instance *VM) error {
|
||||
return fmt.Errorf("nbd endpoint not ready for %s: %w", ep.name, err)
|
||||
}
|
||||
}
|
||||
for _, dev := range blockDevices {
|
||||
if err := checkBlockDevice(dev.path); err != nil {
|
||||
return fmt.Errorf("block device not present for %s: %w", dev.name, err)
|
||||
}
|
||||
}
|
||||
|
||||
processChan := make(chan int, 1)
|
||||
exitChan := make(chan int, 1)
|
||||
@@ -900,11 +912,11 @@ func qmpGreetingTimeout(v *VM) time.Duration {
|
||||
|
||||
// hasNetworkBackedDrive reports whether any of the guest's drives is served
|
||||
// over NBD, and so has to be opened across the object store before QEMU's
|
||||
// monitor answers. Every production guest boots from one; a VM without is a
|
||||
// unit test.
|
||||
// monitor answers. A host block device opens locally and does not count.
|
||||
func hasNetworkBackedDrive(v *VM) bool {
|
||||
return slices.ContainsFunc(v.EBSRequests.Requests, func(req EBSRequest) bool {
|
||||
return req.NBDURI != ""
|
||||
kind, _ := classifyVolumeExport(req)
|
||||
return kind == volumeExportNBD
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1445,8 +1457,9 @@ type driveConfig struct {
|
||||
}
|
||||
|
||||
// buildDrives converts EBS requests into QEMU drive/iothread/device/blockdev
|
||||
// configs. Returns an error if any volume is missing its NBDURI, or if the
|
||||
// EBS hot-plug port pool is exhausted while allocating one for a data volume.
|
||||
// configs. Returns an error if any volume has no valid export (see
|
||||
// classifyVolumeExport), or if the EBS hot-plug port pool is exhausted while
|
||||
// allocating one for a data volume.
|
||||
//
|
||||
// Boot volumes keep the existing anonymous if=none -drive shape exactly as
|
||||
// before — they are not detachable (see ErrVolumeNotDetachable), so the
|
||||
@@ -1471,11 +1484,17 @@ func buildDrives(requests []EBSRequest, cpuCount int, machineType string) (drive
|
||||
|
||||
for i := range requests {
|
||||
v := &requests[i]
|
||||
if v.NBDURI == "" {
|
||||
return driveConfig{}, fmt.Errorf("NBDURI not set for volume %s - was volume mounted?", v.Name)
|
||||
kind, err := classifyVolumeExport(*v)
|
||||
if err != nil {
|
||||
return driveConfig{}, err
|
||||
}
|
||||
|
||||
switch {
|
||||
case kind == volumeExportBlock && (v.Boot || v.EFI):
|
||||
// Boot and EFI keep the legacy -drive shape, which has no
|
||||
// host_device form here; only data volumes take a block export.
|
||||
return driveConfig{}, fmt.Errorf("volume %s: block device export is only supported for data volumes", v.Name)
|
||||
|
||||
case v.Boot:
|
||||
drive := Drive{
|
||||
File: v.NBDURI,
|
||||
@@ -1529,29 +1548,39 @@ func buildDrives(requests []EBSRequest, cpuCount int, machineType string) (drive
|
||||
v.HotplugPort = port
|
||||
}
|
||||
|
||||
serverType, socketPath, nbdHost, nbdPort, err := nbd.ParseNBDURI(v.NBDURI)
|
||||
if err != nil {
|
||||
return driveConfig{}, fmt.Errorf("parse NBDURI for volume %s: %w", v.Name, err)
|
||||
}
|
||||
|
||||
nodeName := VolumeNodeName(v.Name)
|
||||
iothreadID := VolumeIOThreadID(v.Name)
|
||||
bus := HotplugEBSBus(v.HotplugPort)
|
||||
|
||||
var blockdev Blockdev
|
||||
if kind == volumeExportBlock {
|
||||
blockdev = VolumeHostDeviceBlockdev(nodeName, HostDeviceOpts{Path: v.BlockDevice, SizeBytes: v.BlockSizeBytes})
|
||||
} else {
|
||||
serverType, socketPath, nbdHost, nbdPort, err := nbd.ParseNBDURI(v.NBDURI)
|
||||
if err != nil {
|
||||
return driveConfig{}, fmt.Errorf("parse NBDURI for volume %s: %w", v.Name, err)
|
||||
}
|
||||
blockdev = VolumeBlockdev(nodeName, NBDServerOpts{
|
||||
Type: serverType,
|
||||
Path: socketPath,
|
||||
Host: nbdHost,
|
||||
Port: nbdPort,
|
||||
})
|
||||
}
|
||||
|
||||
cfg.IOThreads = append(cfg.IOThreads, IOThread{ID: iothreadID})
|
||||
cfg.Blockdevs = append(cfg.Blockdevs, VolumeBlockdev(nodeName, NBDServerOpts{
|
||||
Type: serverType,
|
||||
Path: socketPath,
|
||||
Host: nbdHost,
|
||||
Port: nbdPort,
|
||||
}))
|
||||
cfg.Blockdevs = append(cfg.Blockdevs, blockdev)
|
||||
cfg.Devices = append(cfg.Devices, VolumeBlkDevice(v.Name, nodeName, iothreadID, bus))
|
||||
// Deliberately no entry in cfg.Drives: the bare anonymous -drive
|
||||
// this case used to emit (no id=, no node-name=) is exactly what
|
||||
// left the volume undetachable.
|
||||
}
|
||||
|
||||
slog.Info("Using NBD URI for drive", "volume", v.Name, "uri", v.NBDURI)
|
||||
if kind == volumeExportBlock {
|
||||
slog.Info("Using block device for drive", "volume", v.Name, "device", v.BlockDevice, "size_bytes", v.BlockSizeBytes)
|
||||
} else {
|
||||
slog.Info("Using NBD URI for drive", "volume", v.Name, "uri", v.NBDURI)
|
||||
}
|
||||
}
|
||||
|
||||
return cfg, nil
|
||||
|
||||
@@ -273,6 +273,83 @@ func TestBuildDrives(t *testing.T) {
|
||||
},
|
||||
wantHotplugPorts: []int{0, 0, 3},
|
||||
},
|
||||
{
|
||||
name: "block data volume emits a raw node over host_device with the same device wiring",
|
||||
requests: []EBSRequest{
|
||||
{Name: "vol-blk", BlockDevice: "/dev/sdx", BlockSizeBytes: 10 << 30, HotplugPort: 2},
|
||||
},
|
||||
cpuCount: 2,
|
||||
wantIOThreads: []IOThread{{ID: "ioth-vol-blk"}},
|
||||
wantBlockdevs: []Blockdev{
|
||||
{Value: "driver=raw,node-name=nbd-vol-blk,size=10737418240,file.driver=host_device,file.filename=/dev/sdx,file.aio=native,file.cache.direct=on"},
|
||||
},
|
||||
wantDevices: []Device{
|
||||
{Value: "virtio-blk-pci,id=vdisk-vol-blk,drive=nbd-vol-blk,iothread=ioth-vol-blk,serial=volblk,bus=hotplug-ebs2,werror=stop,rerror=stop"},
|
||||
},
|
||||
wantHotplugPorts: []int{2},
|
||||
},
|
||||
{
|
||||
name: "NBD boot volume alongside a block data volume",
|
||||
requests: []EBSRequest{
|
||||
{Name: "vol-boot", NBDURI: "nbd:unix:/tmp/boot.sock", Boot: true},
|
||||
{Name: "vol-blk", BlockDevice: "/dev/sdx", BlockSizeBytes: 1 << 20},
|
||||
},
|
||||
cpuCount: 2,
|
||||
wantDrives: []Drive{
|
||||
{File: "nbd:unix:/tmp/boot.sock", Format: "raw", If: "none", Media: "disk", ID: "os", Cache: "none", Werror: "stop", Rerror: "stop", ReconnectDelay: NBDReconnectDelaySeconds},
|
||||
},
|
||||
wantIOThreads: []IOThread{{ID: "ioth-os"}, {ID: "ioth-vol-blk"}},
|
||||
wantBlockdevs: []Blockdev{
|
||||
{Value: "driver=raw,node-name=nbd-vol-blk,size=1048576,file.driver=host_device,file.filename=/dev/sdx,file.aio=native,file.cache.direct=on"},
|
||||
},
|
||||
wantDevices: []Device{
|
||||
{Value: "virtio-blk-pci,drive=os,iothread=ioth-os,num-queues=2,bootindex=1"},
|
||||
{Value: "virtio-blk-pci,id=vdisk-vol-blk,drive=nbd-vol-blk,iothread=ioth-vol-blk,serial=volblk,bus=hotplug-ebs1,werror=stop,rerror=stop"},
|
||||
},
|
||||
wantHotplugPorts: []int{0, 1},
|
||||
},
|
||||
{
|
||||
name: "block boot volume is rejected",
|
||||
requests: []EBSRequest{{Name: "vol-boot", BlockDevice: "/dev/sdx", BlockSizeBytes: 1 << 20, Boot: true}},
|
||||
cpuCount: 2,
|
||||
wantErr: "volume vol-boot: block device export is only supported for data volumes",
|
||||
},
|
||||
{
|
||||
name: "block EFI volume is rejected",
|
||||
requests: []EBSRequest{{Name: "vol-efi", BlockDevice: "/dev/sdx", BlockSizeBytes: 1 << 20, EFI: true}},
|
||||
cpuCount: 2,
|
||||
wantErr: "volume vol-efi: block device export is only supported for data volumes",
|
||||
},
|
||||
{
|
||||
name: "both NBDURI and BlockDevice set is rejected",
|
||||
requests: []EBSRequest{{Name: "vol-both", NBDURI: "nbd:unix:/tmp/x.sock", BlockDevice: "/dev/sdx", BlockSizeBytes: 1 << 20}},
|
||||
cpuCount: 2,
|
||||
wantErr: "volume vol-both has both NBDURI and BlockDevice set",
|
||||
},
|
||||
{
|
||||
name: "NBDURI with a stray BlockSizeBytes is rejected",
|
||||
requests: []EBSRequest{{Name: "vol-stray", NBDURI: "nbd:unix:/tmp/x.sock", BlockSizeBytes: 1 << 20}},
|
||||
cpuCount: 2,
|
||||
wantErr: "volume vol-stray has BlockSizeBytes set without BlockDevice",
|
||||
},
|
||||
{
|
||||
name: "block device with zero size is rejected",
|
||||
requests: []EBSRequest{{Name: "vol-nosize", BlockDevice: "/dev/sdx"}},
|
||||
cpuCount: 2,
|
||||
wantErr: `volume vol-nosize BlockDevice "/dev/sdx" has zero BlockSizeBytes`,
|
||||
},
|
||||
{
|
||||
name: "relative block device path is rejected",
|
||||
requests: []EBSRequest{{Name: "vol-rel", BlockDevice: "dev/sdx", BlockSizeBytes: 1 << 20}},
|
||||
cpuCount: 2,
|
||||
wantErr: `volume vol-rel BlockDevice "dev/sdx" is not an absolute path`,
|
||||
},
|
||||
{
|
||||
name: "block device path with a comma is rejected",
|
||||
requests: []EBSRequest{{Name: "vol-comma", BlockDevice: "/dev/sdx,file.locking=off", BlockSizeBytes: 1 << 20}},
|
||||
cpuCount: 2,
|
||||
wantErr: `volume vol-comma BlockDevice "/dev/sdx,file.locking=off" contains a comma`,
|
||||
},
|
||||
{
|
||||
name: "empty requests",
|
||||
requests: []EBSRequest{},
|
||||
@@ -1008,6 +1085,28 @@ func TestStartQEMU_DirectBoot_InstanceTypeNotFound(t *testing.T) {
|
||||
assert.Empty(t, plumber.setupCalls)
|
||||
}
|
||||
|
||||
// TestStartQEMU_BlockDeviceMissing verifies a block volume's launch readiness
|
||||
// is the device's presence, not an NBD connect.
|
||||
func TestStartQEMU_BlockDeviceMissing(t *testing.T) {
|
||||
m := directBootManager(t, &fakeNetworkPlumber{})
|
||||
missing := filepath.Join(t.TempDir(), "missing-dev")
|
||||
|
||||
instance := &VM{
|
||||
ID: "i-blk-missing",
|
||||
InstanceType: "t3.nano",
|
||||
DirectBoot: true,
|
||||
}
|
||||
instance.EBSRequests.Requests = []EBSRequest{
|
||||
{Name: "vol-blk", BlockDevice: missing, BlockSizeBytes: 1 << 20},
|
||||
}
|
||||
|
||||
err := m.startQEMU(instance)
|
||||
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "block device not present for vol-blk")
|
||||
assert.ErrorIs(t, err, os.ErrNotExist)
|
||||
}
|
||||
|
||||
// scriptedNetworkPlumber returns errors from a pre-loaded slice, one per call.
|
||||
type scriptedNetworkPlumber struct {
|
||||
calls []TapSpec
|
||||
@@ -1271,6 +1370,12 @@ func TestQMPGreetingTimeout(t *testing.T) {
|
||||
assert.Greater(t, qmpNBDGreetingTimeout, 35*time.Second,
|
||||
"the deadline must exceed the 35s cold open measured with one node of three away")
|
||||
|
||||
block := &VM{EBSRequests: EBSRequests{Requests: []EBSRequest{
|
||||
{Name: "vol-data", BlockDevice: "/dev/sdx", BlockSizeBytes: 1 << 20},
|
||||
}}}
|
||||
assert.Equal(t, qmp.DefaultGreetingTimeout, qmpGreetingTimeout(block),
|
||||
"a host block device opens locally, so it keeps the plain default")
|
||||
|
||||
// A small GPU guest (<1 GiB scaling contribution) sits at the floor.
|
||||
smallGPU := &VM{
|
||||
GPUAttachments: []gpu.GPUAttachment{{PCIAddress: "0000:5e:00.0"}},
|
||||
|
||||
@@ -142,6 +142,10 @@ type fakeVolumeMounter struct {
|
||||
unmountErr error
|
||||
unmountOneErr error
|
||||
mountOneURI string
|
||||
// mountOneBlockDevice and mountOneBlockSize, when set, are written onto
|
||||
// the request by MountOne as a block device export.
|
||||
mountOneBlockDevice string
|
||||
mountOneBlockSize uint64
|
||||
// onMount fires synchronously inside Mount before the configured
|
||||
// mountErr is returned. Used by lifecycle tests to simulate a
|
||||
// concurrent terminate flipping VM.Status while Mount is in flight.
|
||||
@@ -181,6 +185,7 @@ func (f *fakeVolumeMounter) MountOne(_ context.Context, _ string, req *EBSReques
|
||||
f.mountedOne = append(f.mountedOne, req.Name)
|
||||
mountOneErr := f.mountOneErr
|
||||
uri := f.mountOneURI
|
||||
blockDevice, blockSize := f.mountOneBlockDevice, f.mountOneBlockSize
|
||||
f.mu.Unlock()
|
||||
if mountOneErr != nil {
|
||||
return mountOneErr
|
||||
@@ -188,6 +193,10 @@ func (f *fakeVolumeMounter) MountOne(_ context.Context, _ string, req *EBSReques
|
||||
if uri != "" {
|
||||
req.NBDURI = uri
|
||||
}
|
||||
if blockDevice != "" {
|
||||
req.BlockDevice = blockDevice
|
||||
req.BlockSizeBytes = blockSize
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -523,14 +523,22 @@ func isInstanceProcessRunning(instance *VM) bool {
|
||||
}
|
||||
|
||||
// AreVolumeSocketsValid dials each NBD Unix socket to confirm viperblock is
|
||||
// still listening. A stat-only check is insufficient since viperblock may restart
|
||||
// leaving stale socket files. TCP and unparseable URIs are treated as valid.
|
||||
// still listening, and checks each block export's device still exists. TCP,
|
||||
// unparseable and invalid exports are treated as valid.
|
||||
func AreVolumeSocketsValid(instance *VM) bool {
|
||||
instance.EBSRequests.Mu.Lock()
|
||||
defer instance.EBSRequests.Mu.Unlock()
|
||||
|
||||
for _, req := range instance.EBSRequests.Requests {
|
||||
if req.NBDURI == "" {
|
||||
kind, _ := classifyVolumeExport(req)
|
||||
if kind == volumeExportBlock {
|
||||
if err := checkBlockDevice(req.BlockDevice); err != nil {
|
||||
slog.Debug("Block device missing", "volume", req.Name, "device", req.BlockDevice, "err", err)
|
||||
return false
|
||||
}
|
||||
continue
|
||||
}
|
||||
if kind != volumeExportNBD {
|
||||
continue
|
||||
}
|
||||
serverType, sockPath, _, _, err := nbd.ParseNBDURI(req.NBDURI)
|
||||
|
||||
@@ -82,6 +82,22 @@ func TestAreVolumeSocketsValid_OneOfManyMissing(t *testing.T) {
|
||||
"a single dead socket is enough to invalidate the whole instance")
|
||||
}
|
||||
|
||||
// TestAreVolumeSocketsValid_BlockDevice checks a block export is judged by its
|
||||
// device node's presence, with no NBD probe.
|
||||
func TestAreVolumeSocketsValid_BlockDevice(t *testing.T) {
|
||||
present := filepath.Join(t.TempDir(), "dev-present")
|
||||
require.NoError(t, os.WriteFile(present, nil, 0o600))
|
||||
missing := filepath.Join(t.TempDir(), "dev-missing")
|
||||
|
||||
blockVM := func(path string) *VM {
|
||||
v := &VM{ID: "i-blk", Status: StateRunning}
|
||||
v.EBSRequests.Requests = []EBSRequest{{Name: "vol-blk", BlockDevice: path, BlockSizeBytes: 1 << 20}}
|
||||
return v
|
||||
}
|
||||
assert.True(t, AreVolumeSocketsValid(blockVM(present)), "a present block device is valid")
|
||||
assert.False(t, AreVolumeSocketsValid(blockVM(missing)), "a vanished block device is the orphan-QEMU signal")
|
||||
}
|
||||
|
||||
// classifyTestManager wires a manager with an in-memory StateStore plus
|
||||
// the InstanceTypes resolver and Resources controller that
|
||||
// classifyRestoredInstances consults. Returns the manager and the
|
||||
|
||||
@@ -119,17 +119,36 @@ func (m *Manager) AttachVolume(ctx context.Context, id, volumeID, device string)
|
||||
return "", fmt.Errorf("mount volume %s: %w", volumeID, err)
|
||||
}
|
||||
|
||||
serverType, socketPath, nbdHost, nbdPort, err := nbd.ParseNBDURI(ebsRequest.NBDURI)
|
||||
if err != nil {
|
||||
slog.ErrorContext(ctx, "AttachVolume: failed to parse NBDURI", "uri", ebsRequest.NBDURI, "err", err)
|
||||
_ = m.rollbackUnmount(ctx, instance.AccountID, ebsRequest)
|
||||
return "", fmt.Errorf("parse NBDURI: %w", err)
|
||||
}
|
||||
serverArg := NBDServerOpts{Type: serverType, Path: socketPath, Host: nbdHost, Port: nbdPort}.QMPArg()
|
||||
|
||||
// Shared with buildDrives' cold-boot path so a relaunched volume's block
|
||||
// graph uses exactly the same names DetachVolume addresses.
|
||||
nodeName := VolumeNodeName(volumeID)
|
||||
|
||||
kind, err := classifyVolumeExport(ebsRequest)
|
||||
if err != nil {
|
||||
slog.ErrorContext(ctx, "AttachVolume: invalid volume export", "volumeId", volumeID, "err", err)
|
||||
_ = m.rollbackUnmount(ctx, instance.AccountID, ebsRequest)
|
||||
return "", fmt.Errorf("volume export: %w", err)
|
||||
}
|
||||
var blockdevArgs map[string]any
|
||||
if kind == volumeExportBlock {
|
||||
blockdevArgs = HostDeviceOpts{Path: ebsRequest.BlockDevice, SizeBytes: ebsRequest.BlockSizeBytes}.QMPArgs(nodeName)
|
||||
} else {
|
||||
serverType, socketPath, nbdHost, nbdPort, err := nbd.ParseNBDURI(ebsRequest.NBDURI)
|
||||
if err != nil {
|
||||
slog.ErrorContext(ctx, "AttachVolume: failed to parse NBDURI", "uri", ebsRequest.NBDURI, "err", err)
|
||||
_ = m.rollbackUnmount(ctx, instance.AccountID, ebsRequest)
|
||||
return "", fmt.Errorf("parse NBDURI: %w", err)
|
||||
}
|
||||
blockdevArgs = map[string]any{
|
||||
"node-name": nodeName,
|
||||
"driver": "nbd",
|
||||
"server": NBDServerOpts{Type: serverType, Path: socketPath, Host: nbdHost, Port: nbdPort}.QMPArg(),
|
||||
"export": "",
|
||||
"read-only": false,
|
||||
"reconnect-delay": NBDReconnectDelaySeconds,
|
||||
}
|
||||
}
|
||||
|
||||
deviceID := VolumeDeviceID(volumeID)
|
||||
iothreadID := VolumeIOThreadID(volumeID)
|
||||
|
||||
@@ -163,15 +182,8 @@ func (m *Manager) AttachVolume(ctx context.Context, id, volumeID, device string)
|
||||
}
|
||||
|
||||
if _, err := sendQMPCommand(ctx, instance.QMPClient, qmp.QMPCommand{
|
||||
Execute: "blockdev-add",
|
||||
Arguments: map[string]any{
|
||||
"node-name": nodeName,
|
||||
"driver": "nbd",
|
||||
"server": serverArg,
|
||||
"export": "",
|
||||
"read-only": false,
|
||||
"reconnect-delay": NBDReconnectDelaySeconds,
|
||||
},
|
||||
Execute: "blockdev-add",
|
||||
Arguments: blockdevArgs,
|
||||
}, instance.ID); err != nil {
|
||||
slog.ErrorContext(ctx, "AttachVolume: QMP blockdev-add failed", "volumeId", volumeID, "err", err)
|
||||
m.delIothreadBestEffort(ctx, instance, iothreadID, volumeID)
|
||||
|
||||
@@ -943,6 +943,91 @@ func TestAttachVolume_NBDURIParseFailure_TriggersUnmountOne(t *testing.T) {
|
||||
assert.Equal(t, []string{"vol-1"}, mounter.unmountedOne)
|
||||
}
|
||||
|
||||
// TestAttachVolume_BlockDeviceBlockdevAdd pins the blockdev-add a block export
|
||||
// sends: a sized raw node over a host_device file node, under the same node
|
||||
// name the NBD path uses, followed by the usual device_add.
|
||||
func TestAttachVolume_BlockDeviceBlockdevAdd(t *testing.T) {
|
||||
recorder := &qmpRecorder{}
|
||||
qmpClient, cancel := newMockQMPClient(t, func(cmd qmp.QMPCommand) map[string]any {
|
||||
recorder.record(cmd)
|
||||
return nil
|
||||
})
|
||||
defer cancel()
|
||||
|
||||
mounter := &fakeVolumeMounter{mountOneBlockDevice: "/dev/sdx", mountOneBlockSize: 10 << 30}
|
||||
m, v := attachVolumeRunningInstance(t, qmpClient, mounter)
|
||||
|
||||
_, err := m.AttachVolume(t.Context(), "i-1", "vol-1", "/dev/sdf")
|
||||
require.NoError(t, err)
|
||||
|
||||
recorder.mu.Lock()
|
||||
var blockdevAdd *qmp.QMPCommand
|
||||
for i := range recorder.commands {
|
||||
if recorder.commands[i].Execute == "blockdev-add" {
|
||||
blockdevAdd = &recorder.commands[i]
|
||||
}
|
||||
}
|
||||
recorder.mu.Unlock()
|
||||
require.NotNil(t, blockdevAdd)
|
||||
got, err := json.Marshal(blockdevAdd.Arguments)
|
||||
require.NoError(t, err)
|
||||
assert.JSONEq(t, `{
|
||||
"node-name":"nbd-vol-1", "driver":"raw", "size":10737418240,
|
||||
"file":{"driver":"host_device", "filename":"/dev/sdx", "aio":"native", "cache":{"direct":true}}
|
||||
}`, string(got))
|
||||
assert.Equal(t, []string{"object-add", "blockdev-add", "device_add"}, recorder.executes()[:3])
|
||||
|
||||
v.EBSRequests.Mu.Lock()
|
||||
defer v.EBSRequests.Mu.Unlock()
|
||||
require.Len(t, v.EBSRequests.Requests, 1)
|
||||
assert.Equal(t, "/dev/sdx", v.EBSRequests.Requests[0].BlockDevice)
|
||||
assert.Equal(t, uint64(10<<30), v.EBSRequests.Requests[0].BlockSizeBytes)
|
||||
assert.Empty(t, v.EBSRequests.Requests[0].NBDURI)
|
||||
}
|
||||
|
||||
// TestAttachVolume_InvalidExport_TriggersUnmountOne covers a mount that
|
||||
// returns both exports: AttachVolume must unmount before any QMP traffic.
|
||||
func TestAttachVolume_InvalidExport_TriggersUnmountOne(t *testing.T) {
|
||||
recorder := &qmpRecorder{}
|
||||
qmpClient, cancel := newMockQMPClient(t, func(cmd qmp.QMPCommand) map[string]any {
|
||||
recorder.record(cmd)
|
||||
return nil
|
||||
})
|
||||
defer cancel()
|
||||
|
||||
mounter := &fakeVolumeMounter{mountOneURI: "nbd:unix:/tmp/test.sock", mountOneBlockDevice: "/dev/sdx", mountOneBlockSize: 1 << 20}
|
||||
m, _ := attachVolumeRunningInstance(t, qmpClient, mounter)
|
||||
|
||||
_, err := m.AttachVolume(t.Context(), "i-1", "vol-1", "/dev/sdf")
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "both NBDURI and BlockDevice set")
|
||||
assert.Empty(t, recorder.executes())
|
||||
assert.Equal(t, []string{"vol-1"}, mounter.unmountedOne)
|
||||
}
|
||||
|
||||
// TestAttachVolume_BlockDeviceBlockdevAddFailure_TriggersUnmountOne checks a
|
||||
// block export unwinds a failed blockdev-add exactly as an NBD one does.
|
||||
func TestAttachVolume_BlockDeviceBlockdevAddFailure_TriggersUnmountOne(t *testing.T) {
|
||||
recorder := &qmpRecorder{}
|
||||
qmpClient, cancel := newMockQMPClient(t, func(cmd qmp.QMPCommand) map[string]any {
|
||||
recorder.record(cmd)
|
||||
if cmd.Execute == "blockdev-add" {
|
||||
return map[string]any{"error": map[string]any{"class": "GenericError", "desc": "Could not open '/dev/sdx'"}}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
defer cancel()
|
||||
|
||||
mounter := &fakeVolumeMounter{mountOneBlockDevice: "/dev/sdx", mountOneBlockSize: 1 << 20}
|
||||
m, _ := attachVolumeRunningInstance(t, qmpClient, mounter)
|
||||
|
||||
_, err := m.AttachVolume(t.Context(), "i-1", "vol-1", "/dev/sdf")
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "blockdev-add")
|
||||
assert.Equal(t, []string{"object-add", "blockdev-add", "object-del"}, recorder.executes())
|
||||
assert.Equal(t, []string{"vol-1"}, mounter.unmountedOne)
|
||||
}
|
||||
|
||||
// TestAttachVolume_ObjectAddFailure_TriggersUnmountOne covers QMP iothread
|
||||
// object-add failure: AttachVolume must unmount and bail before issuing
|
||||
// blockdev-add (no further QMP traffic past the failed object-add).
|
||||
|
||||
@@ -104,8 +104,8 @@ func TestLive_QEMUNBDLifecycle(t *testing.T) {
|
||||
|
||||
pub, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: nodeID})
|
||||
require.NoError(t, err, "publish volume")
|
||||
t.Logf("published volume %s to node %s: %s", pub.VolumeID, pub.NodeID, pub.NBDURI)
|
||||
sockPath := requireUnixSocketPath(t, pub.NBDURI)
|
||||
t.Logf("published volume %s to node %s: %s", pub.VolumeID, pub.NodeID, liveNBDURI(t, pub))
|
||||
sockPath := requireUnixSocketPath(t, liveNBDURI(t, pub))
|
||||
|
||||
writePattern(t, sockPath, pattern, liveVolumeBytes)
|
||||
readAndVerifyPattern(t, sockPath, pattern, liveVolumeBytes)
|
||||
@@ -129,8 +129,8 @@ func TestLive_QEMUNBDLifecycle(t *testing.T) {
|
||||
|
||||
restoredPub, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: restoredVolumeID, NodeID: nodeID})
|
||||
require.NoError(t, err, "publish restored volume")
|
||||
t.Logf("published restored volume %s to node %s: %s", restoredPub.VolumeID, restoredPub.NodeID, restoredPub.NBDURI)
|
||||
restoredSockPath := requireUnixSocketPath(t, restoredPub.NBDURI)
|
||||
t.Logf("published restored volume %s to node %s: %s", restoredPub.VolumeID, restoredPub.NodeID, liveNBDURI(t, restoredPub))
|
||||
restoredSockPath := requireUnixSocketPath(t, liveNBDURI(t, restoredPub))
|
||||
|
||||
// The real proof: bytes written before the snapshot survive both the
|
||||
// snapshot and the restore into an independent volume.
|
||||
@@ -151,6 +151,15 @@ func cleanupBestEffort(t *testing.T, step string, fn func() error) {
|
||||
}
|
||||
}
|
||||
|
||||
// liveNBDURI requires pub to carry an nbd export, which is all qemunbdd
|
||||
// publishes, and returns its URI.
|
||||
func liveNBDURI(t *testing.T, pub *ebsprovider.PublishedVolume) string {
|
||||
t.Helper()
|
||||
require.Equal(t, ebsprovider.ExportKindNBD, pub.Export.Kind, "qemunbdd must publish an nbd export")
|
||||
require.NotNil(t, pub.Export.NBD, "nbd export has no nbd details")
|
||||
return pub.Export.NBD.NBDURI
|
||||
}
|
||||
|
||||
// requireUnixSocketPath asserts nbdURI is a nbd:unix: URI and returns its
|
||||
// socket path, parsed with the same helper the daemon uses to format it
|
||||
// rather than by slicing the string.
|
||||
|
||||
@@ -688,7 +688,7 @@ func (p *Provider) PublishVolume(ctx context.Context, req ebsprovider.PublishVol
|
||||
if pub.nodeID != req.NodeID {
|
||||
return nil, fmt.Errorf("%w: volume %s is published to %s", ebsprovider.ErrVolumeInUse, req.VolumeID, pub.nodeID)
|
||||
}
|
||||
return &ebsprovider.PublishedVolume{VolumeID: req.VolumeID, NodeID: pub.nodeID, NBDURI: nbd.FormatNBDSocketURI(pub.socketPath)}, nil
|
||||
return &ebsprovider.PublishedVolume{VolumeID: req.VolumeID, NodeID: pub.nodeID, Export: ebsprovider.NewNBDExport(nbd.FormatNBDSocketURI(pub.socketPath))}, nil
|
||||
}
|
||||
|
||||
sockPath := p.socketPath(req.VolumeID)
|
||||
@@ -718,7 +718,7 @@ func (p *Provider) PublishVolume(ctx context.Context, req ebsprovider.PublishVol
|
||||
|
||||
pid := readPID(pidPath)
|
||||
p.published[req.VolumeID] = &publication{nodeID: req.NodeID, socketPath: sockPath, pid: pid}
|
||||
return &ebsprovider.PublishedVolume{VolumeID: req.VolumeID, NodeID: req.NodeID, NBDURI: nbd.FormatNBDSocketURI(sockPath)}, nil
|
||||
return &ebsprovider.PublishedVolume{VolumeID: req.VolumeID, NodeID: req.NodeID, Export: ebsprovider.NewNBDExport(nbd.FormatNBDSocketURI(sockPath))}, nil
|
||||
}
|
||||
|
||||
// readPID best-effort reads the PID qemu-nbd's --fork wrote to pidPath. A
|
||||
|
||||
@@ -544,7 +544,7 @@ func TestPublishVolume_ArgvReadWrite(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "vol-pub", pub.VolumeID)
|
||||
assert.Equal(t, "node-1", pub.NodeID)
|
||||
assert.Equal(t, "nbd+unix:///?socket="+p.socketPath("vol-pub"), pub.NBDURI)
|
||||
assert.Equal(t, ebsprovider.NewNBDExport("nbd+unix:///?socket="+p.socketPath("vol-pub")), pub.Export)
|
||||
|
||||
calls := fr.callsSnapshot()
|
||||
require.Len(t, calls, 1)
|
||||
|
||||
Reference in New Issue
Block a user