diff --git a/.golangci.yml b/.golangci.yml index ee5e2867d..ce17c5200 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -26,6 +26,7 @@ linters: - errorlint # catches ==/!= on errors, type assertions on errors, and %v instead of %w — all break on wrapped errors - exptostd # detects golang.org/x/exp/ functions replaceable by stdlib - fatcontext # detects nested contexts in loops + - forbidigo # blocks the legacy JetStream/KV symbols - gocheckcompilerdirectives # validates go compiler directive comments (//go:) - godot # checks that declaration comments end in a period - goprintffuncname # checks that printf-like functions are named with f at the end @@ -60,6 +61,16 @@ linters: embeddedstructfieldcheck: forbid-mutex: true + forbidigo: + analyze-types: true + forbid: + - pattern: '\.JetStream$' + pkg: '^github\.com/nats-io/nats\.go$' + msg: legacy nc.JetStream() is superseded; construct with jetstream.New(nc). + - pattern: '^nats\.(JetStreamContext|KeyValue.*|KeyWatcher|Err(KeyNotFound|KeyExists|BucketNotFound|NoKeysFound|StreamNotFound))$' + pkg: '^github\.com/nats-io/nats\.go$' + msg: legacy nats KV types/sentinels are superseded; use github.com/nats-io/nats.go/jetstream. + depguard: rules: main: diff --git a/cmd/spinifex/cmd/admin.go b/cmd/spinifex/cmd/admin.go index d17aed25a..750a96662 100644 --- a/cmd/spinifex/cmd/admin.go +++ b/cmd/spinifex/cmd/admin.go @@ -2466,7 +2466,9 @@ func initIAMServiceFromConfig() (*handlers_iam.IAMServiceImpl, *config.ClusterCo return nil, nil, nil, nil, fmt.Errorf("load master key: %w", err) } - svc, err := handlers_iam.NewIAMServiceImpl(nc, masterKey, len(cfg.Nodes)) + // Background: this runs at CLI top level, where there is no request to + // inherit a deadline from and the process exits after the one command. + svc, err := handlers_iam.NewIAMServiceImpl(context.Background(), nc, masterKey, len(cfg.Nodes)) if err != nil { nc.Close() return nil, nil, nil, nil, fmt.Errorf("init IAM service: %w", err) @@ -2500,7 +2502,7 @@ func runAccountCreate(cmd *cobra.Command, args []string) { // Create default VPC for the new account (belt-and-suspenders: daemon also // does this via iam.account.created event, but daemon may not be running). nodeConfig := cfg.Nodes[cfg.Node] - vpcSvc, vpcErr := handlers_ec2_vpc.NewVPCServiceImplWithNATS(&nodeConfig, nc) + vpcSvc, vpcErr := handlers_ec2_vpc.NewVPCServiceImplWithNATS(context.Background(), &nodeConfig, nc) if vpcErr != nil { fmt.Fprintf(os.Stderr, "Warning: could not create default VPC service: %v\n", vpcErr) } else if _, vpcErr = vpcSvc.EnsureDefaultVPC(accountID); vpcErr != nil { diff --git a/spinifex/daemon/daemon.go b/spinifex/daemon/daemon.go index 8c3e8e8f4..ff5ee9442 100644 --- a/spinifex/daemon/daemon.go +++ b/spinifex/daemon/daemon.go @@ -57,6 +57,7 @@ import ( handlers_elbv2 "github.com/mulgadc/spinifex/spinifex/handlers/elbv2" handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" "github.com/mulgadc/spinifex/spinifex/instancetypes" + "github.com/mulgadc/spinifex/spinifex/kvutil" "github.com/mulgadc/spinifex/spinifex/network/external" "github.com/mulgadc/spinifex/spinifex/network/external/dhcp" "github.com/mulgadc/spinifex/spinifex/network/host" @@ -66,6 +67,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/utils" "github.com/mulgadc/spinifex/spinifex/vm" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // ResourceManager handles the allocation and tracking of system resources. @@ -1296,8 +1298,8 @@ func (d *Daemon) startCluster() error { } // Remove the obsolete spinifex-dhcp-leases bucket (idempotent). - if js, jsErr := d.natsConn.JetStream(); jsErr == nil { - if err := utils.DeleteKVBucketIfExists(js, "spinifex-dhcp-leases"); err != nil { + if js, jsErr := jetstream.New(d.natsConn); jsErr == nil { + if err := kvutil.DeleteBucketIfExists(d.ctx, js, "spinifex-dhcp-leases"); err != nil { slog.Warn("Failed to delete obsolete spinifex-dhcp-leases KV bucket", "err", err) } } @@ -1333,10 +1335,10 @@ func (d *Daemon) startCluster() error { type snapResult struct { svc *handlers_ec2_snapshot.SnapshotServiceImpl - kv nats.KeyValue + kv jetstream.KeyValue } snap, err := initServiceWithRetry("snapshot service", func() (snapResult, error) { - svc, kv, err := handlers_ec2_snapshot.NewSnapshotServiceImplWithNATS(d.config, d.natsConn) + svc, kv, err := handlers_ec2_snapshot.NewSnapshotServiceImplWithNATS(d.ctx, d.config, d.natsConn) return snapResult{svc, kv}, err }) if err != nil { @@ -1348,42 +1350,42 @@ func (d *Daemon) startCluster() error { d.tagsService = handlers_ec2_tags.NewTagsServiceImpl(d.config) d.eigwService, err = initServiceWithRetry("EIGW service", func() (*handlers_ec2_eigw.EgressOnlyIGWServiceImpl, error) { - return handlers_ec2_eigw.NewEgressOnlyIGWServiceImplWithNATS(d.config, d.natsConn) + return handlers_ec2_eigw.NewEgressOnlyIGWServiceImplWithNATS(d.ctx, d.config, d.natsConn) }) if err != nil { return fmt.Errorf("failed to initialize EIGW service: %w", err) } d.igwService, err = initServiceWithRetry("IGW service", func() (*handlers_ec2_igw.IGWServiceImpl, error) { - return handlers_ec2_igw.NewIGWServiceImplWithNATS(d.config, d.natsConn) + return handlers_ec2_igw.NewIGWServiceImplWithNATS(d.ctx, d.config, d.natsConn) }) if err != nil { return fmt.Errorf("failed to initialize IGW service: %w", err) } d.placementGroupService, err = initServiceWithRetry("placement group service", func() (*handlers_ec2_placementgroup.PlacementGroupServiceImpl, error) { - return handlers_ec2_placementgroup.NewPlacementGroupServiceImplWithNATS(d.config, d.natsConn) + return handlers_ec2_placementgroup.NewPlacementGroupServiceImplWithNATS(d.ctx, d.config, d.natsConn) }) if err != nil { return fmt.Errorf("failed to initialize placement group service: %w", err) } d.launchTemplateService, err = initServiceWithRetry("launch template service", func() (*handlers_ec2_launchtemplate.LaunchTemplateServiceImpl, error) { - return handlers_ec2_launchtemplate.NewLaunchTemplateServiceImplWithNATS(d.config, d.natsConn) + return handlers_ec2_launchtemplate.NewLaunchTemplateServiceImplWithNATS(d.ctx, d.config, d.natsConn) }) if err != nil { return fmt.Errorf("failed to initialize launch template service: %w", err) } d.spotInstanceService, err = initServiceWithRetry("spot instance service", func() (*handlers_ec2_spotinstance.SpotInstanceServiceImpl, error) { - return handlers_ec2_spotinstance.NewSpotInstanceServiceImplWithNATS(d.config, d.natsConn) + return handlers_ec2_spotinstance.NewSpotInstanceServiceImplWithNATS(d.ctx, d.config, d.natsConn) }) if err != nil { return fmt.Errorf("failed to initialize spot instance service: %w", err) } d.vpcService, err = initServiceWithRetry("VPC service", func() (*handlers_ec2_vpc.VPCServiceImpl, error) { - return handlers_ec2_vpc.NewVPCServiceImplWithNATS(d.config, d.natsConn) + return handlers_ec2_vpc.NewVPCServiceImplWithNATS(d.ctx, d.config, d.natsConn) }) if err != nil { return fmt.Errorf("failed to initialize VPC service: %w", err) @@ -1394,7 +1396,7 @@ func (d *Daemon) startCluster() error { d.vpcService.SetDefaultPublicIPMapping(d.hasPublicIPPools()) d.routeTableService, err = initServiceWithRetry("RouteTable service", func() (*handlers_ec2_routetable.RouteTableServiceImpl, error) { - return handlers_ec2_routetable.NewRouteTableServiceImplWithNATS(d.config, d.natsConn) + return handlers_ec2_routetable.NewRouteTableServiceImplWithNATS(d.ctx, d.config, d.natsConn) }) if err != nil { return fmt.Errorf("failed to initialize RouteTable service: %w", err) @@ -1404,7 +1406,7 @@ func (d *Daemon) startCluster() error { d.igwService.SetGatePublisher(d.routeTableService) d.natGatewayService, err = initServiceWithRetry("NatGateway service", func() (*handlers_ec2_natgw.NatGatewayServiceImpl, error) { - return handlers_ec2_natgw.NewNatGatewayServiceImplWithNATS(d.natsConn) + return handlers_ec2_natgw.NewNatGatewayServiceImplWithNATS(d.ctx, d.natsConn) }) if err != nil { return fmt.Errorf("failed to initialize NatGateway service: %w", err) @@ -1414,7 +1416,7 @@ func (d *Daemon) startCluster() error { // mode with a public pool alongside the transit segment). The transit pool // never enters IPAM — its addresses are gateway-LRP plumbing, not EIPs. if d.hasPublicIPPools() { - js, jsErr := d.natsConn.JetStream() + js, jsErr := jetstream.New(d.natsConn) if jsErr != nil { slog.Warn("Failed to get JetStream for external IPAM", "err", jsErr) } else { @@ -1440,7 +1442,7 @@ func (d *Daemon) startCluster() error { anyDHCP = true } } - d.externalIPAM, err = handlers_ec2_vpc.NewExternalIPAM(js, pools) + d.externalIPAM, err = handlers_ec2_vpc.NewExternalIPAM(d.ctx, js, pools) if err != nil { slog.Warn("Failed to initialize external IPAM", "err", err) } else { @@ -1457,7 +1459,7 @@ func (d *Daemon) startCluster() error { // Initialize EIP service if external IPAM is available if d.externalIPAM != nil && d.vpcService != nil { - eipSvc, eipErr := handlers_ec2_eip.NewEIPServiceImpl(d.natsConn, d.externalIPAM, d.vpcService) + eipSvc, eipErr := handlers_ec2_eip.NewEIPServiceImpl(d.ctx, d.natsConn, d.externalIPAM, d.vpcService) if eipErr != nil { slog.Warn("Failed to initialize EIP service", "err", eipErr) } else { @@ -1467,11 +1469,11 @@ func (d *Daemon) startCluster() error { // Inject external IPAM + EIP KV into VPC service so DeleteNetworkInterface // can release auto-assigned public IPs and NAT rules. - eipJS, eipJSErr := d.natsConn.JetStream() + eipJS, eipJSErr := jetstream.New(d.natsConn) if eipJSErr != nil { slog.Warn("Failed to get JetStream for VPC external IPAM injection", "err", eipJSErr) } else { - eipKV, eipKVErr := utils.GetOrCreateKVBucket(eipJS, handlers_ec2_eip.KVBucketEIPs, 10) + eipKV, eipKVErr := kvutil.GetOrCreateBucket(d.ctx, eipJS, handlers_ec2_eip.KVBucketEIPs, 10) if eipKVErr != nil { slog.Warn("Failed to get EIP KV bucket for VPC service", "err", eipKVErr) } else { @@ -1495,7 +1497,7 @@ func (d *Daemon) startCluster() error { } d.accountService, err = initServiceWithRetry("account settings service", func() (*handlers_ec2_account.AccountSettingsServiceImpl, error) { - return handlers_ec2_account.NewAccountSettingsServiceImplWithNATS(d.config, d.natsConn) + return handlers_ec2_account.NewAccountSettingsServiceImplWithNATS(d.ctx, d.config, d.natsConn) }) if err != nil { return fmt.Errorf("failed to initialize account settings service: %w", err) @@ -1547,9 +1549,9 @@ func (d *Daemon) startCluster() error { // reaper. The scheduler is disabled (handlers still serve) when JetStream is // unavailable. d.ecsService = handlers_ecs.NewService(d.natsConn, d.config.Region, d.clusterConfig.AWS.InternalSuffix).WithDeps(d.buildECSServiceDeps()) - if js, jsErr := d.natsConn.JetStream(); jsErr != nil { + if js, jsErr := jetstream.New(d.natsConn); jsErr != nil { slog.Warn("ECS scheduler disabled: JetStream unavailable", "err", jsErr) - } else if _, lbErr := handlers_ecs.InitLeaderBucket(js); lbErr != nil { + } else if _, lbErr := handlers_ecs.InitLeaderBucket(d.ctx, js); lbErr != nil { slog.Warn("ECS scheduler disabled: leader bucket init failed", "err", lbErr) } else { d.ecsScheduler = handlers_ecs.NewScheduler(d.natsConn, d.ecsService, d.node) @@ -1564,7 +1566,7 @@ func (d *Daemon) startCluster() error { } d.acmService, err = initServiceWithRetry("ACM service", func() (*handlers_acm.ACMServiceImpl, error) { - return handlers_acm.NewACMServiceImplWithNATS(d.config, d.natsConn) + return handlers_acm.NewACMServiceImplWithNATS(d.ctx, d.config, d.natsConn) }) if err != nil { return fmt.Errorf("failed to initialize ACM service: %w", err) @@ -1573,7 +1575,7 @@ func (d *Daemon) startCluster() error { // ECR metadata service: owns per-account JetStream KV for repos, tags, // manifest records and upload-state CAS. Disabled (gateway returns NATS // timeouts) when JetStream is unavailable. - if js, jsErr := d.natsConn.JetStream(); jsErr != nil { + if js, jsErr := jetstream.New(d.natsConn); jsErr != nil { slog.Warn("ECR metadata service disabled: JetStream unavailable", "err", jsErr) } else { d.ecrMetaService = handlers_ecr.NewKVMetaService(js) diff --git a/spinifex/daemon/daemon_handlers_instance_tags_test.go b/spinifex/daemon/daemon_handlers_instance_tags_test.go index 22d8a698d..e3b921615 100644 --- a/spinifex/daemon/daemon_handlers_instance_tags_test.go +++ b/spinifex/daemon/daemon_handlers_instance_tags_test.go @@ -13,7 +13,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/objectstore" "github.com/mulgadc/spinifex/spinifex/types" "github.com/mulgadc/spinifex/spinifex/vm" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -54,11 +54,11 @@ func (m *memStoppedStore) DeleteStoppedInstance(id string) error { } // UpdateStoppedInstance mimics the real CAS semantics: a missing record -// returns nats.ErrKeyNotFound instead of resurrecting it. +// returns jetstream.ErrKeyNotFound instead of resurrecting it. func (m *memStoppedStore) UpdateStoppedInstance(id string, mutate func(*vm.VM)) (*vm.VM, error) { v, ok := m.instances[id] if !ok { - return nil, nats.ErrKeyNotFound + return nil, jetstream.ErrKeyNotFound } mutate(v) return v, nil diff --git a/spinifex/daemon/daemon_handlers_instance_test.go b/spinifex/daemon/daemon_handlers_instance_test.go index c4065165a..1ad2dbc9c 100644 --- a/spinifex/daemon/daemon_handlers_instance_test.go +++ b/spinifex/daemon/daemon_handlers_instance_test.go @@ -15,6 +15,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/utils" "github.com/mulgadc/spinifex/spinifex/vm" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -126,7 +127,7 @@ func (f *fakeStateStore) ClaimStoppedInstance(id string) (*vm.VM, error) { // UpdateStoppedInstance mimics the real CAS semantics for tests: mutate runs // under the store lock against the stored value, and a missing record -// returns nats.ErrKeyNotFound (matching JetStreamManager's createIfAbsent=false +// returns jetstream.ErrKeyNotFound (matching JetStreamManager's createIfAbsent=false // behavior) rather than resurrecting it. func (f *fakeStateStore) UpdateStoppedInstance(id string, mutate func(*vm.VM)) (*vm.VM, error) { f.mu.Lock() @@ -136,7 +137,7 @@ func (f *fakeStateStore) UpdateStoppedInstance(id string, mutate func(*vm.VM)) ( } v, ok := f.stopped[id] if !ok { - return nil, nats.ErrKeyNotFound + return nil, jetstream.ErrKeyNotFound } mutate(v) return v, nil diff --git a/spinifex/daemon/daemon_handlers_test.go b/spinifex/daemon/daemon_handlers_test.go index 3252b58a3..962fcdd5d 100644 --- a/spinifex/daemon/daemon_handlers_test.go +++ b/spinifex/daemon/daemon_handlers_test.go @@ -40,6 +40,7 @@ import ( "github.com/mulgadc/viperblock/viperblock" "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -132,7 +133,7 @@ func initAccountServiceForTest(t *testing.T, daemon *Daemon) { require.NoError(t, err) t.Cleanup(func() { nc.Close() }) - svc, err := handlers_ec2_account.NewAccountSettingsServiceImplWithNATS(nil, nc) + svc, err := handlers_ec2_account.NewAccountSettingsServiceImplWithNATS(t.Context(), nil, nc) require.NoError(t, err) daemon.accountService = svc } @@ -2295,11 +2296,11 @@ func createVPCTestDaemon(t *testing.T) *Daemon { testutil.StubVpcdSGResponder(t, nc) - vpcSvc, err := handlers_ec2_vpc.NewVPCServiceImplWithNATS(daemon.config, nc) + vpcSvc, err := handlers_ec2_vpc.NewVPCServiceImplWithNATS(t.Context(), daemon.config, nc) require.NoError(t, err) daemon.vpcService = vpcSvc - igwSvc, err := handlers_ec2_igw.NewIGWServiceImplWithNATS(daemon.config, nc) + igwSvc, err := handlers_ec2_igw.NewIGWServiceImplWithNATS(t.Context(), daemon.config, nc) require.NoError(t, err) daemon.igwService = igwSvc @@ -2611,7 +2612,7 @@ func TestDelegateHandlers_EIGW(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { nc.Close() }) - eigwSvc, err := handlers_ec2_eigw.NewEgressOnlyIGWServiceImplWithNATS(daemon.config, nc) + eigwSvc, err := handlers_ec2_eigw.NewEgressOnlyIGWServiceImplWithNATS(t.Context(), daemon.config, nc) require.NoError(t, err) daemon.eigwService = eigwSvc @@ -3563,14 +3564,14 @@ func TestDelegateHandlers_EIP(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { nc.Close() }) - js, err := nc.JetStream() + js, err := jetstream.New(nc) require.NoError(t, err) - ipam, err := handlers_ec2_vpc.NewExternalIPAM(js, []external.ExternalPoolConfig{ + ipam, err := handlers_ec2_vpc.NewExternalIPAM(t.Context(), js, []external.ExternalPoolConfig{ {Name: "test-pool", RangeStart: "192.168.100.2", RangeEnd: "192.168.100.254", Gateway: "192.168.100.1", PrefixLen: 24}, }) require.NoError(t, err) - eipSvc, err := handlers_ec2_eip.NewEIPServiceImpl(nc, ipam, daemon.vpcService) + eipSvc, err := handlers_ec2_eip.NewEIPServiceImpl(t.Context(), nc, ipam, daemon.vpcService) require.NoError(t, err) daemon.eipService = eipSvc @@ -3756,7 +3757,7 @@ func TestDelegateHandlers_RouteTable(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { nc.Close() }) - rtbSvc, err := handlers_ec2_routetable.NewRouteTableServiceImplWithNATS(daemon.config, nc) + rtbSvc, err := handlers_ec2_routetable.NewRouteTableServiceImplWithNATS(t.Context(), daemon.config, nc) require.NoError(t, err) daemon.routeTableService = rtbSvc @@ -3889,7 +3890,7 @@ func TestDelegateHandlers_PlacementGroup(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { nc.Close() }) - pgSvc, err := handlers_ec2_placementgroup.NewPlacementGroupServiceImplWithNATS(daemon.config, nc) + pgSvc, err := handlers_ec2_placementgroup.NewPlacementGroupServiceImplWithNATS(t.Context(), daemon.config, nc) require.NoError(t, err) daemon.placementGroupService = pgSvc diff --git a/spinifex/daemon/daemon_mgmt_ip.go b/spinifex/daemon/daemon_mgmt_ip.go index d9f7b63fb..44977bc59 100644 --- a/spinifex/daemon/daemon_mgmt_ip.go +++ b/spinifex/daemon/daemon_mgmt_ip.go @@ -8,7 +8,7 @@ import ( "sync" "github.com/mulgadc/spinifex/spinifex/vm" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // mgmtIPAMKeyPrefix namespaces mgmt-IP allocation records within the shared @@ -38,7 +38,7 @@ func updateMgmtIPAMWithRetry(jsManager *JetStreamManager, subnet string, mutate if err == nil { return record, nil } - if !errors.Is(err, nats.ErrKeyExists) { + if !errors.Is(err, jetstream.ErrKeyExists) { return nil, err } lastErr = err @@ -247,7 +247,7 @@ func (a *MgmtIPAllocator) Release(instanceID string) { // createIfAbsent=false: releasing an instance this node never allocated // (or that no node has a record for) must not conjure an empty record - // into existence — nats.ErrKeyNotFound just means there was nothing to + // into existence — jetstream.ErrKeyNotFound just means there was nothing to // release. _, err := updateMgmtIPAMWithRetry(jsManager, subnet, func(r *MgmtIPRecord) { for i, e := range r.Allocated { @@ -257,7 +257,7 @@ func (a *MgmtIPAllocator) Release(instanceID string) { } } }, false) - if err != nil && !errors.Is(err, nats.ErrKeyNotFound) { + if err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { slog.Warn("Failed to release mgmt IP in cluster KV; address remains reserved until reconciled", "instance", instanceID, "err", err) } diff --git a/spinifex/daemon/daemon_mgmt_ip_test.go b/spinifex/daemon/daemon_mgmt_ip_test.go index d58e7b2ef..5b9a4bb11 100644 --- a/spinifex/daemon/daemon_mgmt_ip_test.go +++ b/spinifex/daemon/daemon_mgmt_ip_test.go @@ -1,6 +1,7 @@ package daemon import ( + "context" "fmt" "strings" "sync" @@ -38,7 +39,7 @@ func newTestMgmtJSM(t *testing.T) *JetStreamManager { func cleanupMgmtIPAM(t *testing.T, jsm *JetStreamManager, a *MgmtIPAllocator) { t.Helper() t.Cleanup(func() { - _ = jsm.clusterKV.Delete(mgmtIPAMKeyPrefix + a.subnet) + _ = jsm.clusterKV.Delete(context.Background(), mgmtIPAMKeyPrefix+a.subnet) }) } diff --git a/spinifex/daemon/daemon_system_instance_test.go b/spinifex/daemon/daemon_system_instance_test.go index 62140fa48..0c90874e8 100644 --- a/spinifex/daemon/daemon_system_instance_test.go +++ b/spinifex/daemon/daemon_system_instance_test.go @@ -21,6 +21,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/vm" "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -637,9 +638,9 @@ func TestRefreshSystemInstanceState_RewritesBlobs(t *testing.T) { d.elbv2Service.GatewayURL = "https://10.0.0.1:9999" d.elbv2Service.CACert = "-----BEGIN CERTIFICATE-----\nfake-ca\n-----END CERTIFICATE-----\n" - seedStore, err := handlers_elbv2.NewStore(nc) + seedStore, err := handlers_elbv2.NewStore(t.Context(), nc) require.NoError(t, err) - require.NoError(t, seedStore.PutLoadBalancer(&handlers_elbv2.LoadBalancerRecord{ + require.NoError(t, seedStore.PutLoadBalancer(t.Context(), &handlers_elbv2.LoadBalancerRecord{ LoadBalancerID: "lb-recover", Name: "recover-alb", Scheme: handlers_elbv2.SchemeInternal, @@ -683,9 +684,9 @@ func TestRefreshSystemInstanceState_RewritesBlobs(t *testing.T) { func TestRefreshSystemInstanceState_WriteErrorPropagates(t *testing.T) { d, nc, tmpDir := newRefreshTestDaemon(t) - seedStore, err := handlers_elbv2.NewStore(nc) + seedStore, err := handlers_elbv2.NewStore(t.Context(), nc) require.NoError(t, err) - require.NoError(t, seedStore.PutLoadBalancer(&handlers_elbv2.LoadBalancerRecord{ + require.NoError(t, seedStore.PutLoadBalancer(t.Context(), &handlers_elbv2.LoadBalancerRecord{ LoadBalancerID: "lb-werr", Name: "werr-alb", Scheme: handlers_elbv2.SchemeInternal, @@ -747,9 +748,9 @@ func TestLaunchSystemInstance_NATFailureRollsBackPublicIP(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { jsNC.Close() }) - js, err := jsNC.JetStream() + js, err := jetstream.New(jsNC) require.NoError(t, err) - ipam, err := handlers_ec2_vpc.NewExternalIPAM(js, []external.ExternalPoolConfig{ + ipam, err := handlers_ec2_vpc.NewExternalIPAM(t.Context(), js, []external.ExternalPoolConfig{ {Name: "wan-test", RangeStart: "203.0.113.10", RangeEnd: "203.0.113.20", Gateway: "203.0.113.1", PrefixLen: 24}, }) require.NoError(t, err) @@ -869,15 +870,15 @@ func TestReleaseSystemInstanceEIP_ReleasesEipServiceAllocation(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { jsNC.Close() }) - js, err := jsNC.JetStream() + js, err := jetstream.New(jsNC) require.NoError(t, err) - ipam, err := handlers_ec2_vpc.NewExternalIPAM(js, []external.ExternalPoolConfig{ + ipam, err := handlers_ec2_vpc.NewExternalIPAM(t.Context(), js, []external.ExternalPoolConfig{ {Name: "wan-test", RangeStart: "203.0.113.10", RangeEnd: "203.0.113.20", Gateway: "203.0.113.1", PrefixLen: 24}, }) require.NoError(t, err) d.externalIPAM = ipam - eipSvc, err := handlers_ec2_eip.NewEIPServiceImpl(jsNC, ipam, d.vpcService) + eipSvc, err := handlers_ec2_eip.NewEIPServiceImpl(t.Context(), jsNC, ipam, d.vpcService) require.NoError(t, err) d.eipService = eipSvc diff --git a/spinifex/daemon/daemon_test.go b/spinifex/daemon/daemon_test.go index 84f08f8e7..63249aad0 100644 --- a/spinifex/daemon/daemon_test.go +++ b/spinifex/daemon/daemon_test.go @@ -54,6 +54,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/vm" "github.com/mulgadc/viperblock/viperblock" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -1565,9 +1566,9 @@ func TestInstanceCleanerAdapter_DeleteVolumes_BootVolumeDeletedAfterAttachmentCl jsConn, err := nats.Connect(sharedJSNATSURL) require.NoError(t, err) t.Cleanup(func() { jsConn.Close() }) - js, err := jsConn.JetStream() + js, err := jetstream.New(jsConn) require.NoError(t, err) - snapKV, err := js.CreateKeyValue(&nats.KeyValueConfig{ + snapKV, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{ Bucket: "snap-kv-" + strings.ReplaceAll(t.Name(), "/", "-"), }) require.NoError(t, err) @@ -1659,9 +1660,9 @@ func TestTerminatedTeardownReaper_SelfHealsFailedVolumeTeardown(t *testing.T) { daemon.stateStore = newStateStoreAdapter(jsManager) // DeleteVolume's snapshot-reference guard requires a non-nil snapshotKV. - js, err := daemon.natsConn.JetStream() + js, err := jetstream.New(daemon.natsConn) require.NoError(t, err) - snapKV, err := js.CreateKeyValue(&nats.KeyValueConfig{ + snapKV, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{ Bucket: "snap-kv-" + strings.ReplaceAll(t.Name(), "/", "-"), }) require.NoError(t, err) diff --git a/spinifex/daemon/ecs_deps.go b/spinifex/daemon/ecs_deps.go index daea71bfc..30fa57890 100644 --- a/spinifex/daemon/ecs_deps.go +++ b/spinifex/daemon/ecs_deps.go @@ -46,7 +46,7 @@ func (d *Daemon) buildECSServiceDeps() handlers_ecs.Deps { if d.clusterConfig != nil { clusterSize = len(d.clusterConfig.Nodes) } - if iamSvc, iamErr := handlers_iam.NewIAMServiceImpl(d.natsConn, masterKey, clusterSize); iamErr != nil { + if iamSvc, iamErr := handlers_iam.NewIAMServiceImpl(d.ctx, d.natsConn, masterKey, clusterSize); iamErr != nil { slog.Warn("ECS: IAM service init failed; capacity provisioning disabled", "err", iamErr) } else { deps.IAM = iamSvc diff --git a/spinifex/daemon/eks_deps.go b/spinifex/daemon/eks_deps.go index aca4b7362..1802fb25f 100644 --- a/spinifex/daemon/eks_deps.go +++ b/spinifex/daemon/eks_deps.go @@ -37,7 +37,7 @@ func (d *Daemon) systemRoleEnsurer() handlers_iam.SystemInstanceRoleEnsurer { if d.clusterConfig != nil { clusterSize = len(d.clusterConfig.Nodes) } - iamSvc, iamErr := handlers_iam.NewIAMServiceImpl(d.natsConn, masterKey, clusterSize) + iamSvc, iamErr := handlers_iam.NewIAMServiceImpl(d.ctx, d.natsConn, masterKey, clusterSize) if iamErr != nil { slog.Warn("System role ensurer: IAM service init failed (retried on next launch); system VMs fall back to baked static creds", "err", iamErr) diff --git a/spinifex/daemon/heartbeat_test.go b/spinifex/daemon/heartbeat_test.go index 949403c44..3d8ef7c53 100644 --- a/spinifex/daemon/heartbeat_test.go +++ b/spinifex/daemon/heartbeat_test.go @@ -124,7 +124,7 @@ func TestHeartbeatKVContract(t *testing.T) { // Verify the KV key prefix and the JSON tag names directly. Reading back // via ReadHeartbeat alone would also pass against a typo'd tag. - entry, err := jsm.clusterKV.Get("heartbeat." + h.Node) + entry, err := jsm.clusterKV.Get(t.Context(), "heartbeat."+h.Node) require.NoError(t, err) var raw map[string]any diff --git a/spinifex/daemon/jetstream.go b/spinifex/daemon/jetstream.go index 4dace1be9..cf4b1a881 100644 --- a/spinifex/daemon/jetstream.go +++ b/spinifex/daemon/jetstream.go @@ -1,6 +1,7 @@ package daemon import ( + "context" "encoding/json" "errors" "fmt" @@ -13,6 +14,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/utils" "github.com/mulgadc/spinifex/spinifex/vm" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) const ( @@ -46,10 +48,10 @@ type KVSyncObserver interface { // JetStreamManager manages JetStream KV store operations for instance state. type JetStreamManager struct { - js nats.JetStreamContext - kv nats.KeyValue // spinifex-instance-state - clusterKV nats.KeyValue // spinifex-cluster-state - terminatedKV nats.KeyValue // spinifex-terminated-instances + js jetstream.JetStream + kv jetstream.KeyValue // spinifex-instance-state + clusterKV jetstream.KeyValue // spinifex-cluster-state + terminatedKV jetstream.KeyValue // spinifex-terminated-instances replicas int kvMu sync.Mutex // protects kv during recovery obs KVSyncObserver @@ -68,14 +70,14 @@ func (m *JetStreamManager) KVHealthy() bool { if m.js == nil { return false } - _, err := m.js.AccountInfo() + _, err := m.js.AccountInfo(context.Background()) return err == nil } // NewJetStreamManager creates a new JetStreamManager from a NATS connection. // replicas specifies the number of replicas for the KV bucket (typically matches cluster node count). func NewJetStreamManager(nc *nats.Conn, replicas int) (*JetStreamManager, error) { - js, err := nc.JetStream() + js, err := jetstream.New(nc) if err != nil { return nil, err } @@ -88,13 +90,14 @@ func NewJetStreamManager(nc *nats.Conn, replicas int) (*JetStreamManager, error) // InitKVBucket initializes the KV bucket, creating it if it doesn't exist. func (m *JetStreamManager) InitKVBucket() error { + ctx := context.Background() // Try to get the existing bucket first - kv, err := m.js.KeyValue(InstanceStateBucket) + kv, err := m.js.KeyValue(ctx, InstanceStateBucket) if err != nil { - if errors.Is(err, nats.ErrBucketNotFound) { + if errors.Is(err, jetstream.ErrBucketNotFound) { // Bucket doesn't exist, create it slog.Debug("Creating JetStream KV bucket", "bucket", InstanceStateBucket, "replicas", m.replicas) - kv, err = m.js.CreateKeyValue(&nats.KeyValueConfig{ + kv, err = m.js.CreateKeyValue(ctx, jetstream.KeyValueConfig{ Bucket: InstanceStateBucket, Description: "Spinifex instance state storage", History: 1, // Only keep latest value @@ -111,7 +114,7 @@ func (m *JetStreamManager) InitKVBucket() error { } m.kv = kv - if err := migrate.DefaultRegistry.RunKV(InstanceStateBucket, kv, InstanceStateBucketVersion); err != nil { + if err := migrate.DefaultRegistry.RunKV(ctx, InstanceStateBucket, kv, InstanceStateBucketVersion); err != nil { return fmt.Errorf("migrate %s: %w", InstanceStateBucket, err) } return nil @@ -119,11 +122,12 @@ func (m *JetStreamManager) InitKVBucket() error { // InitClusterStateBucket initializes the cluster-state KV bucket, creating it if it doesn't exist. func (m *JetStreamManager) InitClusterStateBucket() error { - kv, err := m.js.KeyValue(ClusterStateBucket) + ctx := context.Background() + kv, err := m.js.KeyValue(ctx, ClusterStateBucket) if err != nil { - if errors.Is(err, nats.ErrBucketNotFound) { + if errors.Is(err, jetstream.ErrBucketNotFound) { slog.Debug("Creating JetStream KV bucket", "bucket", ClusterStateBucket, "replicas", m.replicas) - kv, err = m.js.CreateKeyValue(&nats.KeyValueConfig{ + kv, err = m.js.CreateKeyValue(ctx, jetstream.KeyValueConfig{ Bucket: ClusterStateBucket, Description: "Spinifex cluster state (heartbeats, shutdown markers, service maps)", History: 1, @@ -141,7 +145,7 @@ func (m *JetStreamManager) InitClusterStateBucket() error { } m.clusterKV = kv - if err := migrate.DefaultRegistry.RunKV(ClusterStateBucket, kv, ClusterStateBucketVersion); err != nil { + if err := migrate.DefaultRegistry.RunKV(ctx, ClusterStateBucket, kv, ClusterStateBucketVersion); err != nil { return fmt.Errorf("migrate %s: %w", ClusterStateBucket, err) } return nil @@ -150,11 +154,12 @@ func (m *JetStreamManager) InitClusterStateBucket() error { // InitTerminatedInstanceBucket initializes the terminated-instances KV bucket with a 1-hour TTL. // JetStream automatically purges keys after 1 hour, matching AWS behavior for terminated instances. func (m *JetStreamManager) InitTerminatedInstanceBucket() error { - kv, err := m.js.KeyValue(TerminatedInstanceBucket) + ctx := context.Background() + kv, err := m.js.KeyValue(ctx, TerminatedInstanceBucket) if err != nil { - if errors.Is(err, nats.ErrBucketNotFound) { + if errors.Is(err, jetstream.ErrBucketNotFound) { slog.Debug("Creating JetStream KV bucket", "bucket", TerminatedInstanceBucket, "replicas", m.replicas) - kv, err = m.js.CreateKeyValue(&nats.KeyValueConfig{ + kv, err = m.js.CreateKeyValue(ctx, jetstream.KeyValueConfig{ Bucket: TerminatedInstanceBucket, Description: "Terminated instances (auto-expire after 1 hour)", History: 1, @@ -172,7 +177,7 @@ func (m *JetStreamManager) InitTerminatedInstanceBucket() error { } m.terminatedKV = kv - if err := migrate.DefaultRegistry.RunKV(TerminatedInstanceBucket, kv, TerminatedInstanceBucketVersion); err != nil { + if err := migrate.DefaultRegistry.RunKV(ctx, TerminatedInstanceBucket, kv, TerminatedInstanceBucketVersion); err != nil { return fmt.Errorf("migrate %s: %w", TerminatedInstanceBucket, err) } return nil @@ -189,8 +194,8 @@ func isStreamUnavailable(err error) bool { if err == nil { return false } - if errors.Is(err, nats.ErrStreamNotFound) || - errors.Is(err, nats.ErrNoStreamResponse) || + if errors.Is(err, jetstream.ErrStreamNotFound) || + errors.Is(err, jetstream.ErrNoStreamResponse) || errors.Is(err, nats.ErrNoResponders) { return true } @@ -201,19 +206,19 @@ func isStreamUnavailable(err error) bool { // underlying JetStream stream was lost during cluster formation. // Returns the recovered KV handle directly so callers avoid a racy re-read. // When a bucket is recreated, the schema version is re-stamped. -func (m *JetStreamManager) recoverBucket(cfg *nats.KeyValueConfig, field *nats.KeyValue, version int) (nats.KeyValue, error) { +func (m *JetStreamManager) recoverBucket(ctx context.Context, cfg jetstream.KeyValueConfig, field *jetstream.KeyValue, version int) (jetstream.KeyValue, error) { m.kvMu.Lock() defer m.kvMu.Unlock() // Try to reconnect to existing bucket first (another goroutine may have recovered it) - kv, err := m.js.KeyValue(cfg.Bucket) + kv, err := m.js.KeyValue(ctx, cfg.Bucket) if err == nil { *field = kv slog.Info("Reconnected to KV bucket", "bucket", cfg.Bucket) return kv, nil } - if !errors.Is(err, nats.ErrBucketNotFound) && !isStreamUnavailable(err) { + if !errors.Is(err, jetstream.ErrBucketNotFound) && !isStreamUnavailable(err) { return nil, err } @@ -221,13 +226,13 @@ func (m *JetStreamManager) recoverBucket(cfg *nats.KeyValueConfig, field *nats.K slog.Warn("KV bucket stream lost, recreating", "bucket", cfg.Bucket, "replicas", m.replicas) cfg.History = 1 cfg.Replicas = m.replicas - kv, err = m.js.CreateKeyValue(cfg) + kv, err = m.js.CreateKeyValue(ctx, cfg) if err != nil { slog.Error("Failed to recreate KV bucket", "bucket", cfg.Bucket, "err", err) return nil, err } - if err := migrate.DefaultRegistry.RunKV(cfg.Bucket, kv, version); err != nil { + if err := migrate.DefaultRegistry.RunKV(ctx, cfg.Bucket, kv, version); err != nil { slog.Error("Failed to run migrations on recreated bucket", "bucket", cfg.Bucket, "err", err) return nil, fmt.Errorf("migrate recreated bucket %s: %w", cfg.Bucket, err) } @@ -237,15 +242,15 @@ func (m *JetStreamManager) recoverBucket(cfg *nats.KeyValueConfig, field *nats.K return kv, nil } -func (m *JetStreamManager) recoverKVBucket() (nats.KeyValue, error) { - return m.recoverBucket(&nats.KeyValueConfig{ +func (m *JetStreamManager) recoverKVBucket(ctx context.Context) (jetstream.KeyValue, error) { + return m.recoverBucket(ctx, jetstream.KeyValueConfig{ Bucket: InstanceStateBucket, Description: "Spinifex instance state storage", }, &m.kv, InstanceStateBucketVersion) } -func (m *JetStreamManager) recoverTerminatedKVBucket() (nats.KeyValue, error) { - return m.recoverBucket(&nats.KeyValueConfig{ +func (m *JetStreamManager) recoverTerminatedKVBucket(ctx context.Context) (jetstream.KeyValue, error) { + return m.recoverBucket(ctx, jetstream.KeyValueConfig{ Bucket: TerminatedInstanceBucket, Description: "Terminated instances (auto-expire after 1 hour)", TTL: 1 * time.Hour, @@ -260,10 +265,10 @@ const maxCASRetries = 5 // isRevisionConflict reports whether err is the nats.go "wrong last // sequence" error that KeyValue.Update/Create return when the key's // revision no longer matches what the caller expected, i.e. a concurrent -// writer won the race. nats.go surfaces this as nats.ErrKeyExists for both -// Update (stale revision) and Create (key already present). +// writer won the race. The jetstream package surfaces this as +// jetstream.ErrKeyExists for both Update (stale revision) and Create (key already present). func isRevisionConflict(err error) bool { - return errors.Is(err, nats.ErrKeyExists) + return errors.Is(err, jetstream.ErrKeyExists) } // casUpdate performs an optimistic-concurrency read-modify-write against kv: @@ -272,20 +277,20 @@ func isRevisionConflict(err error) bool { // Update/Create using the observed revision. A concurrent writer that changes // the key between Get and Update/Create is detected via a revision conflict // and retried, bounded by maxCASRetries. -func casUpdate[T any](kv nats.KeyValue, key string, mutate func(*T), createIfAbsent bool) (*T, error) { +func casUpdate[T any](ctx context.Context, kv jetstream.KeyValue, key string, mutate func(*T), createIfAbsent bool) (*T, error) { var lastErr error for range maxCASRetries { var value T var revision uint64 - entry, err := kv.Get(key) + entry, err := kv.Get(ctx, key) switch { case err == nil: if uerr := json.Unmarshal(entry.Value(), &value); uerr != nil { return nil, uerr } revision = entry.Revision() - case errors.Is(err, nats.ErrKeyNotFound) && createIfAbsent: + case errors.Is(err, jetstream.ErrKeyNotFound) && createIfAbsent: revision = 0 default: return nil, err @@ -299,9 +304,9 @@ func casUpdate[T any](kv nats.KeyValue, key string, mutate func(*T), createIfAbs } if revision == 0 { - _, err = kv.Create(key, data) + _, err = kv.Create(ctx, key, data) } else { - _, err = kv.Update(key, data, revision) + _, err = kv.Update(ctx, key, data, revision) } if err == nil { return &value, nil @@ -322,25 +327,25 @@ func casUpdate[T any](kv nats.KeyValue, key string, mutate func(*T), createIfAbs // counterpart to casUpdate, for values (like vm.VM, which embeds a // sync.Mutex) that cannot be safely decoded into and copied out of a shared // struct value. -func casPut(kv nats.KeyValue, key string, data []byte, createIfAbsent bool) error { +func casPut(ctx context.Context, kv jetstream.KeyValue, key string, data []byte, createIfAbsent bool) error { var lastErr error for range maxCASRetries { var revision uint64 - entry, err := kv.Get(key) + entry, err := kv.Get(ctx, key) switch { case err == nil: revision = entry.Revision() - case errors.Is(err, nats.ErrKeyNotFound) && createIfAbsent: + case errors.Is(err, jetstream.ErrKeyNotFound) && createIfAbsent: revision = 0 default: return err } if revision == 0 { - _, err = kv.Create(key, data) + _, err = kv.Create(ctx, key, data) } else { - _, err = kv.Update(key, data, revision) + _, err = kv.Update(ctx, key, data, revision) } if err == nil { return nil @@ -363,12 +368,12 @@ func casPut(kv nats.KeyValue, key string, data []byte, createIfAbsent bool) erro // detected via a revision conflict and retried, bounded by maxCASRetries, // so a losing racer only fails outright when the key is truly gone // (notFound) or retries are exhausted. -func casClaim[T any](kv nats.KeyValue, key string) (value *T, notFound bool, err error) { +func casClaim[T any](ctx context.Context, kv jetstream.KeyValue, key string) (value *T, notFound bool, err error) { var lastErr error for range maxCASRetries { - entry, gerr := kv.Get(key) + entry, gerr := kv.Get(ctx, key) if gerr != nil { - if errors.Is(gerr, nats.ErrKeyNotFound) { + if errors.Is(gerr, jetstream.ErrKeyNotFound) { return nil, true, nil } return nil, false, gerr @@ -379,7 +384,7 @@ func casClaim[T any](kv nats.KeyValue, key string) (value *T, notFound bool, err return nil, false, uerr } - if derr := kv.Delete(key, nats.LastRevision(entry.Revision())); derr != nil { + if derr := kv.Delete(ctx, key, jetstream.LastRevision(entry.Revision())); derr != nil { if isRevisionConflict(derr) { lastErr = derr continue @@ -422,7 +427,7 @@ func (m *JetStreamManager) WriteHeartbeat(h *Heartbeat) error { if err != nil { return err } - _, err = m.clusterKV.Put("heartbeat."+h.Node, data) + _, err = m.clusterKV.Put(context.Background(), "heartbeat."+h.Node, data) return err } @@ -431,7 +436,7 @@ func (m *JetStreamManager) ReadHeartbeat(nodeID string) (*Heartbeat, error) { if m.clusterKV == nil { return nil, errors.New("cluster state KV not initialized") } - entry, err := m.clusterKV.Get("heartbeat." + nodeID) + entry, err := m.clusterKV.Get(context.Background(), "heartbeat."+nodeID) if err != nil { return nil, err } @@ -461,7 +466,7 @@ func (m *JetStreamManager) WriteClusterShutdown(state *ClusterShutdownState) err if m.clusterKV == nil { return errors.New("cluster state KV not initialized") } - _, err := casUpdate(m.clusterKV, "cluster.shutdown", func(s *ClusterShutdownState) { + _, err := casUpdate(context.Background(), m.clusterKV, "cluster.shutdown", func(s *ClusterShutdownState) { *s = *state }, true) return err @@ -470,12 +475,12 @@ func (m *JetStreamManager) WriteClusterShutdown(state *ClusterShutdownState) err // UpdateClusterShutdown atomically applies mutate to the current cluster // shutdown state (e.g. merging a node's ack into NodesAcked) and writes it // back with optimistic concurrency, retrying on a concurrent writer's -// revision conflict. Returns nats.ErrKeyNotFound if no shutdown is in progress. +// revision conflict. Returns jetstream.ErrKeyNotFound if no shutdown is in progress. func (m *JetStreamManager) UpdateClusterShutdown(mutate func(*ClusterShutdownState)) (*ClusterShutdownState, error) { if m.clusterKV == nil { return nil, errors.New("cluster state KV not initialized") } - return casUpdate(m.clusterKV, "cluster.shutdown", mutate, false) + return casUpdate(context.Background(), m.clusterKV, "cluster.shutdown", mutate, false) } // UpdateMgmtIPAM atomically applies mutate to the mgmt-ipam record for @@ -491,7 +496,7 @@ func (m *JetStreamManager) UpdateMgmtIPAM(subnet string, mutate func(*MgmtIPReco if m.clusterKV == nil { return nil, errors.New("cluster state KV not initialized") } - return casUpdate(m.clusterKV, mgmtIPAMKeyPrefix+subnet, mutate, createIfAbsent) + return casUpdate(context.Background(), m.clusterKV, mgmtIPAMKeyPrefix+subnet, mutate, createIfAbsent) } // ReadClusterShutdown reads the cluster shutdown state from KV. @@ -499,7 +504,7 @@ func (m *JetStreamManager) ReadClusterShutdown() (*ClusterShutdownState, error) if m.clusterKV == nil { return nil, errors.New("cluster state KV not initialized") } - entry, err := m.clusterKV.Get("cluster.shutdown") + entry, err := m.clusterKV.Get(context.Background(), "cluster.shutdown") if err != nil { return nil, err } @@ -515,8 +520,8 @@ func (m *JetStreamManager) DeleteClusterShutdown() error { if m.clusterKV == nil { return errors.New("cluster state KV not initialized") } - err := m.clusterKV.Delete("cluster.shutdown") - if err != nil && !errors.Is(err, nats.ErrKeyNotFound) { + err := m.clusterKV.Delete(context.Background(), "cluster.shutdown") + if err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { return err } return nil @@ -534,7 +539,7 @@ func (m *JetStreamManager) WriteShutdownMarker(nodeID string) error { if err != nil { return fmt.Errorf("marshal shutdown marker: %w", err) } - _, err = m.clusterKV.Put("shutdown."+nodeID, data) + _, err = m.clusterKV.Put(context.Background(), "shutdown."+nodeID, data) return err } @@ -543,8 +548,8 @@ func (m *JetStreamManager) ReadShutdownMarker(nodeID string) (bool, error) { if m.clusterKV == nil { return false, errors.New("cluster state KV not initialized") } - _, err := m.clusterKV.Get("shutdown." + nodeID) - if errors.Is(err, nats.ErrKeyNotFound) { + _, err := m.clusterKV.Get(context.Background(), "shutdown."+nodeID) + if errors.Is(err, jetstream.ErrKeyNotFound) { return false, nil } if err != nil { @@ -558,8 +563,8 @@ func (m *JetStreamManager) DeleteShutdownMarker(nodeID string) error { if m.clusterKV == nil { return errors.New("cluster state KV not initialized") } - err := m.clusterKV.Delete("shutdown." + nodeID) - if err != nil && !errors.Is(err, nats.ErrKeyNotFound) { + err := m.clusterKV.Delete(context.Background(), "shutdown."+nodeID) + if err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { return err } return nil @@ -580,7 +585,7 @@ func (m *JetStreamManager) WriteServiceManifest(nodeID string, services []string if err != nil { return fmt.Errorf("marshal service manifest: %w", err) } - _, err = m.clusterKV.Put("node."+nodeID+".services", data) + _, err = m.clusterKV.Put(context.Background(), "node."+nodeID+".services", data) return err } @@ -597,15 +602,15 @@ func (m *JetStreamManager) WriteState(nodeID string, vms map[string]*vm.VM) erro } key := InstanceStatePrefix + nodeID - _, err = m.kv.Put(key, jsonData) + _, err = m.kv.Put(context.Background(), key, jsonData) if err != nil { if isStreamUnavailable(err) { slog.Warn("KV stream unavailable, attempting recovery", "operation", "WriteState", "key", key, "err", err) - kv, recoverErr := m.recoverKVBucket() + kv, recoverErr := m.recoverKVBucket(context.Background()) if recoverErr != nil { return err } - if _, retryErr := kv.Put(key, jsonData); retryErr != nil { + if _, retryErr := kv.Put(context.Background(), key, jsonData); retryErr != nil { return retryErr } slog.Debug("Wrote state to JetStream KV (after recovery)", "key", key, "instances", len(vms)) @@ -623,43 +628,32 @@ func (m *JetStreamManager) WriteState(nodeID string, vms map[string]*vm.VM) erro // blocks the caller past `timeout` and never returns an error. Used when the // local state file is the source of truth and KV is a best-effort cache; hot // paths marshal under a short-lived lock and commit lock-free. -// -// Note: kv.Put has no context API. On timeout, the in-flight Put goroutine -// continues and completes (or fails) on its own. This leaks at most one -// goroutine per write; under sustained partition the leak is bounded by -// WriteState call cadence (per-state-transition, not a tight loop). func (m *JetStreamManager) WriteStateBytesBestEffort(nodeID string, jsonData []byte, timeout time.Duration) { if m.kv == nil { slog.Debug("KV bucket not initialized, skipping cluster sync", "node", nodeID) return } - key := InstanceStatePrefix + nodeID - done := make(chan error, 1) - go func() { - _, putErr := m.kv.Put(key, jsonData) - done <- putErr - }() + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() - select { - case putErr := <-done: - if putErr != nil { - if m.obs != nil { - m.obs.RecordKVSyncFailure(InstanceStateBucket, putErr) - } - slog.Warn("KV sync failed (best-effort)", "key", key, "err", putErr) + key := InstanceStatePrefix + nodeID + _, err := m.kv.Put(ctx, key, jsonData) + if err != nil { + if m.obs != nil { + m.obs.RecordKVSyncFailure(InstanceStateBucket, err) + } + if errors.Is(err, context.DeadlineExceeded) { + slog.Warn("KV sync timed out (best-effort)", "key", key, "timeout", timeout) return } - if m.obs != nil { - m.obs.RecordKVSyncSuccess(InstanceStateBucket) - } - slog.Debug("Wrote state to KV (best-effort)", "key", key, "bytes", len(jsonData)) - case <-time.After(timeout): - if m.obs != nil { - m.obs.RecordKVSyncFailure(InstanceStateBucket, fmt.Errorf("kv sync timeout after %s", timeout)) - } - slog.Warn("KV sync timed out (best-effort)", "key", key, "timeout", timeout) + slog.Warn("KV sync failed (best-effort)", "key", key, "err", err) + return } + if m.obs != nil { + m.obs.RecordKVSyncSuccess(InstanceStateBucket) + } + slog.Debug("Wrote state to KV (best-effort)", "key", key, "bytes", len(jsonData)) } // marshalInstanceState produces the JSON wire form of vms. @@ -680,22 +674,22 @@ func (m *JetStreamManager) LoadState(nodeID string) (map[string]*vm.VM, error) { } key := InstanceStatePrefix + nodeID - entry, err := m.kv.Get(key) + entry, err := m.kv.Get(context.Background(), key) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { slog.Debug("No existing state in JetStream KV, returning empty state", "key", key) return make(map[string]*vm.VM), nil } if isStreamUnavailable(err) { slog.Warn("KV stream unavailable, attempting recovery", "operation", "LoadState", "key", key, "err", err) - kv, recoverErr := m.recoverKVBucket() + kv, recoverErr := m.recoverKVBucket(context.Background()) if recoverErr != nil { return nil, err } // Retry the read — if we reconnected, data may still exist - entry, err = kv.Get(key) + entry, err = kv.Get(context.Background(), key) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { slog.Warn("No state found after KV recovery", "key", key) return make(map[string]*vm.VM), nil } @@ -728,16 +722,16 @@ func (m *JetStreamManager) DeleteState(nodeID string) error { } key := InstanceStatePrefix + nodeID - err := m.kv.Delete(key) - if err != nil && !errors.Is(err, nats.ErrKeyNotFound) { + err := m.kv.Delete(context.Background(), key) + if err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { if isStreamUnavailable(err) { slog.Warn("KV stream unavailable, attempting recovery", "operation", "DeleteState", "key", key, "err", err) - kv, recoverErr := m.recoverKVBucket() + kv, recoverErr := m.recoverKVBucket(context.Background()) if recoverErr != nil { return err } // Retry — if we reconnected the key may still exist - if retryErr := kv.Delete(key); retryErr != nil && !errors.Is(retryErr, nats.ErrKeyNotFound) { + if retryErr := kv.Delete(context.Background(), key); retryErr != nil && !errors.Is(retryErr, jetstream.ErrKeyNotFound) { return retryErr } return nil @@ -760,14 +754,21 @@ func (m *JetStreamManager) UpdateReplicas(newReplicas int) error { m.replicas = newReplicas + ctx := context.Background() // Iterate all streams and update any KV-backed stream (prefixed "KV_") updated := 0 - for name := range m.js.StreamNames() { + lister := m.js.StreamNames(ctx) + for name := range lister.Name() { if !strings.HasPrefix(name, "KV_") { continue } - info, err := m.js.StreamInfo(name) + stream, err := m.js.Stream(ctx, name) + if err != nil { + slog.Warn("Failed to open stream", "stream", name, "error", err) + continue + } + info, err := stream.Info(ctx) if err != nil { slog.Warn("Failed to get stream info", "stream", name, "error", err) continue @@ -779,7 +780,7 @@ func (m *JetStreamManager) UpdateReplicas(newReplicas int) error { oldReplicas := info.Config.Replicas info.Config.Replicas = newReplicas - if _, err := m.js.UpdateStream(&info.Config); err != nil { + if _, err := m.js.UpdateStream(ctx, info.Config); err != nil { slog.Warn("Failed to update KV bucket replicas", "stream", name, "error", err) continue } @@ -788,6 +789,9 @@ func (m *JetStreamManager) UpdateReplicas(newReplicas int) error { slog.Info("Updated KV bucket replicas", "bucket", bucket, "oldReplicas", oldReplicas, "newReplicas", newReplicas) updated++ } + if err := lister.Err(); err != nil { + return fmt.Errorf("list JetStream streams: %w", err) + } if updated > 0 { slog.Info("KV replication update complete", "bucketsUpdated", updated, "replicas", newReplicas) @@ -813,15 +817,15 @@ func (m *JetStreamManager) WriteStoppedInstance(instanceID string, instance *vm. } key := StoppedInstancePrefix + instanceID - err = casPut(m.kv, key, jsonData, true) + err = casPut(context.Background(), m.kv, key, jsonData, true) if err != nil { if isStreamUnavailable(err) { slog.Warn("KV stream unavailable, attempting recovery", "operation", "WriteStoppedInstance", "key", key, "err", err) - kv, recoverErr := m.recoverKVBucket() + kv, recoverErr := m.recoverKVBucket(context.Background()) if recoverErr != nil { return err } - if retryErr := casPut(kv, key, jsonData, true); retryErr != nil { + if retryErr := casPut(context.Background(), kv, key, jsonData, true); retryErr != nil { return retryErr } slog.Debug("Wrote stopped instance to JetStream KV (after recovery)", "key", key, "instanceId", instanceID) @@ -842,21 +846,21 @@ func (m *JetStreamManager) LoadStoppedInstance(instanceID string) (*vm.VM, error } key := StoppedInstancePrefix + instanceID - entry, err := m.kv.Get(key) + entry, err := m.kv.Get(context.Background(), key) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil, nil } if isStreamUnavailable(err) { slog.Warn("KV stream unavailable, attempting recovery", "operation", "LoadStoppedInstance", "key", key, "err", err) - kv, recoverErr := m.recoverKVBucket() + kv, recoverErr := m.recoverKVBucket(context.Background()) if recoverErr != nil { return nil, err } // Retry — if we reconnected, data may still exist - entry, err = kv.Get(key) + entry, err = kv.Get(context.Background(), key) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil, nil } return nil, err @@ -883,16 +887,16 @@ func (m *JetStreamManager) DeleteStoppedInstance(instanceID string) error { } key := StoppedInstancePrefix + instanceID - err := m.kv.Delete(key) - if err != nil && !errors.Is(err, nats.ErrKeyNotFound) { + err := m.kv.Delete(context.Background(), key) + if err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { if isStreamUnavailable(err) { slog.Warn("KV stream unavailable, attempting recovery", "operation", "DeleteStoppedInstance", "key", key, "err", err) - kv, recoverErr := m.recoverKVBucket() + kv, recoverErr := m.recoverKVBucket(context.Background()) if recoverErr != nil { return err } // Retry — if we reconnected the key may still exist - if retryErr := kv.Delete(key); retryErr != nil && !errors.Is(retryErr, nats.ErrKeyNotFound) { + if retryErr := kv.Delete(context.Background(), key); retryErr != nil && !errors.Is(retryErr, jetstream.ErrKeyNotFound) { return retryErr } return nil @@ -919,15 +923,15 @@ func (m *JetStreamManager) ClaimStoppedInstance(instanceID string) (*vm.VM, erro } key := StoppedInstancePrefix + instanceID - instance, notFound, err := casClaim[vm.VM](m.kv, key) + instance, notFound, err := casClaim[vm.VM](context.Background(), m.kv, key) if err != nil { if isStreamUnavailable(err) { slog.Warn("KV stream unavailable, attempting recovery", "operation", "ClaimStoppedInstance", "key", key, "err", err) - kv, recoverErr := m.recoverKVBucket() + kv, recoverErr := m.recoverKVBucket(context.Background()) if recoverErr != nil { return nil, err } - instance, notFound, err = casClaim[vm.VM](kv, key) + instance, notFound, err = casClaim[vm.VM](context.Background(), kv, key) if err != nil { return nil, err } @@ -947,25 +951,25 @@ func (m *JetStreamManager) ClaimStoppedInstance(instanceID string) (*vm.VM, erro // stopped record for instanceID and writes it back using optimistic // concurrency (CAS), retrying on a concurrent writer's revision conflict. // createIfAbsent is false: if a winning ClaimStoppedInstance deletes the -// record mid-flight, the CAS write fails with nats.ErrKeyNotFound instead of +// record mid-flight, the CAS write fails with jetstream.ErrKeyNotFound instead of // resurrecting it. Used by tag/attribute mutations that read-modify-write a // stopped instance so they cannot race a claim into recreating a stale -// record. Returns nats.ErrKeyNotFound if no record exists. +// record. Returns jetstream.ErrKeyNotFound if no record exists. func (m *JetStreamManager) UpdateStoppedInstance(instanceID string, mutate func(*vm.VM)) (*vm.VM, error) { if m.kv == nil { return nil, errors.New("KV bucket not initialized") } key := StoppedInstancePrefix + instanceID - updated, err := casUpdate(m.kv, key, mutate, false) + updated, err := casUpdate(context.Background(), m.kv, key, mutate, false) if err != nil { if isStreamUnavailable(err) { slog.Warn("KV stream unavailable, attempting recovery", "operation", "UpdateStoppedInstance", "key", key, "err", err) - kv, recoverErr := m.recoverKVBucket() + kv, recoverErr := m.recoverKVBucket(context.Background()) if recoverErr != nil { return nil, err } - return casUpdate(kv, key, mutate, false) + return casUpdate(context.Background(), kv, key, mutate, false) } return nil, err } @@ -978,21 +982,21 @@ func (m *JetStreamManager) ListStoppedInstances() ([]*vm.VM, error) { return nil, errors.New("KV bucket not initialized") } - keys, err := m.kv.Keys() + keys, err := m.kv.Keys(context.Background()) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return nil, nil } if isStreamUnavailable(err) { slog.Warn("KV stream unavailable, attempting recovery", "operation", "ListStoppedInstances", "err", err) - kv, recoverErr := m.recoverKVBucket() + kv, recoverErr := m.recoverKVBucket(context.Background()) if recoverErr != nil { return nil, err } // Retry — if we reconnected, data may still exist - keys, err = kv.Keys() + keys, err = kv.Keys(context.Background()) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return nil, nil } return nil, err @@ -1012,9 +1016,9 @@ func (m *JetStreamManager) ListStoppedInstances() ([]*vm.VM, error) { continue } - entry, err := m.kv.Get(key) + entry, err := m.kv.Get(context.Background(), key) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { continue } return nil, err @@ -1051,15 +1055,15 @@ func (m *JetStreamManager) WriteTerminatedInstance(instanceID string, instance * } key := TerminatedInstancePrefix + instanceID - err = casPut(m.terminatedKV, key, jsonData, true) + err = casPut(context.Background(), m.terminatedKV, key, jsonData, true) if err != nil { if isStreamUnavailable(err) { slog.Warn("KV stream unavailable, attempting recovery", "operation", "WriteTerminatedInstance", "key", key, "err", err) - kv, recoverErr := m.recoverTerminatedKVBucket() + kv, recoverErr := m.recoverTerminatedKVBucket(context.Background()) if recoverErr != nil { return err } - if retryErr := casPut(kv, key, jsonData, true); retryErr != nil { + if retryErr := casPut(context.Background(), kv, key, jsonData, true); retryErr != nil { return retryErr } slog.Debug("Wrote terminated instance to JetStream KV (after recovery)", "key", key, "instanceId", instanceID) @@ -1077,22 +1081,22 @@ func (m *JetStreamManager) WriteTerminatedInstance(instanceID string, instance * // optimistic concurrency (CAS), retrying on a concurrent writer's revision // conflict. Used by the teardown reaper to merge per-dependent Teardown // progress without clobbering marks written by a concurrent update to the -// same record. Returns nats.ErrKeyNotFound if no record exists yet. +// same record. Returns jetstream.ErrKeyNotFound if no record exists yet. func (m *JetStreamManager) UpdateTerminatedInstance(instanceID string, mutate func(*vm.VM)) (*vm.VM, error) { if m.terminatedKV == nil { return nil, errors.New("terminated instance KV bucket not initialized") } key := TerminatedInstancePrefix + instanceID - updated, err := casUpdate(m.terminatedKV, key, mutate, false) + updated, err := casUpdate(context.Background(), m.terminatedKV, key, mutate, false) if err != nil { if isStreamUnavailable(err) { slog.Warn("KV stream unavailable, attempting recovery", "operation", "UpdateTerminatedInstance", "key", key, "err", err) - kv, recoverErr := m.recoverTerminatedKVBucket() + kv, recoverErr := m.recoverTerminatedKVBucket(context.Background()) if recoverErr != nil { return nil, err } - return casUpdate(kv, key, mutate, false) + return casUpdate(context.Background(), kv, key, mutate, false) } return nil, err } @@ -1105,20 +1109,20 @@ func (m *JetStreamManager) ListTerminatedInstances() ([]*vm.VM, error) { return nil, errors.New("terminated instance KV bucket not initialized") } - keys, err := m.terminatedKV.Keys() + keys, err := m.terminatedKV.Keys(context.Background()) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return nil, nil } if isStreamUnavailable(err) { slog.Warn("KV stream unavailable, attempting recovery", "operation", "ListTerminatedInstances", "err", err) - kv, recoverErr := m.recoverTerminatedKVBucket() + kv, recoverErr := m.recoverTerminatedKVBucket(context.Background()) if recoverErr != nil { return nil, err } - keys, err = kv.Keys() + keys, err = kv.Keys(context.Background()) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return nil, nil } return nil, err @@ -1137,9 +1141,9 @@ func (m *JetStreamManager) ListTerminatedInstances() ([]*vm.VM, error) { continue } - entry, err := m.terminatedKV.Get(key) + entry, err := m.terminatedKV.Get(context.Background(), key) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { continue } return nil, err @@ -1164,15 +1168,15 @@ func (m *JetStreamManager) DeleteTerminatedInstance(instanceID string) error { } key := TerminatedInstancePrefix + instanceID - err := m.terminatedKV.Delete(key) - if err != nil && !errors.Is(err, nats.ErrKeyNotFound) { + err := m.terminatedKV.Delete(context.Background(), key) + if err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { if isStreamUnavailable(err) { slog.Warn("KV stream unavailable, attempting recovery", "operation", "DeleteTerminatedInstance", "key", key, "err", err) - kv, recoverErr := m.recoverTerminatedKVBucket() + kv, recoverErr := m.recoverTerminatedKVBucket(context.Background()) if recoverErr != nil { return err } - if retryErr := kv.Delete(key); retryErr != nil && !errors.Is(retryErr, nats.ErrKeyNotFound) { + if retryErr := kv.Delete(context.Background(), key); retryErr != nil && !errors.Is(retryErr, jetstream.ErrKeyNotFound) { return retryErr } return nil @@ -1192,20 +1196,20 @@ func (m *JetStreamManager) LoadTerminatedInstance(instanceID string) (*vm.VM, er } key := TerminatedInstancePrefix + instanceID - entry, err := m.terminatedKV.Get(key) + entry, err := m.terminatedKV.Get(context.Background(), key) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil, nil } if isStreamUnavailable(err) { slog.Warn("KV stream unavailable, attempting recovery", "operation", "LoadTerminatedInstance", "key", key, "err", err) - kv, recoverErr := m.recoverTerminatedKVBucket() + kv, recoverErr := m.recoverTerminatedKVBucket(context.Background()) if recoverErr != nil { return nil, err } - entry, err = kv.Get(key) + entry, err = kv.Get(context.Background(), key) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil, nil } return nil, err diff --git a/spinifex/daemon/jetstream_test.go b/spinifex/daemon/jetstream_test.go index c59f8283a..c2be43803 100644 --- a/spinifex/daemon/jetstream_test.go +++ b/spinifex/daemon/jetstream_test.go @@ -7,9 +7,10 @@ import ( "testing" "time" - "github.com/mulgadc/spinifex/spinifex/utils" + "github.com/mulgadc/spinifex/spinifex/kvutil" "github.com/mulgadc/spinifex/spinifex/vm" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -343,8 +344,10 @@ func TestJetStreamManager_UpdateReplicas(t *testing.T) { require.NoError(t, err) // Verify initial replica count - js, _ := nc.JetStream() - streamInfo, err := js.StreamInfo("KV_" + InstanceStateBucket) + js, _ := jetstream.New(nc) + stream, err := js.Stream(t.Context(), "KV_"+InstanceStateBucket) + require.NoError(t, err) + streamInfo, err := stream.Info(t.Context()) require.NoError(t, err) assert.Equal(t, 1, streamInfo.Config.Replicas, "Should start with 1 replica") @@ -598,7 +601,7 @@ func TestJetStreamManager_UpdateStoppedInstance_ConflictRetry(t *testing.T) { } // TestJetStreamManager_UpdateStoppedInstance_NotFound verifies -// UpdateStoppedInstance surfaces nats.ErrKeyNotFound rather than silently +// UpdateStoppedInstance surfaces jetstream.ErrKeyNotFound rather than silently // creating a record when nothing exists to update. func TestJetStreamManager_UpdateStoppedInstance_NotFound(t *testing.T) { nc, err := nats.Connect(sharedJSNATSURL) @@ -610,7 +613,7 @@ func TestJetStreamManager_UpdateStoppedInstance_NotFound(t *testing.T) { require.NoError(t, jsm.InitKVBucket()) _, err = jsm.UpdateStoppedInstance("i-does-not-exist", func(v *vm.VM) {}) - assert.ErrorIs(t, err, nats.ErrKeyNotFound) + assert.ErrorIs(t, err, jetstream.ErrKeyNotFound) } // TestJetStreamManager_UpdateStoppedInstance_NoResurrectAfterClaim is the @@ -646,7 +649,7 @@ func TestJetStreamManager_UpdateStoppedInstance_NoResurrectAfterClaim(t *testing _, err = jsm.UpdateStoppedInstance(testVM.ID, func(v *vm.VM) { v.InstanceType = "should-not-land" }) - assert.ErrorIs(t, err, nats.ErrKeyNotFound, "a claim that deleted the record must not be resurrected by a losing racer's update") + assert.ErrorIs(t, err, jetstream.ErrKeyNotFound, "a claim that deleted the record must not be resurrected by a losing racer's update") stillGone, err := jsm.LoadStoppedInstance(testVM.ID) require.NoError(t, err) @@ -776,7 +779,7 @@ func TestJetStreamManager_WriteServiceManifest(t *testing.T) { require.NoError(t, err) // Read the KV entry directly and verify JSON contents - entry, err := jsm.clusterKV.Get("node.test-node-svc.services") + entry, err := jsm.clusterKV.Get(t.Context(), "node.test-node-svc.services") require.NoError(t, err) var manifest map[string]any @@ -811,7 +814,7 @@ func TestJetStreamManager_WriteServiceManifest_EmptyServices(t *testing.T) { err = jsm.WriteServiceManifest("empty-svc-node", []string{}, "10.0.0.2:4222", "10.0.0.2:8443") require.NoError(t, err) - entry, err := jsm.clusterKV.Get("node.empty-svc-node.services") + entry, err := jsm.clusterKV.Get(t.Context(), "node.empty-svc-node.services") require.NoError(t, err) var manifest map[string]any @@ -847,9 +850,9 @@ func TestJetStreamManager_WriteServiceManifest_ClusterKVNotInitialized(t *testin // deleteInstanceStateBucket deletes the underlying JetStream stream for the instance-state KV bucket. func deleteInstanceStateBucket(t *testing.T, nc *nats.Conn) { t.Helper() - js, err := nc.JetStream() + js, err := jetstream.New(nc) require.NoError(t, err) - err = js.DeleteStream("KV_" + InstanceStateBucket) + err = js.DeleteStream(t.Context(), "KV_"+InstanceStateBucket) require.NoError(t, err) } @@ -1006,7 +1009,7 @@ func swapToNonJSContext(t *testing.T, jsm *JetStreamManager) { nc, err := nats.Connect(sharedNATSURL) require.NoError(t, err) t.Cleanup(func() { nc.Close() }) - js, err := nc.JetStream() + js, err := jetstream.New(nc) require.NoError(t, err) jsm.js = js } @@ -1137,12 +1140,12 @@ func TestJetStreamManager_ListStoppedInstances_RecoveryFailure(t *testing.T) { func TestIsStreamUnavailable(t *testing.T) { assert.False(t, isStreamUnavailable(nil)) - assert.True(t, isStreamUnavailable(nats.ErrStreamNotFound)) - assert.True(t, isStreamUnavailable(nats.ErrNoStreamResponse)) + assert.True(t, isStreamUnavailable(jetstream.ErrStreamNotFound)) + assert.True(t, isStreamUnavailable(jetstream.ErrNoStreamResponse)) assert.True(t, isStreamUnavailable(nats.ErrNoResponders)) assert.True(t, isStreamUnavailable(errors.New("nats: stream not found"))) assert.False(t, isStreamUnavailable(errors.New("some other error"))) - assert.False(t, isStreamUnavailable(nats.ErrKeyNotFound)) + assert.False(t, isStreamUnavailable(jetstream.ErrKeyNotFound)) } // --- Terminated instance KV tests --- @@ -1376,7 +1379,7 @@ func TestJetStreamManager_UpdateTerminatedInstance_NotFound(t *testing.T) { require.NoError(t, jsm.InitTerminatedInstanceBucket()) _, err = jsm.UpdateTerminatedInstance("i-does-not-exist", func(v *vm.VM) {}) - assert.ErrorIs(t, err, nats.ErrKeyNotFound) + assert.ErrorIs(t, err, jetstream.ErrKeyNotFound) } // TestJetStreamManager_WriteStoppedInstance_OverwritesConcurrentValue verifies @@ -1416,9 +1419,9 @@ func TestJetStreamManager_WriteStoppedInstance_OverwritesConcurrentValue(t *test // deleteTerminatedInstanceBucket deletes the underlying JetStream stream for the terminated-instances KV bucket. func deleteTerminatedInstanceBucket(t *testing.T, nc *nats.Conn) { t.Helper() - js, err := nc.JetStream() + js, err := jetstream.New(nc) require.NoError(t, err) - err = js.DeleteStream("KV_" + TerminatedInstanceBucket) + err = js.DeleteStream(t.Context(), "KV_"+TerminatedInstanceBucket) require.NoError(t, err) } @@ -1504,15 +1507,15 @@ func TestJetStreamManager_InitBuckets_WritesVersion(t *testing.T) { require.NoError(t, jsm.InitClusterStateBucket()) require.NoError(t, jsm.InitTerminatedInstanceBucket()) - v, err := utils.ReadVersion(jsm.kv) + v, err := kvutil.ReadVersion(t.Context(), jsm.kv) require.NoError(t, err) assert.Equal(t, InstanceStateBucketVersion, v) - v, err = utils.ReadVersion(jsm.clusterKV) + v, err = kvutil.ReadVersion(t.Context(), jsm.clusterKV) require.NoError(t, err) assert.Equal(t, ClusterStateBucketVersion, v) - v, err = utils.ReadVersion(jsm.terminatedKV) + v, err = kvutil.ReadVersion(t.Context(), jsm.terminatedKV) require.NoError(t, err) assert.Equal(t, TerminatedInstanceBucketVersion, v) } @@ -1653,7 +1656,7 @@ func TestJetStreamManager_UpdateMgmtIPAM_ConflictRetry(t *testing.T) { require.NoError(t, jsm.InitClusterStateBucket()) const subnet = "10.99.8.0/24" - defer func() { _ = jsm.clusterKV.Delete(mgmtIPAMKeyPrefix + subnet) }() + defer func() { _ = jsm.clusterKV.Delete(t.Context(), mgmtIPAMKeyPrefix+subnet) }() _, err = jsm.UpdateMgmtIPAM(subnet, func(r *MgmtIPRecord) { r.Subnet = subnet @@ -1690,7 +1693,7 @@ func TestJetStreamManager_UpdateMgmtIPAM_ConflictRetry(t *testing.T) { } // TestJetStreamManager_UpdateMgmtIPAM_NotFound verifies createIfAbsent=false -// surfaces nats.ErrKeyNotFound instead of silently creating a record — +// surfaces jetstream.ErrKeyNotFound instead of silently creating a record — // MgmtIPAllocator.Release relies on this to no-op cleanly when there is // nothing to release. func TestJetStreamManager_UpdateMgmtIPAM_NotFound(t *testing.T) { @@ -1703,7 +1706,7 @@ func TestJetStreamManager_UpdateMgmtIPAM_NotFound(t *testing.T) { require.NoError(t, jsm.InitClusterStateBucket()) _, err = jsm.UpdateMgmtIPAM("10.98.8.0/24-nonexistent", func(r *MgmtIPRecord) {}, false) - assert.ErrorIs(t, err, nats.ErrKeyNotFound) + assert.ErrorIs(t, err, jetstream.ErrKeyNotFound) } // TestJetStreamManager_UpdateMgmtIPAM_ClusterKVNotInitialized tests error diff --git a/spinifex/gateway/bedrock/credentials.go b/spinifex/gateway/bedrock/credentials.go index bd965b1ed..442629fce 100644 --- a/spinifex/gateway/bedrock/credentials.go +++ b/spinifex/gateway/bedrock/credentials.go @@ -7,7 +7,8 @@ import ( "sync" handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" - "github.com/nats-io/nats.go" + "github.com/mulgadc/spinifex/spinifex/kvutil" + "github.com/nats-io/nats.go/jetstream" ) // bedrockCredentialsBucket is the cluster-replicated KV bucket holding @@ -26,13 +27,13 @@ func credentialKey(accountID, vendor string) string { // bedrock-credentials JetStream KV bucket, falling back to a platform-wide // default when an account has none. Keys are encrypted at rest. type CredentialStore struct { - js nats.JetStreamContext + js jetstream.JetStream masterKey []byte replicas int platformDefaults map[string]string mu sync.Mutex - kv nats.KeyValue + kv jetstream.KeyValue } var _ CredentialResolver = (*CredentialStore)(nil) @@ -40,7 +41,7 @@ var _ CredentialResolver = (*CredentialStore)(nil) // NewCredentialStore constructs a CredentialStore. platformDefaults maps // vendor to a platform-wide API key used when an account has no key of its // own; pass an empty map to disable platform defaults entirely. -func NewCredentialStore(js nats.JetStreamContext, masterKey []byte, replicas int, platformDefaults map[string]string) *CredentialStore { +func NewCredentialStore(js jetstream.JetStream, masterKey []byte, replicas int, platformDefaults map[string]string) *CredentialStore { return &CredentialStore{ js: js, masterKey: masterKey, @@ -49,23 +50,17 @@ func NewCredentialStore(js nats.JetStreamContext, masterKey []byte, replicas int } } -// bucket lazily opens (or creates) the bedrock-credentials KV bucket, -// caching the handle, mirroring gateway_ecrauth.openSigningBucket. -func (s *CredentialStore) bucket() (nats.KeyValue, error) { +// bucket lazily opens (or creates) the cluster-replicated bedrock-credentials +// KV bucket, caching the handle for subsequent calls. +func (s *CredentialStore) bucket(ctx context.Context) (jetstream.KeyValue, error) { s.mu.Lock() defer s.mu.Unlock() if s.kv != nil { return s.kv, nil } - kv, err := s.js.CreateKeyValue(&nats.KeyValueConfig{ - Bucket: bedrockCredentialsBucket, - History: bedrockCredentialsHistory, - Replicas: s.replicas, - }) + kv, err := kvutil.GetOrCreateBucketWithReplicas(ctx, s.js, bedrockCredentialsBucket, bedrockCredentialsHistory, s.replicas) if err != nil { - if kv, err = s.js.KeyValue(bedrockCredentialsBucket); err != nil { - return nil, fmt.Errorf("open %s bucket: %w", bedrockCredentialsBucket, err) - } + return nil, err } s.kv = kv return kv, nil @@ -75,13 +70,13 @@ func (s *CredentialStore) bucket() (nats.KeyValue, error) { // stored, else the platform default, else ("", false, nil). A nil js is // tolerated so a store built with only platformDefaults (e.g. in tests) can // still serve the default-only path without touching JetStream. -func (s *CredentialStore) Resolve(_ context.Context, accountID, vendor string) (string, bool, error) { +func (s *CredentialStore) Resolve(ctx context.Context, accountID, vendor string) (string, bool, error) { if s.js != nil { - kv, err := s.bucket() + kv, err := s.bucket(ctx) if err != nil { return "", false, err } - entry, err := kv.Get(credentialKey(accountID, vendor)) + entry, err := kv.Get(ctx, credentialKey(accountID, vendor)) switch { case err == nil: plaintext, err := handlers_iam.DecryptSecret(string(entry.Value()), s.masterKey) @@ -89,7 +84,7 @@ func (s *CredentialStore) Resolve(_ context.Context, accountID, vendor string) ( return "", false, fmt.Errorf("decrypt credential for %s/%s: %w", accountID, vendor, err) } return plaintext, true, nil - case !errors.Is(err, nats.ErrKeyNotFound): + case !errors.Is(err, jetstream.ErrKeyNotFound): return "", false, fmt.Errorf("kv get credential for %s/%s: %w", accountID, vendor, err) } } @@ -100,8 +95,8 @@ func (s *CredentialStore) Resolve(_ context.Context, accountID, vendor string) ( } // PutCredential encrypts and stores key as accountID's vendor credential. -func (s *CredentialStore) PutCredential(_ context.Context, accountID, vendor, key string) error { - kv, err := s.bucket() +func (s *CredentialStore) PutCredential(ctx context.Context, accountID, vendor, key string) error { + kv, err := s.bucket(ctx) if err != nil { return err } @@ -109,7 +104,7 @@ func (s *CredentialStore) PutCredential(_ context.Context, accountID, vendor, ke if err != nil { return fmt.Errorf("encrypt credential for %s/%s: %w", accountID, vendor, err) } - if _, err := kv.Put(credentialKey(accountID, vendor), []byte(ciphertext)); err != nil { + if _, err := kv.Put(ctx, credentialKey(accountID, vendor), []byte(ciphertext)); err != nil { return fmt.Errorf("kv put credential for %s/%s: %w", accountID, vendor, err) } return nil diff --git a/spinifex/gateway/bedrock/credentials_test.go b/spinifex/gateway/bedrock/credentials_test.go index 4803d9ef9..362454ed4 100644 --- a/spinifex/gateway/bedrock/credentials_test.go +++ b/spinifex/gateway/bedrock/credentials_test.go @@ -17,7 +17,7 @@ func fixedMasterKey() []byte { } // TestCredentialStore_Resolve_NilJS covers the platform-default and -// vendor-miss paths of a store built with a nil JetStream context (as a +// vendor-miss paths of a store built with a nil JetStream handle (as a // live NATS server is out of scope here). Resolve must consult // platformDefaults without dereferencing js. func TestCredentialStore_Resolve_NilJS(t *testing.T) { @@ -62,8 +62,8 @@ func TestNoopCredentialResolver_ResolvesNothing(t *testing.T) { // bucket (lazy create), credentialKey, PutCredential (encrypt+Put), and the // KV-hit branch of Resolve (Get+decrypt). func TestCredentialStore_PutAndResolve_KV(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) - store := NewCredentialStore(js, fixedMasterKey(), 1, nil) + _, nc, _ := testutil.StartTestJetStream(t) + store := NewCredentialStore(testutil.NewJetStream(t, nc), fixedMasterKey(), 1, nil) ctx := context.Background() require.NoError(t, store.PutCredential(ctx, "000000000001", "anthropic", "sk-test")) @@ -75,10 +75,10 @@ func TestCredentialStore_PutAndResolve_KV(t *testing.T) { } // TestCredentialStore_Resolve_KVMissFallsBackToPlatformDefault covers a KV -// miss (nats.ErrKeyNotFound) that falls through to the platform default. +// miss (jetstream.ErrKeyNotFound) that falls through to the platform default. func TestCredentialStore_Resolve_KVMissFallsBackToPlatformDefault(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) - store := NewCredentialStore(js, fixedMasterKey(), 1, map[string]string{"anthropic": "sk-platform-default"}) + _, nc, _ := testutil.StartTestJetStream(t) + store := NewCredentialStore(testutil.NewJetStream(t, nc), fixedMasterKey(), 1, map[string]string{"anthropic": "sk-platform-default"}) key, ok, err := store.Resolve(context.Background(), "999999999999", "anthropic") require.NoError(t, err) @@ -89,8 +89,8 @@ func TestCredentialStore_Resolve_KVMissFallsBackToPlatformDefault(t *testing.T) // TestCredentialStore_Resolve_KVMissNoPlatformDefault covers a KV miss on an // unknown account with no platform defaults configured: ("", false, nil). func TestCredentialStore_Resolve_KVMissNoPlatformDefault(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) - store := NewCredentialStore(js, fixedMasterKey(), 1, nil) + _, nc, _ := testutil.StartTestJetStream(t) + store := NewCredentialStore(testutil.NewJetStream(t, nc), fixedMasterKey(), 1, nil) key, ok, err := store.Resolve(context.Background(), "999999999999", "anthropic") require.NoError(t, err) diff --git a/spinifex/gateway/ec2.go b/spinifex/gateway/ec2.go index 0595a13b0..e145aaec8 100644 --- a/spinifex/gateway/ec2.go +++ b/spinifex/gateway/ec2.go @@ -110,7 +110,7 @@ var ec2Actions = map[string]EC2Handler{ return gw.checkPolicyResource(r, "iam", "PassRole", roleARN) } launchQuotaCheck := func() error { - return gw.Quota.EnforceLaunch(accountID, aws.StringValue(input.InstanceType), int(aws.Int64Value(input.MaxCount))) + return gw.Quota.EnforceLaunch(ctx, accountID, aws.StringValue(input.InstanceType), int(aws.Int64Value(input.MaxCount))) } reservation, err := gateway_ec2_instance.RunInstances(ctx, input, gw.NATSConn, gw.IAMService, accountID, passRoleCheck, launchQuotaCheck, gw.ExpectedNodes) if err != nil { @@ -118,7 +118,7 @@ var ec2Actions = map[string]EC2Handler{ } // Charge the actual launched vCPUs; a counter write failure is drift for // reconcile to correct, so it must not fail the already-successful launch. - if err := gw.Quota.ChargeLaunch(accountID, &reservation); err != nil { + if err := gw.Quota.ChargeLaunch(ctx, accountID, &reservation); err != nil { slog.WarnContext(ctx, "RunInstances: vcpu quota charge failed, reconcile will correct", "account", accountID, "err", err) } return reservation, nil @@ -166,7 +166,7 @@ var ec2Actions = map[string]EC2Handler{ var delta int if input.InstanceType != nil { resolve := handlers_quota.NATSInstanceTypeResolver(gw.NATSConn, func() int { return gw.ExpectedNodes }) - d, err := gw.Quota.EnforceRetype(resolve, accountID, aws.StringValue(input.InstanceId), aws.StringValue(input.InstanceType.Value)) + d, err := gw.Quota.EnforceRetype(ctx, resolve, accountID, aws.StringValue(input.InstanceId), aws.StringValue(input.InstanceType.Value)) if err != nil { return nil, err } @@ -178,7 +178,7 @@ var ec2Actions = map[string]EC2Handler{ } // Charge the retype's vCPU growth; a counter write failure is drift for // reconcile to correct, so it must not fail the applied retype. - if err := gw.Quota.AddVCPU(accountID, delta); err != nil { + if err := gw.Quota.AddVCPU(ctx, accountID, delta); err != nil { slog.WarnContext(ctx, "ModifyInstanceAttribute: vcpu quota charge failed, reconcile will correct", "account", accountID, "err", err) } return out, nil diff --git a/spinifex/gateway/ec2/instance/RunInstances.go b/spinifex/gateway/ec2/instance/RunInstances.go index fb6a70a71..2e1c21277 100644 --- a/spinifex/gateway/ec2/instance/RunInstances.go +++ b/spinifex/gateway/ec2/instance/RunInstances.go @@ -101,7 +101,7 @@ func RunInstances(ctx context.Context, input *ec2.RunInstancesInput, natsConn *n } // Hash before any mutation so the same token+params always matches. paramHash := clientTokenParamHash(input) - return runInstancesWithClientToken(store, accountID, token, paramHash, func() (ec2.Reservation, error) { + return runInstancesWithClientToken(ctx, store, accountID, token, paramHash, func() (ec2.Reservation, error) { return runInstancesInner(ctx, input, natsConn, iamSvc, accountID, passRoleCheck, launchQuotaCheck, expectedNodes) }) } diff --git a/spinifex/gateway/ec2/instance/clienttoken.go b/spinifex/gateway/ec2/instance/clienttoken.go index 430efdd04..2b7d84126 100644 --- a/spinifex/gateway/ec2/instance/clienttoken.go +++ b/spinifex/gateway/ec2/instance/clienttoken.go @@ -14,6 +14,7 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" "github.com/mulgadc/spinifex/spinifex/awserrors" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) const ( @@ -54,7 +55,7 @@ type tokenRecord struct { // ClientTokenStore implements RunInstances ClientToken idempotency over a TTL KV // bucket. The first caller owns the launch; duplicates replay (done) or poll (in-flight). type ClientTokenStore struct { - kv nats.KeyValue + kv jetstream.KeyValue } var ( @@ -66,20 +67,24 @@ var ( // getClientTokenStore lazily initialises the process-wide client-token store via sync.Once. func getClientTokenStore(ctx context.Context, nc *nats.Conn) (*ClientTokenStore, error) { ctOnce.Do(func() { - js, err := nc.JetStream() + js, err := jetstream.New(nc) if err != nil { errCTStoreInit = fmt.Errorf("clienttoken jetstream: %w", err) return } - ctStore, errCTStoreInit = newClientTokenStore(js) + // The bind happens once per process, so it must not inherit the first + // caller's cancellation: a client that disconnects mid-open would poison + // the store for every later launch. Deadline-free, so the open falls back + // to the JetStream API's own timeout. + ctStore, errCTStoreInit = newClientTokenStore(context.WithoutCancel(ctx), js) }) return ctStore, errCTStoreInit } -func newClientTokenStore(js nats.JetStreamContext) (*ClientTokenStore, error) { - kv, err := js.KeyValue(kvBucketClientTokens) - if errors.Is(err, nats.ErrBucketNotFound) { - kv, err = js.CreateKeyValue(&nats.KeyValueConfig{ +func newClientTokenStore(ctx context.Context, js jetstream.JetStream) (*ClientTokenStore, error) { + kv, err := js.KeyValue(ctx, kvBucketClientTokens) + if errors.Is(err, jetstream.ErrBucketNotFound) { + kv, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{ Bucket: kvBucketClientTokens, History: 1, TTL: clientTokenTTL, @@ -112,7 +117,7 @@ func clientTokenParamHash(input *ec2.RunInstancesInput) string { // Claim attempts to own the launch for this token. // Returns (nil, true, nil) when the caller owns the launch (must Finalize or Abort), // (reservation, false, nil) to replay a completed launch, or an error on mismatch/timeout. -func (c *ClientTokenStore) Claim(accountID, token, paramHash string) (*ec2.Reservation, bool, error) { +func (c *ClientTokenStore) Claim(ctx context.Context, accountID, token, paramHash string) (*ec2.Reservation, bool, error) { key := clientTokenKey(accountID, token) inflight, err := json.Marshal(tokenRecord{ Status: tokenStatusInFlight, @@ -122,21 +127,21 @@ func (c *ClientTokenStore) Claim(accountID, token, paramHash string) (*ec2.Reser if err != nil { return nil, false, fmt.Errorf("clienttoken marshal: %w", err) } - if _, cerr := c.kv.Create(key, inflight); cerr == nil { + if _, cerr := c.kv.Create(ctx, key, inflight); cerr == nil { return nil, true, nil - } else if !errors.Is(cerr, nats.ErrKeyExists) { + } else if !errors.Is(cerr, jetstream.ErrKeyExists) { return nil, false, fmt.Errorf("clienttoken create %s: %w", key, cerr) } deadline := time.Now().Add(clientTokenWaitTimeout) for { - entry, gerr := c.kv.Get(key) + entry, gerr := c.kv.Get(ctx, key) if gerr != nil { - if errors.Is(gerr, nats.ErrKeyNotFound) { + if errors.Is(gerr, jetstream.ErrKeyNotFound) { // Owner aborted or record aged out: race for ownership. - if _, rcerr := c.kv.Create(key, inflight); rcerr == nil { + if _, rcerr := c.kv.Create(ctx, key, inflight); rcerr == nil { return nil, true, nil - } else if !errors.Is(rcerr, nats.ErrKeyExists) { + } else if !errors.Is(rcerr, jetstream.ErrKeyExists) { return nil, false, fmt.Errorf("clienttoken recreate %s: %w", key, rcerr) } continue @@ -161,7 +166,7 @@ func (c *ClientTokenStore) Claim(accountID, token, paramHash string) (*ec2.Reser } // Finalize marks the token done with the reservation so duplicates can replay it. -func (c *ClientTokenStore) Finalize(accountID, token, paramHash string, res *ec2.Reservation) error { +func (c *ClientTokenStore) Finalize(ctx context.Context, accountID, token, paramHash string, res *ec2.Reservation) error { key := clientTokenKey(accountID, token) data, err := json.Marshal(tokenRecord{ Status: tokenStatusDone, @@ -172,7 +177,7 @@ func (c *ClientTokenStore) Finalize(accountID, token, paramHash string, res *ec2 if err != nil { return fmt.Errorf("clienttoken finalize marshal: %w", err) } - if _, err := c.kv.Put(key, data); err != nil { + if _, err := c.kv.Put(ctx, key, data); err != nil { return fmt.Errorf("clienttoken finalize put %s: %w", key, err) } return nil @@ -182,12 +187,13 @@ func (c *ClientTokenStore) Finalize(accountID, token, paramHash string, res *ec2 // claims the token, replays a completed reservation, or (as owner) launches, // finalizes, and aborts on failure. Extracted for unit-testability. func runInstancesWithClientToken( + ctx context.Context, store *ClientTokenStore, accountID, token, paramHash string, launch func() (ec2.Reservation, error), ) (ec2.Reservation, error) { var zero ec2.Reservation - replay, owned, cerr := store.Claim(accountID, token, paramHash) + replay, owned, cerr := store.Claim(ctx, accountID, token, paramHash) if cerr != nil { if errors.Is(cerr, errIdempotentParamMismatch) { return zero, errors.New(awserrors.ErrorIdempotentParameterMismatch) @@ -203,11 +209,16 @@ func runInstancesWithClientToken( } res, rerr := launch() + + // Recording the launch outcome outlives ctx: a caller that went away mid-launch + // is exactly when the record must be settled, and leaving it in-flight parks + // every retry of that token behind the poll deadline until the record ages out. + outcomeCtx := context.WithoutCancel(ctx) if rerr != nil { - store.Abort(accountID, token) + store.Abort(outcomeCtx, accountID, token) return zero, rerr } - if ferr := store.Finalize(accountID, token, paramHash, &res); ferr != nil { + if ferr := store.Finalize(outcomeCtx, accountID, token, paramHash, &res); ferr != nil { // Launch succeeded; finalize failure only weakens future dedup — don't fail. slog.Warn("RunInstances: failed to finalize client-token record", "token", token, "err", ferr) } @@ -215,9 +226,9 @@ func runInstancesWithClientToken( } // Abort drops the in-flight token so a retry can re-launch. -func (c *ClientTokenStore) Abort(accountID, token string) { +func (c *ClientTokenStore) Abort(ctx context.Context, accountID, token string) { key := clientTokenKey(accountID, token) - if err := c.kv.Delete(key); err != nil && !errors.Is(err, nats.ErrKeyNotFound) { + if err := c.kv.Delete(ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { slog.Warn("clienttoken: failed to abort token record", "key", key, "err", err) } } diff --git a/spinifex/gateway/ec2/instance/clienttoken_test.go b/spinifex/gateway/ec2/instance/clienttoken_test.go index 700d810a9..30602effc 100644 --- a/spinifex/gateway/ec2/instance/clienttoken_test.go +++ b/spinifex/gateway/ec2/instance/clienttoken_test.go @@ -1,7 +1,6 @@ package gateway_ec2_instance import ( - "context" "errors" "sync" "sync/atomic" @@ -20,8 +19,8 @@ const ctTestAccount = "111122223333" func newTestClientTokenStore(t *testing.T) *ClientTokenStore { t.Helper() - _, _, js := testutil.StartTestJetStream(t) - store, err := newClientTokenStore(js) + _, nc, _ := testutil.StartTestJetStream(t) + store, err := newClientTokenStore(t.Context(), testutil.NewJetStream(t, nc)) require.NoError(t, err) return store } @@ -32,14 +31,14 @@ func TestClientToken_ReplaysCompletedReservation(t *testing.T) { store := newTestClientTokenStore(t) const tok, hash = "tok-1", "hash-a" - _, owned, err := store.Claim(ctTestAccount, tok, hash) + _, owned, err := store.Claim(t.Context(), ctTestAccount, tok, hash) require.NoError(t, err) require.True(t, owned, "first caller owns the launch") res := &ec2.Reservation{ReservationId: aws.String("r-123")} - require.NoError(t, store.Finalize(ctTestAccount, tok, hash, res)) + require.NoError(t, store.Finalize(t.Context(), ctTestAccount, tok, hash, res)) - replay, owned2, err := store.Claim(ctTestAccount, tok, hash) + replay, owned2, err := store.Claim(t.Context(), ctTestAccount, tok, hash) require.NoError(t, err) assert.False(t, owned2, "duplicate caller does not own the launch") require.NotNil(t, replay) @@ -51,12 +50,12 @@ func TestClientToken_ParamMismatchRejected(t *testing.T) { store := newTestClientTokenStore(t) const tok = "tok-2" - _, owned, err := store.Claim(ctTestAccount, tok, "hash-a") + _, owned, err := store.Claim(t.Context(), ctTestAccount, tok, "hash-a") require.NoError(t, err) require.True(t, owned) - require.NoError(t, store.Finalize(ctTestAccount, tok, "hash-a", &ec2.Reservation{ReservationId: aws.String("r-1")})) + require.NoError(t, store.Finalize(t.Context(), ctTestAccount, tok, "hash-a", &ec2.Reservation{ReservationId: aws.String("r-1")})) - _, _, err = store.Claim(ctTestAccount, tok, "hash-DIFFERENT") + _, _, err = store.Claim(t.Context(), ctTestAccount, tok, "hash-DIFFERENT") assert.ErrorIs(t, err, errIdempotentParamMismatch) } @@ -66,13 +65,13 @@ func TestClientToken_AbortAllowsRelaunch(t *testing.T) { store := newTestClientTokenStore(t) const tok, hash = "tok-3", "hash-a" - _, owned, err := store.Claim(ctTestAccount, tok, hash) + _, owned, err := store.Claim(t.Context(), ctTestAccount, tok, hash) require.NoError(t, err) require.True(t, owned) - store.Abort(ctTestAccount, tok) + store.Abort(t.Context(), ctTestAccount, tok) - _, owned2, err := store.Claim(ctTestAccount, tok, hash) + _, owned2, err := store.Claim(t.Context(), ctTestAccount, tok, hash) require.NoError(t, err) assert.True(t, owned2, "after abort the token is free to re-own") } @@ -93,7 +92,7 @@ func TestClientToken_ConcurrentSingleOwner(t *testing.T) { wg.Add(1) go func(i int) { defer wg.Done() - replay, owned, err := store.Claim(ctTestAccount, tok, hash) + replay, owned, err := store.Claim(t.Context(), ctTestAccount, tok, hash) if err != nil { errs[i] = err return @@ -101,7 +100,7 @@ func TestClientToken_ConcurrentSingleOwner(t *testing.T) { if owned { atomic.AddInt32(&owners, 1) // Simulate the launch then publish the reservation. - errs[i] = store.Finalize(ctTestAccount, tok, hash, res) + errs[i] = store.Finalize(t.Context(), ctTestAccount, tok, hash, res) replays[i] = res return } @@ -147,13 +146,13 @@ func TestClientTokenParamHash_IgnoresTokenReflectsParams(t *testing.T) { func TestRunInstancesWithClientToken_ReplaySkipsLaunch(t *testing.T) { store := newTestClientTokenStore(t) const tok, hash = "rt-1", "h" - _, owned, err := store.Claim(ctTestAccount, tok, hash) + _, owned, err := store.Claim(t.Context(), ctTestAccount, tok, hash) require.NoError(t, err) require.True(t, owned) - require.NoError(t, store.Finalize(ctTestAccount, tok, hash, &ec2.Reservation{ReservationId: aws.String("r-x")})) + require.NoError(t, store.Finalize(t.Context(), ctTestAccount, tok, hash, &ec2.Reservation{ReservationId: aws.String("r-x")})) launched := false - res, err := runInstancesWithClientToken(store, ctTestAccount, tok, hash, func() (ec2.Reservation, error) { + res, err := runInstancesWithClientToken(t.Context(), store, ctTestAccount, tok, hash, func() (ec2.Reservation, error) { launched = true return ec2.Reservation{}, nil }) @@ -172,11 +171,11 @@ func TestRunInstancesWithClientToken_OwnerLaunchesOnceThenReplay(t *testing.T) { return ec2.Reservation{ReservationId: aws.String("r-own")}, nil } - res, err := runInstancesWithClientToken(store, ctTestAccount, tok, hash, launch) + res, err := runInstancesWithClientToken(t.Context(), store, ctTestAccount, tok, hash, launch) require.NoError(t, err) assert.Equal(t, "r-own", aws.StringValue(res.ReservationId)) - res2, err := runInstancesWithClientToken(store, ctTestAccount, tok, hash, launch) + res2, err := runInstancesWithClientToken(t.Context(), store, ctTestAccount, tok, hash, launch) require.NoError(t, err) assert.Equal(t, "r-own", aws.StringValue(res2.ReservationId)) assert.Equal(t, 1, launches, "duplicate must replay, not relaunch") @@ -187,13 +186,13 @@ func TestRunInstancesWithClientToken_LaunchFailureAborts(t *testing.T) { store := newTestClientTokenStore(t) const tok, hash = "rt-3", "h" - _, err := runInstancesWithClientToken(store, ctTestAccount, tok, hash, func() (ec2.Reservation, error) { + _, err := runInstancesWithClientToken(t.Context(), store, ctTestAccount, tok, hash, func() (ec2.Reservation, error) { return ec2.Reservation{}, errors.New("no capacity") }) require.Error(t, err) relaunched := false - res, err := runInstancesWithClientToken(store, ctTestAccount, tok, hash, func() (ec2.Reservation, error) { + res, err := runInstancesWithClientToken(t.Context(), store, ctTestAccount, tok, hash, func() (ec2.Reservation, error) { relaunched = true return ec2.Reservation{ReservationId: aws.String("r-retry")}, nil }) @@ -207,13 +206,13 @@ func TestRunInstancesWithClientToken_LaunchFailureAborts(t *testing.T) { func TestRunInstancesWithClientToken_ParamMismatchMapsAWSError(t *testing.T) { store := newTestClientTokenStore(t) const tok = "rt-4" - _, owned, err := store.Claim(ctTestAccount, tok, "hA") + _, owned, err := store.Claim(t.Context(), ctTestAccount, tok, "hA") require.NoError(t, err) require.True(t, owned) - require.NoError(t, store.Finalize(ctTestAccount, tok, "hA", &ec2.Reservation{ReservationId: aws.String("r")})) + require.NoError(t, store.Finalize(t.Context(), ctTestAccount, tok, "hA", &ec2.Reservation{ReservationId: aws.String("r")})) launched := false - _, err = runInstancesWithClientToken(store, ctTestAccount, tok, "hB", func() (ec2.Reservation, error) { + _, err = runInstancesWithClientToken(t.Context(), store, ctTestAccount, tok, "hB", func() (ec2.Reservation, error) { launched = true return ec2.Reservation{}, nil }) @@ -225,10 +224,10 @@ func TestRunInstancesWithClientToken_ParamMismatchMapsAWSError(t *testing.T) { // instance on subsequent calls. func TestGetClientTokenStore_BindsOnce(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) - s1, err := getClientTokenStore(context.Background(), nc) + s1, err := getClientTokenStore(t.Context(), nc) require.NoError(t, err) require.NotNil(t, s1) - s2, err := getClientTokenStore(context.Background(), nc) + s2, err := getClientTokenStore(t.Context(), nc) require.NoError(t, err) assert.Same(t, s1, s2, "store binds once") } @@ -239,7 +238,7 @@ func TestClientToken_InFlightWaitTimesOut(t *testing.T) { store := newTestClientTokenStore(t) const tok, hash = "wt-1", "h" - _, owned, err := store.Claim(ctTestAccount, tok, hash) + _, owned, err := store.Claim(t.Context(), ctTestAccount, tok, hash) require.NoError(t, err) require.True(t, owned, "owner holds the in-flight record and never finalizes") @@ -247,6 +246,6 @@ func TestClientToken_InFlightWaitTimesOut(t *testing.T) { clientTokenWaitTimeout, clientTokenPollStep = 30*time.Millisecond, 10*time.Millisecond defer func() { clientTokenWaitTimeout, clientTokenPollStep = origTimeout, origStep }() - _, _, err = store.Claim(ctTestAccount, tok, hash) + _, _, err = store.Claim(t.Context(), ctTestAccount, tok, hash) assert.ErrorIs(t, err, errClientTokenWaitTimeout) } diff --git a/spinifex/gateway/ec2/spotinstance/RequestSpotInstances.go b/spinifex/gateway/ec2/spotinstance/RequestSpotInstances.go index 8a0e8c500..d12ca8645 100644 --- a/spinifex/gateway/ec2/spotinstance/RequestSpotInstances.go +++ b/spinifex/gateway/ec2/spotinstance/RequestSpotInstances.go @@ -76,7 +76,7 @@ func RequestSpotInstances(ctx context.Context, input *ec2.RequestSpotInstancesIn // Gate on the per-account vCPU cap exactly like the on-demand path; spot // launches are real VMs and must not slip past the quota. launchQuotaCheck := func() error { - return quota.EnforceLaunch(accountID, aws.StringValue(runInput.InstanceType), int(count)) + return quota.EnforceLaunch(ctx, accountID, aws.StringValue(runInput.InstanceType), int(count)) } // RunInstances normalises runInput in place (e.g. instance profile to ARN), @@ -88,7 +88,7 @@ func RequestSpotInstances(ctx context.Context, input *ec2.RequestSpotInstancesIn // Charge the actual launched vCPUs; a counter write failure is drift for // reconcile to correct, so it must not fail the already-successful launch. - if err := quota.ChargeLaunch(accountID, &reservation); err != nil { + if err := quota.ChargeLaunch(ctx, accountID, &reservation); err != nil { slog.WarnContext(ctx, "RequestSpotInstances: vcpu quota charge failed, reconcile will correct", "account", accountID, "err", err) } diff --git a/spinifex/gateway/ecr/registry_image_test.go b/spinifex/gateway/ecr/registry_image_test.go index ea0088bbb..f388c2429 100644 --- a/spinifex/gateway/ecr/registry_image_test.go +++ b/spinifex/gateway/ecr/registry_image_test.go @@ -157,7 +157,8 @@ func TestListImages_TaggedAndUntagged(t *testing.T) { // canonicalize on meta.Digest to emit a real digest and keep tag association. // The memory store returns real digests and would mask this. func TestListImages_KVDigestForm(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) reg := NewRegistry(objectstore.NewMemoryObjectStore(), ecr.NewKVMetaStore(js), testAccount) repo := "team/app" require.NoError(t, reg.Meta.PutRepo(context.Background(), testAccount, ecr.RepoMeta{Name: repo})) diff --git a/spinifex/gateway/ecr_createdeleterepository_test.go b/spinifex/gateway/ecr_createdeleterepository_test.go index da9ef0f6c..fe68eabc9 100644 --- a/spinifex/gateway/ecr_createdeleterepository_test.go +++ b/spinifex/gateway/ecr_createdeleterepository_test.go @@ -20,7 +20,8 @@ import ( // subjects against an embedded KV-backed service, returning a gateway client. func newRepoLifecycleGateway(t *testing.T) (*GatewayConfig, *nats.Conn) { t.Helper() - _, nc, js := testutil.StartTestJetStream(t) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) svc := handlers_ecr.NewKVMetaService(js) serveECRMeta(t, nc, handlers_ecr.SubjectRepoCreate, svc.RepoCreate) serveECRMeta(t, nc, handlers_ecr.SubjectRepoDescribe, svc.RepoDescribe) diff --git a/spinifex/gateway/ecr_describerepositories_test.go b/spinifex/gateway/ecr_describerepositories_test.go index c2b1eeaa2..55b60ba0e 100644 --- a/spinifex/gateway/ecr_describerepositories_test.go +++ b/spinifex/gateway/ecr_describerepositories_test.go @@ -42,7 +42,8 @@ func serveECRMeta[I any, O any](t *testing.T, nc *nats.Conn, subject string, fn func newDescribeReposGateway(t *testing.T, repos ...string) *GatewayConfig { t.Helper() - _, nc, js := testutil.StartTestJetStream(t) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) svc := handlers_ecr.NewKVMetaService(js) serveECRMeta(t, nc, handlers_ecr.SubjectRepoCreate, svc.RepoCreate) serveECRMeta(t, nc, handlers_ecr.SubjectRepoDescribe, svc.RepoDescribe) diff --git a/spinifex/gateway/ecrapi/lifecyclepolicy_test.go b/spinifex/gateway/ecrapi/lifecyclepolicy_test.go index 63b4b7d37..29f0ad3e3 100644 --- a/spinifex/gateway/ecrapi/lifecyclepolicy_test.go +++ b/spinifex/gateway/ecrapi/lifecyclepolicy_test.go @@ -20,7 +20,8 @@ const validLifecyclePolicy = `{"rules":[{"rulePriority":1,"selection":{"tagStatu // lifecycle metadata subjects served, returning the gateway NATS connection. func newLifecycleTestConn(t *testing.T) *nats.Conn { t.Helper() - _, nc, js := testutil.StartTestJetStream(t) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) svc := handlers_ecr.NewKVMetaService(js) serveMeta(t, nc, handlers_ecr.SubjectRepoCreate, svc.RepoCreate) serveMeta(t, nc, handlers_ecr.SubjectRepoDescribe, svc.RepoDescribe) diff --git a/spinifex/gateway/ecrapi/repositorypolicy_test.go b/spinifex/gateway/ecrapi/repositorypolicy_test.go index 02013e985..7cfc523dd 100644 --- a/spinifex/gateway/ecrapi/repositorypolicy_test.go +++ b/spinifex/gateway/ecrapi/repositorypolicy_test.go @@ -45,7 +45,8 @@ func serveMeta[I any, O any](t *testing.T, nc *nats.Conn, subject string, fn fun // metadata subjects served, returning the gateway NATS connection. func newPolicyTestConn(t *testing.T) *nats.Conn { t.Helper() - _, nc, js := testutil.StartTestJetStream(t) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) svc := handlers_ecr.NewKVMetaService(js) serveMeta(t, nc, handlers_ecr.SubjectRepoCreate, svc.RepoCreate) serveMeta(t, nc, handlers_ecr.SubjectRepoDescribe, svc.RepoDescribe) diff --git a/spinifex/gateway/ecrauth/ecrauth_test.go b/spinifex/gateway/ecrauth/ecrauth_test.go index 3f3ff8458..1adf69b6a 100644 --- a/spinifex/gateway/ecrauth/ecrauth_test.go +++ b/spinifex/gateway/ecrauth/ecrauth_test.go @@ -37,42 +37,46 @@ func samplePrincipal() Principal { } func TestLoadOrCreateSigningKey_CreatesThenReloads(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) - key1, verify1, err := LoadOrCreateSigningKey(js, testMasterKey, 1) + key1, verify1, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey, 1) require.NoError(t, err) require.NotEmpty(t, key1.Kid) require.Contains(t, verify1, key1.Kid) // Second call must reload the same persisted key, not mint a new one. - key2, verify2, err := LoadOrCreateSigningKey(js, testMasterKey, 1) + key2, verify2, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey, 1) require.NoError(t, err) assert.Equal(t, key1.Kid, key2.Kid, "persisted signing key must be reused") assert.Len(t, verify2, 1) } func TestLoadOrCreateSigningKey_StoresEncrypted(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) - key, _, err := LoadOrCreateSigningKey(js, testMasterKey, 1) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + key, _, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey, 1) require.NoError(t, err) - kv, err := js.KeyValue(SigningBucket) + kv, err := js.KeyValue(t.Context(), SigningBucket) require.NoError(t, err) - entry, err := kv.Get(signingKeyName(key.Kid)) + entry, err := kv.Get(t.Context(), signingKeyName(key.Kid)) require.NoError(t, err) assert.NotContains(t, string(entry.Value()), "PRIVATE KEY", "signing key must be encrypted at rest") } func TestLoadOrCreateSigningKey_EmptyMasterKeyRejected(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) - _, _, err := LoadOrCreateSigningKey(js, nil, 1) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + _, _, err := LoadOrCreateSigningKey(t.Context(), js, nil, 1) require.Error(t, err) } func TestIssuerVerifier_RoundTrip(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) - key, verify, err := LoadOrCreateSigningKey(js, testMasterKey, 1) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + key, verify, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey, 1) require.NoError(t, err) iss := NewIssuer(key, testAudience) @@ -88,8 +92,9 @@ func TestIssuerVerifier_RoundTrip(t *testing.T) { } func TestIssuer_MintRequiresAccountID(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) - key, _, err := LoadOrCreateSigningKey(js, testMasterKey, 1) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + key, _, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey, 1) require.NoError(t, err) p := samplePrincipal() @@ -103,8 +108,9 @@ func TestIssuer_MintRequiresAccountID(t *testing.T) { // accessKeyID, an unsupported principalType) must fail to mint at all, since // a token that can't name a lookup key can never be safely rehydrated. func TestMint_RequiresCompleteIdentityPointer(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) - key, _, err := LoadOrCreateSigningKey(js, testMasterKey, 1) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + key, _, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey, 1) require.NoError(t, err) iss := NewIssuer(key, testAudience) @@ -136,8 +142,9 @@ func TestSupportedPrincipalType(t *testing.T) { } func TestVerifier_RejectsWrongAudience(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) - key, verify, err := LoadOrCreateSigningKey(js, testMasterKey, 1) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + key, verify, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey, 1) require.NoError(t, err) tok, _, err := NewIssuer(key, testAudience).Mint(samplePrincipal()) @@ -148,8 +155,9 @@ func TestVerifier_RejectsWrongAudience(t *testing.T) { } func TestVerifier_RejectsUnknownKid(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) - key, _, err := LoadOrCreateSigningKey(js, testMasterKey, 1) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + key, _, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey, 1) require.NoError(t, err) tok, _, err := NewIssuer(key, testAudience).Mint(samplePrincipal()) @@ -161,8 +169,9 @@ func TestVerifier_RejectsUnknownKid(t *testing.T) { } func TestVerifier_RejectsExpiredToken(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) - key, verify, err := LoadOrCreateSigningKey(js, testMasterKey, 1) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + key, verify, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey, 1) require.NoError(t, err) // Hand-mint an already-expired token with the same signing key. @@ -188,8 +197,9 @@ func TestVerifier_RejectsExpiredToken(t *testing.T) { // Mint's own checks, proving Verify itself refuses a claim set missing any // part of the identity pointer even if it were signed by some other path. func TestVerifier_RejectsIncompleteIdentityPointer(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) - key, verify, err := LoadOrCreateSigningKey(js, testMasterKey, 1) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + key, verify, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey, 1) require.NoError(t, err) base := func() Claims { @@ -234,8 +244,9 @@ func TestVerifier_RejectsIncompleteIdentityPointer(t *testing.T) { } func TestVerifier_RejectsNonES256(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) - _, verify, err := LoadOrCreateSigningKey(js, testMasterKey, 1) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + _, verify, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey, 1) require.NoError(t, err) // HS256 token must be refused regardless of signature. diff --git a/spinifex/gateway/ecrauth/rotator.go b/spinifex/gateway/ecrauth/rotator.go index d08a7baff..1a557910c 100644 --- a/spinifex/gateway/ecrauth/rotator.go +++ b/spinifex/gateway/ecrauth/rotator.go @@ -6,7 +6,7 @@ import ( "sort" "time" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // Rotation defaults (ecr-v1 Q3): rotate the active signing key every 30 days, @@ -17,6 +17,10 @@ const ( DefaultRetainFor = 24 * time.Hour DefaultRetainCount = 2 DefaultRotationCheckInterval = time.Hour + + // rotationCycleTimeout bounds the KV work of one rotation cycle, so a wedged + // JetStream stalls a single tick instead of the whole loop until shutdown. + rotationCycleTimeout = 30 * time.Second ) // rotationParams holds the rotation/retention policy. @@ -61,7 +65,7 @@ func planRotation(metas []keyMeta, now time.Time, p rotationParams) (mint bool, // safe to run on every awsgw instance — mint and delete on the shared KV bucket // converge, and a delete of an already-gone key is a no-op. type Rotator struct { - kv nats.KeyValue + kv jetstream.KeyValue masterKey []byte issuer *Issuer verifier *Verifier @@ -73,8 +77,8 @@ type Rotator struct { // NewRotator opens the signing-key bucket and builds a rotator that keeps issuer // and verifier current. replicas matches the cluster size for first-run bucket // creation. -func NewRotator(js nats.JetStreamContext, masterKey []byte, replicas int, issuer *Issuer, verifier *Verifier) (*Rotator, error) { - kv, err := openSigningBucket(js, masterKey, replicas) +func NewRotator(ctx context.Context, js jetstream.JetStream, masterKey []byte, replicas int, issuer *Issuer, verifier *Verifier) (*Rotator, error) { + kv, err := openSigningBucket(ctx, js, masterKey, replicas) if err != nil { return nil, err } @@ -105,15 +109,17 @@ func (r *Rotator) Run(ctx context.Context) { slog.Info("ECR signing-key rotator stopped") return case <-ticker.C: - r.rotateOnce(r.now()) + cycleCtx, cancel := context.WithTimeout(ctx, rotationCycleTimeout) + r.rotateOnce(cycleCtx, r.now()) + cancel() } } } // rotateOnce runs one rotation/prune cycle and refreshes the issuer/verifier. // Errors are logged and skipped so a transient KV fault never crashes the loop. -func (r *Rotator) rotateOnce(now time.Time) { - _, _, metas, err := reloadKeys(r.kv, r.masterKey) +func (r *Rotator) rotateOnce(ctx context.Context, now time.Time) { + _, _, metas, err := reloadKeys(ctx, r.kv, r.masterKey) if err != nil { slog.Warn("ECR signing-key rotation: load keys failed", "err", err) return @@ -121,7 +127,7 @@ func (r *Rotator) rotateOnce(now time.Time) { mint, prune := planRotation(metas, now, r.params) if mint { - newKey, err := generateSigningKey(r.kv, r.masterKey) + newKey, err := generateSigningKey(ctx, r.kv, r.masterKey) if err != nil { slog.Warn("ECR signing-key rotation: mint failed", "err", err) return @@ -129,19 +135,19 @@ func (r *Rotator) rotateOnce(now time.Time) { slog.Info("ECR signing-key rotated", "kid", newKey.Kid) } for _, kid := range prune { - if err := deleteSigningKey(r.kv, kid); err != nil { + if err := deleteSigningKey(ctx, r.kv, kid); err != nil { slog.Warn("ECR signing-key rotation: prune failed", "kid", kid, "err", err) continue } slog.Info("ECR signing-key pruned", "kid", kid) } - r.refresh() + r.refresh(ctx) } // refresh reloads the canonical key set and installs it on the issuer/verifier. -func (r *Rotator) refresh() { - active, verify, _, err := reloadKeys(r.kv, r.masterKey) +func (r *Rotator) refresh(ctx context.Context) { + active, verify, _, err := reloadKeys(ctx, r.kv, r.masterKey) if err != nil { slog.Warn("ECR signing-key rotation: refresh failed", "err", err) return diff --git a/spinifex/gateway/ecrauth/rotator_test.go b/spinifex/gateway/ecrauth/rotator_test.go index 82b164a20..83fa2f288 100644 --- a/spinifex/gateway/ecrauth/rotator_test.go +++ b/spinifex/gateway/ecrauth/rotator_test.go @@ -77,13 +77,14 @@ func TestPlanRotation(t *testing.T) { // always due: a new active key is minted, installed on the issuer, and accepted // by the verifier, while the prior key stays verifiable. func TestRotateOnce_MintsAndSwaps(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) - key, verify, err := LoadOrCreateSigningKey(js, testMasterKey, 1) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + key, verify, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey, 1) require.NoError(t, err) issuer := NewIssuer(key, testAudience) verifier := NewVerifier(verify, testAudience) - kv, err := openSigningBucket(js, testMasterKey, 1) + kv, err := openSigningBucket(t.Context(), js, testMasterKey, 1) require.NoError(t, err) // Token minted under the original key before rotation. @@ -95,7 +96,7 @@ func TestRotateOnce_MintsAndSwaps(t *testing.T) { params: rotationParams{rotateAfter: 0, retainFor: time.Hour, retainCount: 2}, now: func() time.Time { return time.Now().UTC() }, } - r.rotateOnce(time.Now().UTC()) + r.rotateOnce(t.Context(), time.Now().UTC()) // A new active key took over. assert.NotEqual(t, key.Kid, issuer.ActiveKid(), "issuer should sign with the rotated key") @@ -108,7 +109,7 @@ func TestRotateOnce_MintsAndSwaps(t *testing.T) { _, err = verifier.Verify(newTok) require.NoError(t, err, "token from the rotated key must verify") - _, _, metas, err := reloadKeys(kv, testMasterKey) + _, _, metas, err := reloadKeys(t.Context(), kv, testMasterKey) require.NoError(t, err) assert.Len(t, metas, 2, "previous key retained alongside the new active key") } @@ -117,12 +118,13 @@ func TestRotateOnce_MintsAndSwaps(t *testing.T) { // retainFor=0 (and no mint) so the older key is pruned and dropped from the // verifier. func TestRotateOnce_PrunesExpiredKey(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) - first, verify, err := LoadOrCreateSigningKey(js, testMasterKey, 1) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + first, verify, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey, 1) require.NoError(t, err) - kv, err := openSigningBucket(js, testMasterKey, 1) + kv, err := openSigningBucket(t.Context(), js, testMasterKey, 1) require.NoError(t, err) - second, err := generateSigningKey(kv, testMasterKey) + second, err := generateSigningKey(t.Context(), kv, testMasterKey) require.NoError(t, err) require.NotEqual(t, first.Kid, second.Kid) @@ -134,9 +136,9 @@ func TestRotateOnce_PrunesExpiredKey(t *testing.T) { params: rotationParams{rotateAfter: 1000 * time.Hour, retainFor: 0, retainCount: 2}, now: func() time.Time { return time.Now().UTC() }, } - r.rotateOnce(time.Now().UTC()) + r.rotateOnce(t.Context(), time.Now().UTC()) - active, _, metas, err := reloadKeys(kv, testMasterKey) + active, _, metas, err := reloadKeys(t.Context(), kv, testMasterKey) require.NoError(t, err) require.Len(t, metas, 1, "the older key should be pruned") assert.Equal(t, second.Kid, active.Kid, "newest key remains active") @@ -147,12 +149,13 @@ func TestRotateOnce_PrunesExpiredKey(t *testing.T) { // with rotateAfter=0 so it mints on the first tick, then cancels: the active key // changes and Run returns. func TestRotator_RunRotatesThenStops(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) - key, verify, err := LoadOrCreateSigningKey(js, testMasterKey, 1) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + key, verify, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey, 1) require.NoError(t, err) issuer := NewIssuer(key, testAudience) verifier := NewVerifier(verify, testAudience) - kv, err := openSigningBucket(js, testMasterKey, 1) + kv, err := openSigningBucket(t.Context(), js, testMasterKey, 1) require.NoError(t, err) r := &Rotator{ @@ -180,14 +183,15 @@ func TestRotator_RunRotatesThenStops(t *testing.T) { // TestIssuerVerifier_ConcurrentSwap exercises the hot-swap under -race: Mint and // Verify run while SetActiveKey/SetKeys rotate the key set. func TestIssuerVerifier_ConcurrentSwap(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) - k1, v1, err := LoadOrCreateSigningKey(js, testMasterKey, 1) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + k1, v1, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey, 1) require.NoError(t, err) - kv, err := openSigningBucket(js, testMasterKey, 1) + kv, err := openSigningBucket(t.Context(), js, testMasterKey, 1) require.NoError(t, err) - k2, err := generateSigningKey(kv, testMasterKey) + k2, err := generateSigningKey(t.Context(), kv, testMasterKey) require.NoError(t, err) - _, v2, _, err := reloadKeys(kv, testMasterKey) + _, v2, _, err := reloadKeys(t.Context(), kv, testMasterKey) require.NoError(t, err) issuer := NewIssuer(k1, testAudience) diff --git a/spinifex/gateway/ecrauth/signingkey.go b/spinifex/gateway/ecrauth/signingkey.go index 4c5071349..6e89ccc29 100644 --- a/spinifex/gateway/ecrauth/signingkey.go +++ b/spinifex/gateway/ecrauth/signingkey.go @@ -6,6 +6,7 @@ package gateway_ecrauth import ( + "context" "crypto/ecdsa" "crypto/elliptic" "crypto/rand" @@ -19,7 +20,8 @@ import ( "time" handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" - "github.com/nats-io/nats.go" + "github.com/mulgadc/spinifex/spinifex/kvutil" + "github.com/nats-io/nats.go/jetstream" ) const ( @@ -73,17 +75,17 @@ func kidFor(pub *ecdsa.PublicKey) (string, error) { // returns the active signing key. On first run it generates and persists one. // The active key is the newest by KV creation time (ties broken by kid), so all // nodes converge on the same signer and a freshly rotated key takes over. -func LoadOrCreateSigningKey(js nats.JetStreamContext, masterKey []byte, replicas int) (*SigningKey, map[string]*ecdsa.PublicKey, error) { - kv, err := openSigningBucket(js, masterKey, replicas) +func LoadOrCreateSigningKey(ctx context.Context, js jetstream.JetStream, masterKey []byte, replicas int) (*SigningKey, map[string]*ecdsa.PublicKey, error) { + kv, err := openSigningBucket(ctx, js, masterKey, replicas) if err != nil { return nil, nil, err } - active, verify, _, err := reloadKeys(kv, masterKey) + active, verify, _, err := reloadKeys(ctx, kv, masterKey) if err != nil { return nil, nil, err } if active == nil { - active, err = generateSigningKey(kv, masterKey) + active, err = generateSigningKey(ctx, kv, masterKey) if err != nil { return nil, nil, err } @@ -94,32 +96,22 @@ func LoadOrCreateSigningKey(js nats.JetStreamContext, masterKey []byte, replicas // openSigningBucket validates inputs and returns the awsgw-keys KV handle, // creating the cluster-replicated bucket on first use. -func openSigningBucket(js nats.JetStreamContext, masterKey []byte, replicas int) (nats.KeyValue, error) { +func openSigningBucket(ctx context.Context, js jetstream.JetStream, masterKey []byte, replicas int) (jetstream.KeyValue, error) { if js == nil { return nil, errors.New("ecrauth: nil JetStream context") } if len(masterKey) == 0 { return nil, errors.New("ecrauth: empty master key") } - kv, err := js.CreateKeyValue(&nats.KeyValueConfig{ - Bucket: SigningBucket, - History: signingKeyHistory, - Replicas: replicas, - }) - if err != nil { - if kv, err = js.KeyValue(SigningBucket); err != nil { - return nil, fmt.Errorf("open %s bucket: %w", SigningBucket, err) - } - } - return kv, nil + return kvutil.GetOrCreateBucketWithReplicas(ctx, js, SigningBucket, signingKeyHistory, replicas) } // reloadKeys reads every stored signing key, returning the verify set, each // key's metadata, and the active (newest) decrypted key. active is nil for an // empty bucket. Used by both startup and the rotation scheduler. -func reloadKeys(kv nats.KeyValue, masterKey []byte) (active *SigningKey, verify map[string]*ecdsa.PublicKey, metas []keyMeta, err error) { - names, err := kv.Keys() - if err != nil && !errors.Is(err, nats.ErrNoKeysFound) { +func reloadKeys(ctx context.Context, kv jetstream.KeyValue, masterKey []byte) (active *SigningKey, verify map[string]*ecdsa.PublicKey, metas []keyMeta, err error) { + names, err := kvutil.Keys(ctx, kv) + if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) { return nil, nil, nil, fmt.Errorf("list signing keys: %w", err) } @@ -129,7 +121,7 @@ func reloadKeys(kv nats.KeyValue, masterKey []byte) (active *SigningKey, verify if !strings.HasPrefix(name, signingKeyPrefix) { continue } - entry, err := kv.Get(name) + entry, err := kv.Get(ctx, name) if err != nil { return nil, nil, nil, fmt.Errorf("kv get %s: %w", name, err) } @@ -149,8 +141,8 @@ func reloadKeys(kv nats.KeyValue, masterKey []byte) (active *SigningKey, verify // deleteSigningKey removes a rotated-out key from the bucket once its retention // window has elapsed. -func deleteSigningKey(kv nats.KeyValue, kid string) error { - if err := kv.Delete(signingKeyName(kid)); err != nil { +func deleteSigningKey(ctx context.Context, kv jetstream.KeyValue, kid string) error { + if err := kv.Delete(ctx, signingKeyName(kid)); err != nil { return fmt.Errorf("kv delete %s: %w", signingKeyName(kid), err) } return nil @@ -172,7 +164,7 @@ func decodeSigningKey(kid string, ciphertext []byte, masterKey []byte) (*Signing // generateSigningKey creates a fresh ES256 key, persists the encrypted PEM under // jwt-signing/{kid}, and returns it. -func generateSigningKey(kv nats.KeyValue, masterKey []byte) (*SigningKey, error) { +func generateSigningKey(ctx context.Context, kv jetstream.KeyValue, masterKey []byte) (*SigningKey, error) { priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) if err != nil { return nil, fmt.Errorf("generate ES256 key: %w", err) @@ -190,7 +182,7 @@ func generateSigningKey(kv nats.KeyValue, masterKey []byte) (*SigningKey, error) if err != nil { return nil, fmt.Errorf("encrypt signing key: %w", err) } - if _, err := kv.Put(signingKeyName(kid), []byte(ciphertext)); err != nil { + if _, err := kv.Put(ctx, signingKeyName(kid), []byte(ciphertext)); err != nil { return nil, fmt.Errorf("kv put %s: %w", signingKeyName(kid), err) } return &SigningKey{Kid: kid, priv: priv}, nil diff --git a/spinifex/gateway/ecrauth_middleware_test.go b/spinifex/gateway/ecrauth_middleware_test.go index a7ea9500c..8a6f9d965 100644 --- a/spinifex/gateway/ecrauth_middleware_test.go +++ b/spinifex/gateway/ecrauth_middleware_test.go @@ -32,8 +32,9 @@ func ecrTestMasterKey(t *testing.T) []byte { // newECRAuth builds a wired issuer+verifier over a freshly generated signing key. func newECRAuth(t *testing.T) (*gateway_ecrauth.Issuer, *gateway_ecrauth.Verifier) { t.Helper() - _, _, js := testutil.StartTestJetStream(t) - key, verify, err := gateway_ecrauth.LoadOrCreateSigningKey(js, ecrTestMasterKey(t), 1) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + key, verify, err := gateway_ecrauth.LoadOrCreateSigningKey(t.Context(), js, ecrTestMasterKey(t), 1) require.NoError(t, err) return gateway_ecrauth.NewIssuer(key, ecrTestAudience), gateway_ecrauth.NewVerifier(verify, ecrTestAudience) } diff --git a/spinifex/gateway/eks/token_review_test.go b/spinifex/gateway/eks/token_review_test.go index fa38e8cd6..1ebdf8aa8 100644 --- a/spinifex/gateway/eks/token_review_test.go +++ b/spinifex/gateway/eks/token_review_test.go @@ -10,6 +10,7 @@ import ( handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks" "github.com/mulgadc/spinifex/spinifex/testutil" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -56,9 +57,11 @@ func TestWebhookTokenReview_InfraFaultIsServerInternal(t *testing.T) { } func TestWebhookTokenReview_ResolvesIdentity(t *testing.T) { - _, nc, js := testutil.StartTestJetStream(t) - kv := testutil.SeedKV(t, js, handlers_eks.AccountBucketName("111122223333"), nil) - require.NoError(t, handlers_eks.PutAccessEntryRecord(kv, &handlers_eks.AccessEntryRecord{ + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + kv, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: handlers_eks.AccountBucketName("111122223333")}) + require.NoError(t, err) + require.NoError(t, handlers_eks.PutAccessEntryRecord(t.Context(), kv, &handlers_eks.AccessEntryRecord{ ClusterName: "alpha", PrincipalARN: testARN, KubernetesUsername: testARN, diff --git a/spinifex/gateway/oidc_discovery.go b/spinifex/gateway/oidc_discovery.go index c7c5ec598..16edf0f8a 100644 --- a/spinifex/gateway/oidc_discovery.go +++ b/spinifex/gateway/oidc_discovery.go @@ -1,6 +1,7 @@ package gateway import ( + "context" "encoding/json" "errors" "log/slog" @@ -8,7 +9,7 @@ import ( "github.com/go-chi/chi/v5" handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // oidcDiscoveryDocument is the subset of the OpenID Connect Discovery metadata @@ -24,17 +25,17 @@ type oidcDiscoveryDocument struct { } // eksAccountKV opens the read-only per-account EKS bucket. Returns -// nats.ErrBucketNotFound (unwrapped) when the account has no clusters, which +// jetstream.ErrBucketNotFound (unwrapped) when the account has no clusters, which // the OIDC handlers translate to 404. -func (gw *GatewayConfig) eksAccountKV(accountID string) (nats.KeyValue, error) { +func (gw *GatewayConfig) eksAccountKV(ctx context.Context, accountID string) (jetstream.KeyValue, error) { if gw.NATSConn == nil { return nil, errors.New("gateway NATS connection not initialised") } - js, err := gw.NATSConn.JetStream() + js, err := jetstream.New(gw.NATSConn) if err != nil { return nil, err } - return js.KeyValue(handlers_eks.AccountBucketName(accountID)) + return js.KeyValue(ctx, handlers_eks.AccountBucketName(accountID)) } // OIDCDiscoveryDocument serves the cluster's OpenID discovery document. The @@ -45,9 +46,9 @@ func (gw *GatewayConfig) OIDCDiscoveryDocument(w http.ResponseWriter, r *http.Re accountID := chi.URLParam(r, "accountID") clusterName := chi.URLParam(r, "clusterName") - kv, err := gw.eksAccountKV(accountID) + kv, err := gw.eksAccountKV(r.Context(), accountID) if err != nil { - if errors.Is(err, nats.ErrBucketNotFound) { + if errors.Is(err, jetstream.ErrBucketNotFound) { http.NotFound(w, r) return } @@ -56,7 +57,7 @@ func (gw *GatewayConfig) OIDCDiscoveryDocument(w http.ResponseWriter, r *http.Re return } - meta, err := handlers_eks.GetClusterMeta(kv, clusterName) + meta, err := handlers_eks.GetClusterMeta(r.Context(), kv, clusterName) if err != nil || meta == nil || meta.OIDCIssuer == "" { http.NotFound(w, r) return @@ -78,9 +79,9 @@ func (gw *GatewayConfig) OIDCJWKS(w http.ResponseWriter, r *http.Request) { accountID := chi.URLParam(r, "accountID") clusterName := chi.URLParam(r, "clusterName") - kv, err := gw.eksAccountKV(accountID) + kv, err := gw.eksAccountKV(r.Context(), accountID) if err != nil { - if errors.Is(err, nats.ErrBucketNotFound) { + if errors.Is(err, jetstream.ErrBucketNotFound) { http.NotFound(w, r) return } @@ -89,7 +90,7 @@ func (gw *GatewayConfig) OIDCJWKS(w http.ResponseWriter, r *http.Request) { return } - entry, err := kv.Get(handlers_eks.OIDCJWKSKey(clusterName)) + entry, err := kv.Get(r.Context(), handlers_eks.OIDCJWKSKey(clusterName)) if err != nil { http.NotFound(w, r) return diff --git a/spinifex/gateway/oidc_discovery_test.go b/spinifex/gateway/oidc_discovery_test.go index 1d375aa21..757820271 100644 --- a/spinifex/gateway/oidc_discovery_test.go +++ b/spinifex/gateway/oidc_discovery_test.go @@ -20,20 +20,21 @@ const ( // into the per-account EKS bucket, returning a router wired to the same NATS. func seedDiscoveryCluster(t *testing.T) http.Handler { t.Helper() - _, nc, js := testutil.StartTestJetStream(t) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) - kv, err := handlers_eks.GetOrCreateAccountBucket(js, testDiscAccount, 1) + kv, err := handlers_eks.GetOrCreateAccountBucket(t.Context(), js, testDiscAccount, 1) if err != nil { t.Fatalf("account bucket: %v", err) } - if err := handlers_eks.PutClusterMeta(kv, &handlers_eks.ClusterMeta{ + if err := handlers_eks.PutClusterMeta(t.Context(), kv, &handlers_eks.ClusterMeta{ Name: testDiscCluster, Status: handlers_eks.ClusterStatusActive, OIDCIssuer: testDiscIssuer, }); err != nil { t.Fatalf("put cluster meta: %v", err) } - if _, err := kv.Put(handlers_eks.OIDCJWKSKey(testDiscCluster), []byte(`{"keys":[{"kty":"EC","kid":"abc"}]}`)); err != nil { + if _, err := kv.Put(t.Context(), handlers_eks.OIDCJWKSKey(testDiscCluster), []byte(`{"keys":[{"kty":"EC","kid":"abc"}]}`)); err != nil { t.Fatalf("put jwks: %v", err) } diff --git a/spinifex/handlers/acm/service_impl.go b/spinifex/handlers/acm/service_impl.go index 70f0e8696..4a4c4ba91 100644 --- a/spinifex/handlers/acm/service_impl.go +++ b/spinifex/handlers/acm/service_impl.go @@ -39,8 +39,8 @@ var _ ACMService = (*ACMServiceImpl)(nil) // NewACMServiceImplWithNATS builds an ACM service backed by a JetStream KV // store. cfg may be nil (tests); region then falls back to the default. -func NewACMServiceImplWithNATS(cfg *config.Config, nc *nats.Conn) (*ACMServiceImpl, error) { - store, err := NewStore(nc) +func NewACMServiceImplWithNATS(ctx context.Context, cfg *config.Config, nc *nats.Conn) (*ACMServiceImpl, error) { + store, err := NewStore(ctx, nc) if err != nil { return nil, err } @@ -79,7 +79,7 @@ func (s *ACMServiceImpl) ImportCertificate(ctx context.Context, input *acm.Impor certArn = s.mintCertificateArn(accountID) } else { // Re-import: the ARN must already exist and belong to the caller. - existing, gErr := s.store.GetCert(certArn) + existing, gErr := s.store.GetCert(ctx, certArn) if gErr != nil { return nil, errors.New(awserrors.ErrorInternalError) } @@ -105,7 +105,7 @@ func (s *ACMServiceImpl) ImportCertificate(ctx context.Context, input *acm.Impor ImportedAt: time.Now().UTC(), Tags: tagsToMap(input.Tags), } - if err := s.store.PutCert(rec); err != nil { + if err := s.store.PutCert(ctx, rec); err != nil { slog.ErrorContext(ctx, "ImportCertificate: store failed", "err", err) return nil, errors.New(awserrors.ErrorInternalError) } @@ -119,7 +119,7 @@ func (s *ACMServiceImpl) DescribeCertificate(ctx context.Context, input *acm.Des if input == nil || aws.StringValue(input.CertificateArn) == "" { return nil, errors.New(awserrors.ErrorACMInvalidArn) } - rec, err := s.lookupOwned(aws.StringValue(input.CertificateArn), accountID) + rec, err := s.lookupOwned(ctx, aws.StringValue(input.CertificateArn), accountID) if err != nil { return nil, err } @@ -128,7 +128,7 @@ func (s *ACMServiceImpl) DescribeCertificate(ctx context.Context, input *acm.Des // ListCertificates returns summaries for every cert owned by accountID. func (s *ACMServiceImpl) ListCertificates(ctx context.Context, input *acm.ListCertificatesInput, accountID string) (*acm.ListCertificatesOutput, error) { - recs, err := s.store.ListCerts(accountID) + recs, err := s.store.ListCerts(ctx, accountID) if err != nil { return nil, errors.New(awserrors.ErrorInternalError) } @@ -154,10 +154,10 @@ func (s *ACMServiceImpl) DeleteCertificate(ctx context.Context, input *acm.Delet if input == nil || aws.StringValue(input.CertificateArn) == "" { return nil, errors.New(awserrors.ErrorACMInvalidArn) } - if _, err := s.lookupOwned(aws.StringValue(input.CertificateArn), accountID); err != nil { + if _, err := s.lookupOwned(ctx, aws.StringValue(input.CertificateArn), accountID); err != nil { return nil, err } - if _, err := s.store.DeleteCert(aws.StringValue(input.CertificateArn)); err != nil { + if _, err := s.store.DeleteCert(ctx, aws.StringValue(input.CertificateArn)); err != nil { return nil, errors.New(awserrors.ErrorInternalError) } return &acm.DeleteCertificateOutput{}, nil @@ -165,8 +165,8 @@ func (s *ACMServiceImpl) DeleteCertificate(ctx context.Context, input *acm.Delet // lookupOwned fetches a cert by ARN, returning ResourceNotFound when absent or // owned by a different account (no cross-account disclosure). -func (s *ACMServiceImpl) lookupOwned(certArn, accountID string) (*CertRecord, error) { - rec, err := s.store.GetCert(certArn) +func (s *ACMServiceImpl) lookupOwned(ctx context.Context, certArn, accountID string) (*CertRecord, error) { + rec, err := s.store.GetCert(ctx, certArn) if err != nil { return nil, errors.New(awserrors.ErrorInternalError) } @@ -209,7 +209,7 @@ func (s *ACMServiceImpl) ListTagsForCertificate(ctx context.Context, input *acm. if input == nil || aws.StringValue(input.CertificateArn) == "" { return nil, errors.New(awserrors.ErrorACMInvalidArn) } - rec, err := s.lookupOwned(aws.StringValue(input.CertificateArn), accountID) + rec, err := s.lookupOwned(ctx, aws.StringValue(input.CertificateArn), accountID) if err != nil { return nil, err } @@ -221,7 +221,7 @@ func (s *ACMServiceImpl) AddTagsToCertificate(ctx context.Context, input *acm.Ad if input == nil || aws.StringValue(input.CertificateArn) == "" { return nil, errors.New(awserrors.ErrorACMInvalidArn) } - rec, err := s.lookupOwned(aws.StringValue(input.CertificateArn), accountID) + rec, err := s.lookupOwned(ctx, aws.StringValue(input.CertificateArn), accountID) if err != nil { return nil, err } @@ -229,7 +229,7 @@ func (s *ACMServiceImpl) AddTagsToCertificate(ctx context.Context, input *acm.Ad rec.Tags = map[string]string{} } maps.Copy(rec.Tags, tagsToMap(input.Tags)) - if err := s.store.PutCert(rec); err != nil { + if err := s.store.PutCert(ctx, rec); err != nil { return nil, errors.New(awserrors.ErrorInternalError) } return &acm.AddTagsToCertificateOutput{}, nil @@ -242,7 +242,7 @@ func (s *ACMServiceImpl) RemoveTagsFromCertificate(ctx context.Context, input *a if input == nil || aws.StringValue(input.CertificateArn) == "" { return nil, errors.New(awserrors.ErrorACMInvalidArn) } - rec, err := s.lookupOwned(aws.StringValue(input.CertificateArn), accountID) + rec, err := s.lookupOwned(ctx, aws.StringValue(input.CertificateArn), accountID) if err != nil { return nil, err } @@ -256,7 +256,7 @@ func (s *ACMServiceImpl) RemoveTagsFromCertificate(ctx context.Context, input *a } delete(rec.Tags, key) } - if err := s.store.PutCert(rec); err != nil { + if err := s.store.PutCert(ctx, rec); err != nil { return nil, errors.New(awserrors.ErrorInternalError) } return &acm.RemoveTagsFromCertificateOutput{}, nil diff --git a/spinifex/handlers/acm/service_impl_test.go b/spinifex/handlers/acm/service_impl_test.go index 294a08258..ee48511da 100644 --- a/spinifex/handlers/acm/service_impl_test.go +++ b/spinifex/handlers/acm/service_impl_test.go @@ -25,7 +25,7 @@ const testAccountID = "000000000001" func setupACMService(t *testing.T) *ACMServiceImpl { t.Helper() _, nc, _ := testutil.StartTestJetStream(t) - svc, err := NewACMServiceImplWithNATS(nil, nc) + svc, err := NewACMServiceImplWithNATS(t.Context(), nil, nc) require.NoError(t, err) return svc } diff --git a/spinifex/handlers/acm/store.go b/spinifex/handlers/acm/store.go index 1198f2d15..8e70da34f 100644 --- a/spinifex/handlers/acm/store.go +++ b/spinifex/handlers/acm/store.go @@ -1,6 +1,7 @@ package handlers_acm import ( + "context" "encoding/json" "errors" "fmt" @@ -8,8 +9,9 @@ import ( "strings" "time" - "github.com/mulgadc/spinifex/spinifex/utils" + "github.com/mulgadc/spinifex/spinifex/kvutil" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) const ( @@ -42,19 +44,20 @@ type CertRecord struct { // Store provides CRUD for ACM certificate records backed by JetStream KV. type Store struct { - kv nats.KeyValue + kv jetstream.KeyValue } -// NewStore creates an ACM store using the provided NATS connection. -func NewStore(nc *nats.Conn) (*Store, error) { - js, err := nc.JetStream() +// NewStore creates an ACM store using the provided NATS connection. ctx bounds +// the bucket get-or-create only; each operation carries its own. +func NewStore(ctx context.Context, nc *nats.Conn) (*Store, error) { + js, err := jetstream.New(nc) if err != nil { return nil, fmt.Errorf("failed to get JetStream context: %w", err) } - kv, err := utils.GetOrCreateKVBucket(js, KVBucketACM, KVBucketACMVersion) + kv, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketACM, KVBucketACMVersion) if err != nil { - return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketACM, err) + return nil, err } slog.Info("ACM store initialized", "bucket", KVBucketACM) @@ -71,20 +74,20 @@ func certKey(certArn string) string { } // PutCert stores (or replaces) a certificate record. -func (s *Store) PutCert(rec *CertRecord) error { +func (s *Store) PutCert(ctx context.Context, rec *CertRecord) error { data, err := json.Marshal(rec) if err != nil { return fmt.Errorf("marshal cert: %w", err) } - _, err = s.kv.Put(certKey(rec.CertificateArn), data) + _, err = s.kv.Put(ctx, certKey(rec.CertificateArn), data) return err } // GetCert retrieves a certificate by ARN, returning (nil, nil) when absent. -func (s *Store) GetCert(certArn string) (*CertRecord, error) { - entry, err := s.kv.Get(certKey(certArn)) +func (s *Store) GetCert(ctx context.Context, certArn string) (*CertRecord, error) { + entry, err := s.kv.Get(ctx, certKey(certArn)) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil, nil } return nil, err @@ -97,24 +100,24 @@ func (s *Store) GetCert(certArn string) (*CertRecord, error) { } // DeleteCert removes a certificate by ARN. Returns (false, nil) when absent. -func (s *Store) DeleteCert(certArn string) (bool, error) { - if _, err := s.kv.Get(certKey(certArn)); err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { +func (s *Store) DeleteCert(ctx context.Context, certArn string) (bool, error) { + if _, err := s.kv.Get(ctx, certKey(certArn)); err != nil { + if errors.Is(err, jetstream.ErrKeyNotFound) { return false, nil } return false, err } - if err := s.kv.Delete(certKey(certArn)); err != nil { + if err := s.kv.Delete(ctx, certKey(certArn)); err != nil { return false, err } return true, nil } // ListCerts returns all certificate records owned by accountID. -func (s *Store) ListCerts(accountID string) ([]*CertRecord, error) { - keys, err := s.kv.Keys() +func (s *Store) ListCerts(ctx context.Context, accountID string) ([]*CertRecord, error) { + keys, err := s.kv.Keys(ctx) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return nil, nil } return nil, err @@ -124,7 +127,7 @@ func (s *Store) ListCerts(accountID string) ([]*CertRecord, error) { if !strings.HasPrefix(key, KeyPrefixCert) { continue } - entry, err := s.kv.Get(key) + entry, err := s.kv.Get(ctx, key) if err != nil { continue } diff --git a/spinifex/handlers/ec2/account/service_impl.go b/spinifex/handlers/ec2/account/service_impl.go index 80261b452..a536b542a 100644 --- a/spinifex/handlers/ec2/account/service_impl.go +++ b/spinifex/handlers/ec2/account/service_impl.go @@ -10,9 +10,11 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" "github.com/mulgadc/spinifex/spinifex/config" + "github.com/mulgadc/spinifex/spinifex/kvutil" "github.com/mulgadc/spinifex/spinifex/migrate" "github.com/mulgadc/spinifex/spinifex/utils" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) const ( @@ -31,24 +33,24 @@ type AccountSettingsRecord struct { // AccountSettingsServiceImpl implements account settings operations with NATS JetStream persistence. type AccountSettingsServiceImpl struct { config *config.Config - js nats.JetStreamContext - settingsKV nats.KeyValue + js jetstream.JetStream + settingsKV jetstream.KeyValue } var _ AccountSettingsService = (*AccountSettingsServiceImpl)(nil) // NewAccountSettingsServiceImplWithNATS creates an account settings service with NATS JetStream for persistence. -func NewAccountSettingsServiceImplWithNATS(cfg *config.Config, natsConn *nats.Conn) (*AccountSettingsServiceImpl, error) { - js, err := natsConn.JetStream() +func NewAccountSettingsServiceImplWithNATS(ctx context.Context, cfg *config.Config, natsConn *nats.Conn) (*AccountSettingsServiceImpl, error) { + js, err := jetstream.New(natsConn) if err != nil { return nil, fmt.Errorf("failed to get JetStream context: %w", err) } - settingsKV, err := utils.GetOrCreateKVBucket(js, KVBucketAccountSettings, 10) + settingsKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketAccountSettings, 10) if err != nil { return nil, fmt.Errorf("failed to create account settings KV bucket: %w", err) } - if err := migrate.DefaultRegistry.RunKV(KVBucketAccountSettings, settingsKV, KVBucketAccountSettingsVersion); err != nil { + if err := migrate.DefaultRegistry.RunKV(ctx, KVBucketAccountSettings, settingsKV, KVBucketAccountSettingsVersion); err != nil { return nil, fmt.Errorf("migrate %s: %w", KVBucketAccountSettings, err) } @@ -71,10 +73,10 @@ func settingsKey(accountID string) string { } // getSettings retrieves current account settings. -func (s *AccountSettingsServiceImpl) getSettings(accountID string) (*AccountSettingsRecord, error) { - entry, err := s.settingsKV.Get(settingsKey(accountID)) +func (s *AccountSettingsServiceImpl) getSettings(ctx context.Context, accountID string) (*AccountSettingsRecord, error) { + entry, err := s.settingsKV.Get(ctx, settingsKey(accountID)) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return &AccountSettingsRecord{ EbsEncryptionByDefault: false, SerialConsoleAccess: false, @@ -92,13 +94,13 @@ func (s *AccountSettingsServiceImpl) getSettings(accountID string) (*AccountSett } // saveSettings saves current account settings. -func (s *AccountSettingsServiceImpl) saveSettings(settings *AccountSettingsRecord, accountID string) error { +func (s *AccountSettingsServiceImpl) saveSettings(ctx context.Context, settings *AccountSettingsRecord, accountID string) error { data, err := json.Marshal(settings) if err != nil { return err } - _, err = s.settingsKV.Put(settingsKey(accountID), data) + _, err = s.settingsKV.Put(ctx, settingsKey(accountID), data) return err } @@ -106,14 +108,14 @@ func (s *AccountSettingsServiceImpl) saveSettings(settings *AccountSettingsRecor func (s *AccountSettingsServiceImpl) EnableEbsEncryptionByDefault(ctx context.Context, input *ec2.EnableEbsEncryptionByDefaultInput, accountID string) (*ec2.EnableEbsEncryptionByDefaultOutput, error) { slog.InfoContext(ctx, "EnableEbsEncryptionByDefault called", "accountID", accountID) - settings, err := s.getSettings(accountID) + settings, err := s.getSettings(ctx, accountID) if err != nil { return nil, err } settings.EbsEncryptionByDefault = true - if err := s.saveSettings(settings, accountID); err != nil { + if err := s.saveSettings(ctx, settings, accountID); err != nil { return nil, err } @@ -126,14 +128,14 @@ func (s *AccountSettingsServiceImpl) EnableEbsEncryptionByDefault(ctx context.Co func (s *AccountSettingsServiceImpl) DisableEbsEncryptionByDefault(ctx context.Context, input *ec2.DisableEbsEncryptionByDefaultInput, accountID string) (*ec2.DisableEbsEncryptionByDefaultOutput, error) { slog.InfoContext(ctx, "DisableEbsEncryptionByDefault called", "accountID", accountID) - settings, err := s.getSettings(accountID) + settings, err := s.getSettings(ctx, accountID) if err != nil { return nil, err } settings.EbsEncryptionByDefault = false - if err := s.saveSettings(settings, accountID); err != nil { + if err := s.saveSettings(ctx, settings, accountID); err != nil { return nil, err } @@ -146,7 +148,7 @@ func (s *AccountSettingsServiceImpl) DisableEbsEncryptionByDefault(ctx context.C func (s *AccountSettingsServiceImpl) GetEbsEncryptionByDefault(ctx context.Context, input *ec2.GetEbsEncryptionByDefaultInput, accountID string) (*ec2.GetEbsEncryptionByDefaultOutput, error) { slog.InfoContext(ctx, "GetEbsEncryptionByDefault called", "accountID", accountID) - settings, err := s.getSettings(accountID) + settings, err := s.getSettings(ctx, accountID) if err != nil { return nil, err } @@ -160,7 +162,7 @@ func (s *AccountSettingsServiceImpl) GetEbsEncryptionByDefault(ctx context.Conte func (s *AccountSettingsServiceImpl) GetSerialConsoleAccessStatus(ctx context.Context, input *ec2.GetSerialConsoleAccessStatusInput, accountID string) (*ec2.GetSerialConsoleAccessStatusOutput, error) { slog.InfoContext(ctx, "GetSerialConsoleAccessStatus called", "accountID", accountID) - settings, err := s.getSettings(accountID) + settings, err := s.getSettings(ctx, accountID) if err != nil { return nil, err } @@ -174,14 +176,14 @@ func (s *AccountSettingsServiceImpl) GetSerialConsoleAccessStatus(ctx context.Co func (s *AccountSettingsServiceImpl) EnableSerialConsoleAccess(ctx context.Context, input *ec2.EnableSerialConsoleAccessInput, accountID string) (*ec2.EnableSerialConsoleAccessOutput, error) { slog.InfoContext(ctx, "EnableSerialConsoleAccess called", "accountID", accountID) - settings, err := s.getSettings(accountID) + settings, err := s.getSettings(ctx, accountID) if err != nil { return nil, err } settings.SerialConsoleAccess = true - if err := s.saveSettings(settings, accountID); err != nil { + if err := s.saveSettings(ctx, settings, accountID); err != nil { return nil, err } @@ -194,14 +196,14 @@ func (s *AccountSettingsServiceImpl) EnableSerialConsoleAccess(ctx context.Conte func (s *AccountSettingsServiceImpl) DisableSerialConsoleAccess(ctx context.Context, input *ec2.DisableSerialConsoleAccessInput, accountID string) (*ec2.DisableSerialConsoleAccessOutput, error) { slog.InfoContext(ctx, "DisableSerialConsoleAccess called", "accountID", accountID) - settings, err := s.getSettings(accountID) + settings, err := s.getSettings(ctx, accountID) if err != nil { return nil, err } settings.SerialConsoleAccess = false - if err := s.saveSettings(settings, accountID); err != nil { + if err := s.saveSettings(ctx, settings, accountID); err != nil { return nil, err } diff --git a/spinifex/handlers/ec2/account/service_impl_test.go b/spinifex/handlers/ec2/account/service_impl_test.go index 88f09667c..aa85911ec 100644 --- a/spinifex/handlers/ec2/account/service_impl_test.go +++ b/spinifex/handlers/ec2/account/service_impl_test.go @@ -17,7 +17,7 @@ import ( func TestNewAccountSettingsServiceImplWithNATS_NoJetStream(t *testing.T) { _, nc := testutil.StartTestNATS(t) - svc, err := NewAccountSettingsServiceImplWithNATS(nil, nc) + svc, err := NewAccountSettingsServiceImplWithNATS(t.Context(), nil, nc) assert.Error(t, err) assert.Nil(t, svc) } @@ -28,7 +28,7 @@ func setupTestAccountService(t *testing.T) *AccountSettingsServiceImpl { t.Helper() _, nc, _ := testutil.StartTestJetStream(t) - svc, err := NewAccountSettingsServiceImplWithNATS(nil, nc) + svc, err := NewAccountSettingsServiceImplWithNATS(t.Context(), nil, nc) require.NoError(t, err) return svc } diff --git a/spinifex/handlers/ec2/eigw/service_impl.go b/spinifex/handlers/ec2/eigw/service_impl.go index 93309e685..5ea73e37d 100644 --- a/spinifex/handlers/ec2/eigw/service_impl.go +++ b/spinifex/handlers/ec2/eigw/service_impl.go @@ -15,9 +15,11 @@ import ( "github.com/mulgadc/spinifex/spinifex/config" "github.com/mulgadc/spinifex/spinifex/filterutil" handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc" + "github.com/mulgadc/spinifex/spinifex/kvutil" "github.com/mulgadc/spinifex/spinifex/migrate" "github.com/mulgadc/spinifex/spinifex/utils" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // Ensure EgressOnlyIGWServiceImpl implements EgressOnlyIGWService. @@ -40,27 +42,27 @@ type EgressOnlyIGWRecord struct { // EgressOnlyIGWServiceImpl implements Egress-only Internet Gateway operations with NATS JetStream persistence. type EgressOnlyIGWServiceImpl struct { config *config.Config - eigwKV nats.KeyValue - vpcKV nats.KeyValue + eigwKV jetstream.KeyValue + vpcKV jetstream.KeyValue } // NewEgressOnlyIGWServiceImplWithNATS creates an Egress-only Internet Gateway service with NATS JetStream for persistence. -func NewEgressOnlyIGWServiceImplWithNATS(cfg *config.Config, natsConn *nats.Conn) (*EgressOnlyIGWServiceImpl, error) { - js, err := natsConn.JetStream() +func NewEgressOnlyIGWServiceImplWithNATS(ctx context.Context, cfg *config.Config, natsConn *nats.Conn) (*EgressOnlyIGWServiceImpl, error) { + js, err := jetstream.New(natsConn) if err != nil { return nil, fmt.Errorf("failed to get JetStream context: %w", err) } - eigwKV, err := utils.GetOrCreateKVBucket(js, KVBucketEgressOnlyIGW, 10) + eigwKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketEgressOnlyIGW, 10) if err != nil { return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketEgressOnlyIGW, err) } - if err := migrate.DefaultRegistry.RunKV(KVBucketEgressOnlyIGW, eigwKV, KVBucketEgressOnlyIGWVersion); err != nil { + if err := migrate.DefaultRegistry.RunKV(ctx, KVBucketEgressOnlyIGW, eigwKV, KVBucketEgressOnlyIGWVersion); err != nil { return nil, fmt.Errorf("migrate %s: %w", KVBucketEgressOnlyIGW, err) } // Get or create VPC KV bucket for cross-resource ownership validation - vpcKV, err := utils.GetOrCreateKVBucket(js, handlers_ec2_vpc.KVBucketVPCs, 10) + vpcKV, err := kvutil.GetOrCreateBucket(ctx, js, handlers_ec2_vpc.KVBucketVPCs, 10) if err != nil { return nil, fmt.Errorf("failed to get VPC KV bucket: %w", err) } @@ -85,7 +87,7 @@ func (s *EgressOnlyIGWServiceImpl) CreateEgressOnlyInternetGateway(ctx context.C slog.ErrorContext(ctx, "VPC KV unavailable, cannot verify VPC ownership") return nil, errors.New(awserrors.ErrorServerInternal) } - if _, err := s.vpcKV.Get(utils.AccountKey(accountID, *input.VpcId)); err != nil { + if _, err := s.vpcKV.Get(ctx, utils.AccountKey(accountID, *input.VpcId)); err != nil { slog.WarnContext(ctx, "CreateEgressOnlyInternetGateway: VPC not found for account", "vpcId", *input.VpcId, "accountID", accountID) return nil, errors.New(awserrors.ErrorInvalidVpcIDNotFound) } @@ -104,7 +106,7 @@ func (s *EgressOnlyIGWServiceImpl) CreateEgressOnlyInternetGateway(ctx context.C if err != nil { return nil, fmt.Errorf("failed to marshal Egress-only IGW record: %w", err) } - if _, err := s.eigwKV.Put(utils.AccountKey(accountID, eigwID), data); err != nil { + if _, err := s.eigwKV.Put(ctx, utils.AccountKey(accountID, eigwID), data); err != nil { return nil, errors.New(awserrors.ErrorServerInternal) } @@ -125,13 +127,13 @@ func (s *EgressOnlyIGWServiceImpl) DeleteEgressOnlyInternetGateway(ctx context.C key := utils.AccountKey(accountID, eigwID) // Verify the EIGW exists before deleting - if _, err := s.eigwKV.Get(key); err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if _, err := s.eigwKV.Get(ctx, key); err != nil { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil, errors.New(awserrors.ErrorInvalidEgressOnlyInternetGatewayIdNotFound) } return nil, errors.New(awserrors.ErrorServerInternal) } - if err := s.eigwKV.Delete(key); err != nil { + if err := s.eigwKV.Delete(ctx, key); err != nil { return nil, errors.New(awserrors.ErrorServerInternal) } @@ -165,8 +167,8 @@ func (s *EgressOnlyIGWServiceImpl) DescribeEgressOnlyInternetGateways(ctx contex } prefix := accountID + "." - keys, err := s.eigwKV.Keys() - if err != nil && !errors.Is(err, nats.ErrNoKeysFound) { + keys, err := s.eigwKV.Keys(ctx) + if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) { return nil, errors.New(awserrors.ErrorServerInternal) } @@ -178,7 +180,7 @@ func (s *EgressOnlyIGWServiceImpl) DescribeEgressOnlyInternetGateways(ctx contex continue } - entry, err := s.eigwKV.Get(key) + entry, err := s.eigwKV.Get(ctx, key) if err != nil { slog.WarnContext(ctx, "Failed to get Egress-only IGW record", "key", key, "error", err) continue @@ -254,7 +256,7 @@ func (s *EgressOnlyIGWServiceImpl) ApplyRecordTags(input *ec2.CreateTagsInput, a if input == nil { return nil } - return utils.MirrorKVRecordTags(s.eigwKV, accountID, "eigw-", input.Resources, + return utils.MirrorKVRecordTags(context.Background(), s.eigwKV, accountID, "eigw-", input.Resources, func(r *EgressOnlyIGWRecord) *map[string]string { return &r.Tags }, utils.MergeTagsMut(input)) } @@ -265,7 +267,7 @@ func (s *EgressOnlyIGWServiceImpl) RemoveRecordTags(input *ec2.DeleteTagsInput, if input == nil { return nil } - return utils.MirrorKVRecordTags(s.eigwKV, accountID, "eigw-", input.Resources, + return utils.MirrorKVRecordTags(context.Background(), s.eigwKV, accountID, "eigw-", input.Resources, func(r *EgressOnlyIGWRecord) *map[string]string { return &r.Tags }, utils.RemoveTagsMut(input)) } diff --git a/spinifex/handlers/ec2/eigw/service_impl_test.go b/spinifex/handlers/ec2/eigw/service_impl_test.go index 3a747fb6b..def1cac6b 100644 --- a/spinifex/handlers/ec2/eigw/service_impl_test.go +++ b/spinifex/handlers/ec2/eigw/service_impl_test.go @@ -26,7 +26,7 @@ func setupTestEIGWService(t *testing.T) *EgressOnlyIGWServiceImpl { } testutil.SeedKV(t, js, handlers_ec2_vpc.KVBucketVPCs, vpcEntries) - svc, err := NewEgressOnlyIGWServiceImplWithNATS(nil, nc) + svc, err := NewEgressOnlyIGWServiceImplWithNATS(t.Context(), nil, nc) require.NoError(t, err) return svc } @@ -285,7 +285,7 @@ func TestCreateEgressOnlyInternetGateway_CrossAccountVPCRejected(t *testing.T) { require.NoError(t, err) // Create EIGW service (VPC KV will be populated since we created it before EIGW init) - svc, err := NewEgressOnlyIGWServiceImplWithNATS(nil, nc) + svc, err := NewEgressOnlyIGWServiceImplWithNATS(t.Context(), nil, nc) require.NoError(t, err) // Other account tries to create EIGW in testAccountID's VPC — should fail diff --git a/spinifex/handlers/ec2/eip/service_impl.go b/spinifex/handlers/ec2/eip/service_impl.go index 386752a79..c9126ad0e 100644 --- a/spinifex/handlers/ec2/eip/service_impl.go +++ b/spinifex/handlers/ec2/eip/service_impl.go @@ -14,10 +14,12 @@ import ( "github.com/mulgadc/spinifex/spinifex/awserrors" "github.com/mulgadc/spinifex/spinifex/filterutil" handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc" + "github.com/mulgadc/spinifex/spinifex/kvutil" "github.com/mulgadc/spinifex/spinifex/migrate" "github.com/mulgadc/spinifex/spinifex/network/topology" "github.com/mulgadc/spinifex/spinifex/utils" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // Ensure EIPServiceImpl implements EIPService. @@ -25,7 +27,7 @@ var _ EIPService = (*EIPServiceImpl)(nil) // EIPServiceImpl implements Elastic IP operations with NATS JetStream persistence. type EIPServiceImpl struct { - eipKV nats.KeyValue + eipKV jetstream.KeyValue externalIPAM *handlers_ec2_vpc.ExternalIPAM vpcService handlers_ec2_vpc.VPCService natsConn *nats.Conn @@ -41,17 +43,17 @@ type natEvent struct { } // NewEIPServiceImpl creates a new EIP service backed by NATS JetStream KV. -func NewEIPServiceImpl(natsConn *nats.Conn, externalIPAM *handlers_ec2_vpc.ExternalIPAM, vpcService handlers_ec2_vpc.VPCService) (*EIPServiceImpl, error) { - js, err := natsConn.JetStream() +func NewEIPServiceImpl(ctx context.Context, natsConn *nats.Conn, externalIPAM *handlers_ec2_vpc.ExternalIPAM, vpcService handlers_ec2_vpc.VPCService) (*EIPServiceImpl, error) { + js, err := jetstream.New(natsConn) if err != nil { return nil, fmt.Errorf("failed to get JetStream context: %w", err) } - eipKV, err := utils.GetOrCreateKVBucket(js, KVBucketEIPs, 10) + eipKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketEIPs, 10) if err != nil { return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketEIPs, err) } - if err := migrate.DefaultRegistry.RunKV(KVBucketEIPs, eipKV, KVBucketEIPsVersion); err != nil { + if err := migrate.DefaultRegistry.RunKV(ctx, KVBucketEIPs, eipKV, KVBucketEIPsVersion); err != nil { return nil, fmt.Errorf("migrate %s: %w", KVBucketEIPs, err) } @@ -104,7 +106,7 @@ func (s *EIPServiceImpl) AllocateAddress(ctx context.Context, input *ec2.Allocat if err != nil { return nil, fmt.Errorf("failed to marshal EIP record: %w", err) } - if _, err := s.eipKV.Put(utils.AccountKey(accountID, allocID), data); err != nil { + if _, err := s.eipKV.Put(ctx, utils.AccountKey(accountID, allocID), data); err != nil { return nil, errors.New(awserrors.ErrorServerInternal) } @@ -127,7 +129,7 @@ func (s *EIPServiceImpl) ReleaseAddress(ctx context.Context, input *ec2.ReleaseA allocID := *input.AllocationId key := utils.AccountKey(accountID, allocID) - entry, err := s.eipKV.Get(key) + entry, err := s.eipKV.Get(ctx, key) if err != nil { return nil, errors.New(awserrors.ErrorInvalidAllocationIDNotFound) } @@ -148,7 +150,7 @@ func (s *EIPServiceImpl) ReleaseAddress(ctx context.Context, input *ec2.ReleaseA slog.WarnContext(ctx, "Failed to release IP back to IPAM pool", "allocationId", allocID, "ip", record.PublicIp, "pool", record.PoolName, "err", err) } - if err := s.eipKV.Delete(key); err != nil { + if err := s.eipKV.Delete(ctx, key); err != nil { return nil, errors.New(awserrors.ErrorServerInternal) } @@ -166,7 +168,7 @@ func (s *EIPServiceImpl) AssociateAddress(ctx context.Context, input *ec2.Associ allocID := *input.AllocationId key := utils.AccountKey(accountID, allocID) - entry, err := s.eipKV.Get(key) + entry, err := s.eipKV.Get(ctx, key) if err != nil { return nil, errors.New(awserrors.ErrorInvalidAllocationIDNotFound) } @@ -222,7 +224,7 @@ func (s *EIPServiceImpl) AssociateAddress(ctx context.Context, input *ec2.Associ if err != nil { return nil, fmt.Errorf("failed to marshal EIP record: %w", err) } - if _, err := s.eipKV.Update(key, data, entry.Revision()); err != nil { + if _, err := s.eipKV.Update(ctx, key, data, entry.Revision()); err != nil { return nil, errors.New(awserrors.ErrorServerInternal) } @@ -252,7 +254,7 @@ func (s *EIPServiceImpl) DisassociateAddress(ctx context.Context, input *ec2.Dis associationID := *input.AssociationId // Find the EIP record by association ID. - record, key, revision, err := s.findByAssociationID(accountID, associationID) + record, key, revision, err := s.findByAssociationID(ctx, accountID, associationID) if err != nil { return nil, err } @@ -280,7 +282,7 @@ func (s *EIPServiceImpl) DisassociateAddress(ctx context.Context, input *ec2.Dis if err != nil { return nil, fmt.Errorf("failed to marshal EIP record: %w", err) } - if _, err := s.eipKV.Update(key, data, revision); err != nil { + if _, err := s.eipKV.Update(ctx, key, data, revision); err != nil { return nil, errors.New(awserrors.ErrorServerInternal) } @@ -321,8 +323,8 @@ func (s *EIPServiceImpl) DescribeAddresses(ctx context.Context, input *ec2.Descr } prefix := accountID + "." - keys, err := s.eipKV.Keys() - if err != nil && !errors.Is(err, nats.ErrNoKeysFound) { + keys, err := s.eipKV.Keys(ctx) + if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) { return nil, errors.New(awserrors.ErrorServerInternal) } @@ -335,7 +337,7 @@ func (s *EIPServiceImpl) DescribeAddresses(ctx context.Context, input *ec2.Descr continue } - entry, err := s.eipKV.Get(k) + entry, err := s.eipKV.Get(ctx, k) if err != nil { slog.WarnContext(ctx, "Failed to get EIP record", "key", k, "error", err) continue @@ -426,7 +428,7 @@ func (s *EIPServiceImpl) DescribeAddressesAttribute(ctx context.Context, input * continue } key := utils.AccountKey(accountID, *id) - entry, err := s.eipKV.Get(key) + entry, err := s.eipKV.Get(ctx, key) if err != nil { continue // not found, skip } @@ -443,8 +445,8 @@ func (s *EIPServiceImpl) DescribeAddressesAttribute(ctx context.Context, input * } else { // No filter — scan all EIPs for this account. prefix := accountID + "." - keys, err := s.eipKV.Keys() - if err != nil && !errors.Is(err, nats.ErrNoKeysFound) { + keys, err := s.eipKV.Keys(ctx) + if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) { return nil, errors.New(awserrors.ErrorServerInternal) } for _, k := range keys { @@ -454,7 +456,7 @@ func (s *EIPServiceImpl) DescribeAddressesAttribute(ctx context.Context, input * if !strings.HasPrefix(k, prefix) { continue } - entry, err := s.eipKV.Get(k) + entry, err := s.eipKV.Get(ctx, k) if err != nil { slog.WarnContext(ctx, "Failed to get EIP record", "key", k, "error", err) continue @@ -537,9 +539,9 @@ func (s *EIPServiceImpl) lookupENIByInstance(ctx context.Context, accountID, ins } // findByAssociationID scans EIP records to find one matching the given association ID. -func (s *EIPServiceImpl) findByAssociationID(accountID, associationID string) (*EIPRecord, string, uint64, error) { +func (s *EIPServiceImpl) findByAssociationID(ctx context.Context, accountID, associationID string) (*EIPRecord, string, uint64, error) { prefix := accountID + "." - keys, err := s.eipKV.Keys() + keys, err := s.eipKV.Keys(ctx) if err != nil { return nil, "", 0, errors.New(awserrors.ErrorServerInternal) } @@ -552,7 +554,7 @@ func (s *EIPServiceImpl) findByAssociationID(accountID, associationID string) (* continue } - entry, err := s.eipKV.Get(k) + entry, err := s.eipKV.Get(ctx, k) if err != nil { continue } @@ -578,7 +580,7 @@ func (s *EIPServiceImpl) AssociatedPublicIPForInstance(ctx context.Context, acco return "", false } prefix := accountID + "." - keys, err := s.eipKV.Keys() + keys, err := s.eipKV.Keys(ctx) if err != nil { return "", false } @@ -586,7 +588,7 @@ func (s *EIPServiceImpl) AssociatedPublicIPForInstance(ctx context.Context, acco if k == utils.VersionKey || !strings.HasPrefix(k, prefix) { continue } - entry, err := s.eipKV.Get(k) + entry, err := s.eipKV.Get(ctx, k) if err != nil { continue } @@ -607,12 +609,13 @@ func (s *EIPServiceImpl) AssociatedPublicIPForInstance(ctx context.Context, acco // gone from the manager the per-instance release never runs, so an // internet-facing ALB's EIP would otherwise orphan. No-op when nothing matches. func (s *EIPServiceImpl) ReleaseAddressByInstanceID(instanceID string) error { + ctx := context.Background() if instanceID == "" { return nil } - keys, err := s.eipKV.Keys() + keys, err := s.eipKV.Keys(ctx) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return nil } return fmt.Errorf("list eip keys: %w", err) @@ -622,7 +625,7 @@ func (s *EIPServiceImpl) ReleaseAddressByInstanceID(instanceID string) error { if k == utils.VersionKey { continue } - entry, err := s.eipKV.Get(k) + entry, err := s.eipKV.Get(ctx, k) if err != nil { continue } @@ -639,13 +642,13 @@ func (s *EIPServiceImpl) ReleaseAddressByInstanceID(instanceID string) error { continue } if record.AssociationId != "" { - if _, err := s.DisassociateAddress(context.Background(), &ec2.DisassociateAddressInput{ + if _, err := s.DisassociateAddress(ctx, &ec2.DisassociateAddressInput{ AssociationId: aws.String(record.AssociationId), }, accountID); err != nil { errs = append(errs, fmt.Errorf("disassociate %s: %w", record.AllocationId, err)) } } - if _, err := s.ReleaseAddress(context.Background(), &ec2.ReleaseAddressInput{ + if _, err := s.ReleaseAddress(ctx, &ec2.ReleaseAddressInput{ AllocationId: aws.String(record.AllocationId), }, accountID); err != nil { errs = append(errs, fmt.Errorf("release %s: %w", record.AllocationId, err)) @@ -701,7 +704,7 @@ func (s *EIPServiceImpl) ApplyRecordTags(input *ec2.CreateTagsInput, accountID s if input == nil { return nil } - return utils.MirrorKVRecordTags(s.eipKV, accountID, "eipalloc-", input.Resources, + return utils.MirrorKVRecordTags(context.Background(), s.eipKV, accountID, "eipalloc-", input.Resources, func(r *EIPRecord) *map[string]string { return &r.Tags }, utils.MergeTagsMut(input)) } @@ -712,7 +715,7 @@ func (s *EIPServiceImpl) RemoveRecordTags(input *ec2.DeleteTagsInput, accountID if input == nil { return nil } - return utils.MirrorKVRecordTags(s.eipKV, accountID, "eipalloc-", input.Resources, + return utils.MirrorKVRecordTags(context.Background(), s.eipKV, accountID, "eipalloc-", input.Resources, func(r *EIPRecord) *map[string]string { return &r.Tags }, utils.RemoveTagsMut(input)) } diff --git a/spinifex/handlers/ec2/eip/service_impl_test.go b/spinifex/handlers/ec2/eip/service_impl_test.go index f95575682..7f6960ac1 100644 --- a/spinifex/handlers/ec2/eip/service_impl_test.go +++ b/spinifex/handlers/ec2/eip/service_impl_test.go @@ -29,13 +29,14 @@ func testPool() external.ExternalPoolConfig { func setupTestEIP(t *testing.T) (*EIPServiceImpl, *handlers_ec2_vpc.ExternalIPAM) { t.Helper() - _, nc, js := testutil.StartTestJetStream(t) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) pool := testPool() - ipam, err := handlers_ec2_vpc.NewExternalIPAM(js, []external.ExternalPoolConfig{pool}) + ipam, err := handlers_ec2_vpc.NewExternalIPAM(t.Context(), js, []external.ExternalPoolConfig{pool}) require.NoError(t, err) - svc, err := NewEIPServiceImpl(nc, ipam, nil) + svc, err := NewEIPServiceImpl(t.Context(), nc, ipam, nil) require.NoError(t, err) return svc, ipam @@ -118,7 +119,7 @@ func TestEIP_ReleaseWhileAssociated(t *testing.T) { // To test the associated path, we need to manipulate the KV directly. // Get the KV entry and update state to "associated" - entry, err := svc.eipKV.Get(testAccountID + "." + allocID) + entry, err := svc.eipKV.Get(t.Context(), testAccountID+"."+allocID) require.NoError(t, err) var record EIPRecord @@ -130,7 +131,7 @@ func TestEIP_ReleaseWhileAssociated(t *testing.T) { data, err := json.Marshal(record) require.NoError(t, err) - _, err = svc.eipKV.Update(testAccountID+"."+allocID, data, entry.Revision()) + _, err = svc.eipKV.Update(t.Context(), testAccountID+"."+allocID, data, entry.Revision()) require.NoError(t, err) // Now try to release — should fail @@ -150,7 +151,7 @@ func TestEIP_ReleaseByInstanceID_ReclaimsAssociatedEIP(t *testing.T) { // Mark the EIP associated with a system instance, as an internet-facing ALB // leaves it. ENIId stays empty so disassociate skips the VPC-service lookup. - entry, err := svc.eipKV.Get(testAccountID + "." + allocID) + entry, err := svc.eipKV.Get(t.Context(), testAccountID+"."+allocID) require.NoError(t, err) var record EIPRecord require.NoError(t, json.Unmarshal(entry.Value(), &record)) @@ -159,7 +160,7 @@ func TestEIP_ReleaseByInstanceID_ReclaimsAssociatedEIP(t *testing.T) { record.InstanceId = "i-alb-test" data, err := json.Marshal(record) require.NoError(t, err) - _, err = svc.eipKV.Update(testAccountID+"."+allocID, data, entry.Revision()) + _, err = svc.eipKV.Update(t.Context(), testAccountID+"."+allocID, data, entry.Revision()) require.NoError(t, err) // Backstop release by instance disassociates then frees the allocation. @@ -180,7 +181,7 @@ func TestEIP_ReleaseByInstanceID_NoMatchIsNoOp(t *testing.T) { // An instance with no recorded EIP must leave existing allocations untouched. require.NoError(t, svc.ReleaseAddressByInstanceID("i-unrelated")) - _, err = svc.eipKV.Get(testAccountID + "." + allocID) + _, err = svc.eipKV.Get(t.Context(), testAccountID+"."+allocID) assert.NoError(t, err) // Empty instance ID is a no-op too. @@ -316,7 +317,7 @@ func TestEIP_RecordToEC2_WithoutTags(t *testing.T) { func TestEIP_FindByAssociationID_NotFound(t *testing.T) { svc, _ := setupTestEIP(t) - _, _, _, err := svc.findByAssociationID(testAccountID, "eipassoc-nonexistent") + _, _, _, err := svc.findByAssociationID(t.Context(), testAccountID, "eipassoc-nonexistent") assert.Error(t, err) assert.Contains(t, err.Error(), "InvalidAssociationID.NotFound") } @@ -463,7 +464,7 @@ func TestEIP_DescribeAddresses_FilterByInstanceId(t *testing.T) { require.NoError(t, err) // Manually set instance-id on the record - entry, err := svc.eipKV.Get(testAccountID + "." + *out.AllocationId) + entry, err := svc.eipKV.Get(t.Context(), testAccountID+"."+*out.AllocationId) require.NoError(t, err) var record EIPRecord require.NoError(t, json.Unmarshal(entry.Value(), &record)) @@ -471,7 +472,7 @@ func TestEIP_DescribeAddresses_FilterByInstanceId(t *testing.T) { record.State = "associated" data, err := json.Marshal(record) require.NoError(t, err) - _, err = svc.eipKV.Update(testAccountID+"."+*out.AllocationId, data, entry.Revision()) + _, err = svc.eipKV.Update(t.Context(), testAccountID+"."+*out.AllocationId, data, entry.Revision()) require.NoError(t, err) // Allocate another without association diff --git a/spinifex/handlers/ec2/igw/service_impl.go b/spinifex/handlers/ec2/igw/service_impl.go index b30295e95..cdaf055a5 100644 --- a/spinifex/handlers/ec2/igw/service_impl.go +++ b/spinifex/handlers/ec2/igw/service_impl.go @@ -15,10 +15,12 @@ import ( "github.com/mulgadc/spinifex/spinifex/config" "github.com/mulgadc/spinifex/spinifex/filterutil" handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc" + "github.com/mulgadc/spinifex/spinifex/kvutil" "github.com/mulgadc/spinifex/spinifex/migrate" "github.com/mulgadc/spinifex/spinifex/types" "github.com/mulgadc/spinifex/spinifex/utils" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // Ensure IGWServiceImpl implements IGWService. @@ -48,8 +50,8 @@ type GatePublisher interface { // IGWServiceImpl implements Internet Gateway operations with NATS JetStream persistence. type IGWServiceImpl struct { config *config.Config - igwKV nats.KeyValue - vpcKV nats.KeyValue + igwKV jetstream.KeyValue + vpcKV jetstream.KeyValue natsConn *nats.Conn gatePublisher GatePublisher } @@ -61,22 +63,22 @@ func (s *IGWServiceImpl) SetGatePublisher(p GatePublisher) { } // NewIGWServiceImplWithNATS creates an Internet Gateway service with NATS JetStream for persistence. -func NewIGWServiceImplWithNATS(cfg *config.Config, natsConn *nats.Conn) (*IGWServiceImpl, error) { - js, err := natsConn.JetStream() +func NewIGWServiceImplWithNATS(ctx context.Context, cfg *config.Config, natsConn *nats.Conn) (*IGWServiceImpl, error) { + js, err := jetstream.New(natsConn) if err != nil { return nil, fmt.Errorf("failed to get JetStream context: %w", err) } - igwKV, err := utils.GetOrCreateKVBucket(js, KVBucketIGW, 10) + igwKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketIGW, 10) if err != nil { return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketIGW, err) } - if err := migrate.DefaultRegistry.RunKV(KVBucketIGW, igwKV, KVBucketIGWVersion); err != nil { + if err := migrate.DefaultRegistry.RunKV(ctx, KVBucketIGW, igwKV, KVBucketIGWVersion); err != nil { return nil, fmt.Errorf("migrate %s: %w", KVBucketIGW, err) } // Get or create VPC KV bucket for cross-resource ownership validation - vpcKV, err := utils.GetOrCreateKVBucket(js, handlers_ec2_vpc.KVBucketVPCs, 10) + vpcKV, err := kvutil.GetOrCreateBucket(ctx, js, handlers_ec2_vpc.KVBucketVPCs, 10) if err != nil { return nil, fmt.Errorf("failed to get VPC KV bucket: %w", err) } @@ -115,7 +117,7 @@ func (s *IGWServiceImpl) createIGW(ctx context.Context, input *ec2.CreateInterne if err != nil { return nil, fmt.Errorf("failed to marshal IGW record: %w", err) } - if _, err := s.igwKV.Put(utils.AccountKey(accountID, igwID), data); err != nil { + if _, err := s.igwKV.Put(ctx, utils.AccountKey(accountID, igwID), data); err != nil { return nil, errors.New(awserrors.ErrorServerInternal) } @@ -135,12 +137,12 @@ func (s *IGWServiceImpl) DeleteInternetGateway(ctx context.Context, input *ec2.D igwID := *input.InternetGatewayId key := utils.AccountKey(accountID, igwID) - entry, err := s.igwKV.Get(key) + entry, err := s.igwKV.Get(ctx, key) if err != nil { // AWS-faithful: an absent internet gateway is NotFound (provider // tolerates it on destroy); destroy orchestration tolerates it too. // A transient read error stays a server error. - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil, errors.New(awserrors.ErrorInvalidInternetGatewayIDNotFound) } return nil, errors.New(awserrors.ErrorServerInternal) @@ -156,7 +158,7 @@ func (s *IGWServiceImpl) DeleteInternetGateway(ctx context.Context, input *ec2.D return nil, errors.New(awserrors.ErrorDependencyViolation) } - if err := s.igwKV.Delete(key); err != nil { + if err := s.igwKV.Delete(ctx, key); err != nil { return nil, errors.New(awserrors.ErrorServerInternal) } @@ -190,8 +192,8 @@ func (s *IGWServiceImpl) DescribeInternetGateways(ctx context.Context, input *ec } prefix := accountID + "." - keys, err := s.igwKV.Keys() - if err != nil && !errors.Is(err, nats.ErrNoKeysFound) { + keys, err := s.igwKV.Keys(ctx) + if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) { return nil, errors.New(awserrors.ErrorServerInternal) } @@ -205,7 +207,7 @@ func (s *IGWServiceImpl) DescribeInternetGateways(ctx context.Context, input *ec continue } - entry, err := s.igwKV.Get(key) + entry, err := s.igwKV.Get(ctx, key) if err != nil { slog.WarnContext(ctx, "Failed to get IGW record", "key", key, "error", err) continue @@ -287,7 +289,7 @@ func (s *IGWServiceImpl) AttachInternetGateway(ctx context.Context, input *ec2.A vpcID := *input.VpcId key := utils.AccountKey(accountID, igwID) - entry, err := s.igwKV.Get(key) + entry, err := s.igwKV.Get(ctx, key) if err != nil { return nil, errors.New(awserrors.ErrorInvalidInternetGatewayIDNotFound) } @@ -306,7 +308,7 @@ func (s *IGWServiceImpl) AttachInternetGateway(ctx context.Context, input *ec2.A slog.ErrorContext(ctx, "VPC KV unavailable, cannot verify VPC ownership") return nil, errors.New(awserrors.ErrorServerInternal) } - if _, err := s.vpcKV.Get(utils.AccountKey(accountID, vpcID)); err != nil { + if _, err := s.vpcKV.Get(ctx, utils.AccountKey(accountID, vpcID)); err != nil { slog.WarnContext(ctx, "AttachInternetGateway: VPC not found for account", "vpcId", vpcID, "accountID", accountID) return nil, errors.New(awserrors.ErrorInvalidVpcIDNotFound) } @@ -318,7 +320,7 @@ func (s *IGWServiceImpl) AttachInternetGateway(ctx context.Context, input *ec2.A if err != nil { return nil, errors.New(awserrors.ErrorServerInternal) } - if _, err := s.igwKV.Put(key, data); err != nil { + if _, err := s.igwKV.Put(ctx, key, data); err != nil { return nil, errors.New(awserrors.ErrorServerInternal) } @@ -358,7 +360,7 @@ func (s *IGWServiceImpl) DetachInternetGateway(ctx context.Context, input *ec2.D vpcID := *input.VpcId key := utils.AccountKey(accountID, igwID) - entry, err := s.igwKV.Get(key) + entry, err := s.igwKV.Get(ctx, key) if err != nil { return nil, errors.New(awserrors.ErrorInvalidInternetGatewayIDNotFound) } @@ -379,7 +381,7 @@ func (s *IGWServiceImpl) DetachInternetGateway(ctx context.Context, input *ec2.D if err != nil { return nil, errors.New(awserrors.ErrorServerInternal) } - if _, err := s.igwKV.Put(key, data); err != nil { + if _, err := s.igwKV.Put(ctx, key, data); err != nil { return nil, errors.New(awserrors.ErrorServerInternal) } @@ -436,7 +438,7 @@ func (s *IGWServiceImpl) ApplyRecordTags(input *ec2.CreateTagsInput, accountID s if input == nil { return nil } - return utils.MirrorKVRecordTags(s.igwKV, accountID, "igw-", input.Resources, + return utils.MirrorKVRecordTags(context.Background(), s.igwKV, accountID, "igw-", input.Resources, func(r *IGWRecord) *map[string]string { return &r.Tags }, utils.MergeTagsMut(input)) } @@ -447,7 +449,7 @@ func (s *IGWServiceImpl) RemoveRecordTags(input *ec2.DeleteTagsInput, accountID if input == nil { return nil } - return utils.MirrorKVRecordTags(s.igwKV, accountID, "igw-", input.Resources, + return utils.MirrorKVRecordTags(context.Background(), s.igwKV, accountID, "igw-", input.Resources, func(r *IGWRecord) *map[string]string { return &r.Tags }, utils.RemoveTagsMut(input)) } diff --git a/spinifex/handlers/ec2/igw/service_impl_test.go b/spinifex/handlers/ec2/igw/service_impl_test.go index 5980a4257..7f83f9881 100644 --- a/spinifex/handlers/ec2/igw/service_impl_test.go +++ b/spinifex/handlers/ec2/igw/service_impl_test.go @@ -56,7 +56,7 @@ func setupTestIGWService(t *testing.T) (*IGWServiceImpl, *nats.Conn) { } testutil.SeedKV(t, js, handlers_ec2_vpc.KVBucketVPCs, vpcEntries) - svc, err := NewIGWServiceImplWithNATS(nil, nc) + svc, err := NewIGWServiceImplWithNATS(t.Context(), nil, nc) require.NoError(t, err) return svc, nc } @@ -463,14 +463,13 @@ func TestCreateInternetGateway_PublishesNoEvent(t *testing.T) { func TestAttachInternetGateway_CrossAccountVPCRejected(t *testing.T) { svc, nc := setupTestIGWService(t) - // Create VPC KV bucket and add a VPC owned by testAccountID - js, err := nc.JetStream() - require.NoError(t, err) - vpcKV, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: handlers_ec2_vpc.KVBucketVPCs, History: 1}) + // Add a VPC owned by testAccountID to the existing VPC bucket. + js := testutil.NewJetStream(t, nc) + vpcKV, err := js.KeyValue(t.Context(), handlers_ec2_vpc.KVBucketVPCs) require.NoError(t, err) vpcID := "vpc-alpha123" - _, err = vpcKV.Put(utils.AccountKey(testAccountID, vpcID), []byte(`{"vpc_id":"vpc-alpha123","state":"available"}`)) + _, err = vpcKV.Put(t.Context(), utils.AccountKey(testAccountID, vpcID), []byte(`{"vpc_id":"vpc-alpha123","state":"available"}`)) require.NoError(t, err) // Refresh service to pick up the VPC KV bucket diff --git a/spinifex/handlers/ec2/instance/service.go b/spinifex/handlers/ec2/instance/service.go index 679b06162..9f38ce173 100644 --- a/spinifex/handlers/ec2/instance/service.go +++ b/spinifex/handlers/ec2/instance/service.go @@ -88,7 +88,7 @@ type StoppedInstanceStore interface { // UpdateStoppedInstance atomically applies mutate to the current stopped // record under optimistic concurrency (CAS) with createIfAbsent=false, so // a caller racing a winning ClaimStoppedInstance gets a clean - // nats.ErrKeyNotFound instead of resurrecting a deleted record. + // jetstream.ErrKeyNotFound instead of resurrecting a deleted record. UpdateStoppedInstance(instanceID string, mutate func(*vm.VM)) (*vm.VM, error) WriteTerminatedInstance(instanceID string, instance *vm.VM) error // ClaimStoppedInstance atomically removes instanceID's record and diff --git a/spinifex/handlers/ec2/instance/service_impl.go b/spinifex/handlers/ec2/instance/service_impl.go index 842347c64..eee12edea 100644 --- a/spinifex/handlers/ec2/instance/service_impl.go +++ b/spinifex/handlers/ec2/instance/service_impl.go @@ -32,6 +32,7 @@ import ( "github.com/mulgadc/viperblock/viperblock" "github.com/mulgadc/viperblock/viperblock/backends/s3" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // VolumeInfo holds volume information returned from GenerateVolumes @@ -374,14 +375,14 @@ func (s *InstanceServiceImpl) TagStoppedInstance(ctx context.Context, instanceID // CAS the tag mutation into the shared KV record instead of a wholesale // WriteStoppedInstance: createIfAbsent=false means a claim that deletes // the record between the Load above and this write fails cleanly with - // nats.ErrKeyNotFound rather than resurrecting a stale stopped entry. + // jetstream.ErrKeyNotFound rather than resurrecting a stale stopped entry. newTags := instance.Instance.Tags if _, err := s.stoppedStore.UpdateStoppedInstance(instanceID, func(v *vm.VM) { if v.Instance != nil { v.Instance.Tags = newTags } }); err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { slog.WarnContext(ctx, "TagStoppedInstance: instance claimed concurrently, not resurrecting", "instanceId", instanceID) return errors.New(awserrors.ErrorInvalidInstanceIDNotFound) } @@ -2056,7 +2057,7 @@ func (s *InstanceServiceImpl) ModifyInstanceAttribute(ctx context.Context, input // CAS the mutations into the shared KV record instead of a wholesale // WriteStoppedInstance: createIfAbsent=false means a claim that deletes // the record between the Load above and this write fails cleanly with - // nats.ErrKeyNotFound rather than resurrecting a stale stopped entry. + // jetstream.ErrKeyNotFound rather than resurrecting a stale stopped entry. _, err = s.stoppedStore.UpdateStoppedInstance(instanceID, func(v *vm.VM) { if input.InstanceType != nil && input.InstanceType.Value != nil && v.Instance != nil { newType := *input.InstanceType.Value @@ -2085,7 +2086,7 @@ func (s *InstanceServiceImpl) ModifyInstanceAttribute(ctx context.Context, input } }) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { // The record existed moments ago (Load above) but a concurrent // claim removed it: the instance is mid-transition, not gone. slog.WarnContext(ctx, "ModifyInstanceAttribute: instance claimed concurrently, not resurrecting", "instanceId", instanceID) diff --git a/spinifex/handlers/ec2/instance/service_impl_test.go b/spinifex/handlers/ec2/instance/service_impl_test.go index 6916ba3b1..7d36a9b6a 100644 --- a/spinifex/handlers/ec2/instance/service_impl_test.go +++ b/spinifex/handlers/ec2/instance/service_impl_test.go @@ -22,6 +22,7 @@ import ( "github.com/mulgadc/viperblock/viperblock" "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -714,7 +715,7 @@ func (f *fakeStoppedStore) DeleteTerminatedInstance(string) error { return nil } // UpdateStoppedInstance mimics the real CAS semantics: mutate runs under the // store lock against loadByID's entry, and a missing record (e.g. removed by -// a concurrent ClaimStoppedInstance) returns nats.ErrKeyNotFound instead of +// a concurrent ClaimStoppedInstance) returns jetstream.ErrKeyNotFound instead of // resurrecting it — the same createIfAbsent=false contract as JetStreamManager. func (f *fakeStoppedStore) UpdateStoppedInstance(id string, mutate func(*vm.VM)) (*vm.VM, error) { f.mu.Lock() @@ -725,7 +726,7 @@ func (f *fakeStoppedStore) UpdateStoppedInstance(id string, mutate func(*vm.VM)) } v, ok := f.loadByID[id] if !ok { - return nil, nats.ErrKeyNotFound + return nil, jetstream.ErrKeyNotFound } mutate(v) return v, nil diff --git a/spinifex/handlers/ec2/launchtemplate/service_impl.go b/spinifex/handlers/ec2/launchtemplate/service_impl.go index e6607eb01..a85a09b16 100644 --- a/spinifex/handlers/ec2/launchtemplate/service_impl.go +++ b/spinifex/handlers/ec2/launchtemplate/service_impl.go @@ -17,9 +17,11 @@ import ( "github.com/mulgadc/spinifex/spinifex/awserrors" "github.com/mulgadc/spinifex/spinifex/config" "github.com/mulgadc/spinifex/spinifex/filterutil" + "github.com/mulgadc/spinifex/spinifex/kvutil" "github.com/mulgadc/spinifex/spinifex/migrate" "github.com/mulgadc/spinifex/spinifex/utils" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // Ensure LaunchTemplateServiceImpl implements LaunchTemplateService. @@ -48,21 +50,21 @@ const ( type LaunchTemplateServiceImpl struct { config *config.Config natsConn *nats.Conn - kv nats.KeyValue + kv jetstream.KeyValue } // NewLaunchTemplateServiceImplWithNATS creates a launch template service with NATS JetStream. -func NewLaunchTemplateServiceImplWithNATS(cfg *config.Config, natsConn *nats.Conn) (*LaunchTemplateServiceImpl, error) { - js, err := natsConn.JetStream() +func NewLaunchTemplateServiceImplWithNATS(ctx context.Context, cfg *config.Config, natsConn *nats.Conn) (*LaunchTemplateServiceImpl, error) { + js, err := jetstream.New(natsConn) if err != nil { return nil, fmt.Errorf("failed to get JetStream context: %w", err) } - kv, err := utils.GetOrCreateKVBucket(js, KVBucketLaunchTemplates, 10) + kv, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketLaunchTemplates, 10) if err != nil { return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketLaunchTemplates, err) } - if err := migrate.DefaultRegistry.RunKV(KVBucketLaunchTemplates, kv, KVBucketLaunchTemplatesVersion); err != nil { + if err := migrate.DefaultRegistry.RunKV(ctx, KVBucketLaunchTemplates, kv, KVBucketLaunchTemplatesVersion); err != nil { return nil, fmt.Errorf("migrate %s: %w", KVBucketLaunchTemplates, err) } @@ -102,8 +104,8 @@ func versionPrefix(accountID, ltID string) string { // --- record load helpers --- // getHeaderByID reads a header and its KV entry (for CAS) by launch template id. -func (s *LaunchTemplateServiceImpl) getHeaderByID(accountID, ltID string) (*LaunchTemplateHeader, nats.KeyValueEntry, error) { - entry, err := s.kv.Get(headerKey(accountID, ltID)) +func (s *LaunchTemplateServiceImpl) getHeaderByID(ctx context.Context, accountID, ltID string) (*LaunchTemplateHeader, jetstream.KeyValueEntry, error) { + entry, err := s.kv.Get(ctx, headerKey(accountID, ltID)) if err != nil { return nil, nil, errors.New(awserrors.ErrorInvalidLaunchTemplateIdNotFound) } @@ -116,13 +118,13 @@ func (s *LaunchTemplateServiceImpl) getHeaderByID(accountID, ltID string) (*Laun // getHeaderByName resolves a template name to its header via the name index. // A dangling name index (header missing) is treated as not-found. -func (s *LaunchTemplateServiceImpl) getHeaderByName(accountID, name string) (*LaunchTemplateHeader, nats.KeyValueEntry, error) { - entry, err := s.kv.Get(nameKey(accountID, name)) +func (s *LaunchTemplateServiceImpl) getHeaderByName(ctx context.Context, accountID, name string) (*LaunchTemplateHeader, jetstream.KeyValueEntry, error) { + entry, err := s.kv.Get(ctx, nameKey(accountID, name)) if err != nil { return nil, nil, errors.New(awserrors.ErrorInvalidLaunchTemplateNameNotFoundException) } ltID := string(entry.Value()) - h, hentry, err := s.getHeaderByID(accountID, ltID) + h, hentry, err := s.getHeaderByID(ctx, accountID, ltID) if err != nil { return nil, nil, errors.New(awserrors.ErrorInvalidLaunchTemplateNameNotFoundException) } @@ -131,7 +133,7 @@ func (s *LaunchTemplateServiceImpl) getHeaderByName(accountID, name string) (*La // resolveHeader resolves a template by id or name (mutually exclusive) and // returns the header plus its KV entry for CAS operations. -func (s *LaunchTemplateServiceImpl) resolveHeader(accountID string, ltID, name *string) (*LaunchTemplateHeader, nats.KeyValueEntry, error) { +func (s *LaunchTemplateServiceImpl) resolveHeader(ctx context.Context, accountID string, ltID, name *string) (*LaunchTemplateHeader, jetstream.KeyValueEntry, error) { id := aws.StringValue(ltID) nm := aws.StringValue(name) switch { @@ -141,9 +143,9 @@ func (s *LaunchTemplateServiceImpl) resolveHeader(accountID string, ltID, name * if err := validateTemplateID(id); err != nil { return nil, nil, err } - return s.getHeaderByID(accountID, id) + return s.getHeaderByID(ctx, accountID, id) case nm != "": - return s.getHeaderByName(accountID, nm) + return s.getHeaderByName(ctx, accountID, nm) default: return nil, nil, errors.New(awserrors.ErrorMissingParameter) } @@ -151,8 +153,8 @@ func (s *LaunchTemplateServiceImpl) resolveHeader(accountID string, ltID, name * // getVersion loads an immutable version body. A missing body is the read-side // guard that turns a dangling default or deleted version into VersionNotFound. -func (s *LaunchTemplateServiceImpl) getVersion(accountID, ltID string, n int64) (*LaunchTemplateVersionRec, error) { - entry, err := s.kv.Get(versionKey(accountID, ltID, n)) +func (s *LaunchTemplateServiceImpl) getVersion(ctx context.Context, accountID, ltID string, n int64) (*LaunchTemplateVersionRec, error) { + entry, err := s.kv.Get(ctx, versionKey(accountID, ltID, n)) if err != nil { return nil, errors.New(awserrors.ErrorInvalidLaunchTemplateIdVersionNotFound) } @@ -165,10 +167,10 @@ func (s *LaunchTemplateServiceImpl) getVersion(accountID, ltID string, n int64) // listVersionNumbers returns the existing version numbers for a template, // derived by scanning its version keys (sorted ascending). -func (s *LaunchTemplateServiceImpl) listVersionNumbers(accountID, ltID string) ([]int64, error) { - keys, err := s.kv.Keys() +func (s *LaunchTemplateServiceImpl) listVersionNumbers(ctx context.Context, accountID, ltID string) ([]int64, error) { + keys, err := s.kv.Keys(ctx) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return nil, nil } return nil, errors.New(awserrors.ErrorServerInternal) @@ -190,8 +192,8 @@ func (s *LaunchTemplateServiceImpl) listVersionNumbers(accountID, ltID string) ( } // latestVersionNumber returns the highest existing version number, or 0 if none. -func (s *LaunchTemplateServiceImpl) latestVersionNumber(accountID, ltID string) (int64, error) { - nums, err := s.listVersionNumbers(accountID, ltID) +func (s *LaunchTemplateServiceImpl) latestVersionNumber(ctx context.Context, accountID, ltID string) (int64, error) { + nums, err := s.listVersionNumbers(ctx, accountID, ltID) if err != nil { return 0, err } @@ -235,12 +237,12 @@ func latestVersionsFromKeys(keys []string, prefix string) map[string]int64 { // resolveVersionNumber maps a selector ("", $Default, $Latest, or numeric) to a // concrete version number. It does not verify the body exists — callers load the // body and surface VersionNotFound when it is missing. -func (s *LaunchTemplateServiceImpl) resolveVersionNumber(accountID string, h *LaunchTemplateHeader, sel string) (int64, error) { +func (s *LaunchTemplateServiceImpl) resolveVersionNumber(ctx context.Context, accountID string, h *LaunchTemplateHeader, sel string) (int64, error) { switch sel { case "", versionDefault: return h.DefaultVersionNumber, nil case versionLatest: - return s.latestVersionNumber(accountID, h.LaunchTemplateId) + return s.latestVersionNumber(ctx, accountID, h.LaunchTemplateId) default: n, err := strconv.ParseInt(sel, 10, 64) if err != nil { @@ -270,7 +272,7 @@ func (s *LaunchTemplateServiceImpl) CreateLaunchTemplate(ctx context.Context, in } ltID := utils.GenerateResourceID("lt") - if err := s.claimName(accountID, name, ltID); err != nil { + if err := s.claimName(ctx, accountID, name, ltID); err != nil { return nil, err } @@ -282,7 +284,7 @@ func (s *LaunchTemplateServiceImpl) CreateLaunchTemplate(ctx context.Context, in CreatedBy: accountID, Data: data, } - if err := s.putVersion(accountID, ltID, &rec); err != nil { + if err := s.putVersion(ctx, accountID, ltID, &rec); err != nil { return nil, err } @@ -295,7 +297,7 @@ func (s *LaunchTemplateServiceImpl) CreateLaunchTemplate(ctx context.Context, in DefaultVersionNumber: 1, Tags: utils.ExtractTags(input.TagSpecifications, launchTemplateTagResourceType), } - if err := s.putHeader(accountID, &header); err != nil { + if err := s.putHeader(ctx, accountID, &header); err != nil { return nil, err } @@ -310,12 +312,12 @@ func (s *LaunchTemplateServiceImpl) CreateLaunchTemplate(ctx context.Context, in // repair-on-write: if the name already exists but its header is gone (a crash // orphan), reclaim it via a revision-guarded CAS update. Only a name whose header // still exists returns AlreadyExists. -func (s *LaunchTemplateServiceImpl) claimName(accountID, name, ltID string) error { +func (s *LaunchTemplateServiceImpl) claimName(ctx context.Context, accountID, name, ltID string) error { key := nameKey(accountID, name) - if _, err := s.kv.Create(key, []byte(ltID)); err == nil { + if _, err := s.kv.Create(ctx, key, []byte(ltID)); err == nil { return nil } - entry, err := s.kv.Get(key) + entry, err := s.kv.Get(ctx, key) if err != nil { return errors.New(awserrors.ErrorServerInternal) } @@ -323,36 +325,36 @@ func (s *LaunchTemplateServiceImpl) claimName(accountID, name, ltID string) erro // header means the name is taken; any other read error (transient fault, or a // concurrent in-flight create whose header is not yet written) fails closed so // the name is never stolen from a possibly-live template. - _, herr := s.kv.Get(headerKey(accountID, string(entry.Value()))) + _, herr := s.kv.Get(ctx, headerKey(accountID, string(entry.Value()))) switch { case herr == nil: return errors.New(awserrors.ErrorInvalidLaunchTemplateNameAlreadyExistsException) - case !errors.Is(herr, nats.ErrKeyNotFound): + case !errors.Is(herr, jetstream.ErrKeyNotFound): return errors.New(awserrors.ErrorServerInternal) } - if _, err := s.kv.Update(key, []byte(ltID), entry.Revision()); err != nil { + if _, err := s.kv.Update(ctx, key, []byte(ltID), entry.Revision()); err != nil { return errors.New(awserrors.ErrorInvalidLaunchTemplateNameAlreadyExistsException) } return nil } -func (s *LaunchTemplateServiceImpl) putHeader(accountID string, h *LaunchTemplateHeader) error { +func (s *LaunchTemplateServiceImpl) putHeader(ctx context.Context, accountID string, h *LaunchTemplateHeader) error { data, err := json.Marshal(h) if err != nil { return errors.New(awserrors.ErrorServerInternal) } - if _, err := s.kv.Put(headerKey(accountID, h.LaunchTemplateId), data); err != nil { + if _, err := s.kv.Put(ctx, headerKey(accountID, h.LaunchTemplateId), data); err != nil { return errors.New(awserrors.ErrorServerInternal) } return nil } -func (s *LaunchTemplateServiceImpl) putVersion(accountID, ltID string, rec *LaunchTemplateVersionRec) error { +func (s *LaunchTemplateServiceImpl) putVersion(ctx context.Context, accountID, ltID string, rec *LaunchTemplateVersionRec) error { data, err := json.Marshal(rec) if err != nil { return errors.New(awserrors.ErrorServerInternal) } - if _, err := s.kv.Put(versionKey(accountID, ltID, rec.VersionNumber), data); err != nil { + if _, err := s.kv.Put(ctx, versionKey(accountID, ltID, rec.VersionNumber), data); err != nil { return errors.New(awserrors.ErrorServerInternal) } return nil @@ -364,7 +366,7 @@ func (s *LaunchTemplateServiceImpl) CreateLaunchTemplateVersion(ctx context.Cont if input.LaunchTemplateData == nil { return nil, errors.New(awserrors.ErrorMissingParameter) } - header, _, err := s.resolveHeader(accountID, input.LaunchTemplateId, input.LaunchTemplateName) + header, _, err := s.resolveHeader(ctx, accountID, input.LaunchTemplateId, input.LaunchTemplateName) if err != nil { return nil, err } @@ -380,11 +382,11 @@ func (s *LaunchTemplateServiceImpl) CreateLaunchTemplateVersion(ctx context.Cont // SourceVersion clones-and-overrides; without it the new version is the data as given. data := override if src := aws.StringValue(input.SourceVersion); src != "" { - n, err := s.resolveVersionNumber(accountID, header, src) + n, err := s.resolveVersionNumber(ctx, accountID, header, src) if err != nil { return nil, err } - base, err := s.getVersion(accountID, header.LaunchTemplateId, n) + base, err := s.getVersion(ctx, accountID, header.LaunchTemplateId, n) if err != nil { return nil, err } @@ -401,7 +403,7 @@ func (s *LaunchTemplateServiceImpl) CreateLaunchTemplateVersion(ctx context.Cont // Assign the number with an atomic kv.Create in a bounded retry loop: on // collision another writer took n, so rescan and try the next number. for attempt := range maxVersionRetries { - latest, err := s.latestVersionNumber(accountID, header.LaunchTemplateId) + latest, err := s.latestVersionNumber(ctx, accountID, header.LaunchTemplateId) if err != nil { return nil, err } @@ -411,8 +413,8 @@ func (s *LaunchTemplateServiceImpl) CreateLaunchTemplateVersion(ctx context.Cont if err != nil { return nil, errors.New(awserrors.ErrorServerInternal) } - if _, err := s.kv.Create(versionKey(accountID, header.LaunchTemplateId, next), body); err != nil { - if errors.Is(err, nats.ErrKeyExists) { + if _, err := s.kv.Create(ctx, versionKey(accountID, header.LaunchTemplateId, next), body); err != nil { + if errors.Is(err, jetstream.ErrKeyExists) { slog.DebugContext(ctx, "CreateLaunchTemplateVersion: version-number collision, retrying", "attempt", attempt, "next", next) continue } @@ -432,12 +434,12 @@ func (s *LaunchTemplateServiceImpl) CreateLaunchTemplateVersion(ctx context.Cont // --- ModifyLaunchTemplate --- func (s *LaunchTemplateServiceImpl) ModifyLaunchTemplate(ctx context.Context, input *ec2.ModifyLaunchTemplateInput, accountID string) (*ec2.ModifyLaunchTemplateOutput, error) { - header, entry, err := s.resolveHeader(accountID, input.LaunchTemplateId, input.LaunchTemplateName) + header, entry, err := s.resolveHeader(ctx, accountID, input.LaunchTemplateId, input.LaunchTemplateName) if err != nil { return nil, err } if aws.BoolValue(input.DryRun) { - latest, err := s.latestVersionNumber(accountID, header.LaunchTemplateId) + latest, err := s.latestVersionNumber(ctx, accountID, header.LaunchTemplateId) if err != nil { return nil, err } @@ -445,13 +447,13 @@ func (s *LaunchTemplateServiceImpl) ModifyLaunchTemplate(ctx context.Context, in } if sel := aws.StringValue(input.DefaultVersion); sel != "" { - n, err := s.resolveVersionNumber(accountID, header, sel) + n, err := s.resolveVersionNumber(ctx, accountID, header, sel) if err != nil { return nil, err } // Verify the target body exists immediately before the header CAS so a // concurrent delete degrades to VersionNotFound, never a dangling default. - if _, err := s.getVersion(accountID, header.LaunchTemplateId, n); err != nil { + if _, err := s.getVersion(ctx, accountID, header.LaunchTemplateId, n); err != nil { return nil, err } header.DefaultVersionNumber = n @@ -459,12 +461,12 @@ func (s *LaunchTemplateServiceImpl) ModifyLaunchTemplate(ctx context.Context, in if err != nil { return nil, errors.New(awserrors.ErrorServerInternal) } - if _, err := s.kv.Update(headerKey(accountID, header.LaunchTemplateId), data, entry.Revision()); err != nil { + if _, err := s.kv.Update(ctx, headerKey(accountID, header.LaunchTemplateId), data, entry.Revision()); err != nil { return nil, errors.New(awserrors.ErrorServerInternal) } } - latest, err := s.latestVersionNumber(accountID, header.LaunchTemplateId) + latest, err := s.latestVersionNumber(ctx, accountID, header.LaunchTemplateId) if err != nil { return nil, err } @@ -475,19 +477,19 @@ func (s *LaunchTemplateServiceImpl) ModifyLaunchTemplate(ctx context.Context, in // --- DeleteLaunchTemplate --- func (s *LaunchTemplateServiceImpl) DeleteLaunchTemplate(ctx context.Context, input *ec2.DeleteLaunchTemplateInput, accountID string) (*ec2.DeleteLaunchTemplateOutput, error) { - header, _, err := s.resolveHeader(accountID, input.LaunchTemplateId, input.LaunchTemplateName) + header, _, err := s.resolveHeader(ctx, accountID, input.LaunchTemplateId, input.LaunchTemplateName) if err != nil { return nil, err } if aws.BoolValue(input.DryRun) { - latest, err := s.latestVersionNumber(accountID, header.LaunchTemplateId) + latest, err := s.latestVersionNumber(ctx, accountID, header.LaunchTemplateId) if err != nil { return nil, err } return &ec2.DeleteLaunchTemplateOutput{LaunchTemplate: headerToEC2(header, latest)}, nil } - latest, err := s.latestVersionNumber(accountID, header.LaunchTemplateId) + latest, err := s.latestVersionNumber(ctx, accountID, header.LaunchTemplateId) if err != nil { return nil, err } @@ -495,18 +497,18 @@ func (s *LaunchTemplateServiceImpl) DeleteLaunchTemplate(ctx context.Context, in // Delete the header first: the template immediately vanishes from every // describe. Version bodies and the name index are best-effort cleanup. - if err := s.kv.Delete(headerKey(accountID, header.LaunchTemplateId)); err != nil { + if err := s.kv.Delete(ctx, headerKey(accountID, header.LaunchTemplateId)); err != nil { return nil, errors.New(awserrors.ErrorServerInternal) } - if err := s.kv.Delete(nameKey(accountID, header.LaunchTemplateName)); err != nil { + if err := s.kv.Delete(ctx, nameKey(accountID, header.LaunchTemplateName)); err != nil { slog.WarnContext(ctx, "DeleteLaunchTemplate: name index cleanup failed", "name", header.LaunchTemplateName, "err", err) } - nums, err := s.listVersionNumbers(accountID, header.LaunchTemplateId) + nums, err := s.listVersionNumbers(ctx, accountID, header.LaunchTemplateId) if err != nil { slog.WarnContext(ctx, "DeleteLaunchTemplate: version enumeration failed, bodies left in place", "launchTemplateId", header.LaunchTemplateId, "err", err) } for _, n := range nums { - if err := s.kv.Delete(versionKey(accountID, header.LaunchTemplateId, n)); err != nil { + if err := s.kv.Delete(ctx, versionKey(accountID, header.LaunchTemplateId, n)); err != nil { slog.WarnContext(ctx, "DeleteLaunchTemplate: version cleanup failed", "launchTemplateId", header.LaunchTemplateId, "version", n, "err", err) } } @@ -518,7 +520,7 @@ func (s *LaunchTemplateServiceImpl) DeleteLaunchTemplate(ctx context.Context, in // --- DeleteLaunchTemplateVersions --- func (s *LaunchTemplateServiceImpl) DeleteLaunchTemplateVersions(ctx context.Context, input *ec2.DeleteLaunchTemplateVersionsInput, accountID string) (*ec2.DeleteLaunchTemplateVersionsOutput, error) { - header, _, err := s.resolveHeader(accountID, input.LaunchTemplateId, input.LaunchTemplateName) + header, _, err := s.resolveHeader(ctx, accountID, input.LaunchTemplateId, input.LaunchTemplateName) if err != nil { return nil, err } @@ -544,12 +546,12 @@ func (s *LaunchTemplateServiceImpl) DeleteLaunchTemplateVersions(ctx context.Con deleteErrorItem(header, n, awserrors.ErrorInvalidParameterValue, "cannot delete the default version of a launch template")) continue } - if _, err := s.getVersion(accountID, header.LaunchTemplateId, n); err != nil { + if _, err := s.getVersion(ctx, accountID, header.LaunchTemplateId, n); err != nil { out.UnsuccessfullyDeletedLaunchTemplateVersions = append(out.UnsuccessfullyDeletedLaunchTemplateVersions, deleteErrorItem(header, n, awserrors.ErrorInvalidLaunchTemplateIdVersionNotFound, "version does not exist")) continue } - if err := s.kv.Delete(versionKey(accountID, header.LaunchTemplateId, n)); err != nil { + if err := s.kv.Delete(ctx, versionKey(accountID, header.LaunchTemplateId, n)); err != nil { out.UnsuccessfullyDeletedLaunchTemplateVersions = append(out.UnsuccessfullyDeletedLaunchTemplateVersions, deleteErrorItem(header, n, awserrors.ErrorServerInternal, "failed to delete version")) continue @@ -604,8 +606,8 @@ func (s *LaunchTemplateServiceImpl) DescribeLaunchTemplates(ctx context.Context, idSet := stringSet(input.LaunchTemplateIds) prefix := accountID + "." - keys, err := s.kv.Keys() - if err != nil && !errors.Is(err, nats.ErrNoKeysFound) { + keys, err := s.kv.Keys(ctx) + if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) { return nil, errors.New(awserrors.ErrorServerInternal) } @@ -620,7 +622,7 @@ func (s *LaunchTemplateServiceImpl) DescribeLaunchTemplates(ctx context.Context, if !isHeaderKey(k, prefix) { continue } - entry, err := s.kv.Get(k) + entry, err := s.kv.Get(ctx, k) if err != nil { slog.WarnContext(ctx, "DescribeLaunchTemplates: header read failed", "key", k, "err", err) continue @@ -713,19 +715,19 @@ func (s *LaunchTemplateServiceImpl) DescribeLaunchTemplateVersions(ctx context.C return nil, errors.New(awserrors.ErrorInvalidParameterValue) } - header, _, err := s.resolveHeader(accountID, input.LaunchTemplateId, input.LaunchTemplateName) + header, _, err := s.resolveHeader(ctx, accountID, input.LaunchTemplateId, input.LaunchTemplateName) if err != nil { return nil, err } - numbers, err := s.selectVersionNumbers(accountID, header, input) + numbers, err := s.selectVersionNumbers(ctx, accountID, header, input) if err != nil { return nil, err } var versions []*ec2.LaunchTemplateVersion for _, n := range numbers { - rec, err := s.getVersion(accountID, header.LaunchTemplateId, n) + rec, err := s.getVersion(ctx, accountID, header.LaunchTemplateId, n) if err != nil { return nil, err } @@ -742,16 +744,16 @@ func (s *LaunchTemplateServiceImpl) DescribeLaunchTemplateVersions(ctx context.C // selectVersionNumbers resolves the requested Versions/MinVersion/MaxVersion into // a concrete, ordered list of version numbers. Explicit Versions each resolve and // must exist; otherwise all versions within the optional [Min,Max] range. -func (s *LaunchTemplateServiceImpl) selectVersionNumbers(accountID string, header *LaunchTemplateHeader, input *ec2.DescribeLaunchTemplateVersionsInput) ([]int64, error) { +func (s *LaunchTemplateServiceImpl) selectVersionNumbers(ctx context.Context, accountID string, header *LaunchTemplateHeader, input *ec2.DescribeLaunchTemplateVersionsInput) ([]int64, error) { if len(input.Versions) > 0 { seen := make(map[int64]bool) var out []int64 for _, v := range input.Versions { - n, err := s.resolveVersionNumber(accountID, header, aws.StringValue(v)) + n, err := s.resolveVersionNumber(ctx, accountID, header, aws.StringValue(v)) if err != nil { return nil, err } - if _, err := s.getVersion(accountID, header.LaunchTemplateId, n); err != nil { + if _, err := s.getVersion(ctx, accountID, header.LaunchTemplateId, n); err != nil { return nil, err } if !seen[n] { @@ -762,7 +764,7 @@ func (s *LaunchTemplateServiceImpl) selectVersionNumbers(accountID string, heade return out, nil } - all, err := s.listVersionNumbers(accountID, header.LaunchTemplateId) + all, err := s.listVersionNumbers(ctx, accountID, header.LaunchTemplateId) if err != nil { return nil, err } diff --git a/spinifex/handlers/ec2/launchtemplate/service_impl_test.go b/spinifex/handlers/ec2/launchtemplate/service_impl_test.go index e96afbfb4..a1ef49675 100644 --- a/spinifex/handlers/ec2/launchtemplate/service_impl_test.go +++ b/spinifex/handlers/ec2/launchtemplate/service_impl_test.go @@ -21,7 +21,7 @@ const ( func setupTestService(t *testing.T) *LaunchTemplateServiceImpl { t.Helper() _, nc, _ := testutil.StartTestJetStream(t) - svc, err := NewLaunchTemplateServiceImplWithNATS(nil, nc) + svc, err := NewLaunchTemplateServiceImplWithNATS(t.Context(), nil, nc) require.NoError(t, err) return svc } @@ -129,7 +129,7 @@ func TestCreateLaunchTemplate_OrphanNameReclaim(t *testing.T) { lt := createTemplate(t, svc, "orphan", "t3.micro") // Simulate a crash after the name claim but with the header lost. - require.NoError(t, svc.kv.Delete(headerKey(testAccountID, aws.StringValue(lt.LaunchTemplateId)))) + require.NoError(t, svc.kv.Delete(t.Context(), headerKey(testAccountID, aws.StringValue(lt.LaunchTemplateId)))) // The name index still points at the now-orphaned id; create must reclaim it. out, err := svc.CreateLaunchTemplate(context.Background(), &ec2.CreateLaunchTemplateInput{ @@ -459,10 +459,10 @@ func TestDeleteLaunchTemplate_RemovesEverything(t *testing.T) { assert.Equal(t, awserrors.ErrorInvalidLaunchTemplateIdNotFound, err.Error()) // Version bodies and name index gone. - nums, err := svc.listVersionNumbers(testAccountID, id) + nums, err := svc.listVersionNumbers(t.Context(), testAccountID, id) require.NoError(t, err) assert.Empty(t, nums) - _, err = svc.kv.Get(nameKey(testAccountID, "web")) + _, err = svc.kv.Get(t.Context(), nameKey(testAccountID, "web")) require.Error(t, err) // Name is reusable after delete. diff --git a/spinifex/handlers/ec2/natgw/lifecycle_invariants_test.go b/spinifex/handlers/ec2/natgw/lifecycle_invariants_test.go index fce2eff93..a315c1efb 100644 --- a/spinifex/handlers/ec2/natgw/lifecycle_invariants_test.go +++ b/spinifex/handlers/ec2/natgw/lifecycle_invariants_test.go @@ -1,7 +1,6 @@ package handlers_ec2_natgw import ( - "context" "encoding/json" "fmt" "testing" @@ -10,7 +9,6 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" "github.com/mulgadc/spinifex/spinifex/awserrors" handlers_ec2_eip "github.com/mulgadc/spinifex/spinifex/handlers/ec2/eip" - "github.com/mulgadc/spinifex/spinifex/testutil" "github.com/mulgadc/spinifex/spinifex/utils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -24,7 +22,7 @@ import ( func TestRLC1_NatGatewayDeleteNotFoundOnAbsent(t *testing.T) { svc := setupTestService(t) - _, err := svc.DeleteNatGateway(context.Background(), &ec2.DeleteNatGatewayInput{ + _, err := svc.DeleteNatGateway(t.Context(), &ec2.DeleteNatGatewayInput{ NatGatewayId: aws.String("nat-absent00000000"), }, testAccountID) @@ -42,12 +40,12 @@ func TestRLC3_NatGatewayDeletesWhileRouted(t *testing.T) { svc, js := setupTestServiceJS(t) natgwID := createTestNatGateway(t, svc) - testutil.SeedKV(t, js, kvBucketRouteTables, map[string][]byte{ + seedKV(t, js, kvBucketRouteTables, map[string][]byte{ utils.AccountKey(testAccountID, "rtb-routed"): fmt.Appendf(nil, `{"route_table_id":"rtb-routed","vpc_id":"vpc-test1","routes":[{"destination_cidr_block":"0.0.0.0/0","nat_gateway_id":%q}]}`, natgwID), }) - _, err := svc.DeleteNatGateway(context.Background(), &ec2.DeleteNatGatewayInput{NatGatewayId: aws.String(natgwID)}, testAccountID) + _, err := svc.DeleteNatGateway(t.Context(), &ec2.DeleteNatGatewayInput{NatGatewayId: aws.String(natgwID)}, testAccountID) require.NoErrorf(t, err, "ADR-0004 §2: DeleteNatGateway must succeed even while a live route table forwards to it (the route blackholes, AWS-faithful)") }) @@ -56,7 +54,7 @@ func TestRLC3_NatGatewayDeletesWhileRouted(t *testing.T) { svc := setupTestService(t) natgwID := createTestNatGateway(t, svc) - _, err := svc.DeleteNatGateway(context.Background(), &ec2.DeleteNatGatewayInput{NatGatewayId: aws.String(natgwID)}, testAccountID) + _, err := svc.DeleteNatGateway(t.Context(), &ec2.DeleteNatGatewayInput{NatGatewayId: aws.String(natgwID)}, testAccountID) require.NoErrorf(t, err, "ADR-0004 §2: with no route forwarding to it, DeleteNatGateway must succeed") }) @@ -74,7 +72,7 @@ func TestRLC3_NatGatewayDeleteDisassociatesEIPKeepingAllocated(t *testing.T) { require.Equal(t, "associated", assoc.State, "CreateNatGateway must mark the EIP associated so a second NAT GW cannot reuse it") require.NotEmpty(t, assoc.AssociationId) - _, err := svc.DeleteNatGateway(context.Background(), &ec2.DeleteNatGatewayInput{NatGatewayId: aws.String(natgwID)}, testAccountID) + _, err := svc.DeleteNatGateway(t.Context(), &ec2.DeleteNatGatewayInput{NatGatewayId: aws.String(natgwID)}, testAccountID) require.NoError(t, err) eip := readEIP(t, svc, "eipalloc-test1") @@ -84,7 +82,7 @@ func TestRLC3_NatGatewayDeleteDisassociatesEIPKeepingAllocated(t *testing.T) { func createTestNatGateway(t *testing.T, svc *NatGatewayServiceImpl) string { t.Helper() - out, err := svc.CreateNatGateway(context.Background(), &ec2.CreateNatGatewayInput{ + out, err := svc.CreateNatGateway(t.Context(), &ec2.CreateNatGatewayInput{ SubnetId: aws.String("subnet-pub1"), AllocationId: aws.String("eipalloc-test1"), }, testAccountID) @@ -94,7 +92,7 @@ func createTestNatGateway(t *testing.T, svc *NatGatewayServiceImpl) string { func readEIP(t *testing.T, svc *NatGatewayServiceImpl, allocID string) handlers_ec2_eip.EIPRecord { t.Helper() - entry, err := svc.eipKV.Get(utils.AccountKey(testAccountID, allocID)) + entry, err := svc.eipKV.Get(t.Context(), utils.AccountKey(testAccountID, allocID)) require.NoError(t, err) var eip handlers_ec2_eip.EIPRecord require.NoError(t, json.Unmarshal(entry.Value(), &eip)) diff --git a/spinifex/handlers/ec2/natgw/record_tags_test.go b/spinifex/handlers/ec2/natgw/record_tags_test.go index 75ff63ea3..ef04e033b 100644 --- a/spinifex/handlers/ec2/natgw/record_tags_test.go +++ b/spinifex/handlers/ec2/natgw/record_tags_test.go @@ -1,7 +1,6 @@ package handlers_ec2_natgw import ( - "context" "testing" "github.com/aws/aws-sdk-go/aws" @@ -31,7 +30,7 @@ func TestNatGatewayRecordTagsMirror(t *testing.T) { }, }, testAccountID)) - out, err := svc.DescribeNatGateways(context.Background(), &ec2.DescribeNatGatewaysInput{ + out, err := svc.DescribeNatGateways(t.Context(), &ec2.DescribeNatGatewaysInput{ NatGatewayIds: []*string{aws.String(natgwID)}, }, testAccountID) require.NoError(t, err) @@ -47,7 +46,7 @@ func TestNatGatewayRecordTagsMirror(t *testing.T) { }, }, testAccountID)) - out, err = svc.DescribeNatGateways(context.Background(), &ec2.DescribeNatGatewaysInput{ + out, err = svc.DescribeNatGateways(t.Context(), &ec2.DescribeNatGatewaysInput{ NatGatewayIds: []*string{aws.String(natgwID)}, }, testAccountID) require.NoError(t, err) diff --git a/spinifex/handlers/ec2/natgw/service_impl.go b/spinifex/handlers/ec2/natgw/service_impl.go index 6b815e90f..04a3f6a2c 100644 --- a/spinifex/handlers/ec2/natgw/service_impl.go +++ b/spinifex/handlers/ec2/natgw/service_impl.go @@ -15,67 +15,59 @@ import ( "github.com/mulgadc/spinifex/spinifex/filterutil" handlers_ec2_eip "github.com/mulgadc/spinifex/spinifex/handlers/ec2/eip" handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc" + "github.com/mulgadc/spinifex/spinifex/kvutil" "github.com/mulgadc/spinifex/spinifex/migrate" "github.com/mulgadc/spinifex/spinifex/utils" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) var _ NatGatewayService = (*NatGatewayServiceImpl)(nil) // NatGatewayServiceImpl implements NAT Gateway operations with NATS JetStream persistence. type NatGatewayServiceImpl struct { - natgwKV nats.KeyValue - deletedNatgwKV nats.KeyValue - eipKV nats.KeyValue - subnetKV nats.KeyValue - vpcKV nats.KeyValue + natgwKV jetstream.KeyValue + deletedNatgwKV jetstream.KeyValue + eipKV jetstream.KeyValue + subnetKV jetstream.KeyValue + vpcKV jetstream.KeyValue natsConn *nats.Conn } // NewNatGatewayServiceImplWithNATS creates a NAT Gateway service. -func NewNatGatewayServiceImplWithNATS(natsConn *nats.Conn) (*NatGatewayServiceImpl, error) { - js, err := natsConn.JetStream() +func NewNatGatewayServiceImplWithNATS(ctx context.Context, natsConn *nats.Conn) (*NatGatewayServiceImpl, error) { + js, err := jetstream.New(natsConn) if err != nil { return nil, fmt.Errorf("failed to get JetStream context: %w", err) } - natgwKV, err := utils.GetOrCreateKVBucket(js, KVBucketNatGateways, 10) + natgwKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketNatGateways, 10) if err != nil { return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketNatGateways, err) } - if err := migrate.DefaultRegistry.RunKV(KVBucketNatGateways, natgwKV, KVBucketNatGatewaysVersion); err != nil { + if err := migrate.DefaultRegistry.RunKV(ctx, KVBucketNatGateways, natgwKV, KVBucketNatGatewaysVersion); err != nil { return nil, fmt.Errorf("migrate %s: %w", KVBucketNatGateways, err) } - // Deleted NAT Gateways bucket with 1-hour TTL — keys auto-expire. - // Terraform polls DescribeNatGateways after delete and expects state=deleted. - deletedKV, err := js.CreateKeyValue(&nats.KeyValueConfig{ - Bucket: KVBucketDeletedNatGateways, - Description: "Deleted NAT Gateways (auto-expire after 1 hour)", - History: 1, - TTL: 1 * time.Hour, - }) + deletedKV, err := getOrCreateDeletedBucket(ctx, js) if err != nil { - deletedKV, err = js.KeyValue(KVBucketDeletedNatGateways) - if err != nil { - return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketDeletedNatGateways, err) - } + return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketDeletedNatGateways, err) } - if err := migrate.DefaultRegistry.RunKV(KVBucketDeletedNatGateways, deletedKV, KVBucketDeletedNatGatewaysVersion); err != nil { + if err := migrate.DefaultRegistry.RunKV(ctx, KVBucketDeletedNatGateways, deletedKV, KVBucketDeletedNatGatewaysVersion); err != nil { return nil, fmt.Errorf("migrate %s: %w", KVBucketDeletedNatGateways, err) } - eipKV, err := utils.GetOrCreateKVBucket(js, handlers_ec2_eip.KVBucketEIPs, 10) + eipKV, err := kvutil.GetOrCreateBucket(ctx, js, handlers_ec2_eip.KVBucketEIPs, 10) if err != nil { return nil, fmt.Errorf("failed to get EIP KV bucket: %w", err) } - subnetKV, err := utils.GetOrCreateKVBucket(js, handlers_ec2_vpc.KVBucketSubnets, 10) + subnetKV, err := kvutil.GetOrCreateBucket(ctx, js, handlers_ec2_vpc.KVBucketSubnets, 10) if err != nil { return nil, fmt.Errorf("failed to get subnet KV bucket: %w", err) } - vpcKV, err := utils.GetOrCreateKVBucket(js, handlers_ec2_vpc.KVBucketVPCs, 10) + vpcKV, err := kvutil.GetOrCreateBucket(ctx, js, handlers_ec2_vpc.KVBucketVPCs, 10) if err != nil { return nil, fmt.Errorf("failed to get VPC KV bucket: %w", err) } @@ -92,6 +84,25 @@ func NewNatGatewayServiceImplWithNATS(natsConn *nats.Conn) (*NatGatewayServiceIm }, nil } +// getOrCreateDeletedBucket creates the expiring bucket without changing an +// existing bucket's replica or placement configuration. +func getOrCreateDeletedBucket(ctx context.Context, js jetstream.KeyValueManager) (jetstream.KeyValue, error) { + // Terraform polls DescribeNatGateways after delete and expects state=deleted. + kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{ + Bucket: KVBucketDeletedNatGateways, + Description: "Deleted NAT Gateways (auto-expire after 1 hour)", + History: 1, + TTL: time.Hour, + }) + if err == nil { + return kv, nil + } + if !errors.Is(err, jetstream.ErrBucketExists) { + return nil, err + } + return js.KeyValue(ctx, KVBucketDeletedNatGateways) +} + // natGatewayEvent is published on vpc.add-nat-gateway / vpc.delete-nat-gateway topics. type natGatewayEvent struct { VpcId string `json:"vpc_id"` @@ -113,7 +124,7 @@ func (s *NatGatewayServiceImpl) CreateNatGateway(ctx context.Context, input *ec2 allocID := *input.AllocationId // Validate subnet exists and get its VPC - subnetEntry, err := s.subnetKV.Get(utils.AccountKey(accountID, subnetID)) + subnetEntry, err := s.subnetKV.Get(ctx, utils.AccountKey(accountID, subnetID)) if err != nil { return nil, errors.New(awserrors.ErrorInvalidSubnetIDNotFound) } @@ -123,7 +134,7 @@ func (s *NatGatewayServiceImpl) CreateNatGateway(ctx context.Context, input *ec2 } // Validate EIP exists and is not already associated - eipEntry, err := s.eipKV.Get(utils.AccountKey(accountID, allocID)) + eipEntry, err := s.eipKV.Get(ctx, utils.AccountKey(accountID, allocID)) if err != nil { return nil, errors.New(awserrors.ErrorInvalidAllocationIDNotFound) } @@ -152,7 +163,7 @@ func (s *NatGatewayServiceImpl) CreateNatGateway(ctx context.Context, input *ec2 if err != nil { return nil, errors.New(awserrors.ErrorServerInternal) } - if _, err := s.natgwKV.Put(utils.AccountKey(accountID, natgwID), data); err != nil { + if _, err := s.natgwKV.Put(ctx, utils.AccountKey(accountID, natgwID), data); err != nil { slog.ErrorContext(ctx, "Failed to store NAT Gateway", "natGatewayId", natgwID, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } @@ -165,7 +176,7 @@ func (s *NatGatewayServiceImpl) CreateNatGateway(ctx context.Context, input *ec2 eipRecord.VpcId = subnetRecord.VpcId eipRecord.State = "associated" if eipData, err := json.Marshal(eipRecord); err == nil { - if _, err := s.eipKV.Update(utils.AccountKey(accountID, allocID), eipData, eipEntry.Revision()); err != nil { + if _, err := s.eipKV.Update(ctx, utils.AccountKey(accountID, allocID), eipData, eipEntry.Revision()); err != nil { slog.WarnContext(ctx, "CreateNatGateway: failed to mark EIP associated", "natGatewayId", natgwID, "allocationId", allocID, "err", err) } } @@ -187,9 +198,9 @@ func (s *NatGatewayServiceImpl) DeleteNatGateway(ctx context.Context, input *ec2 natgwID := *input.NatGatewayId key := utils.AccountKey(accountID, natgwID) - entry, err := s.natgwKV.Get(key) + entry, err := s.natgwKV.Get(ctx, key) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { // AWS-faithful: an absent NAT gateway is NotFound (provider // tolerates it on destroy); destroy orchestration tolerates it too. return nil, errors.New(awserrors.ErrorInvalidNatGatewayIDNotFound) @@ -215,13 +226,13 @@ func (s *NatGatewayServiceImpl) DeleteNatGateway(ctx context.Context, input *ec2 if err != nil { return nil, errors.New(awserrors.ErrorServerInternal) } - if _, err := s.deletedNatgwKV.Put(key, deleted); err != nil { + if _, err := s.deletedNatgwKV.Put(ctx, key, deleted); err != nil { slog.ErrorContext(ctx, "Failed to write deleted NAT Gateway", "natGatewayId", natgwID, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } // Remove from active bucket - if err := s.natgwKV.Delete(key); err != nil { + if err := s.natgwKV.Delete(ctx, key); err != nil { slog.ErrorContext(ctx, "Failed to delete NAT Gateway from active bucket", "natGatewayId", natgwID, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } @@ -250,12 +261,17 @@ func (s *NatGatewayServiceImpl) publishDeleteEventsForNatGateway(ctx context.Con if s.natsConn == nil { return } - rtbKV, err := utils.GetOrCreateKVBucket(mustJS(s.natsConn), kvBucketRouteTables, 10) + js, err := jetstream.New(s.natsConn) + if err != nil { + slog.WarnContext(ctx, "DeleteNatGateway: JetStream setup failed, SNAT teardown deferred to route delete", "natGatewayId", record.NatGatewayId, "err", err) + return + } + rtbKV, err := kvutil.GetOrCreateBucket(ctx, js, kvBucketRouteTables, 10) if err != nil { slog.WarnContext(ctx, "DeleteNatGateway: route-table scan failed, SNAT teardown deferred to route delete", "natGatewayId", record.NatGatewayId, "err", err) return } - keys, err := rtbKV.Keys() + keys, err := rtbKV.Keys(ctx) if err != nil { return } @@ -264,7 +280,7 @@ func (s *NatGatewayServiceImpl) publishDeleteEventsForNatGateway(ctx context.Con if key == utils.VersionKey || !strings.HasPrefix(key, prefix) { continue } - entry, err := rtbKV.Get(key) + entry, err := rtbKV.Get(ctx, key) if err != nil { continue } @@ -297,7 +313,7 @@ func (s *NatGatewayServiceImpl) publishDeleteEventsForNatGateway(ctx context.Con if assoc.SubnetId == "" { continue } - subnetEntry, err := s.subnetKV.Get(utils.AccountKey(accountID, assoc.SubnetId)) + subnetEntry, err := s.subnetKV.Get(ctx, utils.AccountKey(accountID, assoc.SubnetId)) if err != nil { continue } @@ -310,11 +326,6 @@ func (s *NatGatewayServiceImpl) publishDeleteEventsForNatGateway(ctx context.Con } } -func mustJS(nc *nats.Conn) nats.JetStreamContext { - js, _ := nc.JetStream() - return js -} - // disassociateEIP clears the EIP's association to this NAT gateway, keeping the // allocation (AWS parity). Idempotent and non-fatal: the NAT gateway is already // gone, so a stale association only delays the EIP's reuse. @@ -323,9 +334,9 @@ func (s *NatGatewayServiceImpl) disassociateEIP(ctx context.Context, record *Nat return } key := utils.AccountKey(accountID, record.AllocationId) - entry, err := s.eipKV.Get(key) + entry, err := s.eipKV.Get(ctx, key) if err != nil { - if !errors.Is(err, nats.ErrKeyNotFound) { + if !errors.Is(err, jetstream.ErrKeyNotFound) { slog.WarnContext(ctx, "DeleteNatGateway: EIP read failed during disassociate", "allocationId", record.AllocationId, "err", err) } return @@ -348,7 +359,7 @@ func (s *NatGatewayServiceImpl) disassociateEIP(ctx context.Context, record *Nat if err != nil { return } - if _, err := s.eipKV.Update(key, data, entry.Revision()); err != nil { + if _, err := s.eipKV.Update(ctx, key, data, entry.Revision()); err != nil { slog.WarnContext(ctx, "DeleteNatGateway: failed to disassociate EIP", "allocationId", record.AllocationId, "err", err) } } @@ -376,8 +387,8 @@ func (s *NatGatewayServiceImpl) DescribeNatGateways(ctx context.Context, input * } prefix := accountID + "." - keys, err := s.natgwKV.Keys() - if err != nil && !errors.Is(err, nats.ErrNoKeysFound) { + keys, err := s.natgwKV.Keys(ctx) + if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) { return nil, errors.New(awserrors.ErrorServerInternal) } @@ -389,9 +400,9 @@ func (s *NatGatewayServiceImpl) DescribeNatGateways(ctx context.Context, input * continue } - entry, err := s.natgwKV.Get(key) + entry, err := s.natgwKV.Get(ctx, key) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { continue } slog.ErrorContext(ctx, "Failed to read NAT Gateway", "key", key, "err", err) @@ -421,9 +432,9 @@ func (s *NatGatewayServiceImpl) DescribeNatGateways(ctx context.Context, input * continue } key := utils.AccountKey(accountID, id) - entry, err := s.deletedNatgwKV.Get(key) + entry, err := s.deletedNatgwKV.Get(ctx, key) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil, errors.New(awserrors.ErrorInvalidNatGatewayIDNotFound) } return nil, errors.New(awserrors.ErrorServerInternal) @@ -493,9 +504,10 @@ func (s *NatGatewayServiceImpl) PublishDeleteEvent(vpcId, natGatewayId, publicIp // GetNatGateway retrieves a NAT Gateway record by ID. func (s *NatGatewayServiceImpl) GetNatGateway(accountID, natgwID string) (*NatGatewayRecord, error) { - entry, err := s.natgwKV.Get(utils.AccountKey(accountID, natgwID)) + ctx := context.Background() + entry, err := s.natgwKV.Get(ctx, utils.AccountKey(accountID, natgwID)) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil, errors.New(awserrors.ErrorInvalidNatGatewayIDNotFound) } return nil, errors.New(awserrors.ErrorServerInternal) @@ -538,7 +550,7 @@ func (s *NatGatewayServiceImpl) ApplyRecordTags(input *ec2.CreateTagsInput, acco if input == nil { return nil } - return utils.MirrorKVRecordTags(s.natgwKV, accountID, "nat-", input.Resources, + return utils.MirrorKVRecordTags(context.Background(), s.natgwKV, accountID, "nat-", input.Resources, func(r *NatGatewayRecord) *map[string]string { return &r.Tags }, utils.MergeTagsMut(input)) } @@ -549,7 +561,7 @@ func (s *NatGatewayServiceImpl) RemoveRecordTags(input *ec2.DeleteTagsInput, acc if input == nil { return nil } - return utils.MirrorKVRecordTags(s.natgwKV, accountID, "nat-", input.Resources, + return utils.MirrorKVRecordTags(context.Background(), s.natgwKV, accountID, "nat-", input.Resources, func(r *NatGatewayRecord) *map[string]string { return &r.Tags }, utils.RemoveTagsMut(input)) } diff --git a/spinifex/handlers/ec2/natgw/service_impl_test.go b/spinifex/handlers/ec2/natgw/service_impl_test.go index 9b7ad7dde..525246f9f 100644 --- a/spinifex/handlers/ec2/natgw/service_impl_test.go +++ b/spinifex/handlers/ec2/natgw/service_impl_test.go @@ -1,8 +1,8 @@ package handlers_ec2_natgw import ( - "context" "testing" + "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" @@ -12,7 +12,7 @@ import ( handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc" "github.com/mulgadc/spinifex/spinifex/testutil" "github.com/mulgadc/spinifex/spinifex/utils" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -24,34 +24,74 @@ func setupTestService(t *testing.T) *NatGatewayServiceImpl { return svc } -func setupTestServiceJS(t *testing.T) (*NatGatewayServiceImpl, nats.JetStreamContext) { +func setupTestServiceJS(t *testing.T) (*NatGatewayServiceImpl, jetstream.JetStream) { t.Helper() - _, nc, js := testutil.StartTestJetStream(t) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) // Seed VPC KV - testutil.SeedKV(t, js, handlers_ec2_vpc.KVBucketVPCs, map[string][]byte{ + seedKV(t, js, handlers_ec2_vpc.KVBucketVPCs, map[string][]byte{ utils.AccountKey(testAccountID, "vpc-test1"): []byte(`{"vpc_id":"vpc-test1","cidr_block":"10.0.0.0/16","state":"available"}`), }) // Seed subnet KV (public subnet) - testutil.SeedKV(t, js, handlers_ec2_vpc.KVBucketSubnets, map[string][]byte{ + seedKV(t, js, handlers_ec2_vpc.KVBucketSubnets, map[string][]byte{ utils.AccountKey(testAccountID, "subnet-pub1"): []byte(`{"subnet_id":"subnet-pub1","vpc_id":"vpc-test1","cidr_block":"10.0.1.0/24","state":"available","map_public_ip_on_launch":true}`), }) // Seed EIP KV - testutil.SeedKV(t, js, handlers_ec2_eip.KVBucketEIPs, map[string][]byte{ + seedKV(t, js, handlers_ec2_eip.KVBucketEIPs, map[string][]byte{ utils.AccountKey(testAccountID, "eipalloc-test1"): []byte(`{"allocation_id":"eipalloc-test1","public_ip":"203.0.113.50","state":"allocated"}`), utils.AccountKey(testAccountID, "eipalloc-used"): []byte(`{"allocation_id":"eipalloc-used","public_ip":"203.0.113.51","state":"associated","association_id":"eipassoc-xxx"}`), }) - svc, err := NewNatGatewayServiceImplWithNATS(nc) + svc, err := NewNatGatewayServiceImplWithNATS(t.Context(), nc) require.NoError(t, err) return svc, js } +func seedKV(t *testing.T, js jetstream.JetStream, bucket string, entries map[string][]byte) jetstream.KeyValue { + t.Helper() + kv, err := js.CreateOrUpdateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: bucket, History: 1}) + require.NoError(t, err) + for key, value := range entries { + _, err := kv.Put(t.Context(), key, value) + require.NoError(t, err) + } + return kv +} + +func TestDeletedBucketTTL(t *testing.T) { + _, js := setupTestServiceJS(t) + kv, err := js.KeyValue(t.Context(), KVBucketDeletedNatGateways) + require.NoError(t, err) + status, err := kv.Status(t.Context()) + require.NoError(t, err) + assert.Equal(t, time.Hour, status.TTL()) +} + +func TestGetOrCreateDeletedBucketPreservesExistingConfiguration(t *testing.T) { + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + const existingTTL = 2 * time.Hour + + _, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{ + Bucket: KVBucketDeletedNatGateways, + History: 1, + TTL: existingTTL, + }) + require.NoError(t, err) + + kv, err := getOrCreateDeletedBucket(t.Context(), js) + require.NoError(t, err) + status, err := kv.Status(t.Context()) + require.NoError(t, err) + require.Equal(t, existingTTL, status.TTL()) +} + func TestCreateNatGateway(t *testing.T) { svc := setupTestService(t) - out, err := svc.CreateNatGateway(context.Background(), &ec2.CreateNatGatewayInput{ + out, err := svc.CreateNatGateway(t.Context(), &ec2.CreateNatGatewayInput{ SubnetId: aws.String("subnet-pub1"), AllocationId: aws.String("eipalloc-test1"), }, testAccountID) @@ -72,7 +112,7 @@ func TestCreateNatGateway(t *testing.T) { // makes the tag-filtered describe return empty, leaking the NAT-GW EIP. func TestCreateNatGateway_PersistsTagsForTagFilterDiscovery(t *testing.T) { svc := setupTestService(t) - _, err := svc.CreateNatGateway(context.Background(), &ec2.CreateNatGatewayInput{ + _, err := svc.CreateNatGateway(t.Context(), &ec2.CreateNatGatewayInput{ SubnetId: aws.String("subnet-pub1"), AllocationId: aws.String("eipalloc-test1"), TagSpecifications: []*ec2.TagSpecification{{ @@ -85,7 +125,7 @@ func TestCreateNatGateway_PersistsTagsForTagFilterDiscovery(t *testing.T) { }, testAccountID) require.NoError(t, err) - out, err := svc.DescribeNatGateways(context.Background(), &ec2.DescribeNatGatewaysInput{ + out, err := svc.DescribeNatGateways(t.Context(), &ec2.DescribeNatGatewaysInput{ Filter: []*ec2.Filter{ {Name: aws.String("tag:spinifex:eks-cluster"), Values: aws.StringSlice([]string{"alpha"})}, {Name: aws.String("tag:spinifex:eks-role"), Values: aws.StringSlice([]string{"cp-natgw"})}, @@ -100,7 +140,7 @@ func TestCreateNatGateway_PersistsTagsForTagFilterDiscovery(t *testing.T) { func TestCreateNatGateway_SubnetNotFound(t *testing.T) { svc := setupTestService(t) - _, err := svc.CreateNatGateway(context.Background(), &ec2.CreateNatGatewayInput{ + _, err := svc.CreateNatGateway(t.Context(), &ec2.CreateNatGatewayInput{ SubnetId: aws.String("subnet-nope"), AllocationId: aws.String("eipalloc-test1"), }, testAccountID) @@ -109,7 +149,7 @@ func TestCreateNatGateway_SubnetNotFound(t *testing.T) { func TestCreateNatGateway_EIPNotFound(t *testing.T) { svc := setupTestService(t) - _, err := svc.CreateNatGateway(context.Background(), &ec2.CreateNatGatewayInput{ + _, err := svc.CreateNatGateway(t.Context(), &ec2.CreateNatGatewayInput{ SubnetId: aws.String("subnet-pub1"), AllocationId: aws.String("eipalloc-nope"), }, testAccountID) @@ -118,7 +158,7 @@ func TestCreateNatGateway_EIPNotFound(t *testing.T) { func TestCreateNatGateway_EIPAlreadyAssociated(t *testing.T) { svc := setupTestService(t) - _, err := svc.CreateNatGateway(context.Background(), &ec2.CreateNatGatewayInput{ + _, err := svc.CreateNatGateway(t.Context(), &ec2.CreateNatGatewayInput{ SubnetId: aws.String("subnet-pub1"), AllocationId: aws.String("eipalloc-used"), }, testAccountID) @@ -127,14 +167,14 @@ func TestCreateNatGateway_EIPAlreadyAssociated(t *testing.T) { func TestDeleteNatGateway(t *testing.T) { svc := setupTestService(t) - createOut, err := svc.CreateNatGateway(context.Background(), &ec2.CreateNatGatewayInput{ + createOut, err := svc.CreateNatGateway(t.Context(), &ec2.CreateNatGatewayInput{ SubnetId: aws.String("subnet-pub1"), AllocationId: aws.String("eipalloc-test1"), }, testAccountID) require.NoError(t, err) natgwID := *createOut.NatGateway.NatGatewayId - deleteOut, err := svc.DeleteNatGateway(context.Background(), &ec2.DeleteNatGatewayInput{ + deleteOut, err := svc.DeleteNatGateway(t.Context(), &ec2.DeleteNatGatewayInput{ NatGatewayId: aws.String(natgwID), }, testAccountID) require.NoError(t, err) @@ -147,13 +187,13 @@ func TestDeleteNatGateway(t *testing.T) { func TestDescribeNatGateways(t *testing.T) { svc := setupTestService(t) - createOut, err := svc.CreateNatGateway(context.Background(), &ec2.CreateNatGatewayInput{ + createOut, err := svc.CreateNatGateway(t.Context(), &ec2.CreateNatGatewayInput{ SubnetId: aws.String("subnet-pub1"), AllocationId: aws.String("eipalloc-test1"), }, testAccountID) require.NoError(t, err) - out, err := svc.DescribeNatGateways(context.Background(), &ec2.DescribeNatGatewaysInput{}, testAccountID) + out, err := svc.DescribeNatGateways(t.Context(), &ec2.DescribeNatGatewaysInput{}, testAccountID) require.NoError(t, err) assert.Len(t, out.NatGateways, 1) assert.Equal(t, *createOut.NatGateway.NatGatewayId, *out.NatGateways[0].NatGatewayId) @@ -161,21 +201,21 @@ func TestDescribeNatGateways(t *testing.T) { func TestDescribeNatGateways_ByID(t *testing.T) { svc := setupTestService(t) - createOut, err := svc.CreateNatGateway(context.Background(), &ec2.CreateNatGatewayInput{ + createOut, err := svc.CreateNatGateway(t.Context(), &ec2.CreateNatGatewayInput{ SubnetId: aws.String("subnet-pub1"), AllocationId: aws.String("eipalloc-test1"), }, testAccountID) require.NoError(t, err) natgwID := *createOut.NatGateway.NatGatewayId - out, err := svc.DescribeNatGateways(context.Background(), &ec2.DescribeNatGatewaysInput{ + out, err := svc.DescribeNatGateways(t.Context(), &ec2.DescribeNatGatewaysInput{ NatGatewayIds: []*string{aws.String(natgwID)}, }, testAccountID) require.NoError(t, err) assert.Len(t, out.NatGateways, 1) // Non-existent ID - _, err = svc.DescribeNatGateways(context.Background(), &ec2.DescribeNatGatewaysInput{ + _, err = svc.DescribeNatGateways(t.Context(), &ec2.DescribeNatGatewaysInput{ NatGatewayIds: []*string{aws.String("nat-nope")}, }, testAccountID) assert.EqualError(t, err, awserrors.ErrorInvalidNatGatewayIDNotFound) @@ -183,7 +223,7 @@ func TestDescribeNatGateways_ByID(t *testing.T) { func TestDescribeNatGateways_FilterByState(t *testing.T) { svc := setupTestService(t) - _, err := svc.CreateNatGateway(context.Background(), &ec2.CreateNatGatewayInput{ + _, err := svc.CreateNatGateway(t.Context(), &ec2.CreateNatGatewayInput{ SubnetId: aws.String("subnet-pub1"), AllocationId: aws.String("eipalloc-test1"), }, testAccountID) @@ -191,14 +231,14 @@ func TestDescribeNatGateways_FilterByState(t *testing.T) { name := "state" val := "available" - out, err := svc.DescribeNatGateways(context.Background(), &ec2.DescribeNatGatewaysInput{ + out, err := svc.DescribeNatGateways(t.Context(), &ec2.DescribeNatGatewaysInput{ Filter: []*ec2.Filter{{Name: &name, Values: []*string{&val}}}, }, testAccountID) require.NoError(t, err) assert.Len(t, out.NatGateways, 1) val2 := "deleted" - out, err = svc.DescribeNatGateways(context.Background(), &ec2.DescribeNatGatewaysInput{ + out, err = svc.DescribeNatGateways(t.Context(), &ec2.DescribeNatGatewaysInput{ Filter: []*ec2.Filter{{Name: &name, Values: []*string{&val2}}}, }, testAccountID) require.NoError(t, err) @@ -207,7 +247,7 @@ func TestDescribeNatGateways_FilterByState(t *testing.T) { func TestGetNatGateway(t *testing.T) { svc := setupTestService(t) - createOut, err := svc.CreateNatGateway(context.Background(), &ec2.CreateNatGatewayInput{ + createOut, err := svc.CreateNatGateway(t.Context(), &ec2.CreateNatGatewayInput{ SubnetId: aws.String("subnet-pub1"), AllocationId: aws.String("eipalloc-test1"), }, testAccountID) @@ -230,12 +270,12 @@ func TestGetNatGateway_NotFound(t *testing.T) { func TestCreateNatGateway_MissingParams(t *testing.T) { svc := setupTestService(t) - _, err := svc.CreateNatGateway(context.Background(), &ec2.CreateNatGatewayInput{ + _, err := svc.CreateNatGateway(t.Context(), &ec2.CreateNatGatewayInput{ AllocationId: aws.String("eipalloc-test1"), }, testAccountID) assert.EqualError(t, err, awserrors.ErrorMissingParameter) - _, err = svc.CreateNatGateway(context.Background(), &ec2.CreateNatGatewayInput{ + _, err = svc.CreateNatGateway(t.Context(), &ec2.CreateNatGatewayInput{ SubnetId: aws.String("subnet-pub1"), }, testAccountID) assert.EqualError(t, err, awserrors.ErrorMissingParameter) @@ -243,20 +283,20 @@ func TestCreateNatGateway_MissingParams(t *testing.T) { func TestDescribeNatGateways_DeletedGatewayVisible(t *testing.T) { svc := setupTestService(t) - createOut, err := svc.CreateNatGateway(context.Background(), &ec2.CreateNatGatewayInput{ + createOut, err := svc.CreateNatGateway(t.Context(), &ec2.CreateNatGatewayInput{ SubnetId: aws.String("subnet-pub1"), AllocationId: aws.String("eipalloc-test1"), }, testAccountID) require.NoError(t, err) natgwID := *createOut.NatGateway.NatGatewayId - _, err = svc.DeleteNatGateway(context.Background(), &ec2.DeleteNatGatewayInput{ + _, err = svc.DeleteNatGateway(t.Context(), &ec2.DeleteNatGatewayInput{ NatGatewayId: aws.String(natgwID), }, testAccountID) require.NoError(t, err) // Describe by ID should return the deleted gateway with state=deleted - out, err := svc.DescribeNatGateways(context.Background(), &ec2.DescribeNatGatewaysInput{ + out, err := svc.DescribeNatGateways(t.Context(), &ec2.DescribeNatGatewaysInput{ NatGatewayIds: []*string{aws.String(natgwID)}, }, testAccountID) require.NoError(t, err) @@ -264,12 +304,12 @@ func TestDescribeNatGateways_DeletedGatewayVisible(t *testing.T) { assert.Equal(t, "deleted", *out.NatGateways[0].State) // Describe without filter should NOT include deleted gateways - out, err = svc.DescribeNatGateways(context.Background(), &ec2.DescribeNatGatewaysInput{}, testAccountID) + out, err = svc.DescribeNatGateways(t.Context(), &ec2.DescribeNatGatewaysInput{}, testAccountID) require.NoError(t, err) assert.Empty(t, out.NatGateways) // Non-existent ID should still error - _, err = svc.DescribeNatGateways(context.Background(), &ec2.DescribeNatGatewaysInput{ + _, err = svc.DescribeNatGateways(t.Context(), &ec2.DescribeNatGatewaysInput{ NatGatewayIds: []*string{aws.String("nat-never-existed")}, }, testAccountID) assert.EqualError(t, err, awserrors.ErrorInvalidNatGatewayIDNotFound) @@ -277,7 +317,7 @@ func TestDescribeNatGateways_DeletedGatewayVisible(t *testing.T) { func TestDescribeNatGateways_FilterByNatGatewayId(t *testing.T) { svc := setupTestService(t) - createOut, err := svc.CreateNatGateway(context.Background(), &ec2.CreateNatGatewayInput{ + createOut, err := svc.CreateNatGateway(t.Context(), &ec2.CreateNatGatewayInput{ SubnetId: aws.String("subnet-pub1"), AllocationId: aws.String("eipalloc-test1"), }, testAccountID) @@ -285,7 +325,7 @@ func TestDescribeNatGateways_FilterByNatGatewayId(t *testing.T) { natgwID := *createOut.NatGateway.NatGatewayId // Exact match - out, err := svc.DescribeNatGateways(context.Background(), &ec2.DescribeNatGatewaysInput{ + out, err := svc.DescribeNatGateways(t.Context(), &ec2.DescribeNatGatewaysInput{ Filter: []*ec2.Filter{{Name: aws.String("nat-gateway-id"), Values: []*string{aws.String(natgwID)}}}, }, testAccountID) require.NoError(t, err) @@ -293,14 +333,14 @@ func TestDescribeNatGateways_FilterByNatGatewayId(t *testing.T) { assert.Equal(t, natgwID, *out.NatGateways[0].NatGatewayId) // Non-match - out, err = svc.DescribeNatGateways(context.Background(), &ec2.DescribeNatGatewaysInput{ + out, err = svc.DescribeNatGateways(t.Context(), &ec2.DescribeNatGatewaysInput{ Filter: []*ec2.Filter{{Name: aws.String("nat-gateway-id"), Values: []*string{aws.String("nat-000000")}}}, }, testAccountID) require.NoError(t, err) assert.Empty(t, out.NatGateways) // Wildcard - out, err = svc.DescribeNatGateways(context.Background(), &ec2.DescribeNatGatewaysInput{ + out, err = svc.DescribeNatGateways(t.Context(), &ec2.DescribeNatGatewaysInput{ Filter: []*ec2.Filter{{Name: aws.String("nat-gateway-id"), Values: []*string{aws.String("nat-*")}}}, }, testAccountID) require.NoError(t, err) @@ -309,21 +349,21 @@ func TestDescribeNatGateways_FilterByNatGatewayId(t *testing.T) { func TestDescribeNatGateways_FilterBySubnetId(t *testing.T) { svc := setupTestService(t) - _, err := svc.CreateNatGateway(context.Background(), &ec2.CreateNatGatewayInput{ + _, err := svc.CreateNatGateway(t.Context(), &ec2.CreateNatGatewayInput{ SubnetId: aws.String("subnet-pub1"), AllocationId: aws.String("eipalloc-test1"), }, testAccountID) require.NoError(t, err) // Exact match - out, err := svc.DescribeNatGateways(context.Background(), &ec2.DescribeNatGatewaysInput{ + out, err := svc.DescribeNatGateways(t.Context(), &ec2.DescribeNatGatewaysInput{ Filter: []*ec2.Filter{{Name: aws.String("subnet-id"), Values: []*string{aws.String("subnet-pub1")}}}, }, testAccountID) require.NoError(t, err) assert.Len(t, out.NatGateways, 1) // Non-match - out, err = svc.DescribeNatGateways(context.Background(), &ec2.DescribeNatGatewaysInput{ + out, err = svc.DescribeNatGateways(t.Context(), &ec2.DescribeNatGatewaysInput{ Filter: []*ec2.Filter{{Name: aws.String("subnet-id"), Values: []*string{aws.String("subnet-000000")}}}, }, testAccountID) require.NoError(t, err) @@ -332,7 +372,7 @@ func TestDescribeNatGateways_FilterBySubnetId(t *testing.T) { func TestDescribeNatGateways_FilterByVpcId(t *testing.T) { svc := setupTestService(t) - _, err := svc.CreateNatGateway(context.Background(), &ec2.CreateNatGatewayInput{ + _, err := svc.CreateNatGateway(t.Context(), &ec2.CreateNatGatewayInput{ SubnetId: aws.String("subnet-pub1"), AllocationId: aws.String("eipalloc-test1"), }, testAccountID) @@ -340,14 +380,14 @@ func TestDescribeNatGateways_FilterByVpcId(t *testing.T) { name := "vpc-id" val := "vpc-test1" - out, err := svc.DescribeNatGateways(context.Background(), &ec2.DescribeNatGatewaysInput{ + out, err := svc.DescribeNatGateways(t.Context(), &ec2.DescribeNatGatewaysInput{ Filter: []*ec2.Filter{{Name: &name, Values: []*string{&val}}}, }, testAccountID) require.NoError(t, err) assert.Len(t, out.NatGateways, 1) val2 := "vpc-nope" - out, err = svc.DescribeNatGateways(context.Background(), &ec2.DescribeNatGatewaysInput{ + out, err = svc.DescribeNatGateways(t.Context(), &ec2.DescribeNatGatewaysInput{ Filter: []*ec2.Filter{{Name: &name, Values: []*string{&val2}}}, }, testAccountID) require.NoError(t, err) diff --git a/spinifex/handlers/ec2/placementgroup/service_impl.go b/spinifex/handlers/ec2/placementgroup/service_impl.go index c231c2085..8582a79c1 100644 --- a/spinifex/handlers/ec2/placementgroup/service_impl.go +++ b/spinifex/handlers/ec2/placementgroup/service_impl.go @@ -14,9 +14,11 @@ import ( "github.com/mulgadc/spinifex/spinifex/awserrors" "github.com/mulgadc/spinifex/spinifex/config" "github.com/mulgadc/spinifex/spinifex/filterutil" + "github.com/mulgadc/spinifex/spinifex/kvutil" "github.com/mulgadc/spinifex/spinifex/migrate" "github.com/mulgadc/spinifex/spinifex/utils" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // Ensure PlacementGroupServiceImpl implements PlacementGroupService. @@ -46,21 +48,21 @@ type PlacementGroupRecord struct { type PlacementGroupServiceImpl struct { config *config.Config natsConn *nats.Conn - kv nats.KeyValue + kv jetstream.KeyValue } // NewPlacementGroupServiceImplWithNATS creates a placement group service with NATS JetStream. -func NewPlacementGroupServiceImplWithNATS(cfg *config.Config, natsConn *nats.Conn) (*PlacementGroupServiceImpl, error) { - js, err := natsConn.JetStream() +func NewPlacementGroupServiceImplWithNATS(ctx context.Context, cfg *config.Config, natsConn *nats.Conn) (*PlacementGroupServiceImpl, error) { + js, err := jetstream.New(natsConn) if err != nil { return nil, fmt.Errorf("failed to get JetStream context: %w", err) } - kv, err := utils.GetOrCreateKVBucket(js, KVBucketPlacementGroups, 10) + kv, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketPlacementGroups, 10) if err != nil { return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketPlacementGroups, err) } - if err := migrate.DefaultRegistry.RunKV(KVBucketPlacementGroups, kv, KVBucketPlacementGroupsVersion); err != nil { + if err := migrate.DefaultRegistry.RunKV(ctx, KVBucketPlacementGroups, kv, KVBucketPlacementGroupsVersion); err != nil { return nil, fmt.Errorf("migrate %s: %w", KVBucketPlacementGroups, err) } @@ -113,7 +115,7 @@ func (s *PlacementGroupServiceImpl) CreatePlacementGroup(ctx context.Context, in return nil, errors.New(awserrors.ErrorServerInternal) } // Atomic create-if-not-exists to prevent TOCTOU race on duplicate names - if _, err := s.kv.Create(key, data); err != nil { + if _, err := s.kv.Create(ctx, key, data); err != nil { // Create fails if key already exists return nil, errors.New(awserrors.ErrorInvalidPlacementGroupDuplicate) } @@ -134,7 +136,7 @@ func (s *PlacementGroupServiceImpl) DeletePlacementGroup(ctx context.Context, in groupName := *input.GroupName key := utils.AccountKey(accountID, groupName) - entry, err := s.kv.Get(key) + entry, err := s.kv.Get(ctx, key) if err != nil { return nil, errors.New(awserrors.ErrorInvalidPlacementGroupUnknown) } @@ -153,7 +155,7 @@ func (s *PlacementGroupServiceImpl) DeletePlacementGroup(ctx context.Context, in return nil, errors.New(awserrors.ErrorInvalidPlacementGroupInUse) } - if err := s.kv.Delete(key); err != nil { + if err := s.kv.Delete(ctx, key); err != nil { return nil, errors.New(awserrors.ErrorServerInternal) } @@ -195,8 +197,8 @@ func (s *PlacementGroupServiceImpl) DescribePlacementGroups(ctx context.Context, } prefix := accountID + "." - keys, err := s.kv.Keys() - if err != nil && !errors.Is(err, nats.ErrNoKeysFound) { + keys, err := s.kv.Keys(ctx) + if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) { return nil, errors.New(awserrors.ErrorServerInternal) } @@ -209,7 +211,7 @@ func (s *PlacementGroupServiceImpl) DescribePlacementGroups(ctx context.Context, continue } - entry, err := s.kv.Get(k) + entry, err := s.kv.Get(ctx, k) if err != nil { slog.WarnContext(ctx, "Failed to get placement group record", "key", k, "error", err) continue @@ -313,9 +315,9 @@ func pgMatchesAnyTag(tags map[string]string, values []string, field func(k, v st // GetPlacementGroupRecord reads a placement group record from KV with its revision for CAS operations. // Returns the record and the KV entry (for revision). Exported for use by gateway spread routing. -func (s *PlacementGroupServiceImpl) GetPlacementGroupRecord(accountID, groupName string) (*PlacementGroupRecord, nats.KeyValueEntry, error) { +func (s *PlacementGroupServiceImpl) GetPlacementGroupRecord(ctx context.Context, accountID, groupName string) (*PlacementGroupRecord, jetstream.KeyValueEntry, error) { key := utils.AccountKey(accountID, groupName) - entry, err := s.kv.Get(key) + entry, err := s.kv.Get(ctx, key) if err != nil { return nil, nil, errors.New(awserrors.ErrorInvalidPlacementGroupUnknown) } @@ -330,13 +332,13 @@ func (s *PlacementGroupServiceImpl) GetPlacementGroupRecord(accountID, groupName // UpdatePlacementGroupRecord writes a placement group record using CAS (optimistic concurrency). // Returns nil on success or the error on CAS conflict. -func (s *PlacementGroupServiceImpl) UpdatePlacementGroupRecord(accountID, groupName string, record *PlacementGroupRecord, revision uint64) error { +func (s *PlacementGroupServiceImpl) UpdatePlacementGroupRecord(ctx context.Context, accountID, groupName string, record *PlacementGroupRecord, revision uint64) error { key := utils.AccountKey(accountID, groupName) data, err := json.Marshal(record) if err != nil { return errors.New(awserrors.ErrorServerInternal) } - if _, err := s.kv.Update(key, data, revision); err != nil { + if _, err := s.kv.Update(ctx, key, data, revision); err != nil { return err } return nil @@ -352,7 +354,7 @@ func (s *PlacementGroupServiceImpl) ReserveSpreadNodes(ctx context.Context, inpu } for attempt := range maxCASRetries { - record, entry, err := s.GetPlacementGroupRecord(accountID, input.GroupName) + record, entry, err := s.GetPlacementGroupRecord(ctx, accountID, input.GroupName) if err != nil { return nil, err } @@ -392,7 +394,7 @@ func (s *PlacementGroupServiceImpl) ReserveSpreadNodes(ctx context.Context, inpu } // CAS write — retry on conflict - if err := s.UpdatePlacementGroupRecord(accountID, input.GroupName, record, entry.Revision()); err != nil { + if err := s.UpdatePlacementGroupRecord(ctx, accountID, input.GroupName, record, entry.Revision()); err != nil { slog.DebugContext(ctx, "ReserveSpreadNodes: CAS conflict, retrying", "attempt", attempt, "err", err) continue } @@ -413,7 +415,7 @@ func (s *PlacementGroupServiceImpl) FinalizeSpreadInstances(ctx context.Context, } for attempt := range maxCASRetries { - record, entry, err := s.GetPlacementGroupRecord(accountID, input.GroupName) + record, entry, err := s.GetPlacementGroupRecord(ctx, accountID, input.GroupName) if err != nil { return nil, err } @@ -421,7 +423,7 @@ func (s *PlacementGroupServiceImpl) FinalizeSpreadInstances(ctx context.Context, // Replace placeholder entries with actual instance IDs per node maps.Copy(record.NodeInstances, input.NodeInstances) - if err := s.UpdatePlacementGroupRecord(accountID, input.GroupName, record, entry.Revision()); err != nil { + if err := s.UpdatePlacementGroupRecord(ctx, accountID, input.GroupName, record, entry.Revision()); err != nil { slog.DebugContext(ctx, "FinalizeSpreadInstances: CAS conflict, retrying", "attempt", attempt, "err", err) continue } @@ -447,7 +449,7 @@ func (s *PlacementGroupServiceImpl) ReleaseSpreadNodes(ctx context.Context, inpu } for attempt := range maxCASRetries { - record, entry, err := s.GetPlacementGroupRecord(accountID, input.GroupName) + record, entry, err := s.GetPlacementGroupRecord(ctx, accountID, input.GroupName) if err != nil { return nil, err } @@ -460,7 +462,7 @@ func (s *PlacementGroupServiceImpl) ReleaseSpreadNodes(ctx context.Context, inpu } } - if err := s.UpdatePlacementGroupRecord(accountID, input.GroupName, record, entry.Revision()); err != nil { + if err := s.UpdatePlacementGroupRecord(ctx, accountID, input.GroupName, record, entry.Revision()); err != nil { slog.DebugContext(ctx, "ReleaseSpreadNodes: CAS conflict, retrying", "attempt", attempt, "err", err) continue } @@ -482,7 +484,7 @@ func (s *PlacementGroupServiceImpl) RemoveInstance(ctx context.Context, input *R } for attempt := range maxCASRetries { - record, entry, lookupErr := s.GetPlacementGroupRecord(accountID, input.GroupName) + record, entry, lookupErr := s.GetPlacementGroupRecord(ctx, accountID, input.GroupName) if lookupErr != nil { // Group may have been deleted already — treat as success slog.DebugContext(ctx, "RemoveInstance: group not found, treating as success", "groupName", input.GroupName) @@ -509,7 +511,7 @@ func (s *PlacementGroupServiceImpl) RemoveInstance(ctx context.Context, input *R record.NodeInstances[input.NodeName] = filtered } - if err := s.UpdatePlacementGroupRecord(accountID, input.GroupName, record, entry.Revision()); err != nil { + if err := s.UpdatePlacementGroupRecord(ctx, accountID, input.GroupName, record, entry.Revision()); err != nil { slog.DebugContext(ctx, "RemoveInstance: CAS conflict, retrying", "attempt", attempt, "err", err) continue } @@ -531,7 +533,7 @@ func (s *PlacementGroupServiceImpl) ReserveClusterNode(ctx context.Context, inpu } for attempt := range maxCASRetries { - record, entry, err := s.GetPlacementGroupRecord(accountID, input.GroupName) + record, entry, err := s.GetPlacementGroupRecord(ctx, accountID, input.GroupName) if err != nil { return nil, err } @@ -555,7 +557,7 @@ func (s *PlacementGroupServiceImpl) ReserveClusterNode(ctx context.Context, inpu targetNode := input.EligibleNodes[0] // first = highest capacity (sorted desc by caller) record.NodeInstances[targetNode] = []string{} - if err := s.UpdatePlacementGroupRecord(accountID, input.GroupName, record, entry.Revision()); err != nil { + if err := s.UpdatePlacementGroupRecord(ctx, accountID, input.GroupName, record, entry.Revision()); err != nil { slog.DebugContext(ctx, "ReserveClusterNode: CAS conflict, retrying", "attempt", attempt, "err", err) continue } @@ -576,7 +578,7 @@ func (s *PlacementGroupServiceImpl) FinalizeClusterInstances(ctx context.Context } for attempt := range maxCASRetries { - record, entry, err := s.GetPlacementGroupRecord(accountID, input.GroupName) + record, entry, err := s.GetPlacementGroupRecord(ctx, accountID, input.GroupName) if err != nil { return nil, err } @@ -586,7 +588,7 @@ func (s *PlacementGroupServiceImpl) FinalizeClusterInstances(ctx context.Context record.NodeInstances[node] = append(record.NodeInstances[node], ids...) } - if err := s.UpdatePlacementGroupRecord(accountID, input.GroupName, record, entry.Revision()); err != nil { + if err := s.UpdatePlacementGroupRecord(ctx, accountID, input.GroupName, record, entry.Revision()); err != nil { slog.DebugContext(ctx, "FinalizeClusterInstances: CAS conflict, retrying", "attempt", attempt, "err", err) continue } diff --git a/spinifex/handlers/ec2/placementgroup/service_impl_test.go b/spinifex/handlers/ec2/placementgroup/service_impl_test.go index 083071809..0b93bf1f6 100644 --- a/spinifex/handlers/ec2/placementgroup/service_impl_test.go +++ b/spinifex/handlers/ec2/placementgroup/service_impl_test.go @@ -18,7 +18,7 @@ func setupTestService(t *testing.T) *PlacementGroupServiceImpl { t.Helper() _, nc, _ := testutil.StartTestJetStream(t) - svc, err := NewPlacementGroupServiceImplWithNATS(nil, nc) + svc, err := NewPlacementGroupServiceImplWithNATS(t.Context(), nil, nc) require.NoError(t, err) return svc } @@ -162,10 +162,10 @@ func TestDeletePlacementGroup_InUse(t *testing.T) { createTestGroup(t, svc, "in-use-group", "spread") // Simulate instances by updating the record directly - record, entry, err := svc.GetPlacementGroupRecord(testAccountID, "in-use-group") + record, entry, err := svc.GetPlacementGroupRecord(t.Context(), testAccountID, "in-use-group") require.NoError(t, err) record.NodeInstances["node1"] = []string{"i-123"} - err = svc.UpdatePlacementGroupRecord(testAccountID, "in-use-group", record, entry.Revision()) + err = svc.UpdatePlacementGroupRecord(t.Context(), testAccountID, "in-use-group", record, entry.Revision()) require.NoError(t, err) _, err = svc.DeletePlacementGroup(context.Background(), &ec2.DeletePlacementGroupInput{ @@ -315,18 +315,18 @@ func TestGetAndUpdatePlacementGroupRecord(t *testing.T) { createTestGroup(t, svc, "cas-group", "spread") // Get with revision - record, entry, err := svc.GetPlacementGroupRecord(testAccountID, "cas-group") + record, entry, err := svc.GetPlacementGroupRecord(t.Context(), testAccountID, "cas-group") require.NoError(t, err) assert.Equal(t, "spread", record.Strategy) assert.Empty(t, record.NodeInstances) // Update with CAS record.NodeInstances["node1"] = []string{"i-abc"} - err = svc.UpdatePlacementGroupRecord(testAccountID, "cas-group", record, entry.Revision()) + err = svc.UpdatePlacementGroupRecord(t.Context(), testAccountID, "cas-group", record, entry.Revision()) require.NoError(t, err) // Verify update - record2, _, err := svc.GetPlacementGroupRecord(testAccountID, "cas-group") + record2, _, err := svc.GetPlacementGroupRecord(t.Context(), testAccountID, "cas-group") require.NoError(t, err) assert.Equal(t, []string{"i-abc"}, record2.NodeInstances["node1"]) } @@ -336,25 +336,25 @@ func TestUpdatePlacementGroupRecord_CASConflict(t *testing.T) { createTestGroup(t, svc, "conflict-group", "spread") // Get the record twice (same revision) - record1, entry1, err := svc.GetPlacementGroupRecord(testAccountID, "conflict-group") + record1, entry1, err := svc.GetPlacementGroupRecord(t.Context(), testAccountID, "conflict-group") require.NoError(t, err) - record2, _, err := svc.GetPlacementGroupRecord(testAccountID, "conflict-group") + record2, _, err := svc.GetPlacementGroupRecord(t.Context(), testAccountID, "conflict-group") require.NoError(t, err) // First update succeeds record1.NodeInstances["node1"] = []string{"i-111"} - err = svc.UpdatePlacementGroupRecord(testAccountID, "conflict-group", record1, entry1.Revision()) + err = svc.UpdatePlacementGroupRecord(t.Context(), testAccountID, "conflict-group", record1, entry1.Revision()) require.NoError(t, err) // Second update with stale revision fails record2.NodeInstances["node2"] = []string{"i-222"} - err = svc.UpdatePlacementGroupRecord(testAccountID, "conflict-group", record2, entry1.Revision()) + err = svc.UpdatePlacementGroupRecord(t.Context(), testAccountID, "conflict-group", record2, entry1.Revision()) require.Error(t, err) } func TestGetPlacementGroupRecord_NotFound(t *testing.T) { svc := setupTestService(t) - _, _, err := svc.GetPlacementGroupRecord(testAccountID, "nonexistent") + _, _, err := svc.GetPlacementGroupRecord(t.Context(), testAccountID, "nonexistent") require.Error(t, err) assert.Equal(t, awserrors.ErrorInvalidPlacementGroupUnknown, err.Error()) } @@ -375,7 +375,7 @@ func TestReserveSpreadNodes_Success(t *testing.T) { assert.Len(t, out.ReservedNodes, 3) // Verify placeholders in record - record, _, err := svc.GetPlacementGroupRecord(testAccountID, "reserve-group") + record, _, err := svc.GetPlacementGroupRecord(t.Context(), testAccountID, "reserve-group") require.NoError(t, err) assert.Len(t, record.NodeInstances, 3) for _, node := range out.ReservedNodes { @@ -389,10 +389,10 @@ func TestReserveSpreadNodes_ExcludesOccupiedNodes(t *testing.T) { createTestGroup(t, svc, "occupied-group", "spread") // Pre-occupy node-a - record, entry, err := svc.GetPlacementGroupRecord(testAccountID, "occupied-group") + record, entry, err := svc.GetPlacementGroupRecord(t.Context(), testAccountID, "occupied-group") require.NoError(t, err) record.NodeInstances["node-a"] = []string{"i-existing"} - require.NoError(t, svc.UpdatePlacementGroupRecord(testAccountID, "occupied-group", record, entry.Revision())) + require.NoError(t, svc.UpdatePlacementGroupRecord(t.Context(), testAccountID, "occupied-group", record, entry.Revision())) out, err := svc.ReserveSpreadNodes(context.Background(), &ReserveSpreadNodesInput{ GroupName: "occupied-group", @@ -413,11 +413,11 @@ func TestReserveSpreadNodes_InsufficientNodes(t *testing.T) { createTestGroup(t, svc, "insufficient-group", "spread") // Pre-occupy node-a and node-b - record, entry, err := svc.GetPlacementGroupRecord(testAccountID, "insufficient-group") + record, entry, err := svc.GetPlacementGroupRecord(t.Context(), testAccountID, "insufficient-group") require.NoError(t, err) record.NodeInstances["node-a"] = []string{"i-1"} record.NodeInstances["node-b"] = []string{"i-2"} - require.NoError(t, svc.UpdatePlacementGroupRecord(testAccountID, "insufficient-group", record, entry.Revision())) + require.NoError(t, svc.UpdatePlacementGroupRecord(t.Context(), testAccountID, "insufficient-group", record, entry.Revision())) _, err = svc.ReserveSpreadNodes(context.Background(), &ReserveSpreadNodesInput{ GroupName: "insufficient-group", @@ -481,7 +481,7 @@ func TestFinalizeSpreadInstances_Success(t *testing.T) { require.NoError(t, err) // Verify - record, _, err := svc.GetPlacementGroupRecord(testAccountID, "finalize-group") + record, _, err := svc.GetPlacementGroupRecord(t.Context(), testAccountID, "finalize-group") require.NoError(t, err) assert.Equal(t, []string{"i-aaa"}, record.NodeInstances["node-a"]) assert.Equal(t, []string{"i-bbb"}, record.NodeInstances["node-b"]) @@ -510,7 +510,7 @@ func TestReleaseSpreadNodes_Success(t *testing.T) { require.NoError(t, err) // Verify only node-a remains - record, _, err := svc.GetPlacementGroupRecord(testAccountID, "release-group") + record, _, err := svc.GetPlacementGroupRecord(t.Context(), testAccountID, "release-group") require.NoError(t, err) assert.Len(t, record.NodeInstances, 1) _, ok := record.NodeInstances["node-a"] @@ -538,7 +538,7 @@ func TestReleaseSpreadNodes_AllNodes(t *testing.T) { require.NoError(t, err) // Verify empty - record, _, err := svc.GetPlacementGroupRecord(testAccountID, "release-all-group") + record, _, err := svc.GetPlacementGroupRecord(t.Context(), testAccountID, "release-all-group") require.NoError(t, err) assert.Empty(t, record.NodeInstances) } @@ -550,11 +550,11 @@ func TestRemoveInstance_Success(t *testing.T) { createTestGroup(t, svc, "remove-inst-group", "spread") // Add instances to two nodes - record, entry, err := svc.GetPlacementGroupRecord(testAccountID, "remove-inst-group") + record, entry, err := svc.GetPlacementGroupRecord(t.Context(), testAccountID, "remove-inst-group") require.NoError(t, err) record.NodeInstances["node-a"] = []string{"i-aaa"} record.NodeInstances["node-b"] = []string{"i-bbb"} - require.NoError(t, svc.UpdatePlacementGroupRecord(testAccountID, "remove-inst-group", record, entry.Revision())) + require.NoError(t, svc.UpdatePlacementGroupRecord(t.Context(), testAccountID, "remove-inst-group", record, entry.Revision())) // Remove i-aaa from node-a _, err = svc.RemoveInstance(context.Background(), &RemoveInstanceInput{ @@ -565,7 +565,7 @@ func TestRemoveInstance_Success(t *testing.T) { require.NoError(t, err) // Verify node-a is removed (was the only instance), node-b remains - record, _, err = svc.GetPlacementGroupRecord(testAccountID, "remove-inst-group") + record, _, err = svc.GetPlacementGroupRecord(t.Context(), testAccountID, "remove-inst-group") require.NoError(t, err) assert.Len(t, record.NodeInstances, 1) _, hasA := record.NodeInstances["node-a"] @@ -605,10 +605,10 @@ func TestRemoveInstance_MultipleInstancesOnNode(t *testing.T) { createTestGroup(t, svc, "multi-inst-group", "cluster") // Add multiple instances to same node (cluster scenario) - record, entry, err := svc.GetPlacementGroupRecord(testAccountID, "multi-inst-group") + record, entry, err := svc.GetPlacementGroupRecord(t.Context(), testAccountID, "multi-inst-group") require.NoError(t, err) record.NodeInstances["node-a"] = []string{"i-111", "i-222", "i-333"} - require.NoError(t, svc.UpdatePlacementGroupRecord(testAccountID, "multi-inst-group", record, entry.Revision())) + require.NoError(t, svc.UpdatePlacementGroupRecord(t.Context(), testAccountID, "multi-inst-group", record, entry.Revision())) // Remove i-222 _, err = svc.RemoveInstance(context.Background(), &RemoveInstanceInput{ @@ -619,7 +619,7 @@ func TestRemoveInstance_MultipleInstancesOnNode(t *testing.T) { require.NoError(t, err) // Verify i-111 and i-333 remain - record, _, err = svc.GetPlacementGroupRecord(testAccountID, "multi-inst-group") + record, _, err = svc.GetPlacementGroupRecord(t.Context(), testAccountID, "multi-inst-group") require.NoError(t, err) assert.Equal(t, []string{"i-111", "i-333"}, record.NodeInstances["node-a"]) } @@ -639,7 +639,7 @@ func TestReserveClusterNode_EmptyGroup(t *testing.T) { assert.Equal(t, "node-a", out.TargetNode) // Verify placeholder written - record, _, err := svc.GetPlacementGroupRecord(testAccountID, "cluster-empty") + record, _, err := svc.GetPlacementGroupRecord(t.Context(), testAccountID, "cluster-empty") require.NoError(t, err) assert.Len(t, record.NodeInstances, 1) _, ok := record.NodeInstances["node-a"] @@ -651,10 +651,10 @@ func TestReserveClusterNode_ExistingNode(t *testing.T) { createTestGroup(t, svc, "cluster-existing", "cluster") // Pre-populate with instances on node-b - record, entry, err := svc.GetPlacementGroupRecord(testAccountID, "cluster-existing") + record, entry, err := svc.GetPlacementGroupRecord(t.Context(), testAccountID, "cluster-existing") require.NoError(t, err) record.NodeInstances["node-b"] = []string{"i-existing"} - require.NoError(t, svc.UpdatePlacementGroupRecord(testAccountID, "cluster-existing", record, entry.Revision())) + require.NoError(t, svc.UpdatePlacementGroupRecord(t.Context(), testAccountID, "cluster-existing", record, entry.Revision())) // Even though node-a is in eligible list, should return existing node-b out, err := svc.ReserveClusterNode(context.Background(), &ReserveClusterNodeInput{ @@ -731,7 +731,7 @@ func TestFinalizeClusterInstances_Success(t *testing.T) { require.NoError(t, err) // Verify instances recorded - record, _, err := svc.GetPlacementGroupRecord(testAccountID, "cluster-finalize") + record, _, err := svc.GetPlacementGroupRecord(t.Context(), testAccountID, "cluster-finalize") require.NoError(t, err) assert.Equal(t, []string{"i-c1", "i-c2", "i-c3"}, record.NodeInstances["node-a"]) } @@ -741,10 +741,10 @@ func TestFinalizeClusterInstances_AppendsToExisting(t *testing.T) { createTestGroup(t, svc, "cluster-append", "cluster") // Pre-populate with existing instances - record, entry, err := svc.GetPlacementGroupRecord(testAccountID, "cluster-append") + record, entry, err := svc.GetPlacementGroupRecord(t.Context(), testAccountID, "cluster-append") require.NoError(t, err) record.NodeInstances["node-a"] = []string{"i-existing1", "i-existing2"} - require.NoError(t, svc.UpdatePlacementGroupRecord(testAccountID, "cluster-append", record, entry.Revision())) + require.NoError(t, svc.UpdatePlacementGroupRecord(t.Context(), testAccountID, "cluster-append", record, entry.Revision())) // Finalize with new instances (should append, not overwrite) _, err = svc.FinalizeClusterInstances(context.Background(), &FinalizeClusterInstancesInput{ @@ -756,7 +756,7 @@ func TestFinalizeClusterInstances_AppendsToExisting(t *testing.T) { require.NoError(t, err) // Verify all instances present - record, _, err = svc.GetPlacementGroupRecord(testAccountID, "cluster-append") + record, _, err = svc.GetPlacementGroupRecord(t.Context(), testAccountID, "cluster-append") require.NoError(t, err) assert.Equal(t, []string{"i-existing1", "i-existing2", "i-new1", "i-new2"}, record.NodeInstances["node-a"]) } diff --git a/spinifex/handlers/ec2/routetable/lifecycle_invariants_test.go b/spinifex/handlers/ec2/routetable/lifecycle_invariants_test.go index 7e357099f..ebedcd385 100644 --- a/spinifex/handlers/ec2/routetable/lifecycle_invariants_test.go +++ b/spinifex/handlers/ec2/routetable/lifecycle_invariants_test.go @@ -1,7 +1,6 @@ package handlers_ec2_routetable import ( - "context" "testing" "github.com/aws/aws-sdk-go/aws" @@ -19,7 +18,7 @@ import ( func TestRLC1_RouteTableDeleteNotFoundOnAbsent(t *testing.T) { svc := setupTestService(t) - _, err := svc.DeleteRouteTable(context.Background(), &ec2.DeleteRouteTableInput{ + _, err := svc.DeleteRouteTable(t.Context(), &ec2.DeleteRouteTableInput{ RouteTableId: aws.String("rtb-absent00000000"), }, testAccountID) diff --git a/spinifex/handlers/ec2/routetable/record_tags_test.go b/spinifex/handlers/ec2/routetable/record_tags_test.go index f144737b5..091dad137 100644 --- a/spinifex/handlers/ec2/routetable/record_tags_test.go +++ b/spinifex/handlers/ec2/routetable/record_tags_test.go @@ -1,7 +1,6 @@ package handlers_ec2_routetable import ( - "context" "testing" "github.com/aws/aws-sdk-go/aws" @@ -31,7 +30,7 @@ func TestRouteTableRecordTagsMirror(t *testing.T) { }, }, testAccountID)) - out, err := svc.DescribeRouteTables(context.Background(), &ec2.DescribeRouteTablesInput{ + out, err := svc.DescribeRouteTables(t.Context(), &ec2.DescribeRouteTablesInput{ RouteTableIds: []*string{aws.String(rtbID)}, }, testAccountID) require.NoError(t, err) @@ -48,7 +47,7 @@ func TestRouteTableRecordTagsMirror(t *testing.T) { }, }, testAccountID)) - out, err = svc.DescribeRouteTables(context.Background(), &ec2.DescribeRouteTablesInput{ + out, err = svc.DescribeRouteTables(t.Context(), &ec2.DescribeRouteTablesInput{ RouteTableIds: []*string{aws.String(rtbID)}, }, testAccountID) require.NoError(t, err) diff --git a/spinifex/handlers/ec2/routetable/service_impl.go b/spinifex/handlers/ec2/routetable/service_impl.go index 370ce484e..579734d2a 100644 --- a/spinifex/handlers/ec2/routetable/service_impl.go +++ b/spinifex/handlers/ec2/routetable/service_impl.go @@ -17,9 +17,11 @@ import ( handlers_ec2_igw "github.com/mulgadc/spinifex/spinifex/handlers/ec2/igw" handlers_ec2_natgw "github.com/mulgadc/spinifex/spinifex/handlers/ec2/natgw" handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc" + "github.com/mulgadc/spinifex/spinifex/kvutil" "github.com/mulgadc/spinifex/spinifex/migrate" "github.com/mulgadc/spinifex/spinifex/utils" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // Ensure RouteTableServiceImpl implements RouteTableService. @@ -28,45 +30,45 @@ var _ RouteTableService = (*RouteTableServiceImpl)(nil) // RouteTableServiceImpl implements Route Table operations with NATS JetStream persistence. type RouteTableServiceImpl struct { config *config.Config - rtbKV nats.KeyValue - vpcKV nats.KeyValue - igwKV nats.KeyValue - subnetKV nats.KeyValue - natgwKV nats.KeyValue + rtbKV jetstream.KeyValue + vpcKV jetstream.KeyValue + igwKV jetstream.KeyValue + subnetKV jetstream.KeyValue + natgwKV jetstream.KeyValue natsConn *nats.Conn } // NewRouteTableServiceImplWithNATS creates a Route Table service with NATS JetStream for persistence. -func NewRouteTableServiceImplWithNATS(cfg *config.Config, natsConn *nats.Conn) (*RouteTableServiceImpl, error) { - js, err := natsConn.JetStream() +func NewRouteTableServiceImplWithNATS(ctx context.Context, cfg *config.Config, natsConn *nats.Conn) (*RouteTableServiceImpl, error) { + js, err := jetstream.New(natsConn) if err != nil { return nil, fmt.Errorf("failed to get JetStream context: %w", err) } - rtbKV, err := utils.GetOrCreateKVBucket(js, KVBucketRouteTables, 10) + rtbKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketRouteTables, 10) if err != nil { return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketRouteTables, err) } - if err := migrate.DefaultRegistry.RunKV(KVBucketRouteTables, rtbKV, KVBucketRouteTablesVersion); err != nil { + if err := migrate.DefaultRegistry.RunKV(ctx, KVBucketRouteTables, rtbKV, KVBucketRouteTablesVersion); err != nil { return nil, fmt.Errorf("migrate %s: %w", KVBucketRouteTables, err) } - vpcKV, err := utils.GetOrCreateKVBucket(js, handlers_ec2_vpc.KVBucketVPCs, 10) + vpcKV, err := kvutil.GetOrCreateBucket(ctx, js, handlers_ec2_vpc.KVBucketVPCs, 10) if err != nil { return nil, fmt.Errorf("failed to get VPC KV bucket: %w", err) } - igwKV, err := utils.GetOrCreateKVBucket(js, handlers_ec2_igw.KVBucketIGW, 10) + igwKV, err := kvutil.GetOrCreateBucket(ctx, js, handlers_ec2_igw.KVBucketIGW, 10) if err != nil { return nil, fmt.Errorf("failed to get IGW KV bucket: %w", err) } - subnetKV, err := utils.GetOrCreateKVBucket(js, handlers_ec2_vpc.KVBucketSubnets, 10) + subnetKV, err := kvutil.GetOrCreateBucket(ctx, js, handlers_ec2_vpc.KVBucketSubnets, 10) if err != nil { return nil, fmt.Errorf("failed to get subnet KV bucket: %w", err) } - natgwKV, err := utils.GetOrCreateKVBucket(js, handlers_ec2_natgw.KVBucketNatGateways, 10) + natgwKV, err := kvutil.GetOrCreateBucket(ctx, js, handlers_ec2_natgw.KVBucketNatGateways, 10) if err != nil { return nil, fmt.Errorf("failed to get NAT Gateway KV bucket: %w", err) } @@ -86,9 +88,9 @@ func NewRouteTableServiceImplWithNATS(cfg *config.Config, natsConn *nats.Conn) ( // getRouteTable retrieves a route table record from KV. func (s *RouteTableServiceImpl) getRouteTable(ctx context.Context, accountID, rtbID string) (*RouteTableRecord, error) { - entry, err := s.rtbKV.Get(utils.AccountKey(accountID, rtbID)) + entry, err := s.rtbKV.Get(ctx, utils.AccountKey(accountID, rtbID)) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil, errors.New(awserrors.ErrorInvalidRouteTableIDNotFound) } slog.ErrorContext(ctx, "Failed to read route table from KV", "routeTableId", rtbID, "err", err) @@ -109,7 +111,7 @@ func (s *RouteTableServiceImpl) putRouteTable(ctx context.Context, accountID str slog.ErrorContext(ctx, "Failed to marshal route table record", "routeTableId", record.RouteTableId, "err", err) return errors.New(awserrors.ErrorServerInternal) } - if _, err := s.rtbKV.Put(utils.AccountKey(accountID, record.RouteTableId), data); err != nil { + if _, err := s.rtbKV.Put(ctx, utils.AccountKey(accountID, record.RouteTableId), data); err != nil { slog.ErrorContext(ctx, "Failed to write route table to KV", "routeTableId", record.RouteTableId, "err", err) return errors.New(awserrors.ErrorServerInternal) } @@ -130,9 +132,9 @@ const rtbCASMaxRetries = 16 func (s *RouteTableServiceImpl) mutateRouteTableCAS(ctx context.Context, accountID, rtbID string, mutate func(*RouteTableRecord) (bool, error)) error { key := utils.AccountKey(accountID, rtbID) for range rtbCASMaxRetries { - entry, err := s.rtbKV.Get(key) + entry, err := s.rtbKV.Get(ctx, key) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return errors.New(awserrors.ErrorInvalidRouteTableIDNotFound) } slog.ErrorContext(ctx, "Failed to read route table from KV", "routeTableId", rtbID, "err", err) @@ -158,8 +160,8 @@ func (s *RouteTableServiceImpl) mutateRouteTableCAS(ctx context.Context, account slog.ErrorContext(ctx, "Failed to marshal route table record", "routeTableId", rtbID, "err", err) return errors.New(awserrors.ErrorServerInternal) } - if _, err := s.rtbKV.Update(key, data, entry.Revision()); err != nil { - if errors.Is(err, nats.ErrKeyExists) { + if _, err := s.rtbKV.Update(ctx, key, data, entry.Revision()); err != nil { + if errors.Is(err, jetstream.ErrKeyExists) { continue // CAS conflict — another writer won, re-read and retry. } slog.ErrorContext(ctx, "Failed to write route table to KV", "routeTableId", rtbID, "err", err) @@ -172,7 +174,7 @@ func (s *RouteTableServiceImpl) mutateRouteTableCAS(ctx context.Context, account // getVPCCidr looks up a VPC's CIDR block from the VPC KV bucket. func (s *RouteTableServiceImpl) getVPCCidr(ctx context.Context, accountID, vpcID string) (string, error) { - entry, err := s.vpcKV.Get(utils.AccountKey(accountID, vpcID)) + entry, err := s.vpcKV.Get(ctx, utils.AccountKey(accountID, vpcID)) if err != nil { return "", errors.New(awserrors.ErrorInvalidVpcIDNotFound) } @@ -219,9 +221,9 @@ func (s *RouteTableServiceImpl) subnetsImplicitlyOnMainRT(ctx context.Context, a } prefix := accountID + "." - keys, err := s.subnetKV.Keys() + keys, err := s.subnetKV.Keys(ctx) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return nil, nil } return nil, fmt.Errorf("list subnet keys: %w", err) @@ -231,9 +233,9 @@ func (s *RouteTableServiceImpl) subnetsImplicitlyOnMainRT(ctx context.Context, a if key == utils.VersionKey || !strings.HasPrefix(key, prefix) { continue } - entry, err := s.subnetKV.Get(key) + entry, err := s.subnetKV.Get(ctx, key) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { continue } return nil, fmt.Errorf("read subnet %s: %w", key, err) @@ -362,30 +364,34 @@ func (s *RouteTableServiceImpl) publishSubnetEgressGateDecision(ctx context.Cont // destCidr, resolving each subnet's effective RT. Called on IGW Attach/Detach. // destCidr defaults to 0.0.0.0/0. func (s *RouteTableServiceImpl) PublishGateDecisionsForVPC(accountID, vpcID, destCidr string) { + s.publishGateDecisionsForVPC(context.Background(), accountID, vpcID, destCidr) +} + +func (s *RouteTableServiceImpl) publishGateDecisionsForVPC(ctx context.Context, accountID, vpcID, destCidr string) { if s.natsConn == nil || vpcID == "" { return } if destCidr == "" { destCidr = "0.0.0.0/0" } - subnets, err := s.allSubnetsForVPC(accountID, vpcID) + subnets, err := s.allSubnetsForVPC(ctx, accountID, vpcID) if err != nil { - slog.Warn("PublishGateDecisionsForVPC: enumerate subnets failed", + slog.WarnContext(ctx, "PublishGateDecisionsForVPC: enumerate subnets failed", "vpcId", vpcID, "err", err) return } for _, subnetID := range subnets { - s.publishSubnetEgressGateDecision(context.Background(), accountID, vpcID, subnetID, destCidr) + s.publishSubnetEgressGateDecision(ctx, accountID, vpcID, subnetID, destCidr) } } // allSubnetsForVPC returns every SubnetId in vpcID by scanning the subnet KV. // Order is unspecified. -func (s *RouteTableServiceImpl) allSubnetsForVPC(accountID, vpcID string) ([]string, error) { +func (s *RouteTableServiceImpl) allSubnetsForVPC(ctx context.Context, accountID, vpcID string) ([]string, error) { prefix := accountID + "." - keys, err := s.subnetKV.Keys() + keys, err := s.subnetKV.Keys(ctx) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return nil, nil } return nil, fmt.Errorf("list subnet keys: %w", err) @@ -395,9 +401,9 @@ func (s *RouteTableServiceImpl) allSubnetsForVPC(accountID, vpcID string) ([]str if key == utils.VersionKey || !strings.HasPrefix(key, prefix) { continue } - entry, err := s.subnetKV.Get(key) + entry, err := s.subnetKV.Get(ctx, key) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { continue } return nil, fmt.Errorf("read subnet %s: %w", key, err) @@ -450,9 +456,9 @@ func (s *RouteTableServiceImpl) publishSubnetEgressGateDecisionForRT(ctx context // allRouteTablesForVPC returns all route tables belonging to a VPC. func (s *RouteTableServiceImpl) allRouteTablesForVPC(ctx context.Context, accountID, vpcID string) ([]RouteTableRecord, error) { prefix := accountID + "." - keys, err := s.rtbKV.Keys() + keys, err := s.rtbKV.Keys(ctx) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return nil, nil } return nil, errors.New(awserrors.ErrorServerInternal) @@ -463,9 +469,9 @@ func (s *RouteTableServiceImpl) allRouteTablesForVPC(ctx context.Context, accoun if key == utils.VersionKey || !strings.HasPrefix(key, prefix) { continue } - entry, err := s.rtbKV.Get(key) + entry, err := s.rtbKV.Get(ctx, key) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { continue // deleted between Keys() and Get() } slog.ErrorContext(ctx, "Failed to read route table during VPC scan", "key", key, "err", err) @@ -581,7 +587,7 @@ func (s *RouteTableServiceImpl) DeleteRouteTable(ctx context.Context, input *ec2 } } - if err := s.rtbKV.Delete(utils.AccountKey(accountID, rtbID)); err != nil { + if err := s.rtbKV.Delete(ctx, utils.AccountKey(accountID, rtbID)); err != nil { return nil, errors.New(awserrors.ErrorServerInternal) } @@ -619,8 +625,8 @@ func (s *RouteTableServiceImpl) DescribeRouteTables(ctx context.Context, input * } prefix := accountID + "." - keys, err := s.rtbKV.Keys() - if err != nil && !errors.Is(err, nats.ErrNoKeysFound) { + keys, err := s.rtbKV.Keys(ctx) + if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) { return nil, errors.New(awserrors.ErrorServerInternal) } @@ -632,9 +638,9 @@ func (s *RouteTableServiceImpl) DescribeRouteTables(ctx context.Context, input * continue } - entry, err := s.rtbKV.Get(key) + entry, err := s.rtbKV.Get(ctx, key) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { continue } slog.ErrorContext(ctx, "Failed to read route table during describe", "key", key, "err", err) @@ -701,7 +707,7 @@ func (s *RouteTableServiceImpl) CreateRoute(ctx context.Context, input *ec2.Crea case input.GatewayId != nil && *input.GatewayId != "": igwID := *input.GatewayId // Verify IGW exists and is attached to the same VPC - igwEntry, err := s.igwKV.Get(utils.AccountKey(accountID, igwID)) + igwEntry, err := s.igwKV.Get(ctx, utils.AccountKey(accountID, igwID)) if err != nil { return nil, errors.New(awserrors.ErrorInvalidInternetGatewayIDNotFound) } @@ -729,7 +735,7 @@ func (s *RouteTableServiceImpl) CreateRoute(ctx context.Context, input *ec2.Crea case input.NatGatewayId != nil && *input.NatGatewayId != "": natgwID := *input.NatGatewayId // Verify NAT GW exists and belongs to the same VPC - natgwEntry, err := s.natgwKV.Get(utils.AccountKey(accountID, natgwID)) + natgwEntry, err := s.natgwKV.Get(ctx, utils.AccountKey(accountID, natgwID)) if err != nil { return nil, errors.New(awserrors.ErrorInvalidNatGatewayIDNotFound) } @@ -878,7 +884,7 @@ func (s *RouteTableServiceImpl) ReplaceRoute(ctx context.Context, input *ec2.Rep igwID := *input.GatewayId // Verify IGW exists and is attached to same VPC - igwEntry, err := s.igwKV.Get(utils.AccountKey(accountID, igwID)) + igwEntry, err := s.igwKV.Get(ctx, utils.AccountKey(accountID, igwID)) if err != nil { return nil, errors.New(awserrors.ErrorInvalidInternetGatewayIDNotFound) } @@ -921,7 +927,7 @@ func (s *RouteTableServiceImpl) AssociateRouteTable(ctx context.Context, input * } // Verify subnet exists and belongs to the same VPC - subnetEntry, err := s.subnetKV.Get(utils.AccountKey(accountID, subnetID)) + subnetEntry, err := s.subnetKV.Get(ctx, utils.AccountKey(accountID, subnetID)) if err != nil { return nil, errors.New(awserrors.ErrorInvalidSubnetIDNotFound) } @@ -1005,9 +1011,9 @@ func (s *RouteTableServiceImpl) DisassociateRouteTable(ctx context.Context, inpu // Search all route tables for this account to find the association prefix := accountID + "." - keys, err := s.rtbKV.Keys() + keys, err := s.rtbKV.Keys(ctx) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return nil, errors.New(awserrors.ErrorInvalidAssociationIDNotFound) } slog.ErrorContext(ctx, "Failed to list route table keys", "err", err) @@ -1019,9 +1025,9 @@ func (s *RouteTableServiceImpl) DisassociateRouteTable(ctx context.Context, inpu continue } - entry, err := s.rtbKV.Get(key) + entry, err := s.rtbKV.Get(ctx, key) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { continue } slog.ErrorContext(ctx, "Failed to read route table during disassociate scan", "key", key, "err", err) @@ -1096,9 +1102,9 @@ func (s *RouteTableServiceImpl) ReplaceRouteTableAssociation(ctx context.Context // Find and remove the association from its current route table prefix := accountID + "." - keys, err := s.rtbKV.Keys() + keys, err := s.rtbKV.Keys(ctx) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return nil, errors.New(awserrors.ErrorInvalidAssociationIDNotFound) } slog.ErrorContext(ctx, "Failed to list route table keys", "err", err) @@ -1110,9 +1116,9 @@ func (s *RouteTableServiceImpl) ReplaceRouteTableAssociation(ctx context.Context continue } - entry, err := s.rtbKV.Get(key) + entry, err := s.rtbKV.Get(ctx, key) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { continue } slog.ErrorContext(ctx, "Failed to read route table during replace scan", "key", key, "err", err) @@ -1157,9 +1163,13 @@ func (s *RouteTableServiceImpl) ReplaceRouteTableAssociation(ctx context.Context Main: assoc.Main, }) if err := s.putRouteTable(ctx, accountID, newRecord); err != nil { - // Compensate: restore association to old table to avoid data loss + // Restore the old association even when the request context caused the + // failed write, otherwise cancellation turns a rollback into a no-op. oldRecord.Associations = append(oldRecord.Associations, assoc) - if restoreErr := s.putRouteTable(ctx, accountID, &oldRecord); restoreErr != nil { + restoreCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) + restoreErr := s.putRouteTable(restoreCtx, accountID, &oldRecord) + cancel() + if restoreErr != nil { slog.ErrorContext(ctx, "CRITICAL: ReplaceRouteTableAssociation partial failure, association lost", "associationId", assocID, "oldRouteTableId", oldRecord.RouteTableId, "newRouteTableId", newRtbID, "restoreErr", restoreErr, "originalErr", err) @@ -1342,7 +1352,7 @@ func (s *RouteTableServiceImpl) publishNatGatewayDeleteEvents(ctx context.Contex if s.natsConn == nil { return } - natgwEntry, err := s.natgwKV.Get(utils.AccountKey(accountID, natgwID)) + natgwEntry, err := s.natgwKV.Get(ctx, utils.AccountKey(accountID, natgwID)) if err != nil { slog.WarnContext(ctx, "NAT GW event: natgw lookup failed", "topic", "vpc.delete-nat-gateway", "natGatewayId", natgwID, "err", err) return @@ -1390,7 +1400,7 @@ func (s *RouteTableServiceImpl) publishNatGatewayEventsForAssociation(ctx contex if r.NatGatewayId == "" { continue } - natgwEntry, err := s.natgwKV.Get(utils.AccountKey(accountID, r.NatGatewayId)) + natgwEntry, err := s.natgwKV.Get(ctx, utils.AccountKey(accountID, r.NatGatewayId)) if err != nil { slog.WarnContext(ctx, "NAT GW event: natgw lookup failed", "topic", topic, "natGatewayId", r.NatGatewayId, "err", err) continue @@ -1415,7 +1425,7 @@ func (s *RouteTableServiceImpl) publishNatGatewayEventForSubnet(ctx context.Cont if s.natsConn == nil { return } - subnetEntry, err := s.subnetKV.Get(utils.AccountKey(accountID, subnetID)) + subnetEntry, err := s.subnetKV.Get(ctx, utils.AccountKey(accountID, subnetID)) if err != nil { slog.WarnContext(ctx, "NAT GW event: subnet lookup failed", "topic", topic, "subnetId", subnetID, "err", err) return @@ -1575,7 +1585,7 @@ func (s *RouteTableServiceImpl) ApplyRecordTags(input *ec2.CreateTagsInput, acco if input == nil { return nil } - return utils.MirrorKVRecordTags(s.rtbKV, accountID, "rtb-", input.Resources, + return utils.MirrorKVRecordTags(context.Background(), s.rtbKV, accountID, "rtb-", input.Resources, func(r *RouteTableRecord) *map[string]string { return &r.Tags }, utils.MergeTagsMut(input)) } @@ -1586,7 +1596,7 @@ func (s *RouteTableServiceImpl) RemoveRecordTags(input *ec2.DeleteTagsInput, acc if input == nil { return nil } - return utils.MirrorKVRecordTags(s.rtbKV, accountID, "rtb-", input.Resources, + return utils.MirrorKVRecordTags(context.Background(), s.rtbKV, accountID, "rtb-", input.Resources, func(r *RouteTableRecord) *map[string]string { return &r.Tags }, utils.RemoveTagsMut(input)) } diff --git a/spinifex/handlers/ec2/routetable/service_impl_test.go b/spinifex/handlers/ec2/routetable/service_impl_test.go index d4acffdba..515827a0a 100644 --- a/spinifex/handlers/ec2/routetable/service_impl_test.go +++ b/spinifex/handlers/ec2/routetable/service_impl_test.go @@ -1,7 +1,6 @@ package handlers_ec2_routetable import ( - "context" "encoding/json" "sync" "testing" @@ -16,6 +15,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/testutil" "github.com/mulgadc/spinifex/spinifex/utils" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -24,37 +24,49 @@ const testAccountID = "123456789012" func setupTestService(t *testing.T) *RouteTableServiceImpl { t.Helper() - _, nc, js := testutil.StartTestJetStream(t) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) // Seed VPC KV - testutil.SeedKV(t, js, handlers_ec2_vpc.KVBucketVPCs, map[string][]byte{ + seedKV(t, js, handlers_ec2_vpc.KVBucketVPCs, map[string][]byte{ utils.AccountKey(testAccountID, "vpc-test1"): []byte(`{"vpc_id":"vpc-test1","cidr_block":"10.0.0.0/16","state":"available"}`), }) // Seed IGW KV (attached to vpc-test1) - testutil.SeedKV(t, js, handlers_ec2_igw.KVBucketIGW, map[string][]byte{ + seedKV(t, js, handlers_ec2_igw.KVBucketIGW, map[string][]byte{ utils.AccountKey(testAccountID, "igw-test1"): []byte(`{"internet_gateway_id":"igw-test1","vpc_id":"vpc-test1","state":"available"}`), }) // Seed subnet KV - testutil.SeedKV(t, js, handlers_ec2_vpc.KVBucketSubnets, map[string][]byte{ + seedKV(t, js, handlers_ec2_vpc.KVBucketSubnets, map[string][]byte{ utils.AccountKey(testAccountID, "subnet-test1"): []byte(`{"subnet_id":"subnet-test1","vpc_id":"vpc-test1","cidr_block":"10.0.1.0/24","state":"available"}`), utils.AccountKey(testAccountID, "subnet-priv1"): []byte(`{"subnet_id":"subnet-priv1","vpc_id":"vpc-test1","cidr_block":"10.0.11.0/24","state":"available"}`), }) // Seed NAT Gateway KV - testutil.SeedKV(t, js, handlers_ec2_natgw.KVBucketNatGateways, map[string][]byte{ + seedKV(t, js, handlers_ec2_natgw.KVBucketNatGateways, map[string][]byte{ utils.AccountKey(testAccountID, "nat-test1"): []byte(`{"nat_gateway_id":"nat-test1","vpc_id":"vpc-test1","subnet_id":"subnet-test1","public_ip":"192.168.1.100","state":"available"}`), }) - svc, err := NewRouteTableServiceImplWithNATS(nil, nc) + svc, err := NewRouteTableServiceImplWithNATS(t.Context(), nil, nc) require.NoError(t, err) return svc } +func seedKV(t *testing.T, js jetstream.JetStream, bucket string, entries map[string][]byte) jetstream.KeyValue { + t.Helper() + kv, err := js.CreateOrUpdateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: bucket, History: 1}) + require.NoError(t, err) + for key, value := range entries { + _, err := kv.Put(t.Context(), key, value) + require.NoError(t, err) + } + return kv +} + func createTestRtb(t *testing.T, svc *RouteTableServiceImpl) string { t.Helper() - out, err := svc.CreateRouteTable(context.Background(), &ec2.CreateRouteTableInput{ + out, err := svc.CreateRouteTable(t.Context(), &ec2.CreateRouteTableInput{ VpcId: aws.String("vpc-test1"), }, testAccountID) require.NoError(t, err) @@ -63,7 +75,7 @@ func createTestRtb(t *testing.T, svc *RouteTableServiceImpl) string { func TestCreateRouteTable(t *testing.T) { svc := setupTestService(t) - out, err := svc.CreateRouteTable(context.Background(), &ec2.CreateRouteTableInput{ + out, err := svc.CreateRouteTable(t.Context(), &ec2.CreateRouteTableInput{ VpcId: aws.String("vpc-test1"), }, testAccountID) require.NoError(t, err) @@ -86,7 +98,7 @@ func TestCreateRouteTable(t *testing.T) { // the NAT gateway (and its billable EIP) be reclaimed. func TestCreateRouteTable_PersistsTagsForTagFilterDiscovery(t *testing.T) { svc := setupTestService(t) - _, err := svc.CreateRouteTable(context.Background(), &ec2.CreateRouteTableInput{ + _, err := svc.CreateRouteTable(t.Context(), &ec2.CreateRouteTableInput{ VpcId: aws.String("vpc-test1"), TagSpecifications: []*ec2.TagSpecification{{ ResourceType: aws.String(ec2.ResourceTypeRouteTable), @@ -98,7 +110,7 @@ func TestCreateRouteTable_PersistsTagsForTagFilterDiscovery(t *testing.T) { }, testAccountID) require.NoError(t, err) - out, err := svc.DescribeRouteTables(context.Background(), &ec2.DescribeRouteTablesInput{ + out, err := svc.DescribeRouteTables(t.Context(), &ec2.DescribeRouteTablesInput{ Filters: []*ec2.Filter{ {Name: aws.String("tag:spinifex:eks-cluster"), Values: aws.StringSlice([]string{"alpha"})}, {Name: aws.String("tag:spinifex:eks-role"), Values: aws.StringSlice([]string{"cp-private-rt"})}, @@ -117,7 +129,7 @@ func TestCreateRouteTable_PersistsTagsForTagFilterDiscovery(t *testing.T) { func TestCreateRouteTable_VpcNotFound(t *testing.T) { svc := setupTestService(t) - _, err := svc.CreateRouteTable(context.Background(), &ec2.CreateRouteTableInput{ + _, err := svc.CreateRouteTable(t.Context(), &ec2.CreateRouteTableInput{ VpcId: aws.String("vpc-nonexistent"), }, testAccountID) assert.EqualError(t, err, awserrors.ErrorInvalidVpcIDNotFound) @@ -127,13 +139,13 @@ func TestDeleteRouteTable(t *testing.T) { svc := setupTestService(t) rtbID := createTestRtb(t, svc) - _, err := svc.DeleteRouteTable(context.Background(), &ec2.DeleteRouteTableInput{ + _, err := svc.DeleteRouteTable(t.Context(), &ec2.DeleteRouteTableInput{ RouteTableId: aws.String(rtbID), }, testAccountID) require.NoError(t, err) // Should be gone - _, err = svc.getRouteTable(context.Background(), testAccountID, rtbID) + _, err = svc.getRouteTable(t.Context(), testAccountID, rtbID) assert.EqualError(t, err, awserrors.ErrorInvalidRouteTableIDNotFound) } @@ -142,7 +154,7 @@ func TestDeleteRouteTable_Main(t *testing.T) { record, err := svc.CreateRouteTableForVPC("vpc-test1", "10.0.0.0/16", testAccountID, true, "") require.NoError(t, err) - _, err = svc.DeleteRouteTable(context.Background(), &ec2.DeleteRouteTableInput{ + _, err = svc.DeleteRouteTable(t.Context(), &ec2.DeleteRouteTableInput{ RouteTableId: aws.String(record.RouteTableId), }, testAccountID) assert.EqualError(t, err, awserrors.ErrorDependencyViolation) @@ -153,14 +165,14 @@ func TestDeleteRouteTable_WithAssociations(t *testing.T) { rtbID := createTestRtb(t, svc) // Associate a subnet - _, err := svc.AssociateRouteTable(context.Background(), &ec2.AssociateRouteTableInput{ + _, err := svc.AssociateRouteTable(t.Context(), &ec2.AssociateRouteTableInput{ RouteTableId: aws.String(rtbID), SubnetId: aws.String("subnet-test1"), }, testAccountID) require.NoError(t, err) // Should fail to delete - _, err = svc.DeleteRouteTable(context.Background(), &ec2.DeleteRouteTableInput{ + _, err = svc.DeleteRouteTable(t.Context(), &ec2.DeleteRouteTableInput{ RouteTableId: aws.String(rtbID), }, testAccountID) assert.EqualError(t, err, awserrors.ErrorDependencyViolation) @@ -170,7 +182,7 @@ func TestDescribeRouteTables(t *testing.T) { svc := setupTestService(t) rtbID := createTestRtb(t, svc) - out, err := svc.DescribeRouteTables(context.Background(), &ec2.DescribeRouteTablesInput{}, testAccountID) + out, err := svc.DescribeRouteTables(t.Context(), &ec2.DescribeRouteTablesInput{}, testAccountID) require.NoError(t, err) assert.Len(t, out.RouteTables, 1) assert.Equal(t, rtbID, *out.RouteTables[0].RouteTableId) @@ -182,7 +194,7 @@ func TestDescribeRouteTables_FilterByVpcId(t *testing.T) { name := "vpc-id" val := "vpc-test1" - out, err := svc.DescribeRouteTables(context.Background(), &ec2.DescribeRouteTablesInput{ + out, err := svc.DescribeRouteTables(t.Context(), &ec2.DescribeRouteTablesInput{ Filters: []*ec2.Filter{{Name: &name, Values: []*string{&val}}}, }, testAccountID) require.NoError(t, err) @@ -190,7 +202,7 @@ func TestDescribeRouteTables_FilterByVpcId(t *testing.T) { // Filter by non-existent VPC val2 := "vpc-nope" - out, err = svc.DescribeRouteTables(context.Background(), &ec2.DescribeRouteTablesInput{ + out, err = svc.DescribeRouteTables(t.Context(), &ec2.DescribeRouteTablesInput{ Filters: []*ec2.Filter{{Name: &name, Values: []*string{&val2}}}, }, testAccountID) require.NoError(t, err) @@ -209,7 +221,7 @@ func TestDescribeRouteTables_FilterByMain(t *testing.T) { name := "association.main" val := "true" - out, err := svc.DescribeRouteTables(context.Background(), &ec2.DescribeRouteTablesInput{ + out, err := svc.DescribeRouteTables(t.Context(), &ec2.DescribeRouteTablesInput{ Filters: []*ec2.Filter{{Name: &name, Values: []*string{&val}}}, }, testAccountID) require.NoError(t, err) @@ -222,14 +234,14 @@ func TestDescribeRouteTables_FilterByRouteState(t *testing.T) { createTestRtb(t, svc) // has local route with state=active // Exact match - out, err := svc.DescribeRouteTables(context.Background(), &ec2.DescribeRouteTablesInput{ + out, err := svc.DescribeRouteTables(t.Context(), &ec2.DescribeRouteTablesInput{ Filters: []*ec2.Filter{{Name: aws.String("route.state"), Values: []*string{aws.String("active")}}}, }, testAccountID) require.NoError(t, err) assert.Len(t, out.RouteTables, 1) // Non-match - out, err = svc.DescribeRouteTables(context.Background(), &ec2.DescribeRouteTablesInput{ + out, err = svc.DescribeRouteTables(t.Context(), &ec2.DescribeRouteTablesInput{ Filters: []*ec2.Filter{{Name: aws.String("route.state"), Values: []*string{aws.String("blackhole")}}}, }, testAccountID) require.NoError(t, err) @@ -240,13 +252,13 @@ func TestDescribeRouteTables_FilterByRouteOrigin(t *testing.T) { svc := setupTestService(t) createTestRtb(t, svc) // local route has origin=CreateRouteTable - out, err := svc.DescribeRouteTables(context.Background(), &ec2.DescribeRouteTablesInput{ + out, err := svc.DescribeRouteTables(t.Context(), &ec2.DescribeRouteTablesInput{ Filters: []*ec2.Filter{{Name: aws.String("route.origin"), Values: []*string{aws.String("CreateRouteTable")}}}, }, testAccountID) require.NoError(t, err) assert.Len(t, out.RouteTables, 1) - out, err = svc.DescribeRouteTables(context.Background(), &ec2.DescribeRouteTablesInput{ + out, err = svc.DescribeRouteTables(t.Context(), &ec2.DescribeRouteTablesInput{ Filters: []*ec2.Filter{{Name: aws.String("route.origin"), Values: []*string{aws.String("CreateRoute")}}}, }, testAccountID) require.NoError(t, err) @@ -258,14 +270,14 @@ func TestDescribeRouteTables_FilterByRouteNatGatewayId(t *testing.T) { createTestRtb(t, svc) // no NAT GW route // No match — no routes have a nat-gateway-id - out, err := svc.DescribeRouteTables(context.Background(), &ec2.DescribeRouteTablesInput{ + out, err := svc.DescribeRouteTables(t.Context(), &ec2.DescribeRouteTablesInput{ Filters: []*ec2.Filter{{Name: aws.String("route.nat-gateway-id"), Values: []*string{aws.String("nat-000000")}}}, }, testAccountID) require.NoError(t, err) assert.Empty(t, out.RouteTables) // Wildcard on empty field should not match - out, err = svc.DescribeRouteTables(context.Background(), &ec2.DescribeRouteTablesInput{ + out, err = svc.DescribeRouteTables(t.Context(), &ec2.DescribeRouteTablesInput{ Filters: []*ec2.Filter{{Name: aws.String("route.nat-gateway-id"), Values: []*string{aws.String("nat-*")}}}, }, testAccountID) require.NoError(t, err) @@ -277,21 +289,21 @@ func TestDescribeRouteTables_FilterByOwnerId(t *testing.T) { createTestRtb(t, svc) // Exact match - out, err := svc.DescribeRouteTables(context.Background(), &ec2.DescribeRouteTablesInput{ + out, err := svc.DescribeRouteTables(t.Context(), &ec2.DescribeRouteTablesInput{ Filters: []*ec2.Filter{{Name: aws.String("owner-id"), Values: []*string{aws.String(testAccountID)}}}, }, testAccountID) require.NoError(t, err) assert.Len(t, out.RouteTables, 1) // Non-match - out, err = svc.DescribeRouteTables(context.Background(), &ec2.DescribeRouteTablesInput{ + out, err = svc.DescribeRouteTables(t.Context(), &ec2.DescribeRouteTablesInput{ Filters: []*ec2.Filter{{Name: aws.String("owner-id"), Values: []*string{aws.String("999999999999")}}}, }, testAccountID) require.NoError(t, err) assert.Empty(t, out.RouteTables) // Wildcard - out, err = svc.DescribeRouteTables(context.Background(), &ec2.DescribeRouteTablesInput{ + out, err = svc.DescribeRouteTables(t.Context(), &ec2.DescribeRouteTablesInput{ Filters: []*ec2.Filter{{Name: aws.String("owner-id"), Values: []*string{aws.String("1234*")}}}, }, testAccountID) require.NoError(t, err) @@ -302,7 +314,7 @@ func TestCreateRoute(t *testing.T) { svc := setupTestService(t) rtbID := createTestRtb(t, svc) - _, err := svc.CreateRoute(context.Background(), &ec2.CreateRouteInput{ + _, err := svc.CreateRoute(t.Context(), &ec2.CreateRouteInput{ RouteTableId: aws.String(rtbID), DestinationCidrBlock: aws.String("0.0.0.0/0"), GatewayId: aws.String("igw-test1"), @@ -310,7 +322,7 @@ func TestCreateRoute(t *testing.T) { require.NoError(t, err) // Verify route was added - record, err := svc.getRouteTable(context.Background(), testAccountID, rtbID) + record, err := svc.getRouteTable(t.Context(), testAccountID, rtbID) require.NoError(t, err) assert.Len(t, record.Routes, 2) // local + igw assert.Equal(t, "0.0.0.0/0", record.Routes[1].DestinationCidrBlock) @@ -321,7 +333,7 @@ func TestCreateRoute_DuplicateDestination(t *testing.T) { svc := setupTestService(t) rtbID := createTestRtb(t, svc) - _, err := svc.CreateRoute(context.Background(), &ec2.CreateRouteInput{ + _, err := svc.CreateRoute(t.Context(), &ec2.CreateRouteInput{ RouteTableId: aws.String(rtbID), DestinationCidrBlock: aws.String("0.0.0.0/0"), GatewayId: aws.String("igw-test1"), @@ -329,7 +341,7 @@ func TestCreateRoute_DuplicateDestination(t *testing.T) { require.NoError(t, err) // Duplicate should fail - _, err = svc.CreateRoute(context.Background(), &ec2.CreateRouteInput{ + _, err = svc.CreateRoute(t.Context(), &ec2.CreateRouteInput{ RouteTableId: aws.String(rtbID), DestinationCidrBlock: aws.String("0.0.0.0/0"), GatewayId: aws.String("igw-test1"), @@ -341,20 +353,20 @@ func TestDeleteRoute(t *testing.T) { svc := setupTestService(t) rtbID := createTestRtb(t, svc) - _, err := svc.CreateRoute(context.Background(), &ec2.CreateRouteInput{ + _, err := svc.CreateRoute(t.Context(), &ec2.CreateRouteInput{ RouteTableId: aws.String(rtbID), DestinationCidrBlock: aws.String("0.0.0.0/0"), GatewayId: aws.String("igw-test1"), }, testAccountID) require.NoError(t, err) - _, err = svc.DeleteRoute(context.Background(), &ec2.DeleteRouteInput{ + _, err = svc.DeleteRoute(t.Context(), &ec2.DeleteRouteInput{ RouteTableId: aws.String(rtbID), DestinationCidrBlock: aws.String("0.0.0.0/0"), }, testAccountID) require.NoError(t, err) - record, err := svc.getRouteTable(context.Background(), testAccountID, rtbID) + record, err := svc.getRouteTable(t.Context(), testAccountID, rtbID) require.NoError(t, err) assert.Len(t, record.Routes, 1) // only local remains } @@ -363,7 +375,7 @@ func TestDeleteRoute_LocalRoute(t *testing.T) { svc := setupTestService(t) rtbID := createTestRtb(t, svc) - _, err := svc.DeleteRoute(context.Background(), &ec2.DeleteRouteInput{ + _, err := svc.DeleteRoute(t.Context(), &ec2.DeleteRouteInput{ RouteTableId: aws.String(rtbID), DestinationCidrBlock: aws.String("10.0.0.0/16"), }, testAccountID) @@ -374,7 +386,7 @@ func TestDeleteRoute_NotFound(t *testing.T) { svc := setupTestService(t) rtbID := createTestRtb(t, svc) - _, err := svc.DeleteRoute(context.Background(), &ec2.DeleteRouteInput{ + _, err := svc.DeleteRoute(t.Context(), &ec2.DeleteRouteInput{ RouteTableId: aws.String(rtbID), DestinationCidrBlock: aws.String("192.168.0.0/16"), }, testAccountID) @@ -385,7 +397,7 @@ func TestReplaceRoute(t *testing.T) { svc := setupTestService(t) rtbID := createTestRtb(t, svc) - _, err := svc.CreateRoute(context.Background(), &ec2.CreateRouteInput{ + _, err := svc.CreateRoute(t.Context(), &ec2.CreateRouteInput{ RouteTableId: aws.String(rtbID), DestinationCidrBlock: aws.String("0.0.0.0/0"), GatewayId: aws.String("igw-test1"), @@ -393,7 +405,7 @@ func TestReplaceRoute(t *testing.T) { require.NoError(t, err) // Replace target (same IGW for simplicity — validates the swap logic) - _, err = svc.ReplaceRoute(context.Background(), &ec2.ReplaceRouteInput{ + _, err = svc.ReplaceRoute(t.Context(), &ec2.ReplaceRouteInput{ RouteTableId: aws.String(rtbID), DestinationCidrBlock: aws.String("0.0.0.0/0"), GatewayId: aws.String("igw-test1"), @@ -405,7 +417,7 @@ func TestAssociateRouteTable(t *testing.T) { svc := setupTestService(t) rtbID := createTestRtb(t, svc) - out, err := svc.AssociateRouteTable(context.Background(), &ec2.AssociateRouteTableInput{ + out, err := svc.AssociateRouteTable(t.Context(), &ec2.AssociateRouteTableInput{ RouteTableId: aws.String(rtbID), SubnetId: aws.String("subnet-test1"), }, testAccountID) @@ -418,14 +430,14 @@ func TestAssociateRouteTable_DuplicateSubnet(t *testing.T) { svc := setupTestService(t) rtbID := createTestRtb(t, svc) - _, err := svc.AssociateRouteTable(context.Background(), &ec2.AssociateRouteTableInput{ + _, err := svc.AssociateRouteTable(t.Context(), &ec2.AssociateRouteTableInput{ RouteTableId: aws.String(rtbID), SubnetId: aws.String("subnet-test1"), }, testAccountID) require.NoError(t, err) // Second association should fail - _, err = svc.AssociateRouteTable(context.Background(), &ec2.AssociateRouteTableInput{ + _, err = svc.AssociateRouteTable(t.Context(), &ec2.AssociateRouteTableInput{ RouteTableId: aws.String(rtbID), SubnetId: aws.String("subnet-test1"), }, testAccountID) @@ -449,7 +461,7 @@ func TestAssociateRouteTable_ConcurrentDistinctSubnets(t *testing.T) { wg.Add(1) go func(i int, sn string) { defer wg.Done() - _, errs[i] = svc.AssociateRouteTable(context.Background(), &ec2.AssociateRouteTableInput{ + _, errs[i] = svc.AssociateRouteTable(t.Context(), &ec2.AssociateRouteTableInput{ RouteTableId: aws.String(rtbID), SubnetId: aws.String(sn), }, testAccountID) @@ -461,7 +473,7 @@ func TestAssociateRouteTable_ConcurrentDistinctSubnets(t *testing.T) { require.NoErrorf(t, err, "associate %s", subnets[i]) } - record, err := svc.getRouteTable(context.Background(), testAccountID, rtbID) + record, err := svc.getRouteTable(t.Context(), testAccountID, rtbID) require.NoError(t, err) got := make([]string, 0, len(record.Associations)) for _, a := range record.Associations { @@ -474,19 +486,19 @@ func TestDisassociateRouteTable(t *testing.T) { svc := setupTestService(t) rtbID := createTestRtb(t, svc) - assocOut, err := svc.AssociateRouteTable(context.Background(), &ec2.AssociateRouteTableInput{ + assocOut, err := svc.AssociateRouteTable(t.Context(), &ec2.AssociateRouteTableInput{ RouteTableId: aws.String(rtbID), SubnetId: aws.String("subnet-test1"), }, testAccountID) require.NoError(t, err) - _, err = svc.DisassociateRouteTable(context.Background(), &ec2.DisassociateRouteTableInput{ + _, err = svc.DisassociateRouteTable(t.Context(), &ec2.DisassociateRouteTableInput{ AssociationId: assocOut.AssociationId, }, testAccountID) require.NoError(t, err) // Verify association removed - record, err := svc.getRouteTable(context.Background(), testAccountID, rtbID) + record, err := svc.getRouteTable(t.Context(), testAccountID, rtbID) require.NoError(t, err) assert.Empty(t, record.Associations) } @@ -496,7 +508,7 @@ func TestDisassociateRouteTable_Main(t *testing.T) { record, err := svc.CreateRouteTableForVPC("vpc-test1", "10.0.0.0/16", testAccountID, true, "") require.NoError(t, err) - _, err = svc.DisassociateRouteTable(context.Background(), &ec2.DisassociateRouteTableInput{ + _, err = svc.DisassociateRouteTable(t.Context(), &ec2.DisassociateRouteTableInput{ AssociationId: aws.String(record.Associations[0].AssociationId), }, testAccountID) assert.EqualError(t, err, awserrors.ErrorInvalidParameterValue) @@ -507,13 +519,13 @@ func TestReplaceRouteTableAssociation(t *testing.T) { rtb1ID := createTestRtb(t, svc) rtb2ID := createTestRtb(t, svc) - assocOut, err := svc.AssociateRouteTable(context.Background(), &ec2.AssociateRouteTableInput{ + assocOut, err := svc.AssociateRouteTable(t.Context(), &ec2.AssociateRouteTableInput{ RouteTableId: aws.String(rtb1ID), SubnetId: aws.String("subnet-test1"), }, testAccountID) require.NoError(t, err) - replaceOut, err := svc.ReplaceRouteTableAssociation(context.Background(), &ec2.ReplaceRouteTableAssociationInput{ + replaceOut, err := svc.ReplaceRouteTableAssociation(t.Context(), &ec2.ReplaceRouteTableAssociationInput{ AssociationId: assocOut.AssociationId, RouteTableId: aws.String(rtb2ID), }, testAccountID) @@ -522,12 +534,12 @@ func TestReplaceRouteTableAssociation(t *testing.T) { assert.NotEqual(t, *assocOut.AssociationId, *replaceOut.NewAssociationId) // Verify old table has no associations - oldRecord, err := svc.getRouteTable(context.Background(), testAccountID, rtb1ID) + oldRecord, err := svc.getRouteTable(t.Context(), testAccountID, rtb1ID) require.NoError(t, err) assert.Empty(t, oldRecord.Associations) // Verify new table has the association - newRecord, err := svc.getRouteTable(context.Background(), testAccountID, rtb2ID) + newRecord, err := svc.getRouteTable(t.Context(), testAccountID, rtb2ID) require.NoError(t, err) require.Len(t, newRecord.Associations, 1) assert.Equal(t, "subnet-test1", newRecord.Associations[0].SubnetId) @@ -577,7 +589,7 @@ func TestAssociateRouteTable_PublishesNatGatewayEvent(t *testing.T) { rtbID := createTestRtb(t, svc) // Add NAT GW route BEFORE any subnet is associated (terraform ordering). - _, err = svc.CreateRoute(context.Background(), &ec2.CreateRouteInput{ + _, err = svc.CreateRoute(t.Context(), &ec2.CreateRouteInput{ RouteTableId: aws.String(rtbID), DestinationCidrBlock: aws.String("0.0.0.0/0"), NatGatewayId: aws.String("nat-test1"), @@ -588,7 +600,7 @@ func TestAssociateRouteTable_PublishesNatGatewayEvent(t *testing.T) { _, err = sub.NextMsg(100 * time.Millisecond) assert.Error(t, err, "CreateRoute must not publish when table has no associations") - _, err = svc.AssociateRouteTable(context.Background(), &ec2.AssociateRouteTableInput{ + _, err = svc.AssociateRouteTable(t.Context(), &ec2.AssociateRouteTableInput{ RouteTableId: aws.String(rtbID), SubnetId: aws.String("subnet-priv1"), }, testAccountID) @@ -609,7 +621,7 @@ func TestAssociateRouteTable_NoNatGatewayRoute(t *testing.T) { rtbID := createTestRtb(t, svc) - _, err = svc.AssociateRouteTable(context.Background(), &ec2.AssociateRouteTableInput{ + _, err = svc.AssociateRouteTable(t.Context(), &ec2.AssociateRouteTableInput{ RouteTableId: aws.String(rtbID), SubnetId: aws.String("subnet-priv1"), }, testAccountID) @@ -628,20 +640,20 @@ func TestDisassociateRouteTable_PublishesNatGatewayDeleteEvent(t *testing.T) { defer func() { _ = delSub.Unsubscribe() }() rtbID := createTestRtb(t, svc) - _, err = svc.CreateRoute(context.Background(), &ec2.CreateRouteInput{ + _, err = svc.CreateRoute(t.Context(), &ec2.CreateRouteInput{ RouteTableId: aws.String(rtbID), DestinationCidrBlock: aws.String("0.0.0.0/0"), NatGatewayId: aws.String("nat-test1"), }, testAccountID) require.NoError(t, err) - assocOut, err := svc.AssociateRouteTable(context.Background(), &ec2.AssociateRouteTableInput{ + assocOut, err := svc.AssociateRouteTable(t.Context(), &ec2.AssociateRouteTableInput{ RouteTableId: aws.String(rtbID), SubnetId: aws.String("subnet-priv1"), }, testAccountID) require.NoError(t, err) - _, err = svc.DisassociateRouteTable(context.Background(), &ec2.DisassociateRouteTableInput{ + _, err = svc.DisassociateRouteTable(t.Context(), &ec2.DisassociateRouteTableInput{ AssociationId: assocOut.AssociationId, }, testAccountID) require.NoError(t, err) @@ -662,12 +674,12 @@ func TestDeleteRoute_NATGW_PublishesDeleteForAssociatedSubnets(t *testing.T) { defer func() { _ = delSub.Unsubscribe() }() rtbID := createTestRtb(t, svc) - _, err = svc.AssociateRouteTable(context.Background(), &ec2.AssociateRouteTableInput{ + _, err = svc.AssociateRouteTable(t.Context(), &ec2.AssociateRouteTableInput{ RouteTableId: aws.String(rtbID), SubnetId: aws.String("subnet-priv1"), }, testAccountID) require.NoError(t, err) - _, err = svc.CreateRoute(context.Background(), &ec2.CreateRouteInput{ + _, err = svc.CreateRoute(t.Context(), &ec2.CreateRouteInput{ RouteTableId: aws.String(rtbID), DestinationCidrBlock: aws.String("0.0.0.0/0"), NatGatewayId: aws.String("nat-test1"), @@ -677,7 +689,7 @@ func TestDeleteRoute_NATGW_PublishesDeleteForAssociatedSubnets(t *testing.T) { natgwID, _ := receiveNatGWEvent(t, addSub, "10.0.11.0/24") require.Equal(t, "nat-test1", natgwID) - _, err = svc.DeleteRoute(context.Background(), &ec2.DeleteRouteInput{ + _, err = svc.DeleteRoute(t.Context(), &ec2.DeleteRouteInput{ RouteTableId: aws.String(rtbID), DestinationCidrBlock: aws.String("0.0.0.0/0"), }, testAccountID) @@ -702,14 +714,14 @@ func TestReplaceRouteTableAssociation_PublishesNatGatewayEvents(t *testing.T) { oldRtb := createTestRtb(t, svc) newRtb := createTestRtb(t, svc) - _, err = svc.CreateRoute(context.Background(), &ec2.CreateRouteInput{ + _, err = svc.CreateRoute(t.Context(), &ec2.CreateRouteInput{ RouteTableId: aws.String(oldRtb), DestinationCidrBlock: aws.String("0.0.0.0/0"), NatGatewayId: aws.String("nat-test1"), }, testAccountID) require.NoError(t, err) - assocOut, err := svc.AssociateRouteTable(context.Background(), &ec2.AssociateRouteTableInput{ + assocOut, err := svc.AssociateRouteTable(t.Context(), &ec2.AssociateRouteTableInput{ RouteTableId: aws.String(oldRtb), SubnetId: aws.String("subnet-priv1"), }, testAccountID) @@ -718,7 +730,7 @@ func TestReplaceRouteTableAssociation_PublishesNatGatewayEvents(t *testing.T) { _, err = addSub.NextMsg(time.Second) require.NoError(t, err) - _, err = svc.ReplaceRouteTableAssociation(context.Background(), &ec2.ReplaceRouteTableAssociationInput{ + _, err = svc.ReplaceRouteTableAssociation(t.Context(), &ec2.ReplaceRouteTableAssociationInput{ AssociationId: assocOut.AssociationId, RouteTableId: aws.String(newRtb), }, testAccountID) @@ -760,13 +772,13 @@ func TestCreateRoute_IGW_PublishesAddIGWRouteForAssociatedSubnets(t *testing.T) // Associate subnet FIRST, then CreateRoute — exercises the "route after // association" leg. - _, err = svc.AssociateRouteTable(context.Background(), &ec2.AssociateRouteTableInput{ + _, err = svc.AssociateRouteTable(t.Context(), &ec2.AssociateRouteTableInput{ RouteTableId: aws.String(rtbID), SubnetId: aws.String("subnet-priv1"), }, testAccountID) require.NoError(t, err) - _, err = svc.CreateRoute(context.Background(), &ec2.CreateRouteInput{ + _, err = svc.CreateRoute(t.Context(), &ec2.CreateRouteInput{ RouteTableId: aws.String(rtbID), DestinationCidrBlock: aws.String("0.0.0.0/0"), GatewayId: aws.String("igw-test1"), @@ -788,7 +800,7 @@ func TestAssociateRouteTable_PublishesAddIGWRouteForExistingRoute(t *testing.T) rtbID := createTestRtb(t, svc) - _, err = svc.CreateRoute(context.Background(), &ec2.CreateRouteInput{ + _, err = svc.CreateRoute(t.Context(), &ec2.CreateRouteInput{ RouteTableId: aws.String(rtbID), DestinationCidrBlock: aws.String("0.0.0.0/0"), GatewayId: aws.String("igw-test1"), @@ -799,7 +811,7 @@ func TestAssociateRouteTable_PublishesAddIGWRouteForExistingRoute(t *testing.T) _, err = sub.NextMsg(100 * time.Millisecond) assert.Error(t, err, "CreateRoute(IGW) must not publish when table has no associations") - _, err = svc.AssociateRouteTable(context.Background(), &ec2.AssociateRouteTableInput{ + _, err = svc.AssociateRouteTable(t.Context(), &ec2.AssociateRouteTableInput{ RouteTableId: aws.String(rtbID), SubnetId: aws.String("subnet-priv1"), }, testAccountID) @@ -819,20 +831,20 @@ func TestDeleteRoute_IGW_PublishesDeleteIGWRoute(t *testing.T) { defer func() { _ = delSub.Unsubscribe() }() rtbID := createTestRtb(t, svc) - _, err = svc.AssociateRouteTable(context.Background(), &ec2.AssociateRouteTableInput{ + _, err = svc.AssociateRouteTable(t.Context(), &ec2.AssociateRouteTableInput{ RouteTableId: aws.String(rtbID), SubnetId: aws.String("subnet-priv1"), }, testAccountID) require.NoError(t, err) - _, err = svc.CreateRoute(context.Background(), &ec2.CreateRouteInput{ + _, err = svc.CreateRoute(t.Context(), &ec2.CreateRouteInput{ RouteTableId: aws.String(rtbID), DestinationCidrBlock: aws.String("0.0.0.0/0"), GatewayId: aws.String("igw-test1"), }, testAccountID) require.NoError(t, err) - _, err = svc.DeleteRoute(context.Background(), &ec2.DeleteRouteInput{ + _, err = svc.DeleteRoute(t.Context(), &ec2.DeleteRouteInput{ RouteTableId: aws.String(rtbID), DestinationCidrBlock: aws.String("0.0.0.0/0"), }, testAccountID) @@ -852,20 +864,20 @@ func TestDisassociateRouteTable_PublishesDeleteIGWRoute(t *testing.T) { defer func() { _ = delSub.Unsubscribe() }() rtbID := createTestRtb(t, svc) - _, err = svc.CreateRoute(context.Background(), &ec2.CreateRouteInput{ + _, err = svc.CreateRoute(t.Context(), &ec2.CreateRouteInput{ RouteTableId: aws.String(rtbID), DestinationCidrBlock: aws.String("0.0.0.0/0"), GatewayId: aws.String("igw-test1"), }, testAccountID) require.NoError(t, err) - assocOut, err := svc.AssociateRouteTable(context.Background(), &ec2.AssociateRouteTableInput{ + assocOut, err := svc.AssociateRouteTable(t.Context(), &ec2.AssociateRouteTableInput{ RouteTableId: aws.String(rtbID), SubnetId: aws.String("subnet-priv1"), }, testAccountID) require.NoError(t, err) - _, err = svc.DisassociateRouteTable(context.Background(), &ec2.DisassociateRouteTableInput{ + _, err = svc.DisassociateRouteTable(t.Context(), &ec2.DisassociateRouteTableInput{ AssociationId: assocOut.AssociationId, }, testAccountID) require.NoError(t, err) @@ -905,7 +917,7 @@ func TestCreateRoute_IGW_OnMainRT_FansOutToImplicitSubnets(t *testing.T) { mainRT, err := svc.CreateRouteTableForVPC("vpc-test1", "10.0.0.0/16", testAccountID, true, "") require.NoError(t, err) - _, err = svc.CreateRoute(context.Background(), &ec2.CreateRouteInput{ + _, err = svc.CreateRoute(t.Context(), &ec2.CreateRouteInput{ RouteTableId: aws.String(mainRT.RouteTableId), DestinationCidrBlock: aws.String("0.0.0.0/0"), GatewayId: aws.String("igw-test1"), @@ -931,13 +943,13 @@ func TestCreateRoute_IGW_OnMainRT_SkipsExplicitlyAssociatedSubnets(t *testing.T) require.NoError(t, err) customRtbID := createTestRtb(t, svc) - _, err = svc.AssociateRouteTable(context.Background(), &ec2.AssociateRouteTableInput{ + _, err = svc.AssociateRouteTable(t.Context(), &ec2.AssociateRouteTableInput{ RouteTableId: aws.String(customRtbID), SubnetId: aws.String("subnet-priv1"), }, testAccountID) require.NoError(t, err) - _, err = svc.CreateRoute(context.Background(), &ec2.CreateRouteInput{ + _, err = svc.CreateRoute(t.Context(), &ec2.CreateRouteInput{ RouteTableId: aws.String(mainRT.RouteTableId), DestinationCidrBlock: aws.String("0.0.0.0/0"), GatewayId: aws.String("igw-test1"), @@ -959,7 +971,7 @@ func TestAssociateRouteTable_RemovesMainRTRoutesForJoiningSubnet(t *testing.T) { mainRT, err := svc.CreateRouteTableForVPC("vpc-test1", "10.0.0.0/16", testAccountID, true, "") require.NoError(t, err) - _, err = svc.CreateRoute(context.Background(), &ec2.CreateRouteInput{ + _, err = svc.CreateRoute(t.Context(), &ec2.CreateRouteInput{ RouteTableId: aws.String(mainRT.RouteTableId), DestinationCidrBlock: aws.String("0.0.0.0/0"), GatewayId: aws.String("igw-test1"), @@ -967,7 +979,7 @@ func TestAssociateRouteTable_RemovesMainRTRoutesForJoiningSubnet(t *testing.T) { require.NoError(t, err) customRtbID := createTestRtb(t, svc) - _, err = svc.AssociateRouteTable(context.Background(), &ec2.AssociateRouteTableInput{ + _, err = svc.AssociateRouteTable(t.Context(), &ec2.AssociateRouteTableInput{ RouteTableId: aws.String(customRtbID), SubnetId: aws.String("subnet-priv1"), }, testAccountID) @@ -1020,7 +1032,7 @@ func TestCreateRoute_IGW_PublishesUngateForAssociatedSubnets(t *testing.T) { defer func() { _ = ungate.Unsubscribe() }() rtbID := createTestRtb(t, svc) - _, err = svc.AssociateRouteTable(context.Background(), &ec2.AssociateRouteTableInput{ + _, err = svc.AssociateRouteTable(t.Context(), &ec2.AssociateRouteTableInput{ RouteTableId: aws.String(rtbID), SubnetId: aws.String("subnet-priv1"), }, testAccountID) @@ -1032,7 +1044,7 @@ func TestCreateRoute_IGW_PublishesUngateForAssociatedSubnets(t *testing.T) { } } - _, err = svc.CreateRoute(context.Background(), &ec2.CreateRouteInput{ + _, err = svc.CreateRoute(t.Context(), &ec2.CreateRouteInput{ RouteTableId: aws.String(rtbID), DestinationCidrBlock: aws.String("0.0.0.0/0"), GatewayId: aws.String("igw-test1"), @@ -1054,12 +1066,12 @@ func TestDeleteRoute_IGW_PublishesGateForAssociatedSubnets(t *testing.T) { defer func() { _ = gate.Unsubscribe() }() rtbID := createTestRtb(t, svc) - _, err = svc.AssociateRouteTable(context.Background(), &ec2.AssociateRouteTableInput{ + _, err = svc.AssociateRouteTable(t.Context(), &ec2.AssociateRouteTableInput{ RouteTableId: aws.String(rtbID), SubnetId: aws.String("subnet-priv1"), }, testAccountID) require.NoError(t, err) - _, err = svc.CreateRoute(context.Background(), &ec2.CreateRouteInput{ + _, err = svc.CreateRoute(t.Context(), &ec2.CreateRouteInput{ RouteTableId: aws.String(rtbID), DestinationCidrBlock: aws.String("0.0.0.0/0"), GatewayId: aws.String("igw-test1"), @@ -1071,7 +1083,7 @@ func TestDeleteRoute_IGW_PublishesGateForAssociatedSubnets(t *testing.T) { } } - _, err = svc.DeleteRoute(context.Background(), &ec2.DeleteRouteInput{ + _, err = svc.DeleteRoute(t.Context(), &ec2.DeleteRouteInput{ RouteTableId: aws.String(rtbID), DestinationCidrBlock: aws.String("0.0.0.0/0"), }, testAccountID) @@ -1091,7 +1103,7 @@ func TestAssociateRouteTable_NoDefaultRoute_PublishesGate(t *testing.T) { defer func() { _ = gate.Unsubscribe() }() rtbID := createTestRtb(t, svc) - _, err = svc.AssociateRouteTable(context.Background(), &ec2.AssociateRouteTableInput{ + _, err = svc.AssociateRouteTable(t.Context(), &ec2.AssociateRouteTableInput{ RouteTableId: aws.String(rtbID), SubnetId: aws.String("subnet-priv1"), }, testAccountID) @@ -1112,14 +1124,14 @@ func TestAssociateRouteTable_HasIGWRoute_PublishesUngate(t *testing.T) { defer func() { _ = ungate.Unsubscribe() }() rtbID := createTestRtb(t, svc) - _, err = svc.CreateRoute(context.Background(), &ec2.CreateRouteInput{ + _, err = svc.CreateRoute(t.Context(), &ec2.CreateRouteInput{ RouteTableId: aws.String(rtbID), DestinationCidrBlock: aws.String("0.0.0.0/0"), GatewayId: aws.String("igw-test1"), }, testAccountID) require.NoError(t, err) - _, err = svc.AssociateRouteTable(context.Background(), &ec2.AssociateRouteTableInput{ + _, err = svc.AssociateRouteTable(t.Context(), &ec2.AssociateRouteTableInput{ RouteTableId: aws.String(rtbID), SubnetId: aws.String("subnet-priv1"), }, testAccountID) @@ -1142,13 +1154,13 @@ func TestDisassociateRouteTable_MainHasNoEgress_PublishesGate(t *testing.T) { _, err = svc.CreateRouteTableForVPC("vpc-test1", "10.0.0.0/16", testAccountID, true, "") require.NoError(t, err) customRtbID := createTestRtb(t, svc) - _, err = svc.CreateRoute(context.Background(), &ec2.CreateRouteInput{ + _, err = svc.CreateRoute(t.Context(), &ec2.CreateRouteInput{ RouteTableId: aws.String(customRtbID), DestinationCidrBlock: aws.String("0.0.0.0/0"), GatewayId: aws.String("igw-test1"), }, testAccountID) require.NoError(t, err) - assocOut, err := svc.AssociateRouteTable(context.Background(), &ec2.AssociateRouteTableInput{ + assocOut, err := svc.AssociateRouteTable(t.Context(), &ec2.AssociateRouteTableInput{ RouteTableId: aws.String(customRtbID), SubnetId: aws.String("subnet-priv1"), }, testAccountID) @@ -1159,7 +1171,7 @@ func TestDisassociateRouteTable_MainHasNoEgress_PublishesGate(t *testing.T) { } } - _, err = svc.DisassociateRouteTable(context.Background(), &ec2.DisassociateRouteTableInput{ + _, err = svc.DisassociateRouteTable(t.Context(), &ec2.DisassociateRouteTableInput{ AssociationId: assocOut.AssociationId, }, testAccountID) require.NoError(t, err) @@ -1181,7 +1193,7 @@ func TestCreateRoute_NonDefaultPrefix_NoGateEvent(t *testing.T) { defer func() { _ = ungate.Unsubscribe() }() rtbID := createTestRtb(t, svc) - _, err = svc.AssociateRouteTable(context.Background(), &ec2.AssociateRouteTableInput{ + _, err = svc.AssociateRouteTable(t.Context(), &ec2.AssociateRouteTableInput{ RouteTableId: aws.String(rtbID), SubnetId: aws.String("subnet-priv1"), }, testAccountID) @@ -1193,7 +1205,7 @@ func TestCreateRoute_NonDefaultPrefix_NoGateEvent(t *testing.T) { } } - _, err = svc.CreateRoute(context.Background(), &ec2.CreateRouteInput{ + _, err = svc.CreateRoute(t.Context(), &ec2.CreateRouteInput{ RouteTableId: aws.String(rtbID), DestinationCidrBlock: aws.String("8.8.8.8/32"), GatewayId: aws.String("igw-test1"), @@ -1216,7 +1228,7 @@ func TestCreateRoute_IGW_OnMainRT_FansOutUngateToImplicitSubnets(t *testing.T) { mainRT, err := svc.CreateRouteTableForVPC("vpc-test1", "10.0.0.0/16", testAccountID, true, "") require.NoError(t, err) - _, err = svc.CreateRoute(context.Background(), &ec2.CreateRouteInput{ + _, err = svc.CreateRoute(t.Context(), &ec2.CreateRouteInput{ RouteTableId: aws.String(mainRT.RouteTableId), DestinationCidrBlock: aws.String("0.0.0.0/0"), GatewayId: aws.String("igw-test1"), @@ -1236,7 +1248,7 @@ func TestDisassociateRouteTable_RestoresMainRTRoutesForDepartingSubnet(t *testin mainRT, err := svc.CreateRouteTableForVPC("vpc-test1", "10.0.0.0/16", testAccountID, true, "") require.NoError(t, err) - _, err = svc.CreateRoute(context.Background(), &ec2.CreateRouteInput{ + _, err = svc.CreateRoute(t.Context(), &ec2.CreateRouteInput{ RouteTableId: aws.String(mainRT.RouteTableId), DestinationCidrBlock: aws.String("0.0.0.0/0"), GatewayId: aws.String("igw-test1"), @@ -1244,7 +1256,7 @@ func TestDisassociateRouteTable_RestoresMainRTRoutesForDepartingSubnet(t *testin require.NoError(t, err) customRtbID := createTestRtb(t, svc) - assocOut, err := svc.AssociateRouteTable(context.Background(), &ec2.AssociateRouteTableInput{ + assocOut, err := svc.AssociateRouteTable(t.Context(), &ec2.AssociateRouteTableInput{ RouteTableId: aws.String(customRtbID), SubnetId: aws.String("subnet-priv1"), }, testAccountID) @@ -1258,7 +1270,7 @@ func TestDisassociateRouteTable_RestoresMainRTRoutesForDepartingSubnet(t *testin } } - _, err = svc.DisassociateRouteTable(context.Background(), &ec2.DisassociateRouteTableInput{ + _, err = svc.DisassociateRouteTable(t.Context(), &ec2.DisassociateRouteTableInput{ AssociationId: assocOut.AssociationId, }, testAccountID) require.NoError(t, err) @@ -1298,13 +1310,13 @@ func TestPublishGateDecisionsForVPC_UngatesSubnetWithIGWRoute(t *testing.T) { // Custom RT with 0.0.0.0/0 -> IGW, associated to subnet-test1. rtbID := createTestRtb(t, svc) - _, err = svc.CreateRoute(context.Background(), &ec2.CreateRouteInput{ + _, err = svc.CreateRoute(t.Context(), &ec2.CreateRouteInput{ RouteTableId: aws.String(rtbID), DestinationCidrBlock: aws.String("0.0.0.0/0"), GatewayId: aws.String("igw-test1"), }, testAccountID) require.NoError(t, err) - _, err = svc.AssociateRouteTable(context.Background(), &ec2.AssociateRouteTableInput{ + _, err = svc.AssociateRouteTable(t.Context(), &ec2.AssociateRouteTableInput{ RouteTableId: aws.String(rtbID), SubnetId: aws.String("subnet-test1"), }, testAccountID) @@ -1353,15 +1365,15 @@ func TestEffectiveRouteTable_DeterministicWithDuplicateMain(t *testing.T) { }, CreatedAt: now, } - require.NoError(t, svc.putRouteTable(context.Background(), testAccountID, &orphan)) - require.NoError(t, svc.putRouteTable(context.Background(), testAccountID, &real_)) + require.NoError(t, svc.putRouteTable(t.Context(), testAccountID, &orphan)) + require.NoError(t, svc.putRouteTable(t.Context(), testAccountID, &real_)) - rt, err := svc.effectiveRouteTable(context.Background(), testAccountID, "vpc-test1", "subnet-priv1") + rt, err := svc.effectiveRouteTable(t.Context(), testAccountID, "vpc-test1", "subnet-priv1") require.NoError(t, err) require.NotNil(t, rt) assert.Equal(t, "rtb-aaareal", rt.RouteTableId, "must prefer main RT with more routes") - main, err := svc.mainRouteTable(context.Background(), testAccountID, "vpc-test1") + main, err := svc.mainRouteTable(t.Context(), testAccountID, "vpc-test1") require.NoError(t, err) require.NotNil(t, main) assert.Equal(t, "rtb-aaareal", main.RouteTableId, "mainRouteTable must use same tiebreak") diff --git a/spinifex/handlers/ec2/snapshot/service_impl.go b/spinifex/handlers/ec2/snapshot/service_impl.go index 374580dc7..9a38b3621 100644 --- a/spinifex/handlers/ec2/snapshot/service_impl.go +++ b/spinifex/handlers/ec2/snapshot/service_impl.go @@ -22,12 +22,14 @@ import ( "github.com/mulgadc/spinifex/spinifex/awserrors" "github.com/mulgadc/spinifex/spinifex/config" "github.com/mulgadc/spinifex/spinifex/filterutil" + "github.com/mulgadc/spinifex/spinifex/kvutil" "github.com/mulgadc/spinifex/spinifex/migrate" "github.com/mulgadc/spinifex/spinifex/objectstore" "github.com/mulgadc/spinifex/spinifex/utils" "github.com/mulgadc/viperblock/viperblock" vbs3 "github.com/mulgadc/viperblock/viperblock/backends/s3" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // Ensure SnapshotServiceImpl implements SnapshotService. @@ -36,6 +38,7 @@ var _ SnapshotService = (*SnapshotServiceImpl)(nil) const ( KVBucketVolumeSnapshots = "spinifex-volume-snapshots" KVBucketVolumeSnapshotsVersion = 1 + snapshotCleanupTimeout = 5 * time.Second ) // SnapshotServiceImpl implements SnapshotService with S3-backed storage. @@ -43,7 +46,7 @@ type SnapshotServiceImpl struct { config *config.Config store objectstore.ObjectStore natsConn *nats.Conn - snapKV nats.KeyValue + snapKV jetstream.KeyValue mutex sync.RWMutex } @@ -63,7 +66,7 @@ type SnapshotConfig struct { } // NewSnapshotServiceImplWithNATS creates a snapshot service with JetStream KV for volume-snapshot tracking. -func NewSnapshotServiceImplWithNATS(cfg *config.Config, natsConn *nats.Conn) (*SnapshotServiceImpl, nats.KeyValue, error) { +func NewSnapshotServiceImplWithNATS(ctx context.Context, cfg *config.Config, natsConn *nats.Conn) (*SnapshotServiceImpl, jetstream.KeyValue, error) { store := objectstore.NewS3ObjectStoreFromConfig( cfg.Predastore.Host, cfg.Predastore.Region, @@ -71,16 +74,16 @@ func NewSnapshotServiceImplWithNATS(cfg *config.Config, natsConn *nats.Conn) (*S cfg.Predastore.SecretKey, ) - js, err := natsConn.JetStream() + js, err := jetstream.New(natsConn) if err != nil { - return nil, nil, fmt.Errorf("failed to get JetStream context: %w", err) + return nil, nil, fmt.Errorf("create JetStream: %w", err) } - kv, err := utils.GetOrCreateKVBucket(js, KVBucketVolumeSnapshots, 10) + kv, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketVolumeSnapshots, 10) if err != nil { - return nil, nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketVolumeSnapshots, err) + return nil, nil, fmt.Errorf("create KV bucket %s: %w", KVBucketVolumeSnapshots, err) } - if err := migrate.DefaultRegistry.RunKV(KVBucketVolumeSnapshots, kv, KVBucketVolumeSnapshotsVersion); err != nil { + if err := migrate.DefaultRegistry.RunKV(ctx, KVBucketVolumeSnapshots, kv, KVBucketVolumeSnapshotsVersion); err != nil { return nil, nil, fmt.Errorf("migrate %s: %w", KVBucketVolumeSnapshots, err) } @@ -96,7 +99,7 @@ func NewSnapshotServiceImplWithNATS(cfg *config.Config, natsConn *nats.Conn) (*S // NewSnapshotServiceImplWithStore creates a snapshot service with a custom ObjectStore (for testing). // An optional snapshotKV can be provided for KV-backed volume-snapshot tracking. -func NewSnapshotServiceImplWithStore(cfg *config.Config, store objectstore.ObjectStore, natsConn *nats.Conn, snapshotKV ...nats.KeyValue) *SnapshotServiceImpl { +func NewSnapshotServiceImplWithStore(cfg *config.Config, store objectstore.ObjectStore, natsConn *nats.Conn, snapshotKV ...jetstream.KeyValue) *SnapshotServiceImpl { svc := &SnapshotServiceImpl{ config: cfg, store: store, @@ -271,14 +274,16 @@ func (s *SnapshotServiceImpl) CreateSnapshot(ctx context.Context, input *ec2.Cre // Track the volume→snapshot dependency in KV before persisting to S3. // This ensures we never have an untracked snapshot in S3. - if err := s.addSnapshotRef(volumeID, snapshotID); err != nil { + if err := s.addSnapshotRef(ctx, volumeID, snapshotID); err != nil { slog.ErrorContext(ctx, "CreateSnapshot failed to add snapshot ref to KV", "snapshotId", snapshotID, "volumeId", volumeID, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } if err := s.putSnapshotConfig(snapshotID, snapshotCfg); err != nil { slog.ErrorContext(ctx, "CreateSnapshot failed to write config", "snapshotId", snapshotID, "err", err) - _ = s.removeSnapshotRef(volumeID, snapshotID) // best-effort cleanup + if cleanupErr := s.removeSnapshotRefForCleanup(ctx, volumeID, snapshotID); cleanupErr != nil { + slog.Error("CreateSnapshot failed to roll back snapshot reference", "snapshotId", snapshotID, "volumeId", volumeID, "err", cleanupErr) + } return nil, errors.New(awserrors.ErrorServerInternal) } @@ -590,8 +595,8 @@ func (s *SnapshotServiceImpl) DeleteSnapshot(ctx context.Context, input *ec2.Del // Remove from KV after S3 cleanup. Failure is logged but not fatal — // a phantom entry safely blocks volume deletion rather than allowing it. - if err := s.removeSnapshotRef(cfg.VolumeID, snapshotID); err != nil { - slog.WarnContext(ctx, "DeleteSnapshot failed to remove snapshot ref from KV", "snapshotId", snapshotID, "volumeId", cfg.VolumeID, "err", err) + if err := s.removeSnapshotRefForCleanup(ctx, cfg.VolumeID, snapshotID); err != nil { + slog.Warn("DeleteSnapshot failed to remove snapshot ref from KV", "snapshotId", snapshotID, "volumeId", cfg.VolumeID, "err", err) } slog.InfoContext(ctx, "DeleteSnapshot completed", "snapshotId", snapshotID) @@ -645,14 +650,16 @@ func (s *SnapshotServiceImpl) CopySnapshot(ctx context.Context, input *ec2.CopyS maps.Copy(newCfg.Tags, sourceCfg.Tags) // Track the volume→snapshot dependency in KV before persisting to S3. - if err := s.addSnapshotRef(sourceCfg.VolumeID, newSnapshotID); err != nil { + if err := s.addSnapshotRef(ctx, sourceCfg.VolumeID, newSnapshotID); err != nil { slog.ErrorContext(ctx, "CopySnapshot failed to add snapshot ref to KV", "snapshotId", newSnapshotID, "volumeId", sourceCfg.VolumeID, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } if err := s.putSnapshotConfig(newSnapshotID, newCfg); err != nil { slog.ErrorContext(ctx, "CopySnapshot failed to write config", "snapshotId", newSnapshotID, "err", err) - _ = s.removeSnapshotRef(sourceCfg.VolumeID, newSnapshotID) // best-effort cleanup + if cleanupErr := s.removeSnapshotRefForCleanup(ctx, sourceCfg.VolumeID, newSnapshotID); cleanupErr != nil { + slog.Error("CopySnapshot failed to roll back snapshot reference", "snapshotId", newSnapshotID, "volumeId", sourceCfg.VolumeID, "err", cleanupErr) + } return nil, errors.New(awserrors.ErrorServerInternal) } @@ -665,7 +672,7 @@ func (s *SnapshotServiceImpl) CopySnapshot(ctx context.Context, input *ec2.CopyS // addSnapshotRef adds snapshotID to the volume's snapshot list in KV. // Uses CAS (Create/Update with revision) to prevent lost updates under concurrency. -func (s *SnapshotServiceImpl) addSnapshotRef(volumeID, snapshotID string) error { +func (s *SnapshotServiceImpl) addSnapshotRef(ctx context.Context, volumeID, snapshotID string) error { if s.snapKV == nil { slog.Debug("addSnapshotRef: snapshotKV is nil, skipping", "volumeId", volumeID, "snapshotId", snapshotID) return nil @@ -673,11 +680,11 @@ func (s *SnapshotServiceImpl) addSnapshotRef(volumeID, snapshotID string) error const maxRetries = 5 for attempt := range maxRetries { - entry, err := s.snapKV.Get(volumeID) + entry, err := s.snapKV.Get(ctx, volumeID) var snapshots []string if err != nil { - if !errors.Is(err, nats.ErrKeyNotFound) { + if !errors.Is(err, jetstream.ErrKeyNotFound) { return fmt.Errorf("addSnapshotRef: failed to get KV key %s: %w", volumeID, err) } // Key doesn't exist yet — create with just this snapshot @@ -685,7 +692,7 @@ func (s *SnapshotServiceImpl) addSnapshotRef(volumeID, snapshotID string) error if err != nil { return fmt.Errorf("addSnapshotRef: failed to marshal snapshot list: %w", err) } - if _, err := s.snapKV.Create(volumeID, data); err != nil { + if _, err := s.snapKV.Create(ctx, volumeID, data); err != nil { if attempt < maxRetries-1 { continue // concurrent Create/Update — retry } @@ -706,7 +713,7 @@ func (s *SnapshotServiceImpl) addSnapshotRef(volumeID, snapshotID string) error return fmt.Errorf("addSnapshotRef: failed to marshal snapshot list: %w", err) } - if _, err := s.snapKV.Update(volumeID, data, entry.Revision()); err != nil { + if _, err := s.snapKV.Update(ctx, volumeID, data, entry.Revision()); err != nil { if attempt < maxRetries-1 { continue // concurrent update — retry } @@ -720,10 +727,18 @@ func (s *SnapshotServiceImpl) addSnapshotRef(volumeID, snapshotID string) error return fmt.Errorf("addSnapshotRef: exhausted retries for KV key %s", volumeID) } +// removeSnapshotRefForCleanup removes a reference with a bounded context that +// survives request cancellation after the snapshot operation has committed. +func (s *SnapshotServiceImpl) removeSnapshotRefForCleanup(ctx context.Context, volumeID, snapshotID string) error { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), snapshotCleanupTimeout) + defer cancel() + return s.removeSnapshotRef(cleanupCtx, volumeID, snapshotID) +} + // removeSnapshotRef removes snapshotID from the volume's snapshot list in KV. // Deletes the key if the list becomes empty. // Uses CAS (Update with revision) to prevent lost updates under concurrency. -func (s *SnapshotServiceImpl) removeSnapshotRef(volumeID, snapshotID string) error { +func (s *SnapshotServiceImpl) removeSnapshotRef(ctx context.Context, volumeID, snapshotID string) error { if s.snapKV == nil { slog.Debug("removeSnapshotRef: snapshotKV is nil, skipping", "volumeId", volumeID, "snapshotId", snapshotID) return nil @@ -731,9 +746,9 @@ func (s *SnapshotServiceImpl) removeSnapshotRef(volumeID, snapshotID string) err const maxRetries = 5 for attempt := range maxRetries { - entry, err := s.snapKV.Get(volumeID) + entry, err := s.snapKV.Get(ctx, volumeID) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil } return fmt.Errorf("removeSnapshotRef: failed to get KV key %s: %w", volumeID, err) @@ -752,7 +767,7 @@ func (s *SnapshotServiceImpl) removeSnapshotRef(volumeID, snapshotID string) err } if len(filtered) == 0 { - if err := s.snapKV.Delete(volumeID); err != nil && !errors.Is(err, nats.ErrKeyNotFound) { + if err := s.snapKV.Delete(ctx, volumeID); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { return fmt.Errorf("removeSnapshotRef: failed to delete KV key %s: %w", volumeID, err) } } else { @@ -760,7 +775,7 @@ func (s *SnapshotServiceImpl) removeSnapshotRef(volumeID, snapshotID string) err if err != nil { return fmt.Errorf("removeSnapshotRef: failed to marshal snapshot list: %w", err) } - if _, err := s.snapKV.Update(volumeID, data, entry.Revision()); err != nil { + if _, err := s.snapKV.Update(ctx, volumeID, data, entry.Revision()); err != nil { if attempt < maxRetries-1 { continue // concurrent update — retry } @@ -776,14 +791,14 @@ func (s *SnapshotServiceImpl) removeSnapshotRef(volumeID, snapshotID string) err } // volumeHasSnapshots returns true if the volume has any snapshots in KV. -func (s *SnapshotServiceImpl) volumeHasSnapshots(volumeID string) (bool, error) { +func (s *SnapshotServiceImpl) volumeHasSnapshots(ctx context.Context, volumeID string) (bool, error) { if s.snapKV == nil { return false, nil } - entry, err := s.snapKV.Get(volumeID) + entry, err := s.snapKV.Get(ctx, volumeID) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return false, nil } return false, fmt.Errorf("volumeHasSnapshots: failed to get KV key %s: %w", volumeID, err) diff --git a/spinifex/handlers/ec2/snapshot/service_impl_test.go b/spinifex/handlers/ec2/snapshot/service_impl_test.go index c05444486..d55d89712 100644 --- a/spinifex/handlers/ec2/snapshot/service_impl_test.go +++ b/spinifex/handlers/ec2/snapshot/service_impl_test.go @@ -8,7 +8,6 @@ import ( "net/http/httptest" "strings" "testing" - "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" @@ -16,9 +15,9 @@ import ( "github.com/mulgadc/spinifex/spinifex/awserrors" "github.com/mulgadc/spinifex/spinifex/config" "github.com/mulgadc/spinifex/spinifex/objectstore" + "github.com/mulgadc/spinifex/spinifex/testutil" "github.com/mulgadc/viperblock/viperblock" - "github.com/nats-io/nats-server/v2/server" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -588,30 +587,12 @@ func TestCopySnapshot_PreservesTags(t *testing.T) { } // setupTestNATSKV creates a NATS JetStream test server and returns a KV bucket for testing. -func setupTestNATSKV(t *testing.T) nats.KeyValue { +func setupTestNATSKV(t *testing.T) jetstream.KeyValue { t.Helper() - opts := &server.Options{ - Host: "127.0.0.1", - Port: -1, - JetStream: true, - StoreDir: t.TempDir(), - NoLog: true, - NoSigs: true, - } - ns, err := server.NewServer(opts) - require.NoError(t, err) - go ns.Start() - require.True(t, ns.ReadyForConnections(5*time.Second)) - t.Cleanup(func() { ns.Shutdown() }) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) - nc, err := nats.Connect(ns.ClientURL()) - require.NoError(t, err) - t.Cleanup(func() { nc.Close() }) - - js, err := nc.JetStream() - require.NoError(t, err) - - kv, err := js.CreateKeyValue(&nats.KeyValueConfig{ + kv, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{ Bucket: KVBucketVolumeSnapshots, }) require.NoError(t, err) @@ -622,10 +603,10 @@ func TestAddSnapshotRef(t *testing.T) { kv := setupTestNATSKV(t) svc := &SnapshotServiceImpl{snapKV: kv} - require.NoError(t, svc.addSnapshotRef("vol-1", "snap-a")) - require.NoError(t, svc.addSnapshotRef("vol-1", "snap-b")) + require.NoError(t, svc.addSnapshotRef(t.Context(), "vol-1", "snap-a")) + require.NoError(t, svc.addSnapshotRef(t.Context(), "vol-1", "snap-b")) - entry, err := kv.Get("vol-1") + entry, err := kv.Get(t.Context(), "vol-1") require.NoError(t, err) var snapshots []string require.NoError(t, json.Unmarshal(entry.Value(), &snapshots)) @@ -636,23 +617,23 @@ func TestRemoveSnapshotRef(t *testing.T) { kv := setupTestNATSKV(t) svc := &SnapshotServiceImpl{snapKV: kv} - require.NoError(t, svc.addSnapshotRef("vol-1", "snap-a")) - require.NoError(t, svc.addSnapshotRef("vol-1", "snap-b")) + require.NoError(t, svc.addSnapshotRef(t.Context(), "vol-1", "snap-a")) + require.NoError(t, svc.addSnapshotRef(t.Context(), "vol-1", "snap-b")) // Remove one - require.NoError(t, svc.removeSnapshotRef("vol-1", "snap-a")) + require.NoError(t, svc.removeSnapshotRef(t.Context(), "vol-1", "snap-a")) - entry, err := kv.Get("vol-1") + entry, err := kv.Get(t.Context(), "vol-1") require.NoError(t, err) var snapshots []string require.NoError(t, json.Unmarshal(entry.Value(), &snapshots)) assert.Equal(t, []string{"snap-b"}, snapshots) // Remove last — key should be deleted - require.NoError(t, svc.removeSnapshotRef("vol-1", "snap-b")) + require.NoError(t, svc.removeSnapshotRef(t.Context(), "vol-1", "snap-b")) - _, err = kv.Get("vol-1") - assert.ErrorIs(t, err, nats.ErrKeyNotFound) + _, err = kv.Get(t.Context(), "vol-1") + assert.ErrorIs(t, err, jetstream.ErrKeyNotFound) } func TestRemoveSnapshotRef_NonExistentKey(t *testing.T) { @@ -660,7 +641,20 @@ func TestRemoveSnapshotRef_NonExistentKey(t *testing.T) { svc := &SnapshotServiceImpl{snapKV: kv} // Should not error on non-existent key - require.NoError(t, svc.removeSnapshotRef("vol-nonexistent", "snap-x")) + require.NoError(t, svc.removeSnapshotRef(t.Context(), "vol-nonexistent", "snap-x")) +} + +func TestRemoveSnapshotRefForCleanupSurvivesCancellation(t *testing.T) { + kv := setupTestNATSKV(t) + svc := &SnapshotServiceImpl{snapKV: kv} + require.NoError(t, svc.addSnapshotRef(t.Context(), "vol-1", "snap-a")) + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + require.NoError(t, svc.removeSnapshotRefForCleanup(ctx, "vol-1", "snap-a")) + + _, err := kv.Get(t.Context(), "vol-1") + require.ErrorIs(t, err, jetstream.ErrKeyNotFound) } func TestVolumeHasSnapshots(t *testing.T) { @@ -668,19 +662,19 @@ func TestVolumeHasSnapshots(t *testing.T) { svc := &SnapshotServiceImpl{snapKV: kv} // No entry → false - has, err := svc.volumeHasSnapshots("vol-1") + has, err := svc.volumeHasSnapshots(t.Context(), "vol-1") require.NoError(t, err) assert.False(t, has) // Add one → true - require.NoError(t, svc.addSnapshotRef("vol-1", "snap-a")) - has, err = svc.volumeHasSnapshots("vol-1") + require.NoError(t, svc.addSnapshotRef(t.Context(), "vol-1", "snap-a")) + has, err = svc.volumeHasSnapshots(t.Context(), "vol-1") require.NoError(t, err) assert.True(t, has) // Remove it → false - require.NoError(t, svc.removeSnapshotRef("vol-1", "snap-a")) - has, err = svc.volumeHasSnapshots("vol-1") + require.NoError(t, svc.removeSnapshotRef(t.Context(), "vol-1", "snap-a")) + has, err = svc.volumeHasSnapshots(t.Context(), "vol-1") require.NoError(t, err) assert.False(t, has) } @@ -689,9 +683,9 @@ func TestKVNilFallback(t *testing.T) { svc := &SnapshotServiceImpl{snapKV: nil} // All methods should be no-ops when KV is nil - require.NoError(t, svc.addSnapshotRef("vol-1", "snap-a")) - require.NoError(t, svc.removeSnapshotRef("vol-1", "snap-a")) - has, err := svc.volumeHasSnapshots("vol-1") + require.NoError(t, svc.addSnapshotRef(t.Context(), "vol-1", "snap-a")) + require.NoError(t, svc.removeSnapshotRef(t.Context(), "vol-1", "snap-a")) + has, err := svc.volumeHasSnapshots(t.Context(), "vol-1") require.NoError(t, err) assert.False(t, has) } @@ -716,12 +710,12 @@ func TestCreateSnapshot_WritesKVEntry(t *testing.T) { require.NoError(t, err) // Verify KV entry exists - has, err := svc.volumeHasSnapshots(volumeID) + has, err := svc.volumeHasSnapshots(t.Context(), volumeID) require.NoError(t, err) assert.True(t, has) // Verify snapshot ID is in the list - entry, err := kv.Get(volumeID) + entry, err := kv.Get(t.Context(), volumeID) require.NoError(t, err) var snapshots []string require.NoError(t, json.Unmarshal(entry.Value(), &snapshots)) @@ -754,7 +748,7 @@ func TestDeleteSnapshot_RemovesKVEntry(t *testing.T) { require.NoError(t, err) // KV should now be empty for this volume - has, err := svc.volumeHasSnapshots(volumeID) + has, err := svc.volumeHasSnapshots(t.Context(), volumeID) require.NoError(t, err) assert.False(t, has) } @@ -784,7 +778,7 @@ func TestCopySnapshot_AddsKVEntry(t *testing.T) { require.NoError(t, err) // Both snapshot IDs should be in KV - entry, err := kv.Get(volumeID) + entry, err := kv.Get(t.Context(), volumeID) require.NoError(t, err) var snapshots []string require.NoError(t, json.Unmarshal(entry.Value(), &snapshots)) diff --git a/spinifex/handlers/ec2/spotinstance/service_impl.go b/spinifex/handlers/ec2/spotinstance/service_impl.go index 634720a8a..20ccfaf4a 100644 --- a/spinifex/handlers/ec2/spotinstance/service_impl.go +++ b/spinifex/handlers/ec2/spotinstance/service_impl.go @@ -14,9 +14,11 @@ import ( "github.com/mulgadc/spinifex/spinifex/awserrors" "github.com/mulgadc/spinifex/spinifex/config" "github.com/mulgadc/spinifex/spinifex/filterutil" + "github.com/mulgadc/spinifex/spinifex/kvutil" "github.com/mulgadc/spinifex/spinifex/migrate" "github.com/mulgadc/spinifex/spinifex/utils" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // Ensure SpotInstanceServiceImpl implements SpotInstanceService. @@ -28,30 +30,30 @@ var _ SpotInstanceService = (*SpotInstanceServiceImpl)(nil) type SpotInstanceServiceImpl struct { config *config.Config natsConn *nats.Conn - activeKV nats.KeyValue - terminalKV nats.KeyValue + activeKV jetstream.KeyValue + terminalKV jetstream.KeyValue } // NewSpotInstanceServiceImplWithNATS creates a spot instance service with NATS JetStream. -func NewSpotInstanceServiceImplWithNATS(cfg *config.Config, natsConn *nats.Conn) (*SpotInstanceServiceImpl, error) { - js, err := natsConn.JetStream() +func NewSpotInstanceServiceImplWithNATS(ctx context.Context, cfg *config.Config, natsConn *nats.Conn) (*SpotInstanceServiceImpl, error) { + js, err := jetstream.New(natsConn) if err != nil { return nil, fmt.Errorf("failed to get JetStream context: %w", err) } - activeKV, err := utils.GetOrCreateKVBucket(js, KVBucketSpotRequests, 10) + activeKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketSpotRequests, 10) if err != nil { return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketSpotRequests, err) } - if err := migrate.DefaultRegistry.RunKV(KVBucketSpotRequests, activeKV, KVBucketSpotRequestsVersion); err != nil { + if err := migrate.DefaultRegistry.RunKV(ctx, KVBucketSpotRequests, activeKV, KVBucketSpotRequestsVersion); err != nil { return nil, fmt.Errorf("migrate %s: %w", KVBucketSpotRequests, err) } - terminalKV, err := getOrCreateTerminalBucket(js) + terminalKV, err := getOrCreateTerminalBucket(ctx, js) if err != nil { return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketSpotRequestsTerminal, err) } - if err := migrate.DefaultRegistry.RunKV(KVBucketSpotRequestsTerminal, terminalKV, KVBucketSpotRequestsVersion); err != nil { + if err := migrate.DefaultRegistry.RunKV(ctx, KVBucketSpotRequestsTerminal, terminalKV, KVBucketSpotRequestsVersion); err != nil { return nil, fmt.Errorf("migrate %s: %w", KVBucketSpotRequestsTerminal, err) } @@ -66,22 +68,27 @@ func NewSpotInstanceServiceImplWithNATS(cfg *config.Config, natsConn *nats.Conn) }, nil } -// getOrCreateTerminalBucket creates the terminal bucket with a TTL, mirroring -// the terminated-instances bucket; GetOrCreateKVBucket cannot set a TTL. -func getOrCreateTerminalBucket(js nats.JetStreamContext) (nats.KeyValue, error) { - kv, err := js.KeyValue(KVBucketSpotRequestsTerminal) +// getOrCreateTerminalBucket opens the terminal bucket without changing its +// existing replica or placement configuration, creating it only when absent. +func getOrCreateTerminalBucket(ctx context.Context, js jetstream.KeyValueManager) (jetstream.KeyValue, error) { + kv, err := js.KeyValue(ctx, KVBucketSpotRequestsTerminal) if err == nil { return kv, nil } - if !errors.Is(err, nats.ErrBucketNotFound) { + if !errors.Is(err, jetstream.ErrBucketNotFound) { return nil, err } - return js.CreateKeyValue(&nats.KeyValueConfig{ + + kv, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{ Bucket: KVBucketSpotRequestsTerminal, Description: "Terminal Spot Instance Requests (auto-expire after 1 hour)", History: 1, TTL: spotTerminalTTL, }) + if errors.Is(err, jetstream.ErrBucketExists) { + return js.KeyValue(ctx, KVBucketSpotRequestsTerminal) + } + return kv, err } // PutSpotInstanceRequests stores each request in the active bucket. @@ -96,7 +103,7 @@ func (s *SpotInstanceServiceImpl) PutSpotInstanceRequests(ctx context.Context, i if err != nil { return nil, errors.New(awserrors.ErrorServerInternal) } - if _, err := s.activeKV.Put(utils.AccountKey(accountID, sirID), data); err != nil { + if _, err := s.activeKV.Put(ctx, utils.AccountKey(accountID, sirID), data); err != nil { slog.ErrorContext(ctx, "PutSpotInstanceRequests: KV put failed", "sirId", sirID, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } @@ -133,11 +140,11 @@ func (s *SpotInstanceServiceImpl) DescribeSpotInstanceRequests(ctx context.Conte } } - active, err := s.listRequests(s.activeKV, accountID) + active, err := s.listRequests(ctx, s.activeKV, accountID) if err != nil { return nil, err } - terminal, err := s.listRequests(s.terminalKV, accountID) + terminal, err := s.listRequests(ctx, s.terminalKV, accountID) if err != nil { return nil, err } @@ -183,15 +190,15 @@ func (s *SpotInstanceServiceImpl) CancelSpotInstanceRequests(ctx context.Context sirID := aws.StringValue(idPtr) key := utils.AccountKey(accountID, sirID) - record, err := s.readRecord(s.activeKV, key) + record, err := s.readRecord(ctx, s.activeKV, key) if err == nil { record.Request.State = aws.String(ec2.SpotInstanceStateCancelled) setSpotStatus(record.Request, SpotStatusCodeCanceledInstanceRunning, "Spot Instance request cancelled; the instance is still running.") - if err := s.moveToTerminal(key, record); err != nil { + if err := s.moveToTerminal(ctx, key, record); err != nil { return nil, err } - } else if !errors.Is(err, nats.ErrKeyNotFound) { + } else if !errors.Is(err, jetstream.ErrKeyNotFound) { return nil, errors.New(awserrors.ErrorServerInternal) } @@ -213,9 +220,9 @@ func (s *SpotInstanceServiceImpl) CloseForInstance(ctx context.Context, instance return nil } - keys, err := s.activeKV.Keys() + keys, err := s.activeKV.Keys(ctx) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return nil } return errors.New(awserrors.ErrorServerInternal) @@ -226,7 +233,7 @@ func (s *SpotInstanceServiceImpl) CloseForInstance(ctx context.Context, instance if key == utils.VersionKey || !strings.HasPrefix(key, prefix) { continue } - record, err := s.readRecord(s.activeKV, key) + record, err := s.readRecord(ctx, s.activeKV, key) if err != nil { continue } @@ -237,7 +244,7 @@ func (s *SpotInstanceServiceImpl) CloseForInstance(ctx context.Context, instance record.Request.State = aws.String(ec2.SpotInstanceStateClosed) setSpotStatus(record.Request, SpotStatusCodeInstanceTerminatedByUser, "Spot Instance request closed; the instance was terminated by the user.") - if err := s.moveToTerminal(key, record); err != nil { + if err := s.moveToTerminal(ctx, key, record); err != nil { return err } slog.InfoContext(ctx, "CloseForInstance moved SIR to terminal", "instanceId", instanceID, @@ -249,10 +256,10 @@ func (s *SpotInstanceServiceImpl) CloseForInstance(ctx context.Context, instance } // listRequests reads all account-scoped requests from a bucket. -func (s *SpotInstanceServiceImpl) listRequests(kv nats.KeyValue, accountID string) ([]*ec2.SpotInstanceRequest, error) { - keys, err := kv.Keys() +func (s *SpotInstanceServiceImpl) listRequests(ctx context.Context, kv jetstream.KeyValue, accountID string) ([]*ec2.SpotInstanceRequest, error) { + keys, err := kv.Keys(ctx) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return nil, nil } return nil, errors.New(awserrors.ErrorServerInternal) @@ -264,7 +271,7 @@ func (s *SpotInstanceServiceImpl) listRequests(kv nats.KeyValue, accountID strin if key == utils.VersionKey || !strings.HasPrefix(key, prefix) { continue } - record, err := s.readRecord(kv, key) + record, err := s.readRecord(ctx, kv, key) if err != nil { slog.Warn("Failed to read spot request record", "key", key, "err", err) continue @@ -275,9 +282,9 @@ func (s *SpotInstanceServiceImpl) listRequests(kv nats.KeyValue, accountID strin } // readRecord fetches and unmarshals a single record. The KV Get error (e.g. -// nats.ErrKeyNotFound) is returned unwrapped so callers can match on it. -func (s *SpotInstanceServiceImpl) readRecord(kv nats.KeyValue, key string) (*SpotRequestRecord, error) { - entry, err := kv.Get(key) +// jetstream.ErrKeyNotFound) is returned unwrapped so callers can match on it. +func (s *SpotInstanceServiceImpl) readRecord(ctx context.Context, kv jetstream.KeyValue, key string) (*SpotRequestRecord, error) { + entry, err := kv.Get(ctx, key) if err != nil { return nil, err } @@ -289,16 +296,16 @@ func (s *SpotInstanceServiceImpl) readRecord(kv nats.KeyValue, key string) (*Spo } // moveToTerminal writes the record to the terminal bucket and deletes it from active. -func (s *SpotInstanceServiceImpl) moveToTerminal(key string, record *SpotRequestRecord) error { +func (s *SpotInstanceServiceImpl) moveToTerminal(ctx context.Context, key string, record *SpotRequestRecord) error { data, err := json.Marshal(record) if err != nil { return errors.New(awserrors.ErrorServerInternal) } - if _, err := s.terminalKV.Put(key, data); err != nil { + if _, err := s.terminalKV.Put(ctx, key, data); err != nil { slog.Error("moveToTerminal: terminal put failed", "key", key, "err", err) return errors.New(awserrors.ErrorServerInternal) } - if err := s.activeKV.Delete(key); err != nil { + if err := s.activeKV.Delete(ctx, key); err != nil { slog.Error("moveToTerminal: active delete failed", "key", key, "err", err) return errors.New(awserrors.ErrorServerInternal) } diff --git a/spinifex/handlers/ec2/spotinstance/service_impl_test.go b/spinifex/handlers/ec2/spotinstance/service_impl_test.go index 3ec3c6d6a..61701a072 100644 --- a/spinifex/handlers/ec2/spotinstance/service_impl_test.go +++ b/spinifex/handlers/ec2/spotinstance/service_impl_test.go @@ -1,13 +1,14 @@ package handlers_ec2_spotinstance import ( - "context" "testing" + "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" "github.com/mulgadc/spinifex/spinifex/awserrors" "github.com/mulgadc/spinifex/spinifex/testutil" + "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -18,11 +19,43 @@ func setupTestService(t *testing.T) *SpotInstanceServiceImpl { t.Helper() _, nc, _ := testutil.StartTestJetStream(t) - svc, err := NewSpotInstanceServiceImplWithNATS(nil, nc) + svc, err := NewSpotInstanceServiceImplWithNATS(t.Context(), nil, nc) require.NoError(t, err) return svc } +func TestTerminalBucketTTL(t *testing.T) { + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + + _, err := NewSpotInstanceServiceImplWithNATS(t.Context(), nil, nc) + require.NoError(t, err) + kv, err := js.KeyValue(t.Context(), KVBucketSpotRequestsTerminal) + require.NoError(t, err) + status, err := kv.Status(t.Context()) + require.NoError(t, err) + assert.Equal(t, spotTerminalTTL, status.TTL()) +} + +func TestGetOrCreateTerminalBucketPreservesExistingConfiguration(t *testing.T) { + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + const existingTTL = 2 * time.Hour + + _, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{ + Bucket: KVBucketSpotRequestsTerminal, + History: 1, + TTL: existingTTL, + }) + require.NoError(t, err) + + kv, err := getOrCreateTerminalBucket(t.Context(), js) + require.NoError(t, err) + status, err := kv.Status(t.Context()) + require.NoError(t, err) + require.Equal(t, existingTTL, status.TTL()) +} + // makeSIR builds an active/fulfilled Spot Instance Request for tests. func makeSIR(sirID, instanceID string) *ec2.SpotInstanceRequest { return &ec2.SpotInstanceRequest{ @@ -42,13 +75,13 @@ func makeSIR(sirID, instanceID string) *ec2.SpotInstanceRequest { func putSIRs(t *testing.T, svc *SpotInstanceServiceImpl, reqs ...*ec2.SpotInstanceRequest) { t.Helper() - _, err := svc.PutSpotInstanceRequests(context.Background(), &PutSpotRequestsInput{Requests: reqs}, testAccountID) + _, err := svc.PutSpotInstanceRequests(t.Context(), &PutSpotRequestsInput{Requests: reqs}, testAccountID) require.NoError(t, err) } func describeAll(t *testing.T, svc *SpotInstanceServiceImpl, accountID string) []*ec2.SpotInstanceRequest { t.Helper() - out, err := svc.DescribeSpotInstanceRequests(context.Background(), &ec2.DescribeSpotInstanceRequestsInput{}, accountID) + out, err := svc.DescribeSpotInstanceRequests(t.Context(), &ec2.DescribeSpotInstanceRequestsInput{}, accountID) require.NoError(t, err) return out.SpotInstanceRequests } @@ -73,7 +106,7 @@ func TestPutDescribe_RoundTrip(t *testing.T) { func TestPutSpotInstanceRequests_MissingID(t *testing.T) { svc := setupTestService(t) - _, err := svc.PutSpotInstanceRequests(context.Background(), &PutSpotRequestsInput{ + _, err := svc.PutSpotInstanceRequests(t.Context(), &PutSpotRequestsInput{ Requests: []*ec2.SpotInstanceRequest{{InstanceId: aws.String("i-noid")}}, }, testAccountID) require.Error(t, err) @@ -99,7 +132,7 @@ func TestDescribe_FilterByID(t *testing.T) { svc := setupTestService(t) putSIRs(t, svc, makeSIR("sir-aaa", "i-aaa"), makeSIR("sir-bbb", "i-bbb")) - out, err := svc.DescribeSpotInstanceRequests(context.Background(), &ec2.DescribeSpotInstanceRequestsInput{ + out, err := svc.DescribeSpotInstanceRequests(t.Context(), &ec2.DescribeSpotInstanceRequestsInput{ SpotInstanceRequestIds: []*string{aws.String("sir-aaa")}, }, testAccountID) require.NoError(t, err) @@ -111,7 +144,7 @@ func TestDescribe_UnknownID(t *testing.T) { svc := setupTestService(t) putSIRs(t, svc, makeSIR("sir-aaa", "i-aaa")) - _, err := svc.DescribeSpotInstanceRequests(context.Background(), &ec2.DescribeSpotInstanceRequestsInput{ + _, err := svc.DescribeSpotInstanceRequests(t.Context(), &ec2.DescribeSpotInstanceRequestsInput{ SpotInstanceRequestIds: []*string{aws.String("sir-ghost")}, }, testAccountID) require.Error(t, err) @@ -123,12 +156,12 @@ func TestDescribe_FilterByState(t *testing.T) { putSIRs(t, svc, makeSIR("sir-aaa", "i-aaa"), makeSIR("sir-bbb", "i-bbb")) // Cancel one so it becomes terminal/cancelled. - _, err := svc.CancelSpotInstanceRequests(context.Background(), &ec2.CancelSpotInstanceRequestsInput{ + _, err := svc.CancelSpotInstanceRequests(t.Context(), &ec2.CancelSpotInstanceRequestsInput{ SpotInstanceRequestIds: []*string{aws.String("sir-bbb")}, }, testAccountID) require.NoError(t, err) - out, err := svc.DescribeSpotInstanceRequests(context.Background(), &ec2.DescribeSpotInstanceRequestsInput{ + out, err := svc.DescribeSpotInstanceRequests(t.Context(), &ec2.DescribeSpotInstanceRequestsInput{ Filters: []*ec2.Filter{{Name: aws.String("state"), Values: []*string{aws.String("active")}}}, }, testAccountID) require.NoError(t, err) @@ -140,7 +173,7 @@ func TestDescribe_FilterByInstanceID(t *testing.T) { svc := setupTestService(t) putSIRs(t, svc, makeSIR("sir-aaa", "i-aaa"), makeSIR("sir-bbb", "i-bbb")) - out, err := svc.DescribeSpotInstanceRequests(context.Background(), &ec2.DescribeSpotInstanceRequestsInput{ + out, err := svc.DescribeSpotInstanceRequests(t.Context(), &ec2.DescribeSpotInstanceRequestsInput{ Filters: []*ec2.Filter{{Name: aws.String("instance-id"), Values: []*string{aws.String("i-bbb")}}}, }, testAccountID) require.NoError(t, err) @@ -154,7 +187,7 @@ func TestDescribe_FilterByLaunchImageID(t *testing.T) { other.LaunchSpecification.ImageId = aws.String("ami-99999999") putSIRs(t, svc, makeSIR("sir-aaa", "i-aaa"), other) - out, err := svc.DescribeSpotInstanceRequests(context.Background(), &ec2.DescribeSpotInstanceRequestsInput{ + out, err := svc.DescribeSpotInstanceRequests(t.Context(), &ec2.DescribeSpotInstanceRequestsInput{ Filters: []*ec2.Filter{{Name: aws.String("launch.image-id"), Values: []*string{aws.String("ami-12345678")}}}, }, testAccountID) require.NoError(t, err) @@ -168,7 +201,7 @@ func TestDescribe_FilterByTag(t *testing.T) { tagged.Tags = []*ec2.Tag{{Key: aws.String("env"), Value: aws.String("prod")}} putSIRs(t, svc, tagged, makeSIR("sir-bbb", "i-bbb")) - out, err := svc.DescribeSpotInstanceRequests(context.Background(), &ec2.DescribeSpotInstanceRequestsInput{ + out, err := svc.DescribeSpotInstanceRequests(t.Context(), &ec2.DescribeSpotInstanceRequestsInput{ Filters: []*ec2.Filter{{Name: aws.String("tag:env"), Values: []*string{aws.String("prod")}}}, }, testAccountID) require.NoError(t, err) @@ -176,7 +209,7 @@ func TestDescribe_FilterByTag(t *testing.T) { assert.Equal(t, "sir-aaa", aws.StringValue(out.SpotInstanceRequests[0].SpotInstanceRequestId)) // tag-key matches any value for the key. - out, err = svc.DescribeSpotInstanceRequests(context.Background(), &ec2.DescribeSpotInstanceRequestsInput{ + out, err = svc.DescribeSpotInstanceRequests(t.Context(), &ec2.DescribeSpotInstanceRequestsInput{ Filters: []*ec2.Filter{{Name: aws.String("tag-key"), Values: []*string{aws.String("env")}}}, }, testAccountID) require.NoError(t, err) @@ -186,7 +219,7 @@ func TestDescribe_FilterByTag(t *testing.T) { func TestDescribe_InvalidFilter(t *testing.T) { svc := setupTestService(t) - _, err := svc.DescribeSpotInstanceRequests(context.Background(), &ec2.DescribeSpotInstanceRequestsInput{ + _, err := svc.DescribeSpotInstanceRequests(t.Context(), &ec2.DescribeSpotInstanceRequestsInput{ Filters: []*ec2.Filter{{Name: aws.String("bogus-filter"), Values: []*string{aws.String("x")}}}, }, testAccountID) require.Error(t, err) @@ -199,7 +232,7 @@ func TestCancel_MovesActiveToTerminal(t *testing.T) { svc := setupTestService(t) putSIRs(t, svc, makeSIR("sir-aaa", "i-aaa")) - out, err := svc.CancelSpotInstanceRequests(context.Background(), &ec2.CancelSpotInstanceRequestsInput{ + out, err := svc.CancelSpotInstanceRequests(t.Context(), &ec2.CancelSpotInstanceRequestsInput{ SpotInstanceRequestIds: []*string{aws.String("sir-aaa")}, }, testAccountID) require.NoError(t, err) @@ -219,11 +252,11 @@ func TestCancel_Idempotent(t *testing.T) { putSIRs(t, svc, makeSIR("sir-aaa", "i-aaa")) in := &ec2.CancelSpotInstanceRequestsInput{SpotInstanceRequestIds: []*string{aws.String("sir-aaa")}} - _, err := svc.CancelSpotInstanceRequests(context.Background(), in, testAccountID) + _, err := svc.CancelSpotInstanceRequests(t.Context(), in, testAccountID) require.NoError(t, err) // Second cancel of an already-terminal request succeeds without duplicating. - out, err := svc.CancelSpotInstanceRequests(context.Background(), in, testAccountID) + out, err := svc.CancelSpotInstanceRequests(t.Context(), in, testAccountID) require.NoError(t, err) require.Len(t, out.CancelledSpotInstanceRequests, 1) @@ -234,7 +267,7 @@ func TestCancel_Idempotent(t *testing.T) { func TestCancel_AbsentIDIdempotent(t *testing.T) { svc := setupTestService(t) - out, err := svc.CancelSpotInstanceRequests(context.Background(), &ec2.CancelSpotInstanceRequestsInput{ + out, err := svc.CancelSpotInstanceRequests(t.Context(), &ec2.CancelSpotInstanceRequestsInput{ SpotInstanceRequestIds: []*string{aws.String("sir-ghost")}, }, testAccountID) require.NoError(t, err) @@ -244,7 +277,7 @@ func TestCancel_AbsentIDIdempotent(t *testing.T) { func TestCancel_MissingIDs(t *testing.T) { svc := setupTestService(t) - _, err := svc.CancelSpotInstanceRequests(context.Background(), &ec2.CancelSpotInstanceRequestsInput{}, testAccountID) + _, err := svc.CancelSpotInstanceRequests(t.Context(), &ec2.CancelSpotInstanceRequestsInput{}, testAccountID) require.Error(t, err) assert.Equal(t, awserrors.ErrorMissingParameter, err.Error()) } @@ -255,9 +288,9 @@ func TestCloseForInstance_MovesMatching(t *testing.T) { svc := setupTestService(t) putSIRs(t, svc, makeSIR("sir-aaa", "i-aaa"), makeSIR("sir-bbb", "i-bbb")) - require.NoError(t, svc.CloseForInstance(context.Background(), "i-bbb", testAccountID)) + require.NoError(t, svc.CloseForInstance(t.Context(), "i-bbb", testAccountID)) - out, err := svc.DescribeSpotInstanceRequests(context.Background(), &ec2.DescribeSpotInstanceRequestsInput{ + out, err := svc.DescribeSpotInstanceRequests(t.Context(), &ec2.DescribeSpotInstanceRequestsInput{ SpotInstanceRequestIds: []*string{aws.String("sir-bbb")}, }, testAccountID) require.NoError(t, err) @@ -266,7 +299,7 @@ func TestCloseForInstance_MovesMatching(t *testing.T) { assert.Equal(t, SpotStatusCodeInstanceTerminatedByUser, aws.StringValue(out.SpotInstanceRequests[0].Status.Code)) // sir-aaa is untouched and still active. - out, err = svc.DescribeSpotInstanceRequests(context.Background(), &ec2.DescribeSpotInstanceRequestsInput{ + out, err = svc.DescribeSpotInstanceRequests(t.Context(), &ec2.DescribeSpotInstanceRequestsInput{ SpotInstanceRequestIds: []*string{aws.String("sir-aaa")}, }, testAccountID) require.NoError(t, err) @@ -277,7 +310,7 @@ func TestCloseForInstance_NoMatchNoOp(t *testing.T) { svc := setupTestService(t) putSIRs(t, svc, makeSIR("sir-aaa", "i-aaa")) - require.NoError(t, svc.CloseForInstance(context.Background(), "i-nomatch", testAccountID)) + require.NoError(t, svc.CloseForInstance(t.Context(), "i-nomatch", testAccountID)) got := describeAll(t, svc, testAccountID) require.Len(t, got, 1) @@ -286,7 +319,7 @@ func TestCloseForInstance_NoMatchNoOp(t *testing.T) { func TestCloseForInstance_EmptyInstanceID(t *testing.T) { svc := setupTestService(t) - require.NoError(t, svc.CloseForInstance(context.Background(), "", testAccountID)) + require.NoError(t, svc.CloseForInstance(t.Context(), "", testAccountID)) } // --- Merge across buckets --- @@ -295,11 +328,11 @@ func TestDescribe_MergesBothBuckets(t *testing.T) { svc := setupTestService(t) putSIRs(t, svc, makeSIR("sir-active", "i-1"), makeSIR("sir-cancel", "i-2"), makeSIR("sir-close", "i-3")) - _, err := svc.CancelSpotInstanceRequests(context.Background(), &ec2.CancelSpotInstanceRequestsInput{ + _, err := svc.CancelSpotInstanceRequests(t.Context(), &ec2.CancelSpotInstanceRequestsInput{ SpotInstanceRequestIds: []*string{aws.String("sir-cancel")}, }, testAccountID) require.NoError(t, err) - require.NoError(t, svc.CloseForInstance(context.Background(), "i-3", testAccountID)) + require.NoError(t, svc.CloseForInstance(t.Context(), "i-3", testAccountID)) states := map[string]string{} for _, r := range describeAll(t, svc, testAccountID) { diff --git a/spinifex/handlers/ec2/volume/service_impl.go b/spinifex/handlers/ec2/volume/service_impl.go index d0b8b549c..df64c3bdd 100644 --- a/spinifex/handlers/ec2/volume/service_impl.go +++ b/spinifex/handlers/ec2/volume/service_impl.go @@ -29,6 +29,7 @@ import ( "github.com/mulgadc/viperblock/viperblock" s3backend "github.com/mulgadc/viperblock/viperblock/backends/s3" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) const ( @@ -57,13 +58,13 @@ type VolumeServiceImpl struct { store objectstore.ObjectStore bucketName string natsConn *nats.Conn - snapshotKV nats.KeyValue + snapshotKV jetstream.KeyValue } // NewVolumeServiceImpl creates a new daemon-side volume service. // snapshotKV is optional — when non-nil, DeleteVolume uses O(1) KV lookup // instead of scanning all snapshots in S3. -func NewVolumeServiceImpl(cfg *config.Config, natsConn *nats.Conn, snapshotKV nats.KeyValue) *VolumeServiceImpl { +func NewVolumeServiceImpl(cfg *config.Config, natsConn *nats.Conn, snapshotKV jetstream.KeyValue) *VolumeServiceImpl { store := objectstore.NewS3ObjectStoreFromConfig( cfg.Predastore.Host, cfg.Predastore.Region, @@ -81,7 +82,7 @@ func NewVolumeServiceImpl(cfg *config.Config, natsConn *nats.Conn, snapshotKV na } // NewVolumeServiceImplWithStore creates a volume service with a custom ObjectStore (for testing). -func NewVolumeServiceImplWithStore(cfg *config.Config, store objectstore.ObjectStore, natsConn *nats.Conn, snapshotKV ...nats.KeyValue) *VolumeServiceImpl { +func NewVolumeServiceImplWithStore(cfg *config.Config, store objectstore.ObjectStore, natsConn *nats.Conn, snapshotKV ...jetstream.KeyValue) *VolumeServiceImpl { bucketName := "" if cfg != nil { bucketName = cfg.Predastore.Bucket @@ -1562,7 +1563,7 @@ func (s *VolumeServiceImpl) DeleteVolume(ctx context.Context, input *ec2.DeleteV // Check if any snapshots reference this volume via JetStream KV. // Snapshot-backed clones read chunk files from the source volume's // S3 prefix via ReadFrom(). Deleting the source would break all clones. - if err := s.checkVolumeHasNoSnapshots(volumeID); err != nil { + if err := s.checkVolumeHasNoSnapshots(ctx, volumeID); err != nil { return nil, err } @@ -1676,29 +1677,29 @@ func (s *VolumeServiceImpl) getSnapshotMetadata(ctx context.Context, snapshotID // checkVolumeHasNoSnapshots checks if a volume has dependent snapshots // using the JetStream KV index. -func (s *VolumeServiceImpl) checkVolumeHasNoSnapshots(volumeID string) error { +func (s *VolumeServiceImpl) checkVolumeHasNoSnapshots(ctx context.Context, volumeID string) error { if s.snapshotKV == nil { - slog.Error("checkVolumeHasNoSnapshots: snapshotKV is nil", "volumeId", volumeID) + slog.ErrorContext(ctx, "checkVolumeHasNoSnapshots: snapshotKV is nil", "volumeId", volumeID) return errors.New(awserrors.ErrorServerInternal) } - has, err := s.volumeHasSnapshotsKV(volumeID) + has, err := s.volumeHasSnapshotsKV(ctx, volumeID) if err != nil { - slog.Error("checkVolumeHasNoSnapshots: KV lookup failed", "volumeId", volumeID, "err", err) + slog.ErrorContext(ctx, "checkVolumeHasNoSnapshots: KV lookup failed", "volumeId", volumeID, "err", err) return errors.New(awserrors.ErrorServerInternal) } if has { - slog.Error("DeleteVolume blocked: volume has snapshots", "volumeId", volumeID) + slog.ErrorContext(ctx, "DeleteVolume blocked: volume has snapshots", "volumeId", volumeID) return errors.New(awserrors.ErrorVolumeInUse) } return nil } // volumeHasSnapshotsKV checks the JetStream KV index for snapshot references. -func (s *VolumeServiceImpl) volumeHasSnapshotsKV(volumeID string) (bool, error) { - entry, err := s.snapshotKV.Get(volumeID) +func (s *VolumeServiceImpl) volumeHasSnapshotsKV(ctx context.Context, volumeID string) (bool, error) { + entry, err := s.snapshotKV.Get(ctx, volumeID) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return false, nil } return false, err diff --git a/spinifex/handlers/ec2/volume/service_impl_test.go b/spinifex/handlers/ec2/volume/service_impl_test.go index 1380cdb5a..5f550849f 100644 --- a/spinifex/handlers/ec2/volume/service_impl_test.go +++ b/spinifex/handlers/ec2/volume/service_impl_test.go @@ -14,10 +14,12 @@ import ( "github.com/mulgadc/spinifex/spinifex/awserrors" "github.com/mulgadc/spinifex/spinifex/config" "github.com/mulgadc/spinifex/spinifex/objectstore" + "github.com/mulgadc/spinifex/spinifex/testutil" "github.com/mulgadc/spinifex/spinifex/types" "github.com/mulgadc/viperblock/viperblock" "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -510,30 +512,12 @@ func TestCreateVolume_FromSnapshot_CorruptMetadata(t *testing.T) { } // setupTestVolumeKV creates a NATS JetStream test server and returns a KV bucket. -func setupTestVolumeKV(t *testing.T) nats.KeyValue { +func setupTestVolumeKV(t *testing.T) jetstream.KeyValue { t.Helper() - opts := &server.Options{ - Host: "127.0.0.1", - Port: -1, - JetStream: true, - StoreDir: t.TempDir(), - NoLog: true, - NoSigs: true, - } - ns, err := server.NewServer(opts) - require.NoError(t, err) - go ns.Start() - require.True(t, ns.ReadyForConnections(5*time.Second)) - t.Cleanup(func() { ns.Shutdown() }) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) - nc, err := nats.Connect(ns.ClientURL()) - require.NoError(t, err) - t.Cleanup(func() { nc.Close() }) - - js, err := nc.JetStream() - require.NoError(t, err) - - kv, err := js.CreateKeyValue(&nats.KeyValueConfig{ + kv, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{ Bucket: "spinifex-volume-snapshots", }) require.NoError(t, err) @@ -574,7 +558,7 @@ func TestDeleteVolume_BlockedByKV(t *testing.T) { // Put a snapshot ref in KV data, err := json.Marshal([]string{"snap-001"}) require.NoError(t, err) - _, err = kv.Put(volumeID, data) + _, err = kv.Put(t.Context(), volumeID, data) require.NoError(t, err) // DeleteVolume should be blocked @@ -1479,7 +1463,7 @@ func TestDeleteVolumeOnTerminate_SurfacesDeleteFailure(t *testing.T) { // clear has already succeeded. snapData, err := json.Marshal([]string{"snap-001"}) require.NoError(t, err) - _, err = kv.Put(volumeID, snapData) + _, err = kv.Put(t.Context(), volumeID, snapData) require.NoError(t, err) err = svc.DeleteVolumeOnTerminate(context.Background(), volumeID, "") diff --git a/spinifex/handlers/ec2/vpc/eni.go b/spinifex/handlers/ec2/vpc/eni.go index fab6b436e..2b445b2f9 100644 --- a/spinifex/handlers/ec2/vpc/eni.go +++ b/spinifex/handlers/ec2/vpc/eni.go @@ -15,7 +15,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/filterutil" "github.com/mulgadc/spinifex/spinifex/network/topology" "github.com/mulgadc/spinifex/spinifex/utils" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) const ( @@ -91,7 +91,7 @@ func (s *VPCServiceImpl) CreateNetworkInterface(ctx context.Context, input *ec2. subnetId := *input.SubnetId // Verify subnet exists and belongs to this account - subnetEntry, err := s.subnetKV.Get(utils.AccountKey(accountID, subnetId)) + subnetEntry, err := s.subnetKV.Get(ctx, utils.AccountKey(accountID, subnetId)) if err != nil { return nil, errors.New(awserrors.ErrorInvalidSubnetIDNotFound) } @@ -109,13 +109,13 @@ func (s *VPCServiceImpl) CreateNetworkInterface(ctx context.Context, input *ec2. } if len(sgIdsIn) == 0 { // AWS attaches the per-VPC default SG when the caller omits Groups. - defaultSGId, err := s.FindDefaultSGForVPC(accountID, subnet.VpcId) + defaultSGId, err := s.findDefaultSGForVPC(ctx, accountID, subnet.VpcId) if err != nil || defaultSGId == "" { return nil, errors.New(awserrors.ErrorServerInternal) } sgIdsIn = []string{defaultSGId} } - if err := s.validateSGAttachment(accountID, sgIdsIn, subnet.VpcId); err != nil { + if err := s.validateSGAttachment(ctx, accountID, sgIdsIn, subnet.VpcId); err != nil { return nil, err } @@ -127,7 +127,7 @@ func (s *VPCServiceImpl) CreateNetworkInterface(ctx context.Context, input *ec2. // TODO: validate the requested IP is in the subnet range and not already allocated privateIP = *input.PrivateIpAddress } else { - ip, err := s.ipam.AllocateIP(subnetId, subnet.CidrBlock, PurposeENIPrimary, eniId) + ip, err := s.ipam.AllocateIP(ctx, subnetId, subnet.CidrBlock, PurposeENIPrimary, eniId) if err != nil { return nil, errors.New(awserrors.ErrorServerInternal) } @@ -160,7 +160,7 @@ func (s *VPCServiceImpl) CreateNetworkInterface(ctx context.Context, input *ec2. if err != nil { return nil, fmt.Errorf("failed to marshal ENI record: %w", err) } - if _, err := s.eniKV.Put(utils.AccountKey(accountID, eniId), data); err != nil { + if _, err := s.eniKV.Put(ctx, utils.AccountKey(accountID, eniId), data); err != nil { return nil, errors.New(awserrors.ErrorServerInternal) } @@ -205,9 +205,9 @@ func (s *VPCServiceImpl) deleteNetworkInterface(ctx context.Context, eniId, acco key := utils.AccountKey(accountID, eniId) // Get the ENI record - entry, err := s.eniKV.Get(key) + entry, err := s.eniKV.Get(ctx, key) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { // Internal teardown (force) tolerates an already-gone ENI so // instance terminate converges (ADR-0003 §2); the public API is // AWS-faithful and returns NotFound. @@ -232,7 +232,7 @@ func (s *VPCServiceImpl) deleteNetworkInterface(ctx context.Context, eniId, acco } // Release the private IP back to the IPAM pool - if err := s.ipam.ReleaseIP(record.SubnetId, record.PrivateIpAddress); err != nil { + if err := s.ipam.ReleaseIP(ctx, record.SubnetId, record.PrivateIpAddress); err != nil { slog.WarnContext(ctx, "Failed to release IP during ENI delete", "eni", eniId, "ip", record.PrivateIpAddress, "err", err) } @@ -255,7 +255,7 @@ func (s *VPCServiceImpl) deleteNetworkInterface(ctx context.Context, eniId, acco } } - if err := s.eniKV.Delete(key); err != nil { + if err := s.eniKV.Delete(ctx, key); err != nil { return nil, errors.New(awserrors.ErrorServerInternal) } @@ -286,7 +286,7 @@ func (s *VPCServiceImpl) ModifyNetworkInterfaceAttribute(ctx context.Context, in eniId := *input.NetworkInterfaceId key := utils.AccountKey(accountID, eniId) - entry, err := s.eniKV.Get(key) + entry, err := s.eniKV.Get(ctx, key) if err != nil { return nil, errors.New(awserrors.ErrorInvalidNetworkInterfaceIDNotFound) } @@ -305,7 +305,7 @@ func (s *VPCServiceImpl) ModifyNetworkInterfaceAttribute(ctx context.Context, in sgIds = append(sgIds, *id) } } - if err := s.validateSGAttachment(accountID, sgIds, record.VpcId); err != nil { + if err := s.validateSGAttachment(ctx, accountID, sgIds, record.VpcId); err != nil { return nil, err } record.SecurityGroupIds = sgIds @@ -321,7 +321,7 @@ func (s *VPCServiceImpl) ModifyNetworkInterfaceAttribute(ctx context.Context, in slog.ErrorContext(ctx, "ModifyNetworkInterfaceAttribute: failed to marshal ENI record", "eniId", eniId, "accountID", accountID, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } - if _, err := s.eniKV.Update(key, data, entry.Revision()); err != nil { + if _, err := s.eniKV.Update(ctx, key, data, entry.Revision()); err != nil { slog.ErrorContext(ctx, "ModifyNetworkInterfaceAttribute: KV update failed", "eniId", eniId, "accountID", accountID, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } @@ -371,8 +371,8 @@ func (s *VPCServiceImpl) DescribeNetworkInterfaces(ctx context.Context, input *e } prefix := accountID + "." - keys, err := s.eniKV.Keys() - if err != nil && !errors.Is(err, nats.ErrNoKeysFound) { + keys, err := s.eniKV.Keys(ctx) + if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) { return nil, errors.New(awserrors.ErrorServerInternal) } @@ -384,7 +384,7 @@ func (s *VPCServiceImpl) DescribeNetworkInterfaces(ctx context.Context, input *e continue } - entry, err := s.eniKV.Get(key) + entry, err := s.eniKV.Get(ctx, key) if err != nil { slog.WarnContext(ctx, "Failed to get ENI record", "key", key, "error", err) continue @@ -504,8 +504,12 @@ func eniMatchesFilters(record *ENIRecord, filters map[string][]string) bool { // AttachENI marks an ENI as attached to an instance (internal use by RunInstances). // accountID scopes the lookup to the correct KV key. func (s *VPCServiceImpl) AttachENI(accountID, eniId, instanceId string, deviceIndex int64) (string, error) { + return s.attachENI(context.Background(), accountID, eniId, instanceId, deviceIndex) +} + +func (s *VPCServiceImpl) attachENI(ctx context.Context, accountID, eniId, instanceId string, deviceIndex int64) (string, error) { key := utils.AccountKey(accountID, eniId) - entry, err := s.eniKV.Get(key) + entry, err := s.eniKV.Get(ctx, key) if err != nil { return "", errors.New(awserrors.ErrorInvalidNetworkInterfaceIDNotFound) } @@ -529,11 +533,11 @@ func (s *VPCServiceImpl) AttachENI(accountID, eniId, instanceId string, deviceIn if err != nil { return "", fmt.Errorf("failed to marshal ENI record: %w", err) } - if _, err := s.eniKV.Update(key, data, entry.Revision()); err != nil { + if _, err := s.eniKV.Update(ctx, key, data, entry.Revision()); err != nil { return "", errors.New(awserrors.ErrorServerInternal) } - slog.Info("ENI attached", "eniId", eniId, "instanceId", instanceId, "attachmentId", attachmentId) + slog.InfoContext(ctx, "ENI attached", "eniId", eniId, "instanceId", instanceId, "attachmentId", attachmentId) return attachmentId, nil } @@ -541,7 +545,7 @@ func (s *VPCServiceImpl) AttachENI(accountID, eniId, instanceId string, deviceIn // accountID scopes the lookup to the correct KV key. func (s *VPCServiceImpl) DetachENI(ctx context.Context, accountID, eniId string) error { key := utils.AccountKey(accountID, eniId) - entry, err := s.eniKV.Get(key) + entry, err := s.eniKV.Get(ctx, key) if err != nil { return errors.New(awserrors.ErrorInvalidNetworkInterfaceIDNotFound) } @@ -560,7 +564,7 @@ func (s *VPCServiceImpl) DetachENI(ctx context.Context, accountID, eniId string) if err != nil { return fmt.Errorf("failed to marshal ENI record: %w", err) } - if _, err := s.eniKV.Update(key, data, entry.Revision()); err != nil { + if _, err := s.eniKV.Update(ctx, key, data, entry.Revision()); err != nil { return errors.New(awserrors.ErrorServerInternal) } @@ -572,8 +576,12 @@ func (s *VPCServiceImpl) DetachENI(ctx context.Context, accountID, eniId string) // hot-plug handlers that need the record's MAC + AttachmentId before // running the QMP pipeline. func (s *VPCServiceImpl) GetENIRecord(accountID, eniId string) (ENIRecord, error) { + return s.getENIRecord(context.Background(), accountID, eniId) +} + +func (s *VPCServiceImpl) getENIRecord(ctx context.Context, accountID, eniId string) (ENIRecord, error) { key := utils.AccountKey(accountID, eniId) - entry, err := s.eniKV.Get(key) + entry, err := s.eniKV.Get(ctx, key) if err != nil { return ENIRecord{}, errors.New(awserrors.ErrorInvalidNetworkInterfaceIDNotFound) } @@ -588,8 +596,12 @@ func (s *VPCServiceImpl) GetENIRecord(accountID, eniId string) (ENIRecord, error // the result under the same revision token (compare-and-swap). fn must // not block on external I/O; the KV update slot is contended. func (s *VPCServiceImpl) UpdateENI(accountID, eniId string, fn func(*ENIRecord)) error { + return s.updateENI(context.Background(), accountID, eniId, fn) +} + +func (s *VPCServiceImpl) updateENI(ctx context.Context, accountID, eniId string, fn func(*ENIRecord)) error { key := utils.AccountKey(accountID, eniId) - entry, err := s.eniKV.Get(key) + entry, err := s.eniKV.Get(ctx, key) if err != nil { return errors.New(awserrors.ErrorInvalidNetworkInterfaceIDNotFound) } @@ -602,7 +614,7 @@ func (s *VPCServiceImpl) UpdateENI(accountID, eniId string, fn func(*ENIRecord)) if err != nil { return fmt.Errorf("marshal ENI record: %w", err) } - if _, err := s.eniKV.Update(key, data, entry.Revision()); err != nil { + if _, err := s.eniKV.Update(ctx, key, data, entry.Revision()); err != nil { return errors.New(awserrors.ErrorServerInternal) } return nil @@ -612,9 +624,13 @@ func (s *VPCServiceImpl) UpdateENI(accountID, eniId string, fn func(*ENIRecord)) // instanceID. Used by the hot-plug reconciler to converge a node's instances // against KV on restart. func (s *VPCServiceImpl) ListInstanceENIs(accountID, instanceID string) ([]ENIRecord, error) { - keys, err := s.eniKV.Keys() + return s.listInstanceENIs(context.Background(), accountID, instanceID) +} + +func (s *VPCServiceImpl) listInstanceENIs(ctx context.Context, accountID, instanceID string) ([]ENIRecord, error) { + keys, err := s.eniKV.Keys(ctx) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return nil, nil } return nil, errors.New(awserrors.ErrorServerInternal) @@ -625,7 +641,7 @@ func (s *VPCServiceImpl) ListInstanceENIs(accountID, instanceID string) ([]ENIRe if key == utils.VersionKey || !strings.HasPrefix(key, prefix) { continue } - entry, err := s.eniKV.Get(key) + entry, err := s.eniKV.Get(ctx, key) if err != nil { continue } @@ -643,8 +659,12 @@ func (s *VPCServiceImpl) ListInstanceENIs(accountID, instanceID string) ([]ENIRe // FindENIByAttachment scans the ENI bucket for the record with the given // AttachmentId. Used by DetachNetworkInterface which identifies by attachment ID. func (s *VPCServiceImpl) FindENIByAttachment(accountID, attachmentId string) (ENIRecord, error) { - keys, err := s.eniKV.Keys() - if err != nil && !errors.Is(err, nats.ErrNoKeysFound) { + return s.findENIByAttachment(context.Background(), accountID, attachmentId) +} + +func (s *VPCServiceImpl) findENIByAttachment(ctx context.Context, accountID, attachmentId string) (ENIRecord, error) { + keys, err := s.eniKV.Keys(ctx) + if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) { return ENIRecord{}, errors.New(awserrors.ErrorServerInternal) } prefix := accountID + "." @@ -652,7 +672,7 @@ func (s *VPCServiceImpl) FindENIByAttachment(accountID, attachmentId string) (EN if key == utils.VersionKey || !strings.HasPrefix(key, prefix) { continue } - entry, err := s.eniKV.Get(key) + entry, err := s.eniKV.Get(ctx, key) if err != nil { continue } @@ -669,8 +689,12 @@ func (s *VPCServiceImpl) FindENIByAttachment(accountID, attachmentId string) (EN // UpdateENIPublicIP updates the PublicIpAddress and PublicIpPool on an ENI record. func (s *VPCServiceImpl) UpdateENIPublicIP(accountID, eniId, publicIP, poolName string) error { + return s.updateENIPublicIP(context.Background(), accountID, eniId, publicIP, poolName) +} + +func (s *VPCServiceImpl) updateENIPublicIP(ctx context.Context, accountID, eniId, publicIP, poolName string) error { key := utils.AccountKey(accountID, eniId) - entry, err := s.eniKV.Get(key) + entry, err := s.eniKV.Get(ctx, key) if err != nil { return fmt.Errorf("ENI %s not found: %w", eniId, err) } @@ -687,11 +711,11 @@ func (s *VPCServiceImpl) UpdateENIPublicIP(accountID, eniId, publicIP, poolName if err != nil { return fmt.Errorf("marshal ENI record: %w", err) } - if _, err := s.eniKV.Update(key, data, entry.Revision()); err != nil { + if _, err := s.eniKV.Update(ctx, key, data, entry.Revision()); err != nil { return fmt.Errorf("update ENI record: %w", err) } - slog.Info("Updated ENI with public IP", "eniId", eniId, "publicIp", publicIP, "pool", poolName) + slog.InfoContext(ctx, "Updated ENI with public IP", "eniId", eniId, "publicIp", publicIP, "pool", poolName) return nil } @@ -832,7 +856,7 @@ func (s *VPCServiceImpl) publishNATEvent(topic, vpcId, externalIP, logicalIP, po // or MissingParameter per AWS contract. const sgPerInterfaceLimit = 5 -func (s *VPCServiceImpl) validateSGAttachment(accountID string, sgIds []string, vpcId string) error { +func (s *VPCServiceImpl) validateSGAttachment(ctx context.Context, accountID string, sgIds []string, vpcId string) error { if len(sgIds) == 0 { return errors.New(awserrors.ErrorMissingParameter) } @@ -840,13 +864,13 @@ func (s *VPCServiceImpl) validateSGAttachment(accountID string, sgIds []string, return errors.New(awserrors.ErrorSecurityGroupsPerInterfaceLimitExceeded) } - if err := s.requireVPCExists(accountID, vpcId); err != nil { + if err := s.requireVPCExists(ctx, accountID, vpcId); err != nil { return err } // Each SG must exist in the caller's account and belong to the same VPC. for _, sgId := range sgIds { - sgEntry, err := s.sgKV.Get(utils.AccountKey(accountID, sgId)) + sgEntry, err := s.sgKV.Get(ctx, utils.AccountKey(accountID, sgId)) if err != nil { return errors.New(awserrors.ErrorInvalidGroupNotFound) } @@ -868,9 +892,9 @@ func (s *VPCServiceImpl) isEIPOwned(ctx context.Context, eniId, accountID string if s.eipKV == nil { return false, nil } - keys, err := s.eipKV.Keys() + keys, err := s.eipKV.Keys(ctx) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return false, nil } return false, fmt.Errorf("eipKV.Keys: %w", err) @@ -880,7 +904,7 @@ func (s *VPCServiceImpl) isEIPOwned(ctx context.Context, eniId, accountID string if len(k) < len(prefix) || k[:len(prefix)] != prefix { continue } - entry, err := s.eipKV.Get(k) + entry, err := s.eipKV.Get(ctx, k) if err != nil { return false, fmt.Errorf("eipKV.Get(%s): %w", k, err) } diff --git a/spinifex/handlers/ec2/vpc/eni_test.go b/spinifex/handlers/ec2/vpc/eni_test.go index a24228bde..af22fe0ba 100644 --- a/spinifex/handlers/ec2/vpc/eni_test.go +++ b/spinifex/handlers/ec2/vpc/eni_test.go @@ -13,6 +13,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/awserrors" "github.com/mulgadc/spinifex/spinifex/testutil" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -935,27 +936,28 @@ func TestDescribeNetworkInterfaces_FilterByAttachmentStatus(t *testing.T) { // --- isEIPOwned tests --- // setupEIPTestService creates a VPC service with an EIP KV bucket wired in. -func setupEIPTestService(t *testing.T) (*VPCServiceImpl, nats.KeyValue) { +func setupEIPTestService(t *testing.T) (*VPCServiceImpl, jetstream.KeyValue) { t.Helper() - _, nc, js := testutil.StartTestJetStream(t) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) - svc, err := NewVPCServiceImplWithNATS(nil, nc) + svc, err := NewVPCServiceImplWithNATS(t.Context(), nil, nc) require.NoError(t, err) - kv, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: "eip-test"}) + kv, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: "eip-test"}) require.NoError(t, err) svc.eipKV = kv return svc, kv } -func putEIPRecord(t *testing.T, kv nats.KeyValue, key, eniID string) { +func putEIPRecord(t *testing.T, kv jetstream.KeyValue, key, eniID string) { t.Helper() data, err := json.Marshal(struct { ENIId string `json:"eni_id"` }{ENIId: eniID}) require.NoError(t, err) - _, err = kv.Put(key, data) + _, err = kv.Put(t.Context(), key, data) require.NoError(t, err) } @@ -1004,7 +1006,7 @@ func TestIsEIPOwned_WrongAccountPrefix(t *testing.T) { func TestIsEIPOwned_MalformedJSON(t *testing.T) { svc, kv := setupEIPTestService(t) - _, err := kv.Put(testAccountID+".eipalloc-001", []byte("not json")) + _, err := kv.Put(t.Context(), testAccountID+".eipalloc-001", []byte("not json")) require.NoError(t, err) owned, err := svc.isEIPOwned(context.Background(), "eni-111", testAccountID) @@ -1030,7 +1032,7 @@ func TestValidateSGAttachment_OK(t *testing.T) { vpcID := createTestVPC(t, svc, "10.0.0.0/16") sg := createTestSG(t, svc, vpcID, "ok") - err := svc.validateSGAttachment(testAccountID, []string{sg}, vpcID) + err := svc.validateSGAttachment(t.Context(), testAccountID, []string{sg}, vpcID) assert.NoError(t, err) } @@ -1038,7 +1040,7 @@ func TestValidateSGAttachment_EmptyList(t *testing.T) { svc := setupTestVPCService(t) vpcID := createTestVPC(t, svc, "10.0.0.0/16") - err := svc.validateSGAttachment(testAccountID, nil, vpcID) + err := svc.validateSGAttachment(t.Context(), testAccountID, nil, vpcID) assert.ErrorContains(t, err, "MissingParameter") } @@ -1050,13 +1052,13 @@ func TestValidateSGAttachment_TooMany(t *testing.T) { sgs[i] = createTestSG(t, svc, vpcID, fmt.Sprintf("sg-%d", i)) } - err := svc.validateSGAttachment(testAccountID, sgs, vpcID) + err := svc.validateSGAttachment(t.Context(), testAccountID, sgs, vpcID) assert.ErrorContains(t, err, "SecurityGroupsPerInterfaceLimitExceeded") } func TestValidateSGAttachment_UnknownVPC(t *testing.T) { svc := setupTestVPCService(t) - err := svc.validateSGAttachment(testAccountID, []string{"sg-aaa"}, "vpc-missing") + err := svc.validateSGAttachment(t.Context(), testAccountID, []string{"sg-aaa"}, "vpc-missing") assert.ErrorContains(t, err, "InvalidVpcID.NotFound") } @@ -1064,7 +1066,7 @@ func TestValidateSGAttachment_SGNotFound(t *testing.T) { svc := setupTestVPCService(t) vpcID := createTestVPC(t, svc, "10.0.0.0/16") - err := svc.validateSGAttachment(testAccountID, []string{"sg-doesntexist"}, vpcID) + err := svc.validateSGAttachment(t.Context(), testAccountID, []string{"sg-doesntexist"}, vpcID) assert.ErrorContains(t, err, "InvalidGroup.NotFound") } @@ -1074,7 +1076,7 @@ func TestValidateSGAttachment_CrossVPC(t *testing.T) { vpcB := createTestVPC(t, svc, "10.1.0.0/16") sgInA := createTestSG(t, svc, vpcA, "in-a") - err := svc.validateSGAttachment(testAccountID, []string{sgInA}, vpcB) + err := svc.validateSGAttachment(t.Context(), testAccountID, []string{sgInA}, vpcB) assert.ErrorContains(t, err, "InvalidParameterValue") } diff --git a/spinifex/handlers/ec2/vpc/external_ipam.go b/spinifex/handlers/ec2/vpc/external_ipam.go index 0fd7bde74..b15b0a4b8 100644 --- a/spinifex/handlers/ec2/vpc/external_ipam.go +++ b/spinifex/handlers/ec2/vpc/external_ipam.go @@ -10,7 +10,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/awserrors" "github.com/mulgadc/spinifex/spinifex/network/external" "github.com/mulgadc/spinifex/spinifex/network/external/dhcp" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // Backward-compatible aliases so callers (daemon, EIP, EC2 handlers, tests) @@ -29,7 +29,7 @@ const ( // ExternalIPAM is the AWS-facing entry point for external IP allocation, // dispatching to StaticPoolAllocator or dhcp.DHCPPoolAllocator per pool name. type ExternalIPAM struct { - kv nats.KeyValue + kv jetstream.KeyValue pools []external.ExternalPoolConfig static *external.StaticPoolAllocator perPool map[string]external.Allocator // dhcp overrides; static pools fall through to `static` @@ -38,15 +38,15 @@ type ExternalIPAM struct { // NewExternalIPAM creates a new ExternalIPAM. Static pools wire through // external.StaticPoolAllocator; DHCP-sourced pools wait for EnableDHCP // to install the per-pool dhcp.DHCPPoolAllocator. -func NewExternalIPAM(js nats.JetStreamContext, pools []external.ExternalPoolConfig) (*ExternalIPAM, error) { +func NewExternalIPAM(ctx context.Context, js jetstream.JetStream, pools []external.ExternalPoolConfig) (*ExternalIPAM, error) { staticPools := filterStatic(pools) var ( alloc *external.StaticPoolAllocator - kv nats.KeyValue + kv jetstream.KeyValue ) if len(staticPools) > 0 { var err error - alloc, err = external.NewStaticPoolAllocator(js, staticPools) + alloc, err = external.NewStaticPoolAllocator(ctx, js, staticPools) if err != nil { return nil, err } @@ -56,7 +56,7 @@ func NewExternalIPAM(js nats.JetStreamContext, pools []external.ExternalPoolConf } // NewExternalIPAMWithKV creates an ExternalIPAM with an existing KV bucket (for testing). -func NewExternalIPAMWithKV(kv nats.KeyValue, pools []external.ExternalPoolConfig) *ExternalIPAM { +func NewExternalIPAMWithKV(kv jetstream.KeyValue, pools []external.ExternalPoolConfig) *ExternalIPAM { alloc := external.NewStaticPoolAllocatorWithKV(kv, filterStatic(pools)) return &ExternalIPAM{kv: kv, pools: pools, static: alloc, perPool: map[string]external.Allocator{}} } @@ -128,10 +128,14 @@ func (m *ExternalIPAM) ReleaseIP(ctx context.Context, poolName, ip, ownerENIID s // GetPoolRecord returns the current IPAM record for a pool. DHCP-sourced // pools have no static record — the per-AZ lease bucket is authoritative. func (m *ExternalIPAM) GetPoolRecord(poolName string) (*ExternalIPAMRecord, error) { + return m.getPoolRecord(context.Background(), poolName) +} + +func (m *ExternalIPAM) getPoolRecord(ctx context.Context, poolName string) (*ExternalIPAMRecord, error) { if m.static == nil { return nil, fmt.Errorf("pool record unavailable: no static allocator") } - return m.static.GetPoolRecord(poolName) + return m.static.GetPoolRecord(ctx, poolName) } func (m *ExternalIPAM) allocatorFor(poolName string) (external.Allocator, error) { diff --git a/spinifex/handlers/ec2/vpc/external_ipam_test.go b/spinifex/handlers/ec2/vpc/external_ipam_test.go index 24c1799cd..3ded19b2d 100644 --- a/spinifex/handlers/ec2/vpc/external_ipam_test.go +++ b/spinifex/handlers/ec2/vpc/external_ipam_test.go @@ -14,9 +14,10 @@ import ( func setupTestExternalIPAM(t *testing.T, pools []external.ExternalPoolConfig) *ExternalIPAM { t.Helper() - _, _, js := testutil.StartTestJetStream(t) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) - ipam, err := NewExternalIPAM(js, pools) + ipam, err := NewExternalIPAM(t.Context(), js, pools) require.NoError(t, err) return ipam } @@ -338,10 +339,11 @@ func TestExternalIPAM_RangeValidation(t *testing.T) { func TestExternalIPAM_InitFromConfig(t *testing.T) { pool := testPool() // Create IPAM twice — second init should be idempotent - _, _, js := testutil.StartTestJetStream(t) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) // First init - ipam1, err := NewExternalIPAM(js, []external.ExternalPoolConfig{pool}) + ipam1, err := NewExternalIPAM(t.Context(), js, []external.ExternalPoolConfig{pool}) require.NoError(t, err) // Allocate an IP @@ -350,7 +352,7 @@ func TestExternalIPAM_InitFromConfig(t *testing.T) { assert.Equal(t, "192.168.1.151", ip) // Second init (simulating restart) — should not lose allocation - ipam2, err := NewExternalIPAM(js, []external.ExternalPoolConfig{pool}) + ipam2, err := NewExternalIPAM(t.Context(), js, []external.ExternalPoolConfig{pool}) require.NoError(t, err) record, err := ipam2.GetPoolRecord("wan") diff --git a/spinifex/handlers/ec2/vpc/ipam.go b/spinifex/handlers/ec2/vpc/ipam.go index 40121f4da..9042abfc2 100644 --- a/spinifex/handlers/ec2/vpc/ipam.go +++ b/spinifex/handlers/ec2/vpc/ipam.go @@ -1,15 +1,16 @@ package handlers_ec2_vpc import ( + "context" "encoding/json" "errors" "fmt" "log/slog" "net/netip" + "github.com/mulgadc/spinifex/spinifex/kvutil" "github.com/mulgadc/spinifex/spinifex/migrate" - "github.com/mulgadc/spinifex/spinifex/utils" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) const ( @@ -34,32 +35,32 @@ type IPAMRecord struct { // IPAM manages IP address allocation for VPC subnets using NATS KV with CAS. type IPAM struct { - kv nats.KeyValue + kv jetstream.KeyValue } // NewIPAM creates a new IPAM instance backed by NATS JetStream KV. -func NewIPAM(js nats.JetStreamContext) (*IPAM, error) { - kv, err := utils.GetOrCreateKVBucket(js, KVBucketIPAM, 5) +func NewIPAM(ctx context.Context, js jetstream.JetStream) (*IPAM, error) { + kv, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketIPAM, 5) if err != nil { return nil, fmt.Errorf("failed to create IPAM KV bucket: %w", err) } - if err := migrate.DefaultRegistry.RunKVWithJetStream(KVBucketIPAM, kv, js, KVBucketIPAMVersion); err != nil { + if err := migrate.DefaultRegistry.RunKVWithJetStream(ctx, KVBucketIPAM, kv, js, KVBucketIPAMVersion); err != nil { return nil, fmt.Errorf("migrate %s: %w", KVBucketIPAM, err) } return &IPAM{kv: kv}, nil } // NewIPAMWithKV creates an IPAM with an existing KV bucket (for testing). -func NewIPAMWithKV(kv nats.KeyValue) *IPAM { +func NewIPAMWithKV(kv jetstream.KeyValue) *IPAM { return &IPAM{kv: kv} } // AllocateIP allocates an IP from the subnet, reserving the first 4 and last // addresses per AWS convention. Uses CAS for conflict-free multi-node allocation. -func (m *IPAM) AllocateIP(subnetId, cidrBlock, purpose, ownerID string) (string, error) { +func (m *IPAM) AllocateIP(ctx context.Context, subnetId, cidrBlock, purpose, ownerID string) (string, error) { for attempt := range 5 { - record, revision, err := m.getRecord(subnetId) - if err != nil && !errors.Is(err, nats.ErrKeyNotFound) { + record, revision, err := m.getRecord(ctx, subnetId) + if err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { return "", fmt.Errorf("get IPAM record: %w", err) } @@ -85,9 +86,9 @@ func (m *IPAM) AllocateIP(subnetId, cidrBlock, purpose, ownerID string) (string, } if revision == 0 { - _, err = m.kv.Create(subnetId, data) + _, err = m.kv.Create(ctx, subnetId, data) } else { - _, err = m.kv.Update(subnetId, data, revision) + _, err = m.kv.Update(ctx, subnetId, data, revision) } if err != nil { @@ -103,9 +104,9 @@ func (m *IPAM) AllocateIP(subnetId, cidrBlock, purpose, ownerID string) (string, } // ReleaseIP releases a previously allocated IP address back to the subnet pool. -func (m *IPAM) ReleaseIP(subnetId, ip string) error { +func (m *IPAM) ReleaseIP(ctx context.Context, subnetId, ip string) error { for attempt := range 5 { - record, revision, err := m.getRecord(subnetId) + record, revision, err := m.getRecord(ctx, subnetId) if err != nil { return fmt.Errorf("get IPAM record for release: %w", err) } @@ -130,7 +131,7 @@ func (m *IPAM) ReleaseIP(subnetId, ip string) error { return fmt.Errorf("marshal IPAM record: %w", err) } - if _, err := m.kv.Update(subnetId, data, revision); err != nil { + if _, err := m.kv.Update(ctx, subnetId, data, revision); err != nil { slog.Debug("IPAM release CAS conflict, retrying", "subnet", subnetId, "attempt", attempt) continue } @@ -143,10 +144,10 @@ func (m *IPAM) ReleaseIP(subnetId, ip string) error { } // AllocatedIPs returns the list of allocated IP entries for a subnet. -func (m *IPAM) AllocatedIPs(subnetId string) ([]IPEntry, error) { - record, _, err := m.getRecord(subnetId) +func (m *IPAM) AllocatedIPs(ctx context.Context, subnetId string) ([]IPEntry, error) { + record, _, err := m.getRecord(ctx, subnetId) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil, nil } return nil, err @@ -157,11 +158,11 @@ func (m *IPAM) AllocatedIPs(subnetId string) ([]IPEntry, error) { return record.Allocated, nil } -func (m *IPAM) getRecord(subnetId string) (*IPAMRecord, uint64, error) { - entry, err := m.kv.Get(subnetId) +func (m *IPAM) getRecord(ctx context.Context, subnetId string) (*IPAMRecord, uint64, error) { + entry, err := m.kv.Get(ctx, subnetId) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { - return nil, 0, nats.ErrKeyNotFound + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil, 0, jetstream.ErrKeyNotFound } return nil, 0, err } diff --git a/spinifex/handlers/ec2/vpc/ipam_test.go b/spinifex/handlers/ec2/vpc/ipam_test.go index d9e1e2ab2..7f4a46c90 100644 --- a/spinifex/handlers/ec2/vpc/ipam_test.go +++ b/spinifex/handlers/ec2/vpc/ipam_test.go @@ -6,16 +6,17 @@ import ( "testing" "github.com/mulgadc/spinifex/spinifex/testutil" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func setupTestIPAM(t *testing.T) *IPAM { t.Helper() - _, _, js := testutil.StartTestJetStream(t) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) - ipam, err := NewIPAM(js) + ipam, err := NewIPAM(t.Context(), js) require.NoError(t, err) return ipam } @@ -23,7 +24,7 @@ func setupTestIPAM(t *testing.T) *IPAM { func TestIPAM_AllocateFirst(t *testing.T) { ipam := setupTestIPAM(t) - ip, err := ipam.AllocateIP("subnet-1", "10.0.1.0/24", PurposeENIPrimary, "eni-1") + ip, err := ipam.AllocateIP(t.Context(), "subnet-1", "10.0.1.0/24", PurposeENIPrimary, "eni-1") require.NoError(t, err) // First allocable IP is .4 (skip .0 network, .1 gateway, .2 DNS, .3 reserved) assert.Equal(t, "10.0.1.4", ip) @@ -32,15 +33,15 @@ func TestIPAM_AllocateFirst(t *testing.T) { func TestIPAM_AllocateSequential(t *testing.T) { ipam := setupTestIPAM(t) - ip1, err := ipam.AllocateIP("subnet-seq", "10.0.2.0/24", PurposeENIPrimary, "eni-seq") + ip1, err := ipam.AllocateIP(t.Context(), "subnet-seq", "10.0.2.0/24", PurposeENIPrimary, "eni-seq") require.NoError(t, err) assert.Equal(t, "10.0.2.4", ip1) - ip2, err := ipam.AllocateIP("subnet-seq", "10.0.2.0/24", PurposeENIPrimary, "eni-seq") + ip2, err := ipam.AllocateIP(t.Context(), "subnet-seq", "10.0.2.0/24", PurposeENIPrimary, "eni-seq") require.NoError(t, err) assert.Equal(t, "10.0.2.5", ip2) - ip3, err := ipam.AllocateIP("subnet-seq", "10.0.2.0/24", PurposeENIPrimary, "eni-seq") + ip3, err := ipam.AllocateIP(t.Context(), "subnet-seq", "10.0.2.0/24", PurposeENIPrimary, "eni-seq") require.NoError(t, err) assert.Equal(t, "10.0.2.6", ip3) } @@ -48,20 +49,20 @@ func TestIPAM_AllocateSequential(t *testing.T) { func TestIPAM_Release(t *testing.T) { ipam := setupTestIPAM(t) - ip1, err := ipam.AllocateIP("subnet-rel", "10.0.3.0/24", PurposeENIPrimary, "eni-rel") + ip1, err := ipam.AllocateIP(t.Context(), "subnet-rel", "10.0.3.0/24", PurposeENIPrimary, "eni-rel") require.NoError(t, err) assert.Equal(t, "10.0.3.4", ip1) - ip2, err := ipam.AllocateIP("subnet-rel", "10.0.3.0/24", PurposeENIPrimary, "eni-rel") + ip2, err := ipam.AllocateIP(t.Context(), "subnet-rel", "10.0.3.0/24", PurposeENIPrimary, "eni-rel") require.NoError(t, err) assert.Equal(t, "10.0.3.5", ip2) // Release first IP - err = ipam.ReleaseIP("subnet-rel", "10.0.3.4") + err = ipam.ReleaseIP(t.Context(), "subnet-rel", "10.0.3.4") require.NoError(t, err) // Next allocation should reuse the released IP - ip3, err := ipam.AllocateIP("subnet-rel", "10.0.3.0/24", PurposeENIPrimary, "eni-rel") + ip3, err := ipam.AllocateIP(t.Context(), "subnet-rel", "10.0.3.0/24", PurposeENIPrimary, "eni-rel") require.NoError(t, err) assert.Equal(t, "10.0.3.4", ip3) } @@ -69,17 +70,17 @@ func TestIPAM_Release(t *testing.T) { func TestIPAM_ReleaseNotAllocated(t *testing.T) { ipam := setupTestIPAM(t) - _, err := ipam.AllocateIP("subnet-rna", "10.0.4.0/24", PurposeENIPrimary, "eni-rna") + _, err := ipam.AllocateIP(t.Context(), "subnet-rna", "10.0.4.0/24", PurposeENIPrimary, "eni-rna") require.NoError(t, err) - err = ipam.ReleaseIP("subnet-rna", "10.0.4.99") + err = ipam.ReleaseIP(t.Context(), "subnet-rna", "10.0.4.99") assert.ErrorContains(t, err, "not allocated") } func TestIPAM_ReleaseNoRecord(t *testing.T) { ipam := setupTestIPAM(t) - err := ipam.ReleaseIP("subnet-nonexistent", "10.0.0.4") + err := ipam.ReleaseIP(t.Context(), "subnet-nonexistent", "10.0.0.4") assert.Error(t, err) } @@ -92,7 +93,7 @@ func TestIPAM_Exhaustion(t *testing.T) { var allocated []string for i := range 11 { - ip, err := ipam.AllocateIP(subnetId, cidr, PurposeENIPrimary, "eni-test") + ip, err := ipam.AllocateIP(t.Context(), subnetId, cidr, PurposeENIPrimary, "eni-test") require.NoError(t, err, "allocation %d should succeed", i) allocated = append(allocated, ip) } @@ -102,7 +103,7 @@ func TestIPAM_Exhaustion(t *testing.T) { assert.Equal(t, "10.0.5.14", allocated[len(allocated)-1]) // Next allocation should fail — subnet exhausted - _, err := ipam.AllocateIP(subnetId, cidr, PurposeENIPrimary, "eni-test") + _, err := ipam.AllocateIP(t.Context(), subnetId, cidr, PurposeENIPrimary, "eni-test") assert.ErrorContains(t, err, "exhausted") } @@ -114,7 +115,7 @@ func TestIPAM_ReservedIPs(t *testing.T) { // First 4 allocations should be .4, .5, .6, .7 (skipping .0-.3) for i := 4; i <= 7; i++ { - ip, err := ipam.AllocateIP(subnetId, cidr, PurposeENIPrimary, "eni-test") + ip, err := ipam.AllocateIP(t.Context(), subnetId, cidr, PurposeENIPrimary, "eni-test") require.NoError(t, err) expected := "10.0.6." + itoa(i) assert.Equal(t, expected, ip) @@ -125,15 +126,15 @@ func TestIPAM_AllocatedIPs(t *testing.T) { ipam := setupTestIPAM(t) // No allocations yet - ips, err := ipam.AllocatedIPs("subnet-empty") + ips, err := ipam.AllocatedIPs(t.Context(), "subnet-empty") require.NoError(t, err) assert.Nil(t, ips) // Allocate some IPs - _, _ = ipam.AllocateIP("subnet-list", "10.0.7.0/24", PurposeENIPrimary, "eni-list") - _, _ = ipam.AllocateIP("subnet-list", "10.0.7.0/24", PurposeENIPrimary, "eni-list") + _, _ = ipam.AllocateIP(t.Context(), "subnet-list", "10.0.7.0/24", PurposeENIPrimary, "eni-list") + _, _ = ipam.AllocateIP(t.Context(), "subnet-list", "10.0.7.0/24", PurposeENIPrimary, "eni-list") - ips, err = ipam.AllocatedIPs("subnet-list") + ips, err = ipam.AllocatedIPs(t.Context(), "subnet-list") require.NoError(t, err) assert.Len(t, ips, 2) assert.Equal(t, "10.0.7.4", ips[0].IP) @@ -145,16 +146,16 @@ func TestIPAM_AllocatedIPs(t *testing.T) { func TestIPAM_MultipleSubnets(t *testing.T) { ipam := setupTestIPAM(t) - ip1, err := ipam.AllocateIP("subnet-a", "10.0.10.0/24", PurposeENIPrimary, "eni-a") + ip1, err := ipam.AllocateIP(t.Context(), "subnet-a", "10.0.10.0/24", PurposeENIPrimary, "eni-a") require.NoError(t, err) assert.Equal(t, "10.0.10.4", ip1) - ip2, err := ipam.AllocateIP("subnet-b", "10.0.20.0/24", PurposeENIPrimary, "eni-b") + ip2, err := ipam.AllocateIP(t.Context(), "subnet-b", "10.0.20.0/24", PurposeENIPrimary, "eni-b") require.NoError(t, err) assert.Equal(t, "10.0.20.4", ip2) // Each subnet tracks independently - ip3, err := ipam.AllocateIP("subnet-a", "10.0.10.0/24", PurposeENIPrimary, "eni-a") + ip3, err := ipam.AllocateIP(t.Context(), "subnet-a", "10.0.10.0/24", PurposeENIPrimary, "eni-a") require.NoError(t, err) assert.Equal(t, "10.0.10.5", ip3) } @@ -163,15 +164,16 @@ func TestIPAM_LargerSubnet(t *testing.T) { ipam := setupTestIPAM(t) // /20 subnet — first allocable is still .4 - ip, err := ipam.AllocateIP("subnet-big", "172.16.0.0/20", PurposeENIPrimary, "eni-big") + ip, err := ipam.AllocateIP(t.Context(), "subnet-big", "172.16.0.0/20", PurposeENIPrimary, "eni-big") require.NoError(t, err) assert.Equal(t, "172.16.0.4", ip) } func TestIPAM_NewIPAMWithKV(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) - kv, err := js.CreateKeyValue(&nats.KeyValueConfig{ + kv, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{ Bucket: "test-ipam-kv", }) require.NoError(t, err) @@ -180,14 +182,14 @@ func TestIPAM_NewIPAMWithKV(t *testing.T) { require.NotNil(t, ipam) // Should work the same as regular IPAM - ip, err := ipam.AllocateIP("subnet-kv", "10.0.1.0/24", PurposeENIPrimary, "eni-kv") + ip, err := ipam.AllocateIP(t.Context(), "subnet-kv", "10.0.1.0/24", PurposeENIPrimary, "eni-kv") require.NoError(t, err) assert.Equal(t, "10.0.1.4", ip) } func TestIPAM_InvalidCIDR(t *testing.T) { ipam := setupTestIPAM(t) - _, err := ipam.AllocateIP("subnet-bad", "not-a-cidr", PurposeENIPrimary, "eni-bad") + _, err := ipam.AllocateIP(t.Context(), "subnet-bad", "not-a-cidr", PurposeENIPrimary, "eni-bad") assert.Error(t, err) assert.Contains(t, err.Error(), "parse CIDR") } diff --git a/spinifex/handlers/ec2/vpc/security_group.go b/spinifex/handlers/ec2/vpc/security_group.go index 7c91b550c..070ec9f33 100644 --- a/spinifex/handlers/ec2/vpc/security_group.go +++ b/spinifex/handlers/ec2/vpc/security_group.go @@ -17,7 +17,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/awserrors" "github.com/mulgadc/spinifex/spinifex/filterutil" "github.com/mulgadc/spinifex/spinifex/utils" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // sgIDRegex must stay in lockstep with utils.GenerateResourceID("sg"). @@ -123,15 +123,15 @@ func (s *VPCServiceImpl) CreateSecurityGroup(ctx context.Context, input *ec2.Cre return nil, errors.New(awserrors.ErrorInvalidGroupReserved) } - if err := s.requireVPCExists(accountID, vpcId); err != nil { + if err := s.requireVPCExists(ctx, accountID, vpcId); err != nil { return nil, err } // Check for duplicate group name in the same VPC and enforce the per-VPC // SG quota in the same bucket walk. prefix := accountID + "." - sgKeys, err := s.sgKV.Keys() - if err != nil && !errors.Is(err, nats.ErrNoKeysFound) { + sgKeys, err := s.sgKV.Keys(ctx) + if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) { return nil, errors.New(awserrors.ErrorServerInternal) } sgsInVPC := 0 @@ -142,11 +142,11 @@ func (s *VPCServiceImpl) CreateSecurityGroup(ctx context.Context, input *ec2.Cre if !strings.HasPrefix(k, prefix) { continue } - entry, err := s.sgKV.Get(k) + entry, err := s.sgKV.Get(ctx, k) if err != nil { // Fail closed — a transient read error must not let a duplicate // SG name slip past, nor undercount the per-VPC quota. - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { continue } slog.WarnContext(ctx, "CreateSecurityGroup: SG read failed", "key", k, "err", err) @@ -196,7 +196,7 @@ func (s *VPCServiceImpl) CreateSecurityGroup(ctx context.Context, input *ec2.Cre if err != nil { return nil, fmt.Errorf("failed to marshal security group record: %w", err) } - if _, err := s.sgKV.Put(utils.AccountKey(accountID, groupId), data); err != nil { + if _, err := s.sgKV.Put(ctx, utils.AccountKey(accountID, groupId), data); err != nil { return nil, errors.New(awserrors.ErrorServerInternal) } @@ -226,12 +226,12 @@ func (s *VPCServiceImpl) DeleteSecurityGroup(ctx context.Context, input *ec2.Del groupId := *input.GroupId key := utils.AccountKey(accountID, groupId) - entry, err := s.sgKV.Get(key) + entry, err := s.sgKV.Get(ctx, key) if err != nil { // AWS-faithful: an absent security group is InvalidGroup.NotFound, not // success. Destroy orchestration tolerates it via awserrors.IsNotFound; // a transient read error stays a server error. - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil, errors.New(awserrors.ErrorInvalidGroupNotFound) } return nil, errors.New(awserrors.ErrorServerInternal) @@ -250,7 +250,7 @@ func (s *VPCServiceImpl) DeleteSecurityGroup(ctx context.Context, input *ec2.Del return nil, err } - if err := s.sgKV.Delete(key); err != nil { + if err := s.sgKV.Delete(ctx, key); err != nil { return nil, errors.New(awserrors.ErrorServerInternal) } @@ -270,12 +270,12 @@ func (s *VPCServiceImpl) DeleteSecurityGroup(ctx context.Context, input *ec2.Del // validateSGRuleReferences returns InvalidGroup.NotFound if any SourceSG in // rules is missing or belongs to a different VPC (cross-VPC refs not // supported; AWS uses the same error for both cases). -func (s *VPCServiceImpl) validateSGRuleReferences(accountID, ownerVpcId string, rules []SGRule) error { +func (s *VPCServiceImpl) validateSGRuleReferences(ctx context.Context, accountID, ownerVpcId string, rules []SGRule) error { for _, r := range rules { if r.SourceSG == "" { continue } - entry, err := s.sgKV.Get(utils.AccountKey(accountID, r.SourceSG)) + entry, err := s.sgKV.Get(ctx, utils.AccountKey(accountID, r.SourceSG)) if err != nil { return errors.New(awserrors.ErrorInvalidGroupNotFound) } @@ -296,15 +296,15 @@ func (s *VPCServiceImpl) validateSGRuleReferences(accountID, ownerVpcId string, func (s *VPCServiceImpl) checkSGDependencies(ctx context.Context, accountID, groupId string) error { prefix := accountID + "." - eniKeys, err := s.eniKV.Keys() - if err != nil && !errors.Is(err, nats.ErrNoKeysFound) { + eniKeys, err := s.eniKV.Keys(ctx) + if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) { return errors.New(awserrors.ErrorServerInternal) } for _, k := range eniKeys { if k == utils.VersionKey || !strings.HasPrefix(k, prefix) { continue } - entry, err := s.eniKV.Get(k) + entry, err := s.eniKV.Get(ctx, k) if err != nil { // Fail closed — a transient read error must not let us delete an // SG that's actually still attached. @@ -321,15 +321,15 @@ func (s *VPCServiceImpl) checkSGDependencies(ctx context.Context, accountID, gro } } - sgKeys, err := s.sgKV.Keys() - if err != nil && !errors.Is(err, nats.ErrNoKeysFound) { + sgKeys, err := s.sgKV.Keys(ctx) + if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) { return errors.New(awserrors.ErrorServerInternal) } for _, k := range sgKeys { if k == utils.VersionKey || !strings.HasPrefix(k, prefix) { continue } - entry, err := s.sgKV.Get(k) + entry, err := s.sgKV.Get(ctx, k) if err != nil { slog.WarnContext(ctx, "checkSGDependencies: SG read failed", "key", k, "err", err) return errors.New(awserrors.ErrorServerInternal) @@ -383,8 +383,8 @@ func (s *VPCServiceImpl) DescribeSecurityGroups(ctx context.Context, input *ec2. } prefix := accountID + "." - keys, err := s.sgKV.Keys() - if err != nil && !errors.Is(err, nats.ErrNoKeysFound) { + keys, err := s.sgKV.Keys(ctx) + if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) { return nil, errors.New(awserrors.ErrorServerInternal) } @@ -396,7 +396,7 @@ func (s *VPCServiceImpl) DescribeSecurityGroups(ctx context.Context, input *ec2. continue } - entry, err := s.sgKV.Get(key) + entry, err := s.sgKV.Get(ctx, key) if err != nil { slog.WarnContext(ctx, "Failed to get security group record", "key", key, "error", err) continue @@ -519,8 +519,8 @@ func (s *VPCServiceImpl) DescribeSecurityGroupRules(ctx context.Context, input * } prefix := accountID + "." - keys, err := s.sgKV.Keys() - if err != nil && !errors.Is(err, nats.ErrNoKeysFound) { + keys, err := s.sgKV.Keys(ctx) + if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) { return nil, errors.New(awserrors.ErrorServerInternal) } @@ -531,7 +531,7 @@ func (s *VPCServiceImpl) DescribeSecurityGroupRules(ctx context.Context, input * if key == utils.VersionKey || !strings.HasPrefix(key, prefix) { continue } - entry, err := s.sgKV.Get(key) + entry, err := s.sgKV.Get(ctx, key) if err != nil { slog.ErrorContext(ctx, "DescribeSecurityGroupRules: SG read failed", "key", key, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) @@ -645,7 +645,7 @@ func (s *VPCServiceImpl) AuthorizeSecurityGroupIngress(ctx context.Context, inpu groupId := *input.GroupId key := utils.AccountKey(accountID, groupId) - entry, err := s.sgKV.Get(key) + entry, err := s.sgKV.Get(ctx, key) if err != nil { return nil, errors.New(awserrors.ErrorInvalidGroupNotFound) } @@ -655,7 +655,7 @@ func (s *VPCServiceImpl) AuthorizeSecurityGroupIngress(ctx context.Context, inpu return nil, errors.New(awserrors.ErrorServerInternal) } - if err := s.requireVPCExists(accountID, record.VpcId); err != nil { + if err := s.requireVPCExists(ctx, accountID, record.VpcId); err != nil { return nil, err } @@ -664,7 +664,7 @@ func (s *VPCServiceImpl) AuthorizeSecurityGroupIngress(ctx context.Context, inpu slog.WarnContext(ctx, "AuthorizeSecurityGroupIngress: invalid rule", "groupId", groupId, "err", err) return nil, errors.New(awserrors.ErrorInvalidParameterValue) } - if err := s.validateSGRuleReferences(accountID, record.VpcId, newRules); err != nil { + if err := s.validateSGRuleReferences(ctx, accountID, record.VpcId, newRules); err != nil { return nil, err } existing := make(map[string]struct{}, len(record.IngressRules)) @@ -685,7 +685,7 @@ func (s *VPCServiceImpl) AuthorizeSecurityGroupIngress(ctx context.Context, inpu if err != nil { return nil, fmt.Errorf("failed to marshal security group record: %w", err) } - if _, err := s.sgKV.Update(key, data, entry.Revision()); err != nil { + if _, err := s.sgKV.Update(ctx, key, data, entry.Revision()); err != nil { return nil, errors.New(awserrors.ErrorServerInternal) } @@ -720,7 +720,7 @@ func (s *VPCServiceImpl) AuthorizeSecurityGroupEgress(ctx context.Context, input groupId := *input.GroupId key := utils.AccountKey(accountID, groupId) - entry, err := s.sgKV.Get(key) + entry, err := s.sgKV.Get(ctx, key) if err != nil { return nil, errors.New(awserrors.ErrorInvalidGroupNotFound) } @@ -730,7 +730,7 @@ func (s *VPCServiceImpl) AuthorizeSecurityGroupEgress(ctx context.Context, input return nil, errors.New(awserrors.ErrorServerInternal) } - if err := s.requireVPCExists(accountID, record.VpcId); err != nil { + if err := s.requireVPCExists(ctx, accountID, record.VpcId); err != nil { return nil, err } @@ -739,7 +739,7 @@ func (s *VPCServiceImpl) AuthorizeSecurityGroupEgress(ctx context.Context, input slog.WarnContext(ctx, "AuthorizeSecurityGroupEgress: invalid rule", "groupId", groupId, "err", err) return nil, errors.New(awserrors.ErrorInvalidParameterValue) } - if err := s.validateSGRuleReferences(accountID, record.VpcId, newRules); err != nil { + if err := s.validateSGRuleReferences(ctx, accountID, record.VpcId, newRules); err != nil { return nil, err } existing := make(map[string]struct{}, len(record.EgressRules)) @@ -760,7 +760,7 @@ func (s *VPCServiceImpl) AuthorizeSecurityGroupEgress(ctx context.Context, input if err != nil { return nil, fmt.Errorf("failed to marshal security group record: %w", err) } - if _, err := s.sgKV.Update(key, data, entry.Revision()); err != nil { + if _, err := s.sgKV.Update(ctx, key, data, entry.Revision()); err != nil { return nil, errors.New(awserrors.ErrorServerInternal) } @@ -795,7 +795,7 @@ func (s *VPCServiceImpl) RevokeSecurityGroupIngress(ctx context.Context, input * groupId := *input.GroupId key := utils.AccountKey(accountID, groupId) - entry, err := s.sgKV.Get(key) + entry, err := s.sgKV.Get(ctx, key) if err != nil { return nil, errors.New(awserrors.ErrorInvalidGroupNotFound) } @@ -827,7 +827,7 @@ func (s *VPCServiceImpl) RevokeSecurityGroupIngress(ctx context.Context, input * if err != nil { return nil, fmt.Errorf("failed to marshal security group record: %w", err) } - if _, err := s.sgKV.Update(key, data, entry.Revision()); err != nil { + if _, err := s.sgKV.Update(ctx, key, data, entry.Revision()); err != nil { return nil, errors.New(awserrors.ErrorServerInternal) } @@ -857,7 +857,7 @@ func (s *VPCServiceImpl) RevokeSecurityGroupEgress(ctx context.Context, input *e groupId := *input.GroupId key := utils.AccountKey(accountID, groupId) - entry, err := s.sgKV.Get(key) + entry, err := s.sgKV.Get(ctx, key) if err != nil { return nil, errors.New(awserrors.ErrorInvalidGroupNotFound) } @@ -889,7 +889,7 @@ func (s *VPCServiceImpl) RevokeSecurityGroupEgress(ctx context.Context, input *e if err != nil { return nil, fmt.Errorf("failed to marshal security group record: %w", err) } - if _, err := s.sgKV.Update(key, data, entry.Revision()); err != nil { + if _, err := s.sgKV.Update(ctx, key, data, entry.Revision()); err != nil { return nil, errors.New(awserrors.ErrorServerInternal) } @@ -1171,7 +1171,7 @@ func (s *VPCServiceImpl) createDefaultSecurityGroupInternal(ctx context.Context, if err != nil { return "", fmt.Errorf("marshal default security group: %w", err) } - if _, err := s.sgKV.Put(utils.AccountKey(accountID, groupId), data); err != nil { + if _, err := s.sgKV.Put(ctx, utils.AccountKey(accountID, groupId), data); err != nil { return "", fmt.Errorf("store default security group: %w", err) } @@ -1193,7 +1193,7 @@ func (s *VPCServiceImpl) createDefaultSecurityGroupInternal(ctx context.Context, // per-VPC default SG. Surfaces vpcd tear-down failures to the caller. func (s *VPCServiceImpl) deleteSecurityGroupInternal(ctx context.Context, accountID, groupId string) error { key := utils.AccountKey(accountID, groupId) - entry, err := s.sgKV.Get(key) + entry, err := s.sgKV.Get(ctx, key) if err != nil { return fmt.Errorf("read default security group: %w", err) } @@ -1201,7 +1201,7 @@ func (s *VPCServiceImpl) deleteSecurityGroupInternal(ctx context.Context, accoun if err := json.Unmarshal(entry.Value(), &record); err != nil { return fmt.Errorf("unmarshal default security group: %w", err) } - if err := s.sgKV.Delete(key); err != nil { + if err := s.sgKV.Delete(ctx, key); err != nil { return fmt.Errorf("delete default security group: %w", err) } return s.requestSGEvent("vpc.delete-sg", SGEvent{ @@ -1214,10 +1214,14 @@ func (s *VPCServiceImpl) deleteSecurityGroupInternal(ctx context.Context, accoun // IsDefault=true and the given VPC. Returns "" if none found (only happens // when CreateVpc failed mid-flow and left the VPC without a default SG). func (s *VPCServiceImpl) FindDefaultSGForVPC(accountID, vpcId string) (string, error) { + return s.findDefaultSGForVPC(context.Background(), accountID, vpcId) +} + +func (s *VPCServiceImpl) findDefaultSGForVPC(ctx context.Context, accountID, vpcId string) (string, error) { prefix := accountID + "." - keys, err := s.sgKV.Keys() + keys, err := s.sgKV.Keys(ctx) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return "", nil } return "", err @@ -1226,7 +1230,7 @@ func (s *VPCServiceImpl) FindDefaultSGForVPC(accountID, vpcId string) (string, e if k == utils.VersionKey || !strings.HasPrefix(k, prefix) { continue } - entry, err := s.sgKV.Get(k) + entry, err := s.sgKV.Get(ctx, k) if err != nil { continue } diff --git a/spinifex/handlers/ec2/vpc/security_group_test.go b/spinifex/handlers/ec2/vpc/security_group_test.go index 5e8fcbf68..2fdadab93 100644 --- a/spinifex/handlers/ec2/vpc/security_group_test.go +++ b/spinifex/handlers/ec2/vpc/security_group_test.go @@ -1118,14 +1118,14 @@ func TestDescribeSecurityGroups_FilterNoResults(t *testing.T) { // IsDefault=true (production code keys off IsDefault, not GroupName). func findDefaultSGInVPC(t *testing.T, svc *VPCServiceImpl, vpcID string) string { t.Helper() - keys, err := svc.sgKV.Keys() + keys, err := svc.sgKV.Keys(t.Context()) require.NoError(t, err) var matches []string for _, k := range keys { if k == utils.VersionKey { continue } - entry, err := svc.sgKV.Get(k) + entry, err := svc.sgKV.Get(t.Context(), k) require.NoError(t, err) var rec SecurityGroupRecord require.NoError(t, json.Unmarshal(entry.Value(), &rec)) @@ -1611,7 +1611,7 @@ func TestFindDefaultSGForVPC_SkipsMalformedRecord(t *testing.T) { svc := setupTestVPCService(t) vpcID := createTestVPC(t, svc, "10.0.0.0/16") - _, err := svc.sgKV.Put(testAccountID+".sg-corrupt0000000", []byte("{not json")) + _, err := svc.sgKV.Put(t.Context(), testAccountID+".sg-corrupt0000000", []byte("{not json")) require.NoError(t, err) // CreateVpc above already provisioned the default SG; FindDefaultSGForVPC @@ -1651,7 +1651,7 @@ func TestDeleteSecurityGroup_FailsClosedOnCorruptENI(t *testing.T) { // Inject a malformed ENI record into the bucket; checkSGDependencies must // reject the delete rather than silently treating it as "no dependent ENI". - _, err := svc.eniKV.Put(testAccountID+".eni-corrupt", []byte("{not json")) + _, err := svc.eniKV.Put(t.Context(), testAccountID+".eni-corrupt", []byte("{not json")) require.NoError(t, err) _, err = svc.DeleteSecurityGroup(context.Background(), &ec2.DeleteSecurityGroupInput{ @@ -1669,7 +1669,7 @@ func seedAvailableVPC(t *testing.T, svc *VPCServiceImpl, vpcID string) { rec := VPCRecord{VpcId: vpcID, CidrBlock: "10.0.0.0/16", State: "available"} data, err := json.Marshal(rec) require.NoError(t, err) - _, err = svc.vpcKV.Put(utils.AccountKey(testAccountID, vpcID), data) + _, err = svc.vpcKV.Put(t.Context(), utils.AccountKey(testAccountID, vpcID), data) require.NoError(t, err) } @@ -1894,7 +1894,7 @@ func TestAuthorizeSecurityGroupIngress_AssignsRuleIDs(t *testing.T) { }, testAccountID) require.NoError(t, err) - entry, err := svc.sgKV.Get(utils.AccountKey(testAccountID, sgID)) + entry, err := svc.sgKV.Get(t.Context(), utils.AccountKey(testAccountID, sgID)) require.NoError(t, err) var record SecurityGroupRecord require.NoError(t, json.Unmarshal(entry.Value(), &record)) diff --git a/spinifex/handlers/ec2/vpc/service_impl.go b/spinifex/handlers/ec2/vpc/service_impl.go index b655b3b2a..acbf547ac 100644 --- a/spinifex/handlers/ec2/vpc/service_impl.go +++ b/spinifex/handlers/ec2/vpc/service_impl.go @@ -16,9 +16,11 @@ import ( "github.com/mulgadc/spinifex/spinifex/awserrors" "github.com/mulgadc/spinifex/spinifex/config" "github.com/mulgadc/spinifex/spinifex/filterutil" + "github.com/mulgadc/spinifex/spinifex/kvutil" "github.com/mulgadc/spinifex/spinifex/migrate" "github.com/mulgadc/spinifex/spinifex/utils" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // Ensure VPCServiceImpl implements VPCService. @@ -74,17 +76,17 @@ type SubnetRecord struct { type VPCServiceImpl struct { config *config.Config natsConn *nats.Conn - vpcKV nats.KeyValue - subnetKV nats.KeyValue - vniKV nats.KeyValue - eniKV nats.KeyValue - sgKV nats.KeyValue - rtbKV nats.KeyValue // route table bucket for auto-creating main route table + vpcKV jetstream.KeyValue + subnetKV jetstream.KeyValue + vniKV jetstream.KeyValue + eniKV jetstream.KeyValue + sgKV jetstream.KeyValue + rtbKV jetstream.KeyValue // route table bucket for auto-creating main route table ipam *IPAM // Optional: injected after construction for public IP cleanup in DeleteNetworkInterface. externalIPAM *ExternalIPAM - eipKV nats.KeyValue + eipKV jetstream.KeyValue // disableDefaultPublicIP seeds default subnets with MapPublicIpOnLaunch=false // (non-pool external modes have no public IPs to assign). Zero value keeps @@ -100,7 +102,7 @@ func (s *VPCServiceImpl) SetDefaultPublicIPMapping(enabled bool) { // SetExternalIPAM injects external IPAM and EIP KV store references so that // DeleteNetworkInterface can release auto-assigned public IPs and NAT rules. -func (s *VPCServiceImpl) SetExternalIPAM(ipam *ExternalIPAM, eipKV nats.KeyValue) { +func (s *VPCServiceImpl) SetExternalIPAM(ipam *ExternalIPAM, eipKV jetstream.KeyValue) { s.externalIPAM = ipam s.eipKV = eipKV } @@ -116,58 +118,58 @@ func (s *VPCServiceImpl) localAZ() string { } // NewVPCServiceImplWithNATS creates a VPC service with NATS JetStream for persistence. -func NewVPCServiceImplWithNATS(cfg *config.Config, natsConn *nats.Conn) (*VPCServiceImpl, error) { - js, err := natsConn.JetStream() +func NewVPCServiceImplWithNATS(ctx context.Context, cfg *config.Config, natsConn *nats.Conn) (*VPCServiceImpl, error) { + js, err := jetstream.New(natsConn) if err != nil { - return nil, fmt.Errorf("failed to get JetStream context: %w", err) + return nil, fmt.Errorf("create JetStream client: %w", err) } - vpcKV, err := utils.GetOrCreateKVBucket(js, KVBucketVPCs, 10) + vpcKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketVPCs, 10) if err != nil { return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketVPCs, err) } - if err := migrate.DefaultRegistry.RunKV(KVBucketVPCs, vpcKV, KVBucketVPCsVersion); err != nil { + if err := migrate.DefaultRegistry.RunKV(ctx, KVBucketVPCs, vpcKV, KVBucketVPCsVersion); err != nil { return nil, fmt.Errorf("migrate %s: %w", KVBucketVPCs, err) } - subnetKV, err := utils.GetOrCreateKVBucket(js, KVBucketSubnets, 10) + subnetKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketSubnets, 10) if err != nil { return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketSubnets, err) } - if err := migrate.DefaultRegistry.RunKV(KVBucketSubnets, subnetKV, KVBucketSubnetsVersion); err != nil { + if err := migrate.DefaultRegistry.RunKV(ctx, KVBucketSubnets, subnetKV, KVBucketSubnetsVersion); err != nil { return nil, fmt.Errorf("migrate %s: %w", KVBucketSubnets, err) } - vniKV, err := utils.GetOrCreateKVBucket(js, KVBucketVNICounter, 10) + vniKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketVNICounter, 10) if err != nil { return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketVNICounter, err) } - if err := migrate.DefaultRegistry.RunKV(KVBucketVNICounter, vniKV, KVBucketVNICounterVersion); err != nil { + if err := migrate.DefaultRegistry.RunKV(ctx, KVBucketVNICounter, vniKV, KVBucketVNICounterVersion); err != nil { return nil, fmt.Errorf("migrate %s: %w", KVBucketVNICounter, err) } - eniKV, err := utils.GetOrCreateKVBucket(js, KVBucketENIs, 10) + eniKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketENIs, 10) if err != nil { return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketENIs, err) } - if err := migrate.DefaultRegistry.RunKV(KVBucketENIs, eniKV, KVBucketENIsVersion); err != nil { + if err := migrate.DefaultRegistry.RunKV(ctx, KVBucketENIs, eniKV, KVBucketENIsVersion); err != nil { return nil, fmt.Errorf("migrate %s: %w", KVBucketENIs, err) } - sgKV, err := utils.GetOrCreateKVBucket(js, KVBucketSecurityGroups, 10) + sgKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketSecurityGroups, 10) if err != nil { return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketSecurityGroups, err) } - if err := migrate.DefaultRegistry.RunKV(KVBucketSecurityGroups, sgKV, KVBucketSecurityGroupsVersion); err != nil { + if err := migrate.DefaultRegistry.RunKV(ctx, KVBucketSecurityGroups, sgKV, KVBucketSecurityGroupsVersion); err != nil { return nil, fmt.Errorf("migrate %s: %w", KVBucketSecurityGroups, err) } - rtbKV, err := utils.GetOrCreateKVBucket(js, "spinifex-vpc-route-tables", 10) + rtbKV, err := kvutil.GetOrCreateBucket(ctx, js, "spinifex-vpc-route-tables", 10) if err != nil { return nil, fmt.Errorf("failed to create KV bucket spinifex-vpc-route-tables: %w", err) } - ipam, err := NewIPAM(js) + ipam, err := NewIPAM(ctx, js) if err != nil { return nil, fmt.Errorf("failed to initialize IPAM: %w", err) } @@ -193,17 +195,17 @@ func NewVPCServiceImplWithNATS(cfg *config.Config, natsConn *nats.Conn) (*VPCSer } // nextVNI allocates the next VNI using atomic increment on the NATS KV counter. -func (s *VPCServiceImpl) nextVNI() (int64, error) { - entry, err := s.vniKV.Get(vniCounterKey) +func (s *VPCServiceImpl) nextVNI(ctx context.Context) (int64, error) { + entry, err := s.vniKV.Get(ctx, vniCounterKey) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { // First VNI allocation — initialize counter vni := int64(vniStart) data, marshalErr := json.Marshal(vni + 1) if marshalErr != nil { return 0, fmt.Errorf("failed to marshal VNI counter: %w", marshalErr) } - if _, err := s.vniKV.Create(vniCounterKey, data); err != nil { + if _, err := s.vniKV.Create(ctx, vniCounterKey, data); err != nil { return 0, fmt.Errorf("failed to initialize VNI counter: %w", err) } return vni, nil @@ -221,7 +223,7 @@ func (s *VPCServiceImpl) nextVNI() (int64, error) { if marshalErr != nil { return 0, fmt.Errorf("failed to marshal VNI counter: %w", marshalErr) } - if _, err := s.vniKV.Update(vniCounterKey, data, entry.Revision()); err != nil { + if _, err := s.vniKV.Update(ctx, vniCounterKey, data, entry.Revision()); err != nil { return 0, fmt.Errorf("failed to update VNI counter (CAS conflict): %w", err) } @@ -247,7 +249,7 @@ func (s *VPCServiceImpl) CreateVpc(ctx context.Context, input *ec2.CreateVpcInpu } // Allocate VNI for overlay network - vni, err := s.nextVNI() + vni, err := s.nextVNI(ctx) if err != nil { return nil, errors.New(awserrors.ErrorServerInternal) } @@ -271,7 +273,7 @@ func (s *VPCServiceImpl) CreateVpc(ctx context.Context, input *ec2.CreateVpcInpu if err != nil { return nil, fmt.Errorf("failed to marshal VPC record: %w", err) } - if _, err := s.vpcKV.Put(utils.AccountKey(accountID, vpcID), data); err != nil { + if _, err := s.vpcKV.Put(ctx, utils.AccountKey(accountID, vpcID), data); err != nil { return nil, errors.New(awserrors.ErrorServerInternal) } @@ -301,8 +303,8 @@ func (s *VPCServiceImpl) CreateVpc(ctx context.Context, input *ec2.CreateVpcInpu // requireVPCExists returns InvalidVpcID.NotFound if the VPC doesn't exist for // this account. -func (s *VPCServiceImpl) requireVPCExists(accountID, vpcId string) error { - if _, err := s.vpcKV.Get(utils.AccountKey(accountID, vpcId)); err != nil { +func (s *VPCServiceImpl) requireVPCExists(ctx context.Context, accountID, vpcId string) error { + if _, err := s.vpcKV.Get(ctx, utils.AccountKey(accountID, vpcId)); err != nil { return errors.New(awserrors.ErrorInvalidVpcIDNotFound) } return nil @@ -317,11 +319,11 @@ func (s *VPCServiceImpl) DeleteVpc(ctx context.Context, input *ec2.DeleteVpcInpu vpcID := *input.VpcId key := utils.AccountKey(accountID, vpcID) - if _, err := s.vpcKV.Get(key); err != nil { + if _, err := s.vpcKV.Get(ctx, key); err != nil { // AWS-faithful: an absent VPC is NotFound (the tofu/SDK provider // tolerates it on destroy); destroy orchestration tolerates it too. // A transient read error stays a server error. - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil, errors.New(awserrors.ErrorInvalidVpcIDNotFound) } return nil, errors.New(awserrors.ErrorServerInternal) @@ -329,8 +331,8 @@ func (s *VPCServiceImpl) DeleteVpc(ctx context.Context, input *ec2.DeleteVpcInpu // Check for dependent subnets owned by this account prefix := accountID + "." - subnetKeys, err := s.subnetKV.Keys() - if err != nil && !errors.Is(err, nats.ErrNoKeysFound) { + subnetKeys, err := s.subnetKV.Keys(ctx) + if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) { return nil, errors.New(awserrors.ErrorServerInternal) } for _, k := range subnetKeys { @@ -340,13 +342,13 @@ func (s *VPCServiceImpl) DeleteVpc(ctx context.Context, input *ec2.DeleteVpcInpu if !strings.HasPrefix(k, prefix) { continue } - entry, err := s.subnetKV.Get(k) + entry, err := s.subnetKV.Get(ctx, k) if err != nil { // ErrKeyNotFound means the subnet was deleted between Keys() and // Get() — fine to skip. Any other error is fail-closed: a // transient read error must not let DeleteVpc bypass a subnet // dependency it can't see. - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { continue } slog.WarnContext(ctx, "DeleteVpc: subnet read failed", "key", k, "err", err) @@ -365,21 +367,21 @@ func (s *VPCServiceImpl) DeleteVpc(ctx context.Context, input *ec2.DeleteVpcInpu // Reject if any non-default SG remains in this VPC; the cascade only // auto-deletes the default SG (matches AWS DeleteVpc semantics). defaultSGId := "" - sgKeys, err := s.sgKV.Keys() - if err != nil && !errors.Is(err, nats.ErrNoKeysFound) { + sgKeys, err := s.sgKV.Keys(ctx) + if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) { return nil, errors.New(awserrors.ErrorServerInternal) } for _, k := range sgKeys { if k == utils.VersionKey || !strings.HasPrefix(k, prefix) { continue } - entry, err := s.sgKV.Get(k) + entry, err := s.sgKV.Get(ctx, k) if err != nil { // ErrKeyNotFound means the SG was deleted between Keys() and // Get() — fine to skip. Any other error is fail-closed: a // transient read error must not let DeleteVpc orphan a // non-default SG it can't see. - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { continue } slog.WarnContext(ctx, "DeleteVpc: SG read failed", "key", k, "err", err) @@ -411,20 +413,20 @@ func (s *VPCServiceImpl) DeleteVpc(ctx context.Context, input *ec2.DeleteVpcInpu // Reap the VPC's main route table: CreateVpc auto-creates it and // DeleteRouteTable refuses to delete a main RT, so nothing else reclaims it // — without this rtbKV leaks one orphaned main RT per deleted VPC. - rtbID, err := s.findMainRouteTableID(accountID, vpcID) + rtbID, err := s.findMainRouteTableID(ctx, accountID, vpcID) if err != nil { slog.ErrorContext(ctx, "DeleteVpc: main route table lookup failed", "vpcId", vpcID, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } if rtbID != "" { - if err := s.rtbKV.Delete(utils.AccountKey(accountID, rtbID)); err != nil && !errors.Is(err, nats.ErrKeyNotFound) { + if err := s.rtbKV.Delete(ctx, utils.AccountKey(accountID, rtbID)); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { slog.ErrorContext(ctx, "DeleteVpc: main route table reap failed", "vpcId", vpcID, "routeTableId", rtbID, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } slog.InfoContext(ctx, "DeleteVpc: reaped main route table", "vpcId", vpcID, "routeTableId", rtbID) } - if err := s.vpcKV.Delete(key); err != nil { + if err := s.vpcKV.Delete(ctx, key); err != nil { return nil, errors.New(awserrors.ErrorServerInternal) } @@ -463,8 +465,8 @@ func (s *VPCServiceImpl) DescribeVpcs(ctx context.Context, input *ec2.DescribeVp } prefix := accountID + "." - keys, err := s.vpcKV.Keys() - if err != nil && !errors.Is(err, nats.ErrNoKeysFound) { + keys, err := s.vpcKV.Keys(ctx) + if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) { return nil, errors.New(awserrors.ErrorServerInternal) } @@ -476,7 +478,7 @@ func (s *VPCServiceImpl) DescribeVpcs(ctx context.Context, input *ec2.DescribeVp continue } - entry, err := s.vpcKV.Get(key) + entry, err := s.vpcKV.Get(ctx, key) if err != nil { slog.WarnContext(ctx, "Failed to get VPC record", "key", key, "error", err) continue @@ -533,7 +535,7 @@ func (s *VPCServiceImpl) CreateSubnet(ctx context.Context, input *ec2.CreateSubn vpcID := *input.VpcId // Verify VPC exists and belongs to this account - vpcEntry, err := s.vpcKV.Get(utils.AccountKey(accountID, vpcID)) + vpcEntry, err := s.vpcKV.Get(ctx, utils.AccountKey(accountID, vpcID)) if err != nil { return nil, errors.New(awserrors.ErrorInvalidVpcIDNotFound) } @@ -566,8 +568,8 @@ func (s *VPCServiceImpl) CreateSubnet(ctx context.Context, input *ec2.CreateSubn // Check for CIDR conflicts with existing subnets in this VPC (same account) prefix := accountID + "." - subnetKeys, err := s.subnetKV.Keys() - if err != nil && !errors.Is(err, nats.ErrNoKeysFound) { + subnetKeys, err := s.subnetKV.Keys(ctx) + if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) { return nil, errors.New(awserrors.ErrorServerInternal) } @@ -578,7 +580,7 @@ func (s *VPCServiceImpl) CreateSubnet(ctx context.Context, input *ec2.CreateSubn if !strings.HasPrefix(k, prefix) { continue } - entry, err := s.subnetKV.Get(k) + entry, err := s.subnetKV.Get(ctx, k) if err != nil { continue } @@ -627,7 +629,7 @@ func (s *VPCServiceImpl) CreateSubnet(ctx context.Context, input *ec2.CreateSubn if err != nil { return nil, fmt.Errorf("failed to marshal subnet record: %w", err) } - if _, err := s.subnetKV.Put(utils.AccountKey(accountID, subnetID), data); err != nil { + if _, err := s.subnetKV.Put(ctx, utils.AccountKey(accountID, subnetID), data); err != nil { return nil, errors.New(awserrors.ErrorServerInternal) } @@ -651,12 +653,12 @@ func (s *VPCServiceImpl) DeleteSubnet(ctx context.Context, input *ec2.DeleteSubn key := utils.AccountKey(accountID, subnetID) // Read subnet record before deletion (needed for vpcd event) - subnetEntry, err := s.subnetKV.Get(key) + subnetEntry, err := s.subnetKV.Get(ctx, key) if err != nil { // AWS-faithful: an absent subnet is NotFound (provider tolerates it on // destroy); destroy orchestration tolerates it too. A transient read // error stays a server error. - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil, errors.New(awserrors.ErrorInvalidSubnetIDNotFound) } return nil, errors.New(awserrors.ErrorServerInternal) @@ -671,7 +673,7 @@ func (s *VPCServiceImpl) DeleteSubnet(ctx context.Context, input *ec2.DeleteSubn return nil, err } - if err := s.subnetKV.Delete(key); err != nil { + if err := s.subnetKV.Delete(ctx, key); err != nil { return nil, errors.New(awserrors.ErrorServerInternal) } @@ -687,9 +689,9 @@ func (s *VPCServiceImpl) DeleteSubnet(ctx context.Context, input *ec2.DeleteSubn // subnet is a live attachment (rule #3). Fail-closed on a KV read error so a // transient fault never lets a delete orphan a port. func (s *VPCServiceImpl) checkSubnetResidents(ctx context.Context, accountID, subnetID string) error { - keys, err := s.eniKV.Keys() + keys, err := s.eniKV.Keys(ctx) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return nil } slog.ErrorContext(ctx, "DeleteSubnet: ENI scan failed, blocking delete to avoid orphaning ports", "subnetId", subnetID, "err", err) @@ -700,9 +702,9 @@ func (s *VPCServiceImpl) checkSubnetResidents(ctx context.Context, accountID, su if key == utils.VersionKey || !strings.HasPrefix(key, prefix) { continue } - entry, err := s.eniKV.Get(key) + entry, err := s.eniKV.Get(ctx, key) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { continue } return errors.New(awserrors.ErrorServerInternal) @@ -736,8 +738,8 @@ func (s *VPCServiceImpl) DescribeSubnets(ctx context.Context, input *ec2.Describ } prefix := accountID + "." - keys, err := s.subnetKV.Keys() - if err != nil && !errors.Is(err, nats.ErrNoKeysFound) { + keys, err := s.subnetKV.Keys(ctx) + if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) { return nil, errors.New(awserrors.ErrorServerInternal) } @@ -749,7 +751,7 @@ func (s *VPCServiceImpl) DescribeSubnets(ctx context.Context, input *ec2.Describ continue } - entry, err := s.subnetKV.Get(key) + entry, err := s.subnetKV.Get(ctx, key) if err != nil { slog.WarnContext(ctx, "Failed to get subnet record", "key", key, "error", err) continue @@ -815,7 +817,7 @@ func (s *VPCServiceImpl) ApplyRecordTags(input *ec2.CreateTagsInput, accountID s if res == nil { continue } - if err := s.updateRecordTags(accountID, *res, merge); err != nil { + if err := s.updateRecordTags(context.Background(), accountID, *res, merge); err != nil { return err } } @@ -834,7 +836,7 @@ func (s *VPCServiceImpl) RemoveRecordTags(input *ec2.DeleteTagsInput, accountID if res == nil { continue } - if err := s.updateRecordTags(accountID, *res, remove); err != nil { + if err := s.updateRecordTags(context.Background(), accountID, *res, remove); err != nil { return err } } @@ -843,31 +845,31 @@ func (s *VPCServiceImpl) RemoveRecordTags(input *ec2.DeleteTagsInput, accountID // updateRecordTags applies mut to the tag map of the subnet-, vpc-, sg-, or // eni-scoped record identified by resourceID. Other resource ids are a no-op. -func (s *VPCServiceImpl) updateRecordTags(accountID, resourceID string, mut func(map[string]string)) error { +func (s *VPCServiceImpl) updateRecordTags(ctx context.Context, accountID, resourceID string, mut func(map[string]string)) error { switch { case strings.HasPrefix(resourceID, "subnet-"): - return utils.UpdateKVRecordTags(s.subnetKV, accountID, resourceID, func(r *SubnetRecord) { + return utils.UpdateKVRecordTags(ctx, s.subnetKV, accountID, resourceID, func(r *SubnetRecord) { if r.Tags == nil { r.Tags = map[string]string{} } mut(r.Tags) }) case strings.HasPrefix(resourceID, "vpc-"): - return utils.UpdateKVRecordTags(s.vpcKV, accountID, resourceID, func(r *VPCRecord) { + return utils.UpdateKVRecordTags(ctx, s.vpcKV, accountID, resourceID, func(r *VPCRecord) { if r.Tags == nil { r.Tags = map[string]string{} } mut(r.Tags) }) case strings.HasPrefix(resourceID, "sg-"): - return utils.UpdateKVRecordTags(s.sgKV, accountID, resourceID, func(r *SecurityGroupRecord) { + return utils.UpdateKVRecordTags(ctx, s.sgKV, accountID, resourceID, func(r *SecurityGroupRecord) { if r.Tags == nil { r.Tags = map[string]string{} } mut(r.Tags) }) case strings.HasPrefix(resourceID, "eni-"): - return utils.UpdateKVRecordTags(s.eniKV, accountID, resourceID, func(r *ENIRecord) { + return utils.UpdateKVRecordTags(ctx, s.eniKV, accountID, resourceID, func(r *ENIRecord) { if r.Tags == nil { r.Tags = map[string]string{} } @@ -1004,7 +1006,7 @@ func (s *VPCServiceImpl) ModifySubnetAttribute(ctx context.Context, input *ec2.M subnetID := *input.SubnetId key := utils.AccountKey(accountID, subnetID) - entry, err := s.subnetKV.Get(key) + entry, err := s.subnetKV.Get(ctx, key) if err != nil { return nil, errors.New(awserrors.ErrorInvalidSubnetIDNotFound) } @@ -1022,7 +1024,7 @@ func (s *VPCServiceImpl) ModifySubnetAttribute(ctx context.Context, input *ec2.M if err != nil { return nil, fmt.Errorf("failed to marshal subnet record: %w", err) } - if _, err := s.subnetKV.Update(key, data, entry.Revision()); err != nil { + if _, err := s.subnetKV.Update(ctx, key, data, entry.Revision()); err != nil { return nil, errors.New(awserrors.ErrorServerInternal) } @@ -1043,7 +1045,7 @@ func (s *VPCServiceImpl) ModifyVpcAttribute(ctx context.Context, input *ec2.Modi vpcID := *input.VpcId key := utils.AccountKey(accountID, vpcID) - entry, err := s.vpcKV.Get(key) + entry, err := s.vpcKV.Get(ctx, key) if err != nil { return nil, errors.New(awserrors.ErrorInvalidVpcIDNotFound) } @@ -1069,7 +1071,7 @@ func (s *VPCServiceImpl) ModifyVpcAttribute(ctx context.Context, input *ec2.Modi slog.ErrorContext(ctx, "ModifyVpcAttribute: failed to marshal VPC record", "vpcId", vpcID, "accountID", accountID, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } - if _, err := s.vpcKV.Update(key, data, entry.Revision()); err != nil { + if _, err := s.vpcKV.Update(ctx, key, data, entry.Revision()); err != nil { slog.ErrorContext(ctx, "ModifyVpcAttribute: KV update failed", "vpcId", vpcID, "accountID", accountID, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } @@ -1091,7 +1093,7 @@ func (s *VPCServiceImpl) DescribeVpcAttribute(ctx context.Context, input *ec2.De vpcID := *input.VpcId key := utils.AccountKey(accountID, vpcID) - entry, err := s.vpcKV.Get(key) + entry, err := s.vpcKV.Get(ctx, key) if err != nil { return nil, errors.New(awserrors.ErrorInvalidVpcIDNotFound) } @@ -1145,14 +1147,18 @@ type BootstrapIDs struct { // EnsureDefaultVPC creates a default VPC and subnet if none exists for the // account. Safe to call multiple times — no-ops if already present. func (s *VPCServiceImpl) EnsureDefaultVPC(accountID string, bootstrap ...BootstrapIDs) (*DefaultVPCInfo, error) { + return s.ensureDefaultVPC(context.Background(), accountID, bootstrap...) +} + +func (s *VPCServiceImpl) ensureDefaultVPC(ctx context.Context, accountID string, bootstrap ...BootstrapIDs) (*DefaultVPCInfo, error) { if s.vpcKV == nil { return nil, nil // No persistence, skip } // Check if a default VPC already exists for this account prefix := accountID + "." - keys, err := s.vpcKV.Keys() - if err != nil && !errors.Is(err, nats.ErrNoKeysFound) { + keys, err := s.vpcKV.Keys(ctx) + if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) { return nil, fmt.Errorf("list VPCs: %w", err) } @@ -1163,7 +1169,7 @@ func (s *VPCServiceImpl) EnsureDefaultVPC(accountID string, bootstrap ...Bootstr if !strings.HasPrefix(key, prefix) { continue } - entry, err := s.vpcKV.Get(key) + entry, err := s.vpcKV.Get(ctx, key) if err != nil { continue } @@ -1174,7 +1180,7 @@ func (s *VPCServiceImpl) EnsureDefaultVPC(accountID string, bootstrap ...Bootstr if record.IsDefault { slog.Debug("Default VPC already exists", "vpcId", record.VpcId, "accountID", accountID) // Look up the default subnet to return full info - defaultSubnet, _ := s.GetDefaultSubnet(accountID) + defaultSubnet, _ := s.getDefaultSubnet(ctx, accountID) info := &DefaultVPCInfo{VpcId: record.VpcId, Cidr: record.CidrBlock} if defaultSubnet != nil { info.SubnetId = defaultSubnet.SubnetId @@ -1185,7 +1191,7 @@ func (s *VPCServiceImpl) EnsureDefaultVPC(accountID string, bootstrap ...Bootstr } // Create default VPC — use bootstrap IDs if provided for consistency - vni, err := s.nextVNI() + vni, err := s.nextVNI(ctx) if err != nil { return nil, fmt.Errorf("allocate VNI for default VPC: %w", err) } @@ -1211,7 +1217,7 @@ func (s *VPCServiceImpl) EnsureDefaultVPC(accountID string, bootstrap ...Bootstr if err != nil { return nil, fmt.Errorf("marshal default VPC: %w", err) } - if _, err := s.vpcKV.Put(utils.AccountKey(accountID, vpcID), data); err != nil { + if _, err := s.vpcKV.Put(ctx, utils.AccountKey(accountID, vpcID), data); err != nil { return nil, fmt.Errorf("store default VPC: %w", err) } @@ -1244,7 +1250,7 @@ func (s *VPCServiceImpl) EnsureDefaultVPC(accountID string, bootstrap ...Bootstr if err != nil { return nil, fmt.Errorf("marshal default subnet: %w", err) } - if _, err := s.subnetKV.Put(utils.AccountKey(accountID, subnetID), data); err != nil { + if _, err := s.subnetKV.Put(ctx, utils.AccountKey(accountID, subnetID), data); err != nil { return nil, fmt.Errorf("store default subnet: %w", err) } @@ -1252,7 +1258,7 @@ func (s *VPCServiceImpl) EnsureDefaultVPC(accountID string, bootstrap ...Bootstr // Create main route table with local route (written directly to KV to avoid circular import) if s.rtbKV != nil { - if err := s.createMainRouteTable(context.Background(), accountID, vpcID, DefaultVPCCidr); err != nil { + if err := s.createMainRouteTable(ctx, accountID, vpcID, DefaultVPCCidr); err != nil { slog.Error("Failed to create main route table for VPC", "vpcId", vpcID, "err", err) } } @@ -1261,7 +1267,7 @@ func (s *VPCServiceImpl) EnsureDefaultVPC(accountID string, bootstrap ...Bootstr // before vpcd has subscribed to vpc.create-sg, so the synchronous round-trip // will time out on first boot. The SG record is already in KV; vpcd's // reconcile-sgs loop creates the OVN port group on its first scan. - if _, err := s.createDefaultSecurityGroupInternal(context.Background(), accountID, vpcID); err != nil { + if _, err := s.createDefaultSecurityGroupInternal(ctx, accountID, vpcID); err != nil { slog.Warn("Default security group bootstrap deferred to vpcd reconciler", "vpcId", vpcID, "accountID", accountID, "err", err) } @@ -1286,7 +1292,7 @@ func (s *VPCServiceImpl) EnsureDefaultVPC(accountID string, bootstrap ...Bootstr // table KV bucket. Avoids a circular import; idempotent via findMainRouteTableID // so concurrent EnsureDefaultVPC calls do not mint duplicate main RTs. func (s *VPCServiceImpl) createMainRouteTable(ctx context.Context, accountID, vpcID, vpcCidr string) error { - if existing, err := s.findMainRouteTableID(accountID, vpcID); err != nil { + if existing, err := s.findMainRouteTableID(ctx, accountID, vpcID); err != nil { return fmt.Errorf("check existing main route table: %w", err) } else if existing != "" { slog.InfoContext(ctx, "Main route table already exists for VPC, skipping creation", @@ -1340,7 +1346,7 @@ func (s *VPCServiceImpl) createMainRouteTable(ctx context.Context, accountID, vp if err != nil { return fmt.Errorf("marshal main route table: %w", err) } - if _, err := s.rtbKV.Put(utils.AccountKey(accountID, rtbID), data); err != nil { + if _, err := s.rtbKV.Put(ctx, utils.AccountKey(accountID, rtbID), data); err != nil { return fmt.Errorf("store main route table: %w", err) } @@ -1351,11 +1357,11 @@ func (s *VPCServiceImpl) createMainRouteTable(ctx context.Context, accountID, vp // findMainRouteTableID returns the rtb-ID of the main route table for vpcID // in accountID, or "" if none exists. Partial unmarshal — only the fields // needed to identify a main RT, so this stays cheap on hot KV scans. -func (s *VPCServiceImpl) findMainRouteTableID(accountID, vpcID string) (string, error) { +func (s *VPCServiceImpl) findMainRouteTableID(ctx context.Context, accountID, vpcID string) (string, error) { prefix := accountID + "." - keys, err := s.rtbKV.Keys() + keys, err := s.rtbKV.Keys(ctx) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return "", nil } return "", fmt.Errorf("list rtb keys: %w", err) @@ -1364,9 +1370,9 @@ func (s *VPCServiceImpl) findMainRouteTableID(accountID, vpcID string) (string, if key == utils.VersionKey || !strings.HasPrefix(key, prefix) { continue } - entry, err := s.rtbKV.Get(key) + entry, err := s.rtbKV.Get(ctx, key) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { continue } return "", fmt.Errorf("read rtb %s: %w", key, err) @@ -1388,10 +1394,14 @@ func (s *VPCServiceImpl) findMainRouteTableID(accountID, vpcID string) (string, // GetDefaultSubnet returns the default subnet for RunInstances when no SubnetId is specified. func (s *VPCServiceImpl) GetDefaultSubnet(accountID string) (*SubnetRecord, error) { + return s.getDefaultSubnet(context.Background(), accountID) +} + +func (s *VPCServiceImpl) getDefaultSubnet(ctx context.Context, accountID string) (*SubnetRecord, error) { prefix := accountID + "." - keys, err := s.subnetKV.Keys() + keys, err := s.subnetKV.Keys(ctx) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return nil, fmt.Errorf("no default subnet found") } return nil, fmt.Errorf("list subnets: %w", err) @@ -1404,7 +1414,7 @@ func (s *VPCServiceImpl) GetDefaultSubnet(accountID string) (*SubnetRecord, erro if !strings.HasPrefix(key, prefix) { continue } - entry, err := s.subnetKV.Get(key) + entry, err := s.subnetKV.Get(ctx, key) if err != nil { continue } @@ -1422,8 +1432,12 @@ func (s *VPCServiceImpl) GetDefaultSubnet(accountID string) (*SubnetRecord, erro // GetSubnet looks up a SubnetRecord by ID. func (s *VPCServiceImpl) GetSubnet(accountID, subnetId string) (*SubnetRecord, error) { + return s.getSubnet(context.Background(), accountID, subnetId) +} + +func (s *VPCServiceImpl) getSubnet(ctx context.Context, accountID, subnetId string) (*SubnetRecord, error) { key := utils.AccountKey(accountID, subnetId) - entry, err := s.subnetKV.Get(key) + entry, err := s.subnetKV.Get(ctx, key) if err != nil { return nil, fmt.Errorf("subnet %s not found: %w", subnetId, err) } diff --git a/spinifex/handlers/ec2/vpc/service_impl_test.go b/spinifex/handlers/ec2/vpc/service_impl_test.go index af38e4fa1..95dddeaff 100644 --- a/spinifex/handlers/ec2/vpc/service_impl_test.go +++ b/spinifex/handlers/ec2/vpc/service_impl_test.go @@ -21,7 +21,7 @@ func setupTestVPCServiceWithNC(t *testing.T) (*VPCServiceImpl, *nats.Conn) { t.Helper() _, nc, _ := testutil.StartTestJetStream(t) - svc, err := NewVPCServiceImplWithNATS(nil, nc) + svc, err := NewVPCServiceImplWithNATS(t.Context(), nil, nc) require.NoError(t, err) testutil.StubVpcdSGResponder(t, nc) return svc, nc @@ -39,7 +39,7 @@ func setupTestVPCService(t *testing.T) *VPCServiceImpl { func setupTestVPCServiceWithFailingVpcd(t *testing.T, errMsg string) (*VPCServiceImpl, *nats.Conn) { t.Helper() _, nc, _ := testutil.StartTestJetStream(t) - svc, err := NewVPCServiceImplWithNATS(nil, nc) + svc, err := NewVPCServiceImplWithNATS(t.Context(), nil, nc) require.NoError(t, err) testutil.StubVpcdSGFailingResponder(t, nc, errMsg) return svc, nc @@ -586,13 +586,13 @@ func TestCreateMainRouteTable_Idempotent(t *testing.T) { svc := setupTestVPCService(t) vpcID := createTestVPC(t, svc, "10.99.0.0/16") // CreateVpc auto-calls createMainRouteTable - firstID, err := svc.findMainRouteTableID(testAccountID, vpcID) + firstID, err := svc.findMainRouteTableID(t.Context(), testAccountID, vpcID) require.NoError(t, err) require.NotEmpty(t, firstID, "CreateVpc should have created a main route table") require.NoError(t, svc.createMainRouteTable(context.Background(), testAccountID, vpcID, "10.99.0.0/16")) - secondID, err := svc.findMainRouteTableID(testAccountID, vpcID) + secondID, err := svc.findMainRouteTableID(t.Context(), testAccountID, vpcID) require.NoError(t, err) assert.Equal(t, firstID, secondID, "second call must be a no-op") @@ -608,7 +608,7 @@ func TestDeleteVpc_ReapsMainRouteTable(t *testing.T) { svc := setupTestVPCService(t) vpcID := createTestVPC(t, svc, "10.77.0.0/16") // auto-creates the main RT - rtbID, err := svc.findMainRouteTableID(testAccountID, vpcID) + rtbID, err := svc.findMainRouteTableID(t.Context(), testAccountID, vpcID) require.NoError(t, err) require.NotEmpty(t, rtbID, "CreateVpc should have created a main route table") require.Equal(t, 1, countMainRouteTablesForVPC(t, svc, vpcID)) @@ -616,7 +616,7 @@ func TestDeleteVpc_ReapsMainRouteTable(t *testing.T) { _, err = svc.DeleteVpc(context.Background(), &ec2.DeleteVpcInput{VpcId: aws.String(vpcID)}, testAccountID) require.NoError(t, err) - gone, err := svc.findMainRouteTableID(testAccountID, vpcID) + gone, err := svc.findMainRouteTableID(t.Context(), testAccountID, vpcID) require.NoError(t, err) assert.Empty(t, gone, "DeleteVpc must reap the main route table, not leak it") assert.Equal(t, 0, countMainRouteTablesForVPC(t, svc, vpcID)) @@ -626,13 +626,13 @@ func TestDeleteVpc_ReapsMainRouteTable(t *testing.T) { // state in tests. Returns the number of IsMain=true records for vpcID. func countMainRouteTablesForVPC(t *testing.T, svc *VPCServiceImpl, vpcID string) int { t.Helper() - keys, err := svc.rtbKV.Keys() + keys, err := svc.rtbKV.Keys(t.Context()) if err != nil { return 0 } n := 0 for _, key := range keys { - entry, err := svc.rtbKV.Get(key) + entry, err := svc.rtbKV.Get(t.Context(), key) if err != nil { continue } @@ -655,7 +655,7 @@ func countMainRouteTablesForVPC(t *testing.T, svc *VPCServiceImpl, vpcID string) // so subnet and RTB must still land in KV when vpcd is absent. func TestEnsureDefaultVPC_NoVpcdResponder(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) - svc, err := NewVPCServiceImplWithNATS(nil, nc) + svc, err := NewVPCServiceImplWithNATS(t.Context(), nil, nc) require.NoError(t, err) // Intentionally NOT calling StubVpcdSGResponder — vpc.create-sg has no // responder, mirroring the bootstrap race. @@ -676,11 +676,11 @@ func TestEnsureDefaultVPC_NoVpcdResponder(t *testing.T) { assert.Equal(t, info.VpcId, *subDesc.Subnets[0].VpcId) require.NotNil(t, svc.rtbKV) - rtbKeys, err := svc.rtbKV.Keys() + rtbKeys, err := svc.rtbKV.Keys(t.Context()) require.NoError(t, err) foundMainRTB := false for _, k := range rtbKeys { - entry, err := svc.rtbKV.Get(k) + entry, err := svc.rtbKV.Get(t.Context(), k) if err != nil { continue } @@ -700,7 +700,7 @@ func TestEnsureDefaultVPC_NoVpcdResponder(t *testing.T) { // Default SG record is best-effort; KV write happens before the synchronous // vpcd round-trip, so the record should still be present. - sgKeys, err := svc.sgKV.Keys() + sgKeys, err := svc.sgKV.Keys(t.Context()) require.NoError(t, err) assert.NotEmpty(t, sgKeys, "default SG record must persist in KV for vpcd reconciler to converge") } @@ -907,7 +907,7 @@ func TestEnsureDefaultVPC_WithConfigAZ(t *testing.T) { testutil.StubVpcdSGResponder(t, nc) cfg := &config.Config{AZ: "us-west-2b"} - svc, err := NewVPCServiceImplWithNATS(cfg, nc) + svc, err := NewVPCServiceImplWithNATS(t.Context(), cfg, nc) require.NoError(t, err) _, err = svc.EnsureDefaultVPC(testAccountID) diff --git a/spinifex/handlers/ecr/meta.go b/spinifex/handlers/ecr/meta.go index 3f4d2637a..8a1f83d76 100644 --- a/spinifex/handlers/ecr/meta.go +++ b/spinifex/handlers/ecr/meta.go @@ -12,8 +12,8 @@ import ( "sync" "time" - "github.com/mulgadc/spinifex/spinifex/utils" - "github.com/nats-io/nats.go" + "github.com/mulgadc/spinifex/spinifex/kvutil" + "github.com/nats-io/nats.go/jetstream" ) // ErrNotFound is returned by MetaStore reads when the requested record is absent. @@ -109,26 +109,26 @@ type MetaStore interface { // KVMetaStore is the JetStream-KV-backed MetaStore. Per-account buckets are // created lazily on first write and cached per process. type KVMetaStore struct { - js nats.JetStreamContext + js jetstream.JetStream mu sync.Mutex - buckets map[string]nats.KeyValue + buckets map[string]jetstream.KeyValue } var _ MetaStore = (*KVMetaStore)(nil) -// NewKVMetaStore constructs a KVMetaStore over the supplied JetStream context. -func NewKVMetaStore(js nats.JetStreamContext) *KVMetaStore { - return &KVMetaStore{js: js, buckets: make(map[string]nats.KeyValue)} +// NewKVMetaStore constructs a KVMetaStore over the supplied JetStream handle. +func NewKVMetaStore(js jetstream.JetStream) *KVMetaStore { + return &KVMetaStore{js: js, buckets: make(map[string]jetstream.KeyValue)} } -func (s *KVMetaStore) bucket(accountID string) (nats.KeyValue, error) { +func (s *KVMetaStore) bucket(ctx context.Context, accountID string) (jetstream.KeyValue, error) { s.mu.Lock() defer s.mu.Unlock() if kv, ok := s.buckets[accountID]; ok { return kv, nil } name := KVAccountBucket(accountID) - kv, err := utils.GetOrCreateKVBucket(s.js, name, KVBucketAccountHistory) + kv, err := kvutil.GetOrCreateBucket(ctx, s.js, name, KVBucketAccountHistory) if err != nil { return nil, fmt.Errorf("ecr: open account bucket %s: %w", name, err) } @@ -136,8 +136,8 @@ func (s *KVMetaStore) bucket(accountID string) (nats.KeyValue, error) { return kv, nil } -func (s *KVMetaStore) PutRepo(_ context.Context, accountID string, meta RepoMeta) error { - kv, err := s.bucket(accountID) +func (s *KVMetaStore) PutRepo(ctx context.Context, accountID string, meta RepoMeta) error { + kv, err := s.bucket(ctx, accountID) if err != nil { return err } @@ -145,16 +145,16 @@ func (s *KVMetaStore) PutRepo(_ context.Context, accountID string, meta RepoMeta if err != nil { return err } - _, err = kv.Put(KVRepoMetaKey(meta.Name), data) + _, err = kv.Put(ctx, KVRepoMetaKey(meta.Name), data) return err } -func (s *KVMetaStore) GetRepo(_ context.Context, accountID, repo string) (RepoMeta, error) { - kv, err := s.bucket(accountID) +func (s *KVMetaStore) GetRepo(ctx context.Context, accountID, repo string) (RepoMeta, error) { + kv, err := s.bucket(ctx, accountID) if err != nil { return RepoMeta{}, err } - entry, err := kv.Get(KVRepoMetaKey(repo)) + entry, err := kv.Get(ctx, KVRepoMetaKey(repo)) if err != nil { return RepoMeta{}, mapKVErr(err) } @@ -165,14 +165,14 @@ func (s *KVMetaStore) GetRepo(_ context.Context, accountID, repo string) (RepoMe return m, nil } -func (s *KVMetaStore) ListRepos(_ context.Context, accountID string) ([]string, error) { - kv, err := s.bucket(accountID) +func (s *KVMetaStore) ListRepos(ctx context.Context, accountID string) ([]string, error) { + kv, err := s.bucket(ctx, accountID) if err != nil { return nil, err } - keys, err := kv.Keys() + keys, err := kv.Keys(ctx) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return nil, nil } return nil, err @@ -195,17 +195,17 @@ func (s *KVMetaStore) ListRepos(_ context.Context, accountID string) ([]string, // tags, and all manifest records. Predastore blob garbage collection is out of // scope here (deferred); only the per-account KV records are removed. Returns // ErrNotFound when the repository meta is absent. -func (s *KVMetaStore) DeleteRepo(_ context.Context, accountID, repo string) error { - kv, err := s.bucket(accountID) +func (s *KVMetaStore) DeleteRepo(ctx context.Context, accountID, repo string) error { + kv, err := s.bucket(ctx, accountID) if err != nil { return err } - if _, err := kv.Get(KVRepoMetaKey(repo)); err != nil { + if _, err := kv.Get(ctx, KVRepoMetaKey(repo)); err != nil { return mapKVErr(err) } - keys, err := kv.Keys() + keys, err := kv.Keys(ctx) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return nil } return err @@ -215,7 +215,7 @@ func (s *KVMetaStore) DeleteRepo(_ context.Context, accountID, repo string) erro for _, k := range keys { if k == metaKey || k == policyKey || k == lifecycleKey || strings.HasPrefix(k, tagsPrefix) || strings.HasPrefix(k, manifestsPrefix) { - if err := kv.Delete(k); err != nil { + if err := kv.Delete(ctx, k); err != nil { return mapKVErr(err) } } @@ -223,14 +223,14 @@ func (s *KVMetaStore) DeleteRepo(_ context.Context, accountID, repo string) erro return nil } -func (s *KVMetaStore) ListManifests(_ context.Context, accountID, repo string) ([]string, error) { - kv, err := s.bucket(accountID) +func (s *KVMetaStore) ListManifests(ctx context.Context, accountID, repo string) ([]string, error) { + kv, err := s.bucket(ctx, accountID) if err != nil { return nil, err } - keys, err := kv.Keys() + keys, err := kv.Keys(ctx) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return nil, nil } return nil, err @@ -246,127 +246,127 @@ func (s *KVMetaStore) ListManifests(_ context.Context, accountID, repo string) ( return digests, nil } -func (s *KVMetaStore) PutRepoPolicy(_ context.Context, accountID, repo string, policyText []byte) error { - kv, err := s.bucket(accountID) +func (s *KVMetaStore) PutRepoPolicy(ctx context.Context, accountID, repo string, policyText []byte) error { + kv, err := s.bucket(ctx, accountID) if err != nil { return err } - _, err = kv.Put(KVRepoPolicyKey(repo), policyText) + _, err = kv.Put(ctx, KVRepoPolicyKey(repo), policyText) return err } -func (s *KVMetaStore) GetRepoPolicy(_ context.Context, accountID, repo string) ([]byte, error) { - kv, err := s.bucket(accountID) +func (s *KVMetaStore) GetRepoPolicy(ctx context.Context, accountID, repo string) ([]byte, error) { + kv, err := s.bucket(ctx, accountID) if err != nil { return nil, err } - entry, err := kv.Get(KVRepoPolicyKey(repo)) + entry, err := kv.Get(ctx, KVRepoPolicyKey(repo)) if err != nil { return nil, mapKVErr(err) } return entry.Value(), nil } -func (s *KVMetaStore) DeleteRepoPolicy(_ context.Context, accountID, repo string) ([]byte, error) { - kv, err := s.bucket(accountID) +func (s *KVMetaStore) DeleteRepoPolicy(ctx context.Context, accountID, repo string) ([]byte, error) { + kv, err := s.bucket(ctx, accountID) if err != nil { return nil, err } // JetStream KV Delete is silently idempotent, so the value is read first to // surface a real not-found and return the deleted document to the caller. - entry, err := kv.Get(KVRepoPolicyKey(repo)) + entry, err := kv.Get(ctx, KVRepoPolicyKey(repo)) if err != nil { return nil, mapKVErr(err) } - if err := kv.Delete(KVRepoPolicyKey(repo)); err != nil { + if err := kv.Delete(ctx, KVRepoPolicyKey(repo)); err != nil { return nil, mapKVErr(err) } return entry.Value(), nil } -func (s *KVMetaStore) PutLifecyclePolicy(_ context.Context, accountID, repo string, policyText []byte) error { - kv, err := s.bucket(accountID) +func (s *KVMetaStore) PutLifecyclePolicy(ctx context.Context, accountID, repo string, policyText []byte) error { + kv, err := s.bucket(ctx, accountID) if err != nil { return err } - _, err = kv.Put(KVLifecyclePolicyKey(repo), policyText) + _, err = kv.Put(ctx, KVLifecyclePolicyKey(repo), policyText) return err } -func (s *KVMetaStore) GetLifecyclePolicy(_ context.Context, accountID, repo string) ([]byte, error) { - kv, err := s.bucket(accountID) +func (s *KVMetaStore) GetLifecyclePolicy(ctx context.Context, accountID, repo string) ([]byte, error) { + kv, err := s.bucket(ctx, accountID) if err != nil { return nil, err } - entry, err := kv.Get(KVLifecyclePolicyKey(repo)) + entry, err := kv.Get(ctx, KVLifecyclePolicyKey(repo)) if err != nil { return nil, mapKVErr(err) } return entry.Value(), nil } -func (s *KVMetaStore) DeleteLifecyclePolicy(_ context.Context, accountID, repo string) ([]byte, error) { - kv, err := s.bucket(accountID) +func (s *KVMetaStore) DeleteLifecyclePolicy(ctx context.Context, accountID, repo string) ([]byte, error) { + kv, err := s.bucket(ctx, accountID) if err != nil { return nil, err } // JetStream KV Delete is silently idempotent, so the value is read first to // surface a real not-found and return the deleted document to the caller. - entry, err := kv.Get(KVLifecyclePolicyKey(repo)) + entry, err := kv.Get(ctx, KVLifecyclePolicyKey(repo)) if err != nil { return nil, mapKVErr(err) } - if err := kv.Delete(KVLifecyclePolicyKey(repo)); err != nil { + if err := kv.Delete(ctx, KVLifecyclePolicyKey(repo)); err != nil { return nil, mapKVErr(err) } return entry.Value(), nil } -func (s *KVMetaStore) PutTag(_ context.Context, accountID, repo, tag, digest string) error { - kv, err := s.bucket(accountID) +func (s *KVMetaStore) PutTag(ctx context.Context, accountID, repo, tag, digest string) error { + kv, err := s.bucket(ctx, accountID) if err != nil { return err } - _, err = kv.Put(KVTagKey(repo, tag), []byte(digest)) + _, err = kv.Put(ctx, KVTagKey(repo, tag), []byte(digest)) return err } -func (s *KVMetaStore) GetTag(_ context.Context, accountID, repo, tag string) (string, error) { - kv, err := s.bucket(accountID) +func (s *KVMetaStore) GetTag(ctx context.Context, accountID, repo, tag string) (string, error) { + kv, err := s.bucket(ctx, accountID) if err != nil { return "", err } - entry, err := kv.Get(KVTagKey(repo, tag)) + entry, err := kv.Get(ctx, KVTagKey(repo, tag)) if err != nil { return "", mapKVErr(err) } return string(entry.Value()), nil } -func (s *KVMetaStore) DeleteTag(_ context.Context, accountID, repo, tag string) error { - kv, err := s.bucket(accountID) +func (s *KVMetaStore) DeleteTag(ctx context.Context, accountID, repo, tag string) error { + kv, err := s.bucket(ctx, accountID) if err != nil { return err } // JetStream KV Delete is silently idempotent, so existence is checked first // to surface a real not-found to the caller. - if _, err := kv.Get(KVTagKey(repo, tag)); err != nil { + if _, err := kv.Get(ctx, KVTagKey(repo, tag)); err != nil { return mapKVErr(err) } - if err := kv.Delete(KVTagKey(repo, tag)); err != nil { + if err := kv.Delete(ctx, KVTagKey(repo, tag)); err != nil { return mapKVErr(err) } return nil } -func (s *KVMetaStore) ListTags(_ context.Context, accountID, repo string) ([]string, error) { - kv, err := s.bucket(accountID) +func (s *KVMetaStore) ListTags(ctx context.Context, accountID, repo string) ([]string, error) { + kv, err := s.bucket(ctx, accountID) if err != nil { return nil, err } - keys, err := kv.Keys() + keys, err := kv.Keys(ctx) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return nil, nil } return nil, err @@ -382,8 +382,8 @@ func (s *KVMetaStore) ListTags(_ context.Context, accountID, repo string) ([]str return tags, nil } -func (s *KVMetaStore) PutManifestMeta(_ context.Context, accountID, repo string, meta ManifestMeta) error { - kv, err := s.bucket(accountID) +func (s *KVMetaStore) PutManifestMeta(ctx context.Context, accountID, repo string, meta ManifestMeta) error { + kv, err := s.bucket(ctx, accountID) if err != nil { return err } @@ -391,16 +391,16 @@ func (s *KVMetaStore) PutManifestMeta(_ context.Context, accountID, repo string, if err != nil { return err } - _, err = kv.Put(KVManifestKey(repo, meta.Digest), data) + _, err = kv.Put(ctx, KVManifestKey(repo, meta.Digest), data) return err } -func (s *KVMetaStore) GetManifestMeta(_ context.Context, accountID, repo, digest string) (ManifestMeta, error) { - kv, err := s.bucket(accountID) +func (s *KVMetaStore) GetManifestMeta(ctx context.Context, accountID, repo, digest string) (ManifestMeta, error) { + kv, err := s.bucket(ctx, accountID) if err != nil { return ManifestMeta{}, err } - entry, err := kv.Get(KVManifestKey(repo, digest)) + entry, err := kv.Get(ctx, KVManifestKey(repo, digest)) if err != nil { return ManifestMeta{}, mapKVErr(err) } @@ -411,19 +411,19 @@ func (s *KVMetaStore) GetManifestMeta(_ context.Context, accountID, repo, digest return m, nil } -func (s *KVMetaStore) DeleteManifestMeta(_ context.Context, accountID, repo, digest string) error { - kv, err := s.bucket(accountID) +func (s *KVMetaStore) DeleteManifestMeta(ctx context.Context, accountID, repo, digest string) error { + kv, err := s.bucket(ctx, accountID) if err != nil { return err } - if _, err := kv.Get(KVManifestKey(repo, digest)); err != nil { + if _, err := kv.Get(ctx, KVManifestKey(repo, digest)); err != nil { return mapKVErr(err) } - return kv.Delete(KVManifestKey(repo, digest)) + return kv.Delete(ctx, KVManifestKey(repo, digest)) } -func (s *KVMetaStore) PutUpload(_ context.Context, accountID, uploadID string, state UploadState) (uint64, error) { - kv, err := s.bucket(accountID) +func (s *KVMetaStore) PutUpload(ctx context.Context, accountID, uploadID string, state UploadState) (uint64, error) { + kv, err := s.bucket(ctx, accountID) if err != nil { return 0, err } @@ -431,15 +431,15 @@ func (s *KVMetaStore) PutUpload(_ context.Context, accountID, uploadID string, s if err != nil { return 0, err } - return kv.Put(KVUploadKey(uploadID), data) + return kv.Put(ctx, KVUploadKey(uploadID), data) } -func (s *KVMetaStore) GetUpload(_ context.Context, accountID, uploadID string) (UploadState, uint64, error) { - kv, err := s.bucket(accountID) +func (s *KVMetaStore) GetUpload(ctx context.Context, accountID, uploadID string) (UploadState, uint64, error) { + kv, err := s.bucket(ctx, accountID) if err != nil { return UploadState{}, 0, err } - entry, err := kv.Get(KVUploadKey(uploadID)) + entry, err := kv.Get(ctx, KVUploadKey(uploadID)) if err != nil { return UploadState{}, 0, mapKVErr(err) } @@ -450,8 +450,8 @@ func (s *KVMetaStore) GetUpload(_ context.Context, accountID, uploadID string) ( return st, entry.Revision(), nil } -func (s *KVMetaStore) UpdateUpload(_ context.Context, accountID, uploadID string, state UploadState, rev uint64) (uint64, error) { - kv, err := s.bucket(accountID) +func (s *KVMetaStore) UpdateUpload(ctx context.Context, accountID, uploadID string, state UploadState, rev uint64) (uint64, error) { + kv, err := s.bucket(ctx, accountID) if err != nil { return 0, err } @@ -459,9 +459,9 @@ func (s *KVMetaStore) UpdateUpload(_ context.Context, accountID, uploadID string if err != nil { return 0, err } - newRev, err := kv.Update(KVUploadKey(uploadID), data, rev) + newRev, err := kv.Update(ctx, KVUploadKey(uploadID), data, rev) if err != nil { - if errors.Is(err, nats.ErrKeyExists) { + if errors.Is(err, jetstream.ErrKeyExists) { return 0, ErrConflict } return 0, err @@ -469,17 +469,17 @@ func (s *KVMetaStore) UpdateUpload(_ context.Context, accountID, uploadID string return newRev, nil } -func (s *KVMetaStore) DeleteUpload(_ context.Context, accountID, uploadID string) error { - kv, err := s.bucket(accountID) +func (s *KVMetaStore) DeleteUpload(ctx context.Context, accountID, uploadID string) error { + kv, err := s.bucket(ctx, accountID) if err != nil { return err } // JetStream KV Delete is silently idempotent, so existence is checked first // to surface a real not-found to the caller. - if _, err := kv.Get(KVUploadKey(uploadID)); err != nil { + if _, err := kv.Get(ctx, KVUploadKey(uploadID)); err != nil { return mapKVErr(err) } - if err := kv.Delete(KVUploadKey(uploadID)); err != nil { + if err := kv.Delete(ctx, KVUploadKey(uploadID)); err != nil { return mapKVErr(err) } return nil @@ -763,7 +763,7 @@ func (m *MemoryMetaStore) DeleteUpload(_ context.Context, accountID, uploadID st // mapKVErr normalizes JetStream KV errors to the MetaStore error vocabulary. func mapKVErr(err error) error { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return ErrNotFound } return err diff --git a/spinifex/handlers/ecr/meta_kv_test.go b/spinifex/handlers/ecr/meta_kv_test.go index 90818d675..701f7f264 100644 --- a/spinifex/handlers/ecr/meta_kv_test.go +++ b/spinifex/handlers/ecr/meta_kv_test.go @@ -14,7 +14,8 @@ import ( // TestKVMetaStore_FullCycle exercises the JetStream-backed MetaStore against an // embedded JetStream server, covering the production CAS path. func TestKVMetaStore_FullCycle(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) store := NewKVMetaStore(js) const acct = "000000000000" @@ -90,7 +91,8 @@ func TestKVMetaStore_FullCycle(t *testing.T) { // tags, and manifests, leaves a same-prefix nested repo intact, and reports // ErrNotFound for an absent repo. func TestKVMetaStore_DeleteRepo(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) store := NewKVMetaStore(js) const acct = "000000000000" const dig = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" @@ -133,7 +135,8 @@ func TestKVMetaStore_DeleteRepo(t *testing.T) { // JetStream-backed store and proves the policy key shares the repo bucket // without being mistaken for a repository by ListRepos. func TestKVMetaStore_RepoPolicy(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) store := NewKVMetaStore(js) const acct = "000000000000" const policy = `{"Version":"2012-10-17","Statement":[]}` @@ -164,7 +167,8 @@ func TestKVMetaStore_RepoPolicy(t *testing.T) { // against the JetStream-backed store and proves the lifecycle key shares the // repo bucket without being mistaken for a repository by ListRepos. func TestKVMetaStore_LifecyclePolicy(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) store := NewKVMetaStore(js) const acct = "000000000000" const policy = `{"rules":[{"rulePriority":1,"selection":{"tagStatus":"untagged","countType":"sinceImagePushed","countUnit":"days","countNumber":14},"action":{"type":"expire"}}]}` diff --git a/spinifex/handlers/ecr/service_impl.go b/spinifex/handlers/ecr/service_impl.go index c04f350ad..0c3c0ea9d 100644 --- a/spinifex/handlers/ecr/service_impl.go +++ b/spinifex/handlers/ecr/service_impl.go @@ -4,7 +4,7 @@ import ( "context" "errors" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // MetaServiceImpl is the daemon-side MetaService. It owns the backing MetaStore @@ -23,7 +23,7 @@ func NewMetaServiceImpl(store MetaStore) *MetaServiceImpl { } // NewKVMetaService builds a MetaService backed by per-account JetStream KV. -func NewKVMetaService(js nats.JetStreamContext) *MetaServiceImpl { +func NewKVMetaService(js jetstream.JetStream) *MetaServiceImpl { return &MetaServiceImpl{store: NewKVMetaStore(js)} } diff --git a/spinifex/handlers/ecr/service_test.go b/spinifex/handlers/ecr/service_test.go index 7964c4046..889292389 100644 --- a/spinifex/handlers/ecr/service_test.go +++ b/spinifex/handlers/ecr/service_test.go @@ -56,7 +56,8 @@ func TestMetaServiceImpl_FlagMapping(t *testing.T) { // TestNATSMetaStore_RepoPolicy drives the policy passthrough round-trip // (put/get/delete + not-found mapping) end to end over the embedded server. func TestNATSMetaStore_RepoPolicy(t *testing.T) { - _, nc, js := testutil.StartTestJetStream(t) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) svc := NewKVMetaService(js) serveMeta(t, nc, SubjectPolicyPut, svc.PolicyPut) @@ -88,7 +89,8 @@ func TestNATSMetaStore_RepoPolicy(t *testing.T) { // TestNATSMetaStore_LifecyclePolicy drives the lifecycle-policy passthrough // round-trip (put/get/delete + not-found mapping) end to end. func TestNATSMetaStore_LifecyclePolicy(t *testing.T) { - _, nc, js := testutil.StartTestJetStream(t) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) svc := NewKVMetaService(js) serveMeta(t, nc, SubjectLifecyclePut, svc.LifecyclePut) @@ -143,7 +145,8 @@ func serveMeta[I any, O any](t *testing.T, nc *nats.Conn, subject string, fn fun // a KV-backed daemon service over an embedded NATS/JetStream server, proving the // full request/reply path including ErrNotFound/ErrConflict reconstruction. func TestNATSMetaStore_RoundTrip(t *testing.T) { - _, nc, js := testutil.StartTestJetStream(t) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) svc := NewKVMetaService(js) serveMeta(t, nc, SubjectRepoCreate, svc.RepoCreate) diff --git a/spinifex/handlers/ecs/capacity_providers.go b/spinifex/handlers/ecs/capacity_providers.go index 6ec72d9eb..3e08b8055 100644 --- a/spinifex/handlers/ecs/capacity_providers.go +++ b/spinifex/handlers/ecs/capacity_providers.go @@ -13,17 +13,17 @@ import ( // PutClusterCapacityProviders persists the cluster's capacity-provider names // and default strategy. Accepted and stored only: no scheduler coupling, no // scale loop (a separate follow-on binds this to an ASG primitive). -func (s *Service) PutClusterCapacityProviders(_ context.Context, input *ecs.PutClusterCapacityProvidersInput, accountID string) (*ecs.PutClusterCapacityProvidersOutput, error) { +func (s *Service) PutClusterCapacityProviders(ctx context.Context, input *ecs.PutClusterCapacityProvidersInput, accountID string) (*ecs.PutClusterCapacityProvidersOutput, error) { if input == nil { return nil, errors.New(awserrors.ErrorECSInvalidParameter) } cluster := clusterShortName(aws.StringValue(input.Cluster)) - kv, err := s.bucket(accountID) + kv, err := s.bucket(ctx, accountID) if err != nil { return nil, err } var rec ClusterRecord - found, err := getJSON(kv, ClusterMetaKey(cluster), &rec) + found, err := getJSON(ctx, kv, ClusterMetaKey(cluster), &rec) if err != nil { return nil, err } @@ -33,7 +33,7 @@ func (s *Service) PutClusterCapacityProviders(_ context.Context, input *ecs.PutC rec.CapacityProviders = awsStringSlice(input.CapacityProviders) rec.DefaultCapacityProviderStrategy = strategyFromAWS(input.DefaultCapacityProviderStrategy) - if err := putJSON(kv, ClusterMetaKey(cluster), &rec); err != nil { + if err := putJSON(ctx, kv, ClusterMetaKey(cluster), &rec); err != nil { return nil, err } return &ecs.PutClusterCapacityProvidersOutput{Cluster: rec.toAWS()}, nil @@ -41,19 +41,19 @@ func (s *Service) PutClusterCapacityProviders(_ context.Context, input *ecs.PutC // CreateCapacityProvider persists a new capacity provider. Idempotent: // re-creating an existing name returns the existing record. -func (s *Service) CreateCapacityProvider(_ context.Context, input *ecs.CreateCapacityProviderInput, accountID string) (*ecs.CreateCapacityProviderOutput, error) { +func (s *Service) CreateCapacityProvider(ctx context.Context, input *ecs.CreateCapacityProviderInput, accountID string) (*ecs.CreateCapacityProviderOutput, error) { if input == nil || aws.StringValue(input.Name) == "" || input.AutoScalingGroupProvider == nil || aws.StringValue(input.AutoScalingGroupProvider.AutoScalingGroupArn) == "" { return nil, errors.New(awserrors.ErrorECSInvalidParameter) } name := aws.StringValue(input.Name) - kv, err := s.bucket(accountID) + kv, err := s.bucket(ctx, accountID) if err != nil { return nil, err } var existing CapacityProviderRecord - found, err := getJSON(kv, CapacityProviderKey(name), &existing) + found, err := getJSON(ctx, kv, CapacityProviderKey(name), &existing) if err != nil { return nil, err } @@ -69,7 +69,7 @@ func (s *Service) CreateCapacityProvider(_ context.Context, input *ecs.CreateCap Tags: tagsToMap(input.Tags), CreatedAt: time.Now().UTC(), } - if err := putJSON(kv, CapacityProviderKey(name), &rec); err != nil { + if err := putJSON(ctx, kv, CapacityProviderKey(name), &rec); err != nil { return nil, err } return &ecs.CreateCapacityProviderOutput{CapacityProvider: rec.toAWS()}, nil @@ -77,8 +77,8 @@ func (s *Service) CreateCapacityProvider(_ context.Context, input *ecs.CreateCap // DescribeCapacityProviders returns the named providers, or every provider in // the account when none are named. Unknown names surface as failures. -func (s *Service) DescribeCapacityProviders(_ context.Context, input *ecs.DescribeCapacityProvidersInput, accountID string) (*ecs.DescribeCapacityProvidersOutput, error) { - kv, err := s.bucket(accountID) +func (s *Service) DescribeCapacityProviders(ctx context.Context, input *ecs.DescribeCapacityProvidersInput, accountID string) (*ecs.DescribeCapacityProvidersOutput, error) { + kv, err := s.bucket(ctx, accountID) if err != nil { return nil, err } @@ -86,13 +86,13 @@ func (s *Service) DescribeCapacityProviders(_ context.Context, input *ecs.Descri names := awsStringSlice(input.CapacityProviders) if len(names) == 0 { - keys, kerr := keysWithPrefix(kv, CapacityProvidersPrefix()) + keys, kerr := keysWithPrefix(ctx, kv, CapacityProvidersPrefix()) if kerr != nil { return nil, kerr } for _, k := range keys { var rec CapacityProviderRecord - ok, gerr := getJSON(kv, k, &rec) + ok, gerr := getJSON(ctx, kv, k, &rec) if gerr != nil { return nil, gerr } @@ -106,7 +106,7 @@ func (s *Service) DescribeCapacityProviders(_ context.Context, input *ecs.Descri for _, ref := range names { name := capacityProviderShortName(ref) var rec CapacityProviderRecord - ok, gerr := getJSON(kv, CapacityProviderKey(name), &rec) + ok, gerr := getJSON(ctx, kv, CapacityProviderKey(name), &rec) if gerr != nil { return nil, gerr } @@ -122,24 +122,24 @@ func (s *Service) DescribeCapacityProviders(_ context.Context, input *ecs.Descri // DeleteCapacityProvider removes a capacity provider record. Clusters that // still reference the name by CapacityProviders are left untouched (v1 does // not enforce referential integrity here). -func (s *Service) DeleteCapacityProvider(_ context.Context, input *ecs.DeleteCapacityProviderInput, accountID string) (*ecs.DeleteCapacityProviderOutput, error) { +func (s *Service) DeleteCapacityProvider(ctx context.Context, input *ecs.DeleteCapacityProviderInput, accountID string) (*ecs.DeleteCapacityProviderOutput, error) { if input == nil || aws.StringValue(input.CapacityProvider) == "" { return nil, errors.New(awserrors.ErrorECSInvalidParameter) } name := capacityProviderShortName(aws.StringValue(input.CapacityProvider)) - kv, err := s.bucket(accountID) + kv, err := s.bucket(ctx, accountID) if err != nil { return nil, err } var rec CapacityProviderRecord - found, err := getJSON(kv, CapacityProviderKey(name), &rec) + found, err := getJSON(ctx, kv, CapacityProviderKey(name), &rec) if err != nil { return nil, err } if !found { return nil, errors.New(awserrors.ErrorECSInvalidParameter) } - if err := kv.Delete(CapacityProviderKey(name)); err != nil { + if err := kv.Delete(ctx, CapacityProviderKey(name)); err != nil { return nil, err } return &ecs.DeleteCapacityProviderOutput{CapacityProvider: rec.toAWS()}, nil diff --git a/spinifex/handlers/ecs/cluster_teardown.go b/spinifex/handlers/ecs/cluster_teardown.go index 8a70d10a8..e2a40c9a3 100644 --- a/spinifex/handlers/ecs/cluster_teardown.go +++ b/spinifex/handlers/ecs/cluster_teardown.go @@ -7,7 +7,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ecs" "github.com/mulgadc/spinifex/spinifex/awserrors" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // clusterKeyPrefix is the KV subtree holding all of a cluster's records. @@ -23,12 +23,12 @@ func clusterKeyPrefix(cluster string) string { // the cluster with Status INACTIVE. func (s *Service) DeleteCluster(ctx context.Context, input *ecs.DeleteClusterInput, accountID string) (*ecs.DeleteClusterOutput, error) { cluster := clusterShortName(aws.StringValue(input.Cluster)) - kv, err := s.bucket(accountID) + kv, err := s.bucket(ctx, accountID) if err != nil { return nil, err } var rec ClusterRecord - found, err := getJSON(kv, ClusterMetaKey(cluster), &rec) + found, err := getJSON(ctx, kv, ClusterMetaKey(cluster), &rec) if err != nil { return nil, err } @@ -36,7 +36,7 @@ func (s *Service) DeleteCluster(ctx context.Context, input *ecs.DeleteClusterInp return nil, errors.New(awserrors.ErrorECSClusterNotFound) } - tasks, err := s.listTaskRecords(kv, cluster) + tasks, err := s.listTaskRecords(ctx, kv, cluster) if err != nil { return nil, err } @@ -44,7 +44,7 @@ func (s *Service) DeleteCluster(ctx context.Context, input *ecs.DeleteClusterInp s.forceStopTask(ctx, kv, accountID, &tasks[i], "Cluster deleted") } - if err := deleteKeysWithPrefix(kv, clusterKeyPrefix(cluster)); err != nil { + if err := deleteKeysWithPrefix(ctx, kv, clusterKeyPrefix(cluster)); err != nil { return nil, err } @@ -60,12 +60,12 @@ func (s *Service) DeleteCluster(ctx context.Context, input *ecs.DeleteClusterInp func (s *Service) DeregisterContainerInstance(ctx context.Context, input *ecs.DeregisterContainerInstanceInput, accountID string) (*ecs.DeregisterContainerInstanceOutput, error) { cluster := clusterShortName(aws.StringValue(input.Cluster)) id := containerInstanceShortID(aws.StringValue(input.ContainerInstance)) - kv, err := s.bucket(accountID) + kv, err := s.bucket(ctx, accountID) if err != nil { return nil, err } var rec InstanceRecord - found, err := getJSON(kv, InstanceKey(cluster, id), &rec) + found, err := getJSON(ctx, kv, InstanceKey(cluster, id), &rec) if err != nil { return nil, err } @@ -73,7 +73,7 @@ func (s *Service) DeregisterContainerInstance(ctx context.Context, input *ecs.De return nil, errors.New(awserrors.ErrorECSInvalidParameter) } - active, err := s.instanceActiveTasks(kv, cluster, id) + active, err := s.instanceActiveTasks(ctx, kv, cluster, id) if err != nil { return nil, err } @@ -84,10 +84,10 @@ func (s *Service) DeregisterContainerInstance(ctx context.Context, input *ecs.De s.forceStopTask(ctx, kv, accountID, &active[i], "Container instance deregistered") } - if derr := deleteKeysWithPrefix(kv, AssignmentsPrefix(cluster, id)); derr != nil { + if derr := deleteKeysWithPrefix(ctx, kv, AssignmentsPrefix(cluster, id)); derr != nil { return nil, derr } - if derr := kv.Delete(InstanceKey(cluster, id)); derr != nil { + if derr := kv.Delete(ctx, InstanceKey(cluster, id)); derr != nil { return nil, derr } rec.Status = ClusterStatusInactive @@ -104,7 +104,7 @@ func (s *Service) UpdateContainerInstancesState(ctx context.Context, input *ecs. if status != InstanceStatusActive && status != InstanceStatusDraining { return nil, errors.New(awserrors.ErrorECSInvalidParameter) } - kv, err := s.bucket(accountID) + kv, err := s.bucket(ctx, accountID) if err != nil { return nil, err } @@ -112,7 +112,7 @@ func (s *Service) UpdateContainerInstancesState(ctx context.Context, input *ecs. for _, ref := range awsStringSlice(input.ContainerInstances) { id := containerInstanceShortID(ref) var rec InstanceRecord - found, gerr := getJSON(kv, InstanceKey(cluster, id), &rec) + found, gerr := getJSON(ctx, kv, InstanceKey(cluster, id), &rec) if gerr != nil { return nil, gerr } @@ -122,7 +122,7 @@ func (s *Service) UpdateContainerInstancesState(ctx context.Context, input *ecs. } rec.Status = status rec.Reaped = false - if perr := putJSON(kv, InstanceKey(cluster, id), &rec); perr != nil { + if perr := putJSON(ctx, kv, InstanceKey(cluster, id), &rec); perr != nil { return nil, perr } if status == InstanceStatusDraining { @@ -134,8 +134,8 @@ func (s *Service) UpdateContainerInstancesState(ctx context.Context, input *ecs. } // instanceActiveTasks returns a cluster's non-STOPPED tasks placed on instanceID. -func (s *Service) instanceActiveTasks(kv nats.KeyValue, cluster, instanceID string) ([]TaskRecord, error) { - all, err := s.listTaskRecords(kv, cluster) +func (s *Service) instanceActiveTasks(ctx context.Context, kv jetstream.KeyValue, cluster, instanceID string) ([]TaskRecord, error) { + all, err := s.listTaskRecords(ctx, kv, cluster) if err != nil { return nil, err } @@ -151,8 +151,8 @@ func (s *Service) instanceActiveTasks(kv nats.KeyValue, cluster, instanceID stri // drainInstanceServiceTasks force-stops the instance's service-owned tasks on an // intentional DRAINING; the service reconciler then relaunches them on another // instance. Standalone tasks are left running (AWS DRAINING semantics). -func (s *Service) drainInstanceServiceTasks(ctx context.Context, kv nats.KeyValue, accountID, cluster, instanceID string) { - active, err := s.instanceActiveTasks(kv, cluster, instanceID) +func (s *Service) drainInstanceServiceTasks(ctx context.Context, kv jetstream.KeyValue, accountID, cluster, instanceID string) { + active, err := s.instanceActiveTasks(ctx, kv, cluster, instanceID) if err != nil { return } diff --git a/spinifex/handlers/ecs/cluster_teardown_test.go b/spinifex/handlers/ecs/cluster_teardown_test.go index 829f65876..344ed6b3f 100644 --- a/spinifex/handlers/ecs/cluster_teardown_test.go +++ b/spinifex/handlers/ecs/cluster_teardown_test.go @@ -26,7 +26,7 @@ func TestService_DeleteCluster_CascadesAndSweeps(t *testing.T) { list, err := svc.ListClusters(context.Background(), &ecs.ListClustersInput{}, testAccountID) require.NoError(t, err) assert.Empty(t, list.ClusterArns) - keys, err := keysWithPrefix(kv, clusterKeyPrefix("web")) + keys, err := keysWithPrefix(t.Context(), kv, clusterKeyPrefix("web")) require.NoError(t, err) assert.Empty(t, keys) } @@ -81,7 +81,7 @@ func TestService_DeregisterContainerInstance_ForceStopsAndRemoves(t *testing.T) require.NoError(t, err) assert.Equal(t, ClusterStatusInactive, aws.StringValue(out.ContainerInstance.Status)) - active, err := svc.instanceActiveTasks(kv, "web", "i-1") + active, err := svc.instanceActiveTasks(t.Context(), kv, "web", "i-1") require.NoError(t, err) assert.Empty(t, active) @@ -111,7 +111,7 @@ func TestService_UpdateContainerInstancesState_DrainsServiceTasks(t *testing.T) assert.Equal(t, InstanceStatusDraining, aws.StringValue(out.ContainerInstances[0].Status)) // Service-owned tasks on the drained instance are stopped. - live, err := svc.listServiceTasks(kv, "web", "web") + live, err := svc.listServiceTasks(t.Context(), kv, "web", "web") require.NoError(t, err) assert.Empty(t, live) } diff --git a/spinifex/handlers/ecs/deployments_test.go b/spinifex/handlers/ecs/deployments_test.go index 48861317c..b473b4ddb 100644 --- a/spinifex/handlers/ecs/deployments_test.go +++ b/spinifex/handlers/ecs/deployments_test.go @@ -7,26 +7,26 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ecs" "github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) // reloadService re-reads a service record from KV so a test sees writes made by // the reconciler / failure accounting. -func reloadService(t *testing.T, kv nats.KeyValue, cluster, name string) *ServiceRecord { +func reloadService(t *testing.T, kv jetstream.KeyValue, cluster, name string) *ServiceRecord { t.Helper() var rec ServiceRecord - found, err := getJSON(kv, ServiceKey(cluster, name), &rec) + found, err := getJSON(t.Context(), kv, ServiceKey(cluster, name), &rec) require.NoError(t, err) require.True(t, found) return &rec } // driveRunning reports RUNNING for every PENDING task of a service. -func driveRunning(t *testing.T, svc *Service, kv nats.KeyValue, cluster, name string) { +func driveRunning(t *testing.T, svc *Service, kv jetstream.KeyValue, cluster, name string) { t.Helper() - tasks, err := svc.listServiceTasks(kv, cluster, name) + tasks, err := svc.listServiceTasks(t.Context(), kv, cluster, name) require.NoError(t, err) for i := range tasks { if tasks[i].LastStatus != TaskStatusPending { @@ -41,9 +41,9 @@ func driveRunning(t *testing.T, svc *Service, kv nats.KeyValue, cluster, name st // failPending reports STOPPED (never-RUNNING) for every PENDING task, simulating // tasks that fail to start — the deployment circuit-breaker signal. -func failPending(t *testing.T, svc *Service, kv nats.KeyValue, cluster, name string) { +func failPending(t *testing.T, svc *Service, kv jetstream.KeyValue, cluster, name string) { t.Helper() - tasks, err := svc.listServiceTasks(kv, cluster, name) + tasks, err := svc.listServiceTasks(t.Context(), kv, cluster, name) require.NoError(t, err) for i := range tasks { if tasks[i].LastStatus != TaskStatusPending { @@ -73,7 +73,7 @@ func TestDeployment_CreateService_SeedsPrimary(t *testing.T) { assert.Equal(t, defaultMinimumHealthyPercent, rec.MinimumHealthyPercent) assert.Equal(t, defaultMaximumPercent, rec.MaximumPercent) // Every launched task carries the primary deployment id in StartedBy. - tasks, err := svc.listServiceTasks(kv, "web", "web") + tasks, err := svc.listServiceTasks(t.Context(), kv, "web", "web") require.NoError(t, err) require.Len(t, tasks, 2) for i := range tasks { @@ -110,7 +110,7 @@ func TestDeployment_RollingUpdate_ReplacesOldWithNew(t *testing.T) { assert.NotEqual(t, firstDeployID, rec.DeploymentID) // maximumPercent=200 lets 2 new tasks launch alongside the 2 old running ones. newTasks := 0 - tasks, err := svc.listServiceTasks(kv, "web", "web") + tasks, err := svc.listServiceTasks(t.Context(), kv, "web", "web") require.NoError(t, err) for i := range tasks { if deploymentIDFromStartedBy(tasks[i].StartedBy) == rec.DeploymentID { @@ -128,7 +128,7 @@ func TestDeployment_RollingUpdate_ReplacesOldWithNew(t *testing.T) { assert.Equal(t, rec.DeploymentID, rec.primaryDeployment().ID) // Only new-deployment tasks remain live. - tasks, err = svc.listServiceTasks(kv, "web", "web") + tasks, err = svc.listServiceTasks(t.Context(), kv, "web", "web") require.NoError(t, err) require.Len(t, tasks, 2) for i := range tasks { @@ -159,7 +159,7 @@ func TestDeployment_MinimumHealthyPercent_GatesDrain(t *testing.T) { // Ceiling of 3 tasks total: 2 old running + 1 new pending. No old task drained // yet because healthy running (2) must not drop below minCount (2). - tasks, err := svc.listServiceTasks(kv, "web", "web") + tasks, err := svc.listServiceTasks(t.Context(), kv, "web", "web") require.NoError(t, err) assert.Len(t, tasks, 3) rec := reloadService(t, kv, "web", "web") @@ -215,7 +215,7 @@ func TestDeployment_LegacyServiceSynthesizesPrimary(t *testing.T) { DesiredCount: 1, Status: ServiceStatusActive, SchedulingStrategy: SchedulingStrategyReplica, DeploymentID: "legacy-dep", } - require.NoError(t, putJSON(kv, ServiceKey("web", "legacy"), rec)) + require.NoError(t, putJSON(t.Context(), kv, ServiceKey("web", "legacy"), rec)) require.NoError(t, svc.reconcileService(context.Background(), kv, testAccountID, rec)) reloaded := reloadService(t, kv, "web", "legacy") diff --git a/spinifex/handlers/ecs/eip_test.go b/spinifex/handlers/ecs/eip_test.go index eae28a91e..a67d59b2f 100644 --- a/spinifex/handlers/ecs/eip_test.go +++ b/spinifex/handlers/ecs/eip_test.go @@ -37,14 +37,14 @@ func TestAssignTaskPublicIP_EnabledAssociatesAndPersists(t *testing.T) { eips := &stubEIPs{publicIP: "192.0.2.50", allocID: "eipalloc-1"} svc.eips = eips - require.NoError(t, putJSON(kv, ServiceKey("web", "web"), &ServiceRecord{ + require.NoError(t, putJSON(t.Context(), kv, ServiceKey("web", "web"), &ServiceRecord{ Name: "web", Cluster: "web", AssignPublicIP: "ENABLED", })) task := &TaskRecord{ TaskID: "t-1", Cluster: "web", Group: serviceTaskGroup("web"), ENIID: "eni-1", ENIPrivateIP: "172.31.0.8", } - require.NoError(t, putJSON(kv, TaskKey("web", "t-1"), task)) + require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "t-1"), task)) svc.assignTaskPublicIP(context.Background(), kv, testAccountID, task) @@ -53,7 +53,7 @@ func TestAssignTaskPublicIP_EnabledAssociatesAndPersists(t *testing.T) { assert.Equal(t, "eipalloc-1", task.ENIEIPAllocationID) var persisted TaskRecord - found, err := getJSON(kv, TaskKey("web", "t-1"), &persisted) + found, err := getJSON(t.Context(), kv, TaskKey("web", "t-1"), &persisted) require.NoError(t, err) require.True(t, found) assert.Equal(t, "192.0.2.50", persisted.ENIPublicIP) @@ -64,7 +64,7 @@ func TestAssignTaskPublicIP_DisabledNoOp(t *testing.T) { eips := &stubEIPs{publicIP: "192.0.2.50", allocID: "eipalloc-1"} svc.eips = eips - require.NoError(t, putJSON(kv, ServiceKey("web", "web"), &ServiceRecord{ + require.NoError(t, putJSON(t.Context(), kv, ServiceKey("web", "web"), &ServiceRecord{ Name: "web", Cluster: "web", // AssignPublicIP unset })) task := &TaskRecord{ diff --git a/spinifex/handlers/ecs/eni_test.go b/spinifex/handlers/ecs/eni_test.go index cbb2bea6e..6a55ea463 100644 --- a/spinifex/handlers/ecs/eni_test.go +++ b/spinifex/handlers/ecs/eni_test.go @@ -208,13 +208,13 @@ func TestReaper_Awsvpc_ReleasesENI(t *testing.T) { _, err = svc.RunTask(context.Background(), awsvpcRunInput(), testAccountID) require.NoError(t, err) - kv, err := svc.bucket(testAccountID) + kv, err := svc.bucket(t.Context(), testAccountID) require.NoError(t, err) var rec InstanceRecord - _, err = getJSON(kv, InstanceKey("web", "i-1"), &rec) + _, err = getJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec) require.NoError(t, err) rec.LastSeen = time.Now().UTC().Add(-2 * heartbeatTimeout) - require.NoError(t, putJSON(kv, InstanceKey("web", "i-1"), &rec)) + require.NoError(t, putJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec)) sc := NewScheduler(nc, svc, "test-holder") sc.reapBucket(context.Background(), kv, testAccountID, time.Now().UTC()) diff --git a/spinifex/handlers/ecs/enumeration_failure_test.go b/spinifex/handlers/ecs/enumeration_failure_test.go new file mode 100644 index 000000000..708648a4a --- /dev/null +++ b/spinifex/handlers/ecs/enumeration_failure_test.go @@ -0,0 +1,42 @@ +package handlers_ecs + +import ( + "context" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ecs" + "github.com/stretchr/testify/require" +) + +// TestSchedulerPasses_EnumerationFailureIsReported pins the contract every +// leader-only sweep depends on: the KV bucket lister closes its channel the same +// way whether the listing completed or failed, so a pass that ignored the +// terminal error would read an unreachable JetStream as an empty fleet and +// report a clean tick while every account went unswept — instances never +// drained, services never relaunched, stopped tasks never pruned. +func TestSchedulerPasses_EnumerationFailureIsReported(t *testing.T) { + svc, nc := newTestService(t) + sc := NewScheduler(nc, svc, "test-holder") + _, err := svc.CreateCluster(context.Background(), &ecs.CreateClusterInput{ClusterName: aws.String("web")}, testAccountID) + require.NoError(t, err) + + passes := map[string]func(context.Context) error{ + "reap": sc.reap, + "sweepStoppedTasks": sc.sweepStoppedTasks, + "reconcileAllServices": svc.reconcileAllServices, + } + for name, pass := range passes { + t.Run(name+"/reachable", func(t *testing.T) { + require.NoError(t, pass(t.Context()), "a complete enumeration must complete the pass") + }) + } + + // Closing the connection fails the stream-names request behind the listing. + nc.Close() + for name, pass := range passes { + t.Run(name+"/unreachable", func(t *testing.T) { + require.Error(t, pass(t.Context()), "a failed bucket enumeration must not report a completed pass") + }) + } +} diff --git a/spinifex/handlers/ecs/gpu_report.go b/spinifex/handlers/ecs/gpu_report.go index dcbbb12fb..9367c02c2 100644 --- a/spinifex/handlers/ecs/gpu_report.go +++ b/spinifex/handlers/ecs/gpu_report.go @@ -28,15 +28,15 @@ type ReportTaskGPUOutput struct { // container records by name, so DescribeTasks can surface gpuIds. A missing // task or container is a silent no-op — the state-report path already owns // the task's lifecycle; this only enriches it. -func (s *Service) ReportTaskGPU(_ context.Context, input *ReportTaskGPUInput, accountID string) (*ReportTaskGPUOutput, error) { +func (s *Service) ReportTaskGPU(ctx context.Context, input *ReportTaskGPUInput, accountID string) (*ReportTaskGPUOutput, error) { cluster := clusterShortName(input.Cluster) taskID := taskShortID(input.Task) - kv, err := s.bucket(accountID) + kv, err := s.bucket(ctx, accountID) if err != nil { return nil, err } var task TaskRecord - found, err := getJSON(kv, TaskKey(cluster, taskID), &task) + found, err := getJSON(ctx, kv, TaskKey(cluster, taskID), &task) if err != nil { return nil, err } @@ -55,7 +55,7 @@ func (s *Service) ReportTaskGPU(_ context.Context, input *ReportTaskGPUInput, ac } } if changed { - if err := putJSON(kv, TaskKey(cluster, taskID), &task); err != nil { + if err := putJSON(ctx, kv, TaskKey(cluster, taskID), &task); err != nil { return nil, err } } diff --git a/spinifex/handlers/ecs/gpu_report_test.go b/spinifex/handlers/ecs/gpu_report_test.go index 4658280ab..754abac97 100644 --- a/spinifex/handlers/ecs/gpu_report_test.go +++ b/spinifex/handlers/ecs/gpu_report_test.go @@ -15,9 +15,9 @@ import ( // container(s) of an existing task record, by container name. func TestReportTaskGPU_MergesOntoTaskContainers(t *testing.T) { svc, _ := newTestService(t) - kv, err := svc.bucket(testAccountID) + kv, err := svc.bucket(t.Context(), testAccountID) require.NoError(t, err) - require.NoError(t, putJSON(kv, TaskKey("web", "t-1"), &TaskRecord{ + require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "t-1"), &TaskRecord{ TaskID: "t-1", Cluster: "web", Containers: []ContainerState{ {Name: "web", Status: "RUNNING"}, @@ -33,7 +33,7 @@ func TestReportTaskGPU_MergesOntoTaskContainers(t *testing.T) { assert.Equal(t, "OK", out.Acknowledgment) var rec TaskRecord - found, err := getJSON(kv, TaskKey("web", "t-1"), &rec) + found, err := getJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec) require.NoError(t, err) require.True(t, found) for _, c := range rec.Containers { @@ -61,9 +61,9 @@ func TestReportTaskGPU_UnknownTaskNoop(t *testing.T) { // field, once ReportTaskGPU has merged them onto the task record. func TestDescribeTasks_GPUIDsPopulatedFromReport(t *testing.T) { svc, _ := newTestService(t) - kv, err := svc.bucket(testAccountID) + kv, err := svc.bucket(t.Context(), testAccountID) require.NoError(t, err) - require.NoError(t, putJSON(kv, TaskKey("web", "t-1"), &TaskRecord{ + require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "t-1"), &TaskRecord{ TaskID: "t-1", ARN: TaskARN(svc.region, testAccountID, "web", "t-1"), Cluster: "web", Containers: []ContainerState{{Name: "trainer", Status: "RUNNING", GPUIDs: []string{"GPU-aaa", "GPU-bbb"}}}, })) @@ -83,9 +83,9 @@ func TestDescribeTasks_GPUIDsPopulatedFromReport(t *testing.T) { // RUNNING report (or ReportTaskGPU) previously set. func TestRecordTaskState_PreservesGPUIDsAcrossStoppedReport(t *testing.T) { svc, _ := newTestService(t) - kv, err := svc.bucket(testAccountID) + kv, err := svc.bucket(t.Context(), testAccountID) require.NoError(t, err) - require.NoError(t, putJSON(kv, TaskKey("web", "t-1"), &TaskRecord{ + require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "t-1"), &TaskRecord{ TaskID: "t-1", ARN: TaskARN(svc.region, testAccountID, "web", "t-1"), Cluster: "web", ContainerInstanceID: "i-1", Containers: []ContainerState{{Name: "trainer", Status: "PENDING"}}, @@ -99,7 +99,7 @@ func TestRecordTaskState_PreservesGPUIDsAcrossStoppedReport(t *testing.T) { })) var rec TaskRecord - found, err := getJSON(kv, TaskKey("web", "t-1"), &rec) + found, err := getJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec) require.NoError(t, err) require.True(t, found) require.Len(t, rec.Containers, 1) @@ -114,7 +114,7 @@ func TestRecordTaskState_PreservesGPUIDsAcrossStoppedReport(t *testing.T) { Containers: []bus.ContainerStatus{{Name: "trainer", Status: bus.TaskStatusStopped, ExitCode: &exit}}, })) - found, err = getJSON(kv, TaskKey("web", "t-1"), &rec) + found, err = getJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec) require.NoError(t, err) require.True(t, found) require.Len(t, rec.Containers, 1) @@ -135,9 +135,9 @@ func TestRecordTaskState_PreservesGPUIDsAcrossStoppedReport(t *testing.T) { // pinned set normally (e.g. a corrected report, or GPUs reassigned on a retry). func TestRecordTaskState_ReportedGPUIDsOverwritePinned(t *testing.T) { svc, _ := newTestService(t) - kv, err := svc.bucket(testAccountID) + kv, err := svc.bucket(t.Context(), testAccountID) require.NoError(t, err) - require.NoError(t, putJSON(kv, TaskKey("web", "t-1"), &TaskRecord{ + require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "t-1"), &TaskRecord{ TaskID: "t-1", ARN: TaskARN(svc.region, testAccountID, "web", "t-1"), Cluster: "web", ContainerInstanceID: "i-1", Containers: []ContainerState{{Name: "trainer", Status: "RUNNING", GPUIDs: []string{"GPU-aaa"}}}, @@ -150,7 +150,7 @@ func TestRecordTaskState_ReportedGPUIDsOverwritePinned(t *testing.T) { })) var rec TaskRecord - found, err := getJSON(kv, TaskKey("web", "t-1"), &rec) + found, err := getJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec) require.NoError(t, err) require.True(t, found) require.Len(t, rec.Containers, 1) @@ -161,9 +161,9 @@ func TestRecordTaskState_ReportedGPUIDsOverwritePinned(t *testing.T) { // nothing to preserve, and the fallback must not fabricate a value. func TestRecordTaskState_NonGPUContainerStaysEmpty(t *testing.T) { svc, _ := newTestService(t) - kv, err := svc.bucket(testAccountID) + kv, err := svc.bucket(t.Context(), testAccountID) require.NoError(t, err) - require.NoError(t, putJSON(kv, TaskKey("web", "t-1"), &TaskRecord{ + require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "t-1"), &TaskRecord{ TaskID: "t-1", ARN: TaskARN(svc.region, testAccountID, "web", "t-1"), Cluster: "web", ContainerInstanceID: "i-1", Containers: []ContainerState{{Name: "web", Status: "PENDING"}}, @@ -183,7 +183,7 @@ func TestRecordTaskState_NonGPUContainerStaysEmpty(t *testing.T) { })) var rec TaskRecord - found, err := getJSON(kv, TaskKey("web", "t-1"), &rec) + found, err := getJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec) require.NoError(t, err) require.True(t, found) require.Len(t, rec.Containers, 1) diff --git a/spinifex/handlers/ecs/instance_recovery_test.go b/spinifex/handlers/ecs/instance_recovery_test.go index 5b6fd1fc3..c10f31158 100644 --- a/spinifex/handlers/ecs/instance_recovery_test.go +++ b/spinifex/handlers/ecs/instance_recovery_test.go @@ -24,10 +24,10 @@ func reRegister(t *testing.T, svc *Service, cluster, id string) { func instanceStatus(t *testing.T, svc *Service, cluster, id string) InstanceRecord { t.Helper() - kv, err := svc.bucket(testAccountID) + kv, err := svc.bucket(t.Context(), testAccountID) require.NoError(t, err) var rec InstanceRecord - found, err := getJSON(kv, InstanceKey(cluster, id), &rec) + found, err := getJSON(t.Context(), kv, InstanceKey(cluster, id), &rec) require.NoError(t, err) require.True(t, found) return rec @@ -42,11 +42,11 @@ func TestRegister_RestoresReapedInstanceToActive(t *testing.T) { require.NoError(t, err) registerInstance(t, svc, "web", "i-1", 1024, 2048) - kv, err := svc.bucket(testAccountID) + kv, err := svc.bucket(t.Context(), testAccountID) require.NoError(t, err) rec := instanceStatus(t, svc, "web", "i-1") rec.LastSeen = time.Now().UTC().Add(-2 * heartbeatTimeout) - require.NoError(t, putJSON(kv, InstanceKey("web", "i-1"), &rec)) + require.NoError(t, putJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec)) NewScheduler(nc, svc, "test-holder").reapBucket(context.Background(), kv, testAccountID, time.Now().UTC()) @@ -83,10 +83,10 @@ func TestReaper_ReleasesCapacityAndRecoversClean(t *testing.T) { require.Equal(t, 256, seeded.ReservedMemoryMiB) require.Contains(t, seeded.PlacedTasks, taskID) - kv, err := svc.bucket(testAccountID) + kv, err := svc.bucket(t.Context(), testAccountID) require.NoError(t, err) seeded.LastSeen = time.Now().UTC().Add(-2 * heartbeatTimeout) - require.NoError(t, putJSON(kv, InstanceKey("web", "i-1"), &seeded)) + require.NoError(t, putJSON(t.Context(), kv, InstanceKey("web", "i-1"), &seeded)) NewScheduler(nc, svc, "test-holder").reapBucket(context.Background(), kv, testAccountID, time.Now().UTC()) diff --git a/spinifex/handlers/ecs/instances.go b/spinifex/handlers/ecs/instances.go index be1774ca0..c9b90fddc 100644 --- a/spinifex/handlers/ecs/instances.go +++ b/spinifex/handlers/ecs/instances.go @@ -10,19 +10,19 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ecs" "github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // listInstanceRecords returns all container-instance records in a cluster. -func (s *Service) listInstanceRecords(kv nats.KeyValue, cluster string) ([]InstanceRecord, error) { - keys, err := keysWithPrefix(kv, InstancesPrefix(cluster)) +func (s *Service) listInstanceRecords(ctx context.Context, kv jetstream.KeyValue, cluster string) ([]InstanceRecord, error) { + keys, err := keysWithPrefix(ctx, kv, InstancesPrefix(cluster)) if err != nil { return nil, err } out := make([]InstanceRecord, 0, len(keys)) for _, k := range keys { var rec InstanceRecord - found, err := getJSON(kv, k, &rec) + found, err := getJSON(ctx, kv, k, &rec) if err != nil { return nil, err } @@ -36,17 +36,17 @@ func (s *Service) listInstanceRecords(kv nats.KeyValue, cluster string) ([]Insta // RegisterContainerInstance is the AWS-API registration path. In 4e the agent // normally registers over the Layer-2 bus; this keeps API parity by writing the // same record shape from an explicit call. -func (s *Service) RegisterContainerInstance(_ context.Context, input *ecs.RegisterContainerInstanceInput, accountID string) (*ecs.RegisterContainerInstanceOutput, error) { +func (s *Service) RegisterContainerInstance(ctx context.Context, input *ecs.RegisterContainerInstanceInput, accountID string) (*ecs.RegisterContainerInstanceOutput, error) { cluster := clusterShortName(aws.StringValue(input.Cluster)) instanceID := aws.StringValue(input.InstanceIdentityDocument) if instanceID == "" { instanceID = "ci-" + time.Now().UTC().Format("20060102150405") } - kv, err := s.bucket(accountID) + kv, err := s.bucket(ctx, accountID) if err != nil { return nil, err } - rec, err := s.upsertInstance(kv, accountID, cluster, instanceID, func(r *InstanceRecord) { + rec, err := s.upsertInstance(ctx, kv, accountID, cluster, instanceID, func(r *InstanceRecord) { for _, res := range input.TotalResources { switch aws.StringValue(res.Name) { case "CPU": @@ -79,9 +79,9 @@ func (s *Service) RegisterContainerInstance(_ context.Context, input *ecs.Regist } // DescribeContainerInstances returns records for the named container instances. -func (s *Service) DescribeContainerInstances(_ context.Context, input *ecs.DescribeContainerInstancesInput, accountID string) (*ecs.DescribeContainerInstancesOutput, error) { +func (s *Service) DescribeContainerInstances(ctx context.Context, input *ecs.DescribeContainerInstancesInput, accountID string) (*ecs.DescribeContainerInstancesOutput, error) { cluster := clusterShortName(aws.StringValue(input.Cluster)) - kv, err := s.bucket(accountID) + kv, err := s.bucket(ctx, accountID) if err != nil { return nil, err } @@ -89,7 +89,7 @@ func (s *Service) DescribeContainerInstances(_ context.Context, input *ecs.Descr for _, ref := range awsStringSlice(input.ContainerInstances) { id := containerInstanceShortID(ref) var rec InstanceRecord - found, err := getJSON(kv, InstanceKey(cluster, id), &rec) + found, err := getJSON(ctx, kv, InstanceKey(cluster, id), &rec) if err != nil { return nil, err } @@ -103,13 +103,13 @@ func (s *Service) DescribeContainerInstances(_ context.Context, input *ecs.Descr } // ListContainerInstances returns the ARNs of all container instances in a cluster. -func (s *Service) ListContainerInstances(_ context.Context, input *ecs.ListContainerInstancesInput, accountID string) (*ecs.ListContainerInstancesOutput, error) { +func (s *Service) ListContainerInstances(ctx context.Context, input *ecs.ListContainerInstancesInput, accountID string) (*ecs.ListContainerInstancesOutput, error) { cluster := clusterShortName(aws.StringValue(input.Cluster)) - kv, err := s.bucket(accountID) + kv, err := s.bucket(ctx, accountID) if err != nil { return nil, err } - recs, err := s.listInstanceRecords(kv, cluster) + recs, err := s.listInstanceRecords(ctx, kv, cluster) if err != nil { return nil, err } @@ -122,9 +122,9 @@ func (s *Service) ListContainerInstances(_ context.Context, input *ecs.ListConta // upsertInstance reads-or-creates the instance record, applies mutate, and writes // it back. Used by both the AWS register path and the bus register handler. -func (s *Service) upsertInstance(kv nats.KeyValue, accountID, cluster, instanceID string, mutate func(*InstanceRecord)) (*InstanceRecord, error) { +func (s *Service) upsertInstance(ctx context.Context, kv jetstream.KeyValue, accountID, cluster, instanceID string, mutate func(*InstanceRecord)) (*InstanceRecord, error) { var rec InstanceRecord - found, err := getJSON(kv, InstanceKey(cluster, instanceID), &rec) + found, err := getJSON(ctx, kv, InstanceKey(cluster, instanceID), &rec) if err != nil { return nil, err } @@ -142,7 +142,7 @@ func (s *Service) upsertInstance(kv nats.KeyValue, accountID, cluster, instanceI if mutate != nil { mutate(&rec) } - if err := putJSON(kv, InstanceKey(cluster, instanceID), &rec); err != nil { + if err := putJSON(ctx, kv, InstanceKey(cluster, instanceID), &rec); err != nil { return nil, err } return &rec, nil @@ -184,12 +184,12 @@ func (s *Service) instanceToAWS(r *InstanceRecord) *ecs.ContainerInstance { // --- Layer-2 bus event handlers (called by the scheduler) --- // recordRegister upserts a container-instance record from a bus RegisterInstance. -func (s *Service) recordRegister(msg *bus.RegisterInstance) error { - kv, err := s.bucket(msg.AccountID) +func (s *Service) recordRegister(ctx context.Context, msg *bus.RegisterInstance) error { + kv, err := s.bucket(ctx, msg.AccountID) if err != nil { return err } - _, err = s.upsertInstance(kv, msg.AccountID, msg.ClusterName, msg.InstanceID, func(r *InstanceRecord) { + _, err = s.upsertInstance(ctx, kv, msg.AccountID, msg.ClusterName, msg.InstanceID, func(r *InstanceRecord) { r.AZ = msg.AZ r.Hostname = msg.Hostname r.AgentVersion = msg.AgentVersion @@ -203,13 +203,13 @@ func (s *Service) recordRegister(msg *bus.RegisterInstance) error { } // recordHeartbeat refreshes an instance's LastSeen and status from a bus beat. -func (s *Service) recordHeartbeat(msg *bus.Heartbeat) error { - kv, err := s.bucket(msg.AccountID) +func (s *Service) recordHeartbeat(ctx context.Context, msg *bus.Heartbeat) error { + kv, err := s.bucket(ctx, msg.AccountID) if err != nil { return err } var rec InstanceRecord - found, err := getJSON(kv, InstanceKey(msg.ClusterName, msg.InstanceID), &rec) + found, err := getJSON(ctx, kv, InstanceKey(msg.ClusterName, msg.InstanceID), &rec) if err != nil || !found { return err } @@ -217,18 +217,18 @@ func (s *Service) recordHeartbeat(msg *bus.Heartbeat) error { if msg.Status != "" { rec.Status = msg.Status } - return putJSON(kv, InstanceKey(msg.ClusterName, msg.InstanceID), &rec) + return putJSON(ctx, kv, InstanceKey(msg.ClusterName, msg.InstanceID), &rec) } // recordTaskState applies an agent task-state report: it updates the task record // and, on STOPPED, releases the reserved capacity back to the instance. func (s *Service) recordTaskState(ctx context.Context, msg *bus.TaskState) error { - kv, err := s.bucket(msg.AccountID) + kv, err := s.bucket(ctx, msg.AccountID) if err != nil { return err } var task TaskRecord - found, err := getJSON(kv, TaskKey(msg.ClusterName, msg.TaskID), &task) + found, err := getJSON(ctx, kv, TaskKey(msg.ClusterName, msg.TaskID), &task) if err != nil || !found { return err } @@ -266,7 +266,7 @@ func (s *Service) recordTaskState(ctx context.Context, msg *bus.TaskState) error task.StoppedReason = msg.Reason } } - if err := putJSON(kv, TaskKey(msg.ClusterName, msg.TaskID), &task); err != nil { + if err := putJSON(ctx, kv, TaskKey(msg.ClusterName, msg.TaskID), &task); err != nil { return err } @@ -287,13 +287,13 @@ func (s *Service) recordTaskState(ctx context.Context, msg *bus.TaskState) error } s.deregisterServiceTargets(ctx, kv, msg.AccountID, &task) s.releaseTaskPublicIP(ctx, msg.AccountID, &task) - s.reclaimAssignInbox(kv, msg.ClusterName, task.ContainerInstanceID, msg.TaskID) - s.reclaimStopInbox(kv, msg.ClusterName, task.ContainerInstanceID, msg.TaskID) + s.reclaimAssignInbox(ctx, kv, msg.ClusterName, task.ContainerInstanceID, msg.TaskID) + s.reclaimStopInbox(ctx, kv, msg.ClusterName, task.ContainerInstanceID, msg.TaskID) s.reclaimTaskENI(ctx, msg.AccountID, &task) - if perr := putJSON(kv, TaskKey(msg.ClusterName, msg.TaskID), &task); perr != nil { + if perr := putJSON(ctx, kv, TaskKey(msg.ClusterName, msg.TaskID), &task); perr != nil { slog.ErrorContext(ctx, "ECS task STOPPED: persist after EIP release failed", "task", msg.TaskID, "err", perr) } - return s.releaseReservation(kv, msg.ClusterName, task.ContainerInstanceID, msg.TaskID, task.ReservedCPU, task.ReservedMemoryMiB, task.GPU) + return s.releaseReservation(ctx, kv, msg.ClusterName, task.ContainerInstanceID, msg.TaskID, task.ReservedCPU, task.ReservedMemoryMiB, task.GPU) } return nil } @@ -332,14 +332,14 @@ func (s *Service) SubmitTaskStateChange(ctx context.Context, input *ecs.SubmitTa // recordDeploymentFailure increments the failed-task counter on the deployment // that launched a task which stopped before ever running, driving the service's // deployment circuit breaker. No-op for a non-service task or an unknown deployment. -func (s *Service) recordDeploymentFailure(ctx context.Context, kv nats.KeyValue, cluster string, task *TaskRecord) { +func (s *Service) recordDeploymentFailure(ctx context.Context, kv jetstream.KeyValue, cluster string, task *TaskRecord) { name := serviceNameFromGroup(task.Group) depID := deploymentIDFromStartedBy(task.StartedBy) if name == "" || depID == "" { return } var svc ServiceRecord - found, err := getJSON(kv, ServiceKey(cluster, name), &svc) + found, err := getJSON(ctx, kv, ServiceKey(cluster, name), &svc) if err != nil || !found { return } @@ -347,7 +347,7 @@ func (s *Service) recordDeploymentFailure(ctx context.Context, kv nats.KeyValue, if svc.Deployments[i].ID == depID { svc.Deployments[i].FailedTasks++ svc.Deployments[i].UpdatedAt = time.Now().UTC() - if perr := putJSON(kv, ServiceKey(cluster, name), &svc); perr != nil { + if perr := putJSON(ctx, kv, ServiceKey(cluster, name), &svc); perr != nil { slog.ErrorContext(ctx, "ECS deployment failure accounting: persist failed", "service", name, "err", perr) } return @@ -356,9 +356,9 @@ func (s *Service) recordDeploymentFailure(ctx context.Context, kv nats.KeyValue, } // releaseReservation returns a stopped task's capacity to its instance under CAS. -func (s *Service) releaseReservation(kv nats.KeyValue, cluster, instanceID, taskID string, cpu, mem, gpu int) error { +func (s *Service) releaseReservation(ctx context.Context, kv jetstream.KeyValue, cluster, instanceID, taskID string, cpu, mem, gpu int) error { for range reservePlacementRetries { - entry, err := kv.Get(InstanceKey(cluster, instanceID)) + entry, err := kv.Get(ctx, InstanceKey(cluster, instanceID)) if err != nil { return nil //nolint:nilerr // instance gone; nothing to release } @@ -374,7 +374,7 @@ func (s *Service) releaseReservation(kv nats.KeyValue, cluster, instanceID, task if merr != nil { return merr } - if _, uerr := kv.Update(InstanceKey(cluster, instanceID), data, entry.Revision()); uerr == nil { + if _, uerr := kv.Update(ctx, InstanceKey(cluster, instanceID), data, entry.Revision()); uerr == nil { return nil } } diff --git a/spinifex/handlers/ecs/poll.go b/spinifex/handlers/ecs/poll.go index fb20ef114..94eb8fa72 100644 --- a/spinifex/handlers/ecs/poll.go +++ b/spinifex/handlers/ecs/poll.go @@ -4,7 +4,7 @@ import ( "context" "github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // PollAssignmentsInput is the agent's poll request. ContainerInstance names the @@ -34,10 +34,10 @@ type PollAssignmentsOutput struct { // writer of the inbox; the agent is the only reader/acker. Account is // authoritative from accountID (SigV4 caller), so an instance cannot drain // another account's inbox. -func (s *Service) PollAssignments(_ context.Context, input *PollAssignmentsInput, accountID string) (*PollAssignmentsOutput, error) { +func (s *Service) PollAssignments(ctx context.Context, input *PollAssignmentsInput, accountID string) (*PollAssignmentsOutput, error) { cluster := clusterShortName(input.Cluster) instanceID := containerInstanceShortID(input.ContainerInstance) - kv, err := s.bucket(accountID) + kv, err := s.bucket(ctx, accountID) if err != nil { return nil, err } @@ -46,23 +46,23 @@ func (s *Service) PollAssignments(_ context.Context, input *PollAssignmentsInput if taskID == "" { continue } - _ = kv.Delete(AssignmentKey(cluster, instanceID, taskShortID(taskID))) + _ = kv.Delete(ctx, AssignmentKey(cluster, instanceID, taskShortID(taskID))) } for _, taskID := range input.AckStopIDs { if taskID == "" { continue } - _ = kv.Delete(StopKey(cluster, instanceID, taskShortID(taskID))) + _ = kv.Delete(ctx, StopKey(cluster, instanceID, taskShortID(taskID))) } - keys, err := keysWithPrefix(kv, AssignmentsPrefix(cluster, instanceID)) + keys, err := keysWithPrefix(ctx, kv, AssignmentsPrefix(cluster, instanceID)) if err != nil { return nil, err } out := &PollAssignmentsOutput{} for _, key := range keys { var as bus.Assign - found, err := getJSON(kv, key, &as) + found, err := getJSON(ctx, kv, key, &as) if err != nil { return nil, err } @@ -71,13 +71,13 @@ func (s *Service) PollAssignments(_ context.Context, input *PollAssignmentsInput } } - stopKeys, err := keysWithPrefix(kv, StopsPrefix(cluster, instanceID)) + stopKeys, err := keysWithPrefix(ctx, kv, StopsPrefix(cluster, instanceID)) if err != nil { return nil, err } for _, key := range stopKeys { var sd bus.StopDirective - found, err := getJSON(kv, key, &sd) + found, err := getJSON(ctx, kv, key, &sd) if err != nil { return nil, err } @@ -90,18 +90,18 @@ func (s *Service) PollAssignments(_ context.Context, input *PollAssignmentsInput // reclaimAssignInbox removes a task's inbox entry on the STOPPED transition so a // stopped task is never re-delivered to the agent. Best-effort. -func (s *Service) reclaimAssignInbox(kv nats.KeyValue, cluster, instanceID, taskID string) { +func (s *Service) reclaimAssignInbox(ctx context.Context, kv jetstream.KeyValue, cluster, instanceID, taskID string) { if instanceID == "" || taskID == "" { return } - _ = kv.Delete(AssignmentKey(cluster, instanceID, taskID)) + _ = kv.Delete(ctx, AssignmentKey(cluster, instanceID, taskID)) } // reclaimStopInbox removes a task's stop-inbox entry on the STOPPED transition so // a reaped task is never re-delivered as a stop. Best-effort. -func (s *Service) reclaimStopInbox(kv nats.KeyValue, cluster, instanceID, taskID string) { +func (s *Service) reclaimStopInbox(ctx context.Context, kv jetstream.KeyValue, cluster, instanceID, taskID string) { if instanceID == "" || taskID == "" { return } - _ = kv.Delete(StopKey(cluster, instanceID, taskID)) + _ = kv.Delete(ctx, StopKey(cluster, instanceID, taskID)) } diff --git a/spinifex/handlers/ecs/poll_test.go b/spinifex/handlers/ecs/poll_test.go index c6fbf3a2f..352b05a20 100644 --- a/spinifex/handlers/ecs/poll_test.go +++ b/spinifex/handlers/ecs/poll_test.go @@ -16,9 +16,9 @@ import ( func TestPollAssignments_DrainsStopsAndAcks(t *testing.T) { svc, _, kv := serviceTestRig(t) - require.NoError(t, putJSON(kv, AssignmentKey("web", "i-1", "t-1"), + require.NoError(t, putJSON(t.Context(), kv, AssignmentKey("web", "i-1", "t-1"), &bus.Assign{TaskID: "t-1", ClusterName: "web", InstanceID: "i-1"})) - require.NoError(t, putJSON(kv, StopKey("web", "i-1", "t-2"), + require.NoError(t, putJSON(t.Context(), kv, StopKey("web", "i-1", "t-2"), &bus.StopDirective{TaskID: "t-2", Reason: "drain"})) out, err := svc.PollAssignments(context.Background(), &PollAssignmentsInput{ @@ -41,7 +41,7 @@ func TestPollAssignments_DrainsStopsAndAcks(t *testing.T) { assert.Empty(t, out.Assignments) assert.Empty(t, out.Stops) - found, err := getJSON(kv, StopKey("web", "i-1", "t-2"), &bus.StopDirective{}) + found, err := getJSON(t.Context(), kv, StopKey("web", "i-1", "t-2"), &bus.StopDirective{}) require.NoError(t, err) assert.False(t, found) } diff --git a/spinifex/handlers/ecs/scheduler.go b/spinifex/handlers/ecs/scheduler.go index 3d05003cb..f4cce2eab 100644 --- a/spinifex/handlers/ecs/scheduler.go +++ b/spinifex/handlers/ecs/scheduler.go @@ -10,6 +10,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // Scheduler timing. The leader lease lives in spinifex-ecs-leader (60s TTL); the @@ -70,21 +71,27 @@ func (sc *Scheduler) Run(ctx context.Context) { case <-leaseTicker.C: sc.evaluateLeadership(ctx) case <-reaperTicker.C: - if sc.isLeader() { - sc.reap() - } + sc.runIfLeader("instance reap", func() error { return sc.reap(ctx) }) case <-reconcileTicker.C: - if sc.isLeader() { - sc.svc.reconcileAllServices() - } + sc.runIfLeader("service reconcile", func() error { return sc.svc.reconcileAllServices(ctx) }) case <-sweepTicker.C: - if sc.isLeader() { - sc.sweepStoppedTasks() - } + sc.runIfLeader("stopped-task sweep", func() error { return sc.sweepStoppedTasks(ctx) }) } } } +// runIfLeader runs one leader-only pass, logging a pass that could not complete. +// A pass errors only when it could not observe the whole fleet, so this log is +// the operator's signal that a tick was skipped rather than found nothing to do. +func (sc *Scheduler) runIfLeader(pass string, run func() error) { + if !sc.isLeader() { + return + } + if err := run(); err != nil { + slog.Error("ECS scheduler: pass failed", "pass", pass, "err", err) + } +} + func (sc *Scheduler) isLeader() bool { sc.mu.Lock() defer sc.mu.Unlock() @@ -94,7 +101,7 @@ func (sc *Scheduler) isLeader() bool { // evaluateLeadership acquires or refreshes the lease, wiring up (or tearing down) // the bus subscriptions as leadership changes. func (sc *Scheduler) evaluateLeadership(ctx context.Context) { - won := sc.acquireOrRefresh() + won := sc.acquireOrRefresh(ctx) sc.mu.Lock() was := sc.leader sc.leader = won @@ -115,19 +122,19 @@ func (sc *Scheduler) evaluateLeadership(ctx context.Context) { // acquireOrRefresh tries to claim the scheduler lease, refreshing it (resetting // the TTL) when we already hold it. Returns true when we are the leader. -func (sc *Scheduler) acquireOrRefresh() bool { - js, err := sc.nc.JetStream() +func (sc *Scheduler) acquireOrRefresh(ctx context.Context) bool { + js, err := jetstream.New(sc.nc) if err != nil { return false } - kv, err := InitLeaderBucket(js) + kv, err := InitLeaderBucket(ctx, js) if err != nil { return false } - if _, err := kv.Create(schedulerLeaderKey, []byte(sc.holder)); err == nil { + if _, err := kv.Create(ctx, schedulerLeaderKey, []byte(sc.holder)); err == nil { return true } - entry, err := kv.Get(schedulerLeaderKey) + entry, err := kv.Get(ctx, schedulerLeaderKey) if err != nil { return false } @@ -135,7 +142,7 @@ func (sc *Scheduler) acquireOrRefresh() bool { return false // someone else holds it } // We hold it — refresh to reset the TTL. - if _, err := kv.Put(schedulerLeaderKey, []byte(sc.holder)); err != nil { + if _, err := kv.Put(ctx, schedulerLeaderKey, []byte(sc.holder)); err != nil { return false } return true @@ -144,16 +151,20 @@ func (sc *Scheduler) acquireOrRefresh() bool { // relinquish drops subscriptions and deletes the lease key on shutdown. func (sc *Scheduler) relinquish() { sc.unsubscribeBus() - js, err := sc.nc.JetStream() + // Run's ctx is already cancelled by the time it calls relinquish, so the + // release runs on its own context — a captured ctx would fail the delete and + // leave the lease for the TTL to reap. + ctx := context.Background() + js, err := jetstream.New(sc.nc) if err != nil { return } - kv, err := InitLeaderBucket(js) + kv, err := InitLeaderBucket(ctx, js) if err != nil { return } - if entry, gerr := kv.Get(schedulerLeaderKey); gerr == nil && string(entry.Value()) == sc.holder { - _ = kv.Delete(schedulerLeaderKey) + if entry, gerr := kv.Get(ctx, schedulerLeaderKey); gerr == nil && string(entry.Value()) == sc.holder { + _ = kv.Delete(ctx, schedulerLeaderKey) } } @@ -197,7 +208,9 @@ func (sc *Scheduler) onRegister(msg *nats.Msg) { slog.Warn("ECS scheduler: bad register payload", "err", err) return } - if err := sc.svc.recordRegister(&m); err != nil { + // A bus callback outlives the Run context that wired its subscription, so it + // binds its own — a cancelled ctx would abandon the KV write mid-record. + if err := sc.svc.recordRegister(context.Background(), &m); err != nil { slog.Error("ECS scheduler: record register failed", "instance", m.InstanceID, "err", err) return } @@ -209,7 +222,7 @@ func (sc *Scheduler) onHeartbeat(msg *nats.Msg) { if err := json.Unmarshal(msg.Data, &m); err != nil { return } - if err := sc.svc.recordHeartbeat(&m); err != nil { + if err := sc.svc.recordHeartbeat(context.Background(), &m); err != nil { slog.Debug("ECS scheduler: record heartbeat failed", "instance", m.InstanceID, "err", err) } } @@ -227,29 +240,33 @@ func (sc *Scheduler) onTaskState(msg *nats.Msg) { } // reap marks instances that have missed their heartbeat window DRAINING and stops -// their tasks, releasing capacity. Iterates every ECS account bucket. -func (sc *Scheduler) reap() { - ctx := context.Background() - js, err := sc.nc.JetStream() +// their tasks, releasing capacity. Iterates every ECS account bucket. Returns an +// error when the account enumeration could not be completed, so a pass that +// could not see the whole fleet is reported rather than read as "nothing to +// reap" — every unlisted account keeps its dead instances' capacity held. +func (sc *Scheduler) reap(ctx context.Context) error { + js, err := jetstream.New(sc.nc) if err != nil { - return + return err + } + buckets, err := accountBuckets(ctx, sc.nc) + if err != nil { + return err } now := time.Now().UTC() - for bucket := range js.KeyValueStoreNames() { - accountID, ok := accountIDFromBucket(bucket) - if !ok { - continue - } - kv, err := js.KeyValue(bucket) + for _, bucket := range buckets { + kv, err := js.KeyValue(ctx, bucket.name) if err != nil { + slog.Error("ECS scheduler: open bucket failed", "bucket", bucket.name, "err", err) continue } - sc.reapBucket(ctx, kv, accountID, now) + sc.reapBucket(ctx, kv, bucket.accountID, now) } + return nil } -func (sc *Scheduler) reapBucket(ctx context.Context, kv nats.KeyValue, accountID string, now time.Time) { - keys, err := keysWithPrefix(kv, "clusters/") +func (sc *Scheduler) reapBucket(ctx context.Context, kv jetstream.KeyValue, accountID string, now time.Time) { + keys, err := keysWithPrefix(ctx, kv, "clusters/") if err != nil { return } @@ -258,7 +275,7 @@ func (sc *Scheduler) reapBucket(ctx context.Context, kv nats.KeyValue, accountID continue } var inst InstanceRecord - found, err := getJSON(kv, k, &inst) + found, err := getJSON(ctx, kv, k, &inst) if err != nil || !found { continue } @@ -269,7 +286,7 @@ func (sc *Scheduler) reapBucket(ctx context.Context, kv nats.KeyValue, accountID "lastSeen", inst.LastSeen.Format(time.RFC3339)) inst.Status = InstanceStatusDraining inst.Reaped = true - if perr := putJSON(kv, k, &inst); perr != nil { + if perr := putJSON(ctx, kv, k, &inst); perr != nil { continue } sc.stopInstanceTasks(ctx, kv, accountID, inst.Cluster, inst.InstanceID) @@ -278,14 +295,14 @@ func (sc *Scheduler) reapBucket(ctx context.Context, kv nats.KeyValue, accountID // stopInstanceTasks transitions a reaped instance's non-stopped tasks to STOPPED // and reclaims each awsvpc task's ENI (leak guard for a dead agent). -func (sc *Scheduler) stopInstanceTasks(ctx context.Context, kv nats.KeyValue, accountID, cluster, instanceID string) { - keys, err := keysWithPrefix(kv, TasksPrefix(cluster)) +func (sc *Scheduler) stopInstanceTasks(ctx context.Context, kv jetstream.KeyValue, accountID, cluster, instanceID string) { + keys, err := keysWithPrefix(ctx, kv, TasksPrefix(cluster)) if err != nil { return } for _, k := range keys { var task TaskRecord - found, err := getJSON(kv, k, &task) + found, err := getJSON(ctx, kv, k, &task) if err != nil || !found { continue } diff --git a/spinifex/handlers/ecs/service.go b/spinifex/handlers/ecs/service.go index b44563edb..746affa76 100644 --- a/spinifex/handlers/ecs/service.go +++ b/spinifex/handlers/ecs/service.go @@ -18,6 +18,7 @@ import ( "github.com/aws/aws-sdk-go/service/iam" "github.com/mulgadc/spinifex/spinifex/awserrors" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // ECSService is the ECS control-plane contract implemented by the daemon and @@ -151,26 +152,26 @@ func NewService(nc *nats.Conn, region, suffix string) *Service { // defaultCluster is the implicit cluster name when a request omits one. const defaultCluster = "default" -func (s *Service) js() (nats.JetStreamContext, error) { +func (s *Service) js() (jetstream.JetStream, error) { if s.nc == nil { return nil, errors.New("ecs service: nil nats connection") } - return s.nc.JetStream() + return jetstream.New(s.nc) } // bucket returns the per-account KV handle, creating it on first use. -func (s *Service) bucket(accountID string) (nats.KeyValue, error) { +func (s *Service) bucket(ctx context.Context, accountID string) (jetstream.KeyValue, error) { js, err := s.js() if err != nil { return nil, err } - return GetOrCreateAccountBucket(js, accountID) + return GetOrCreateAccountBucket(ctx, js, accountID) } // getJSON reads key into out. Returns (false, nil) when the key is absent. -func getJSON(kv nats.KeyValue, key string, out any) (bool, error) { - entry, err := kv.Get(key) - if errors.Is(err, nats.ErrKeyNotFound) { +func getJSON(ctx context.Context, kv jetstream.KeyValue, key string, out any) (bool, error) { + entry, err := kv.Get(ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { return false, nil } if err != nil { @@ -183,12 +184,12 @@ func getJSON(kv nats.KeyValue, key string, out any) (bool, error) { } // putJSON marshals v and writes it at key. -func putJSON(kv nats.KeyValue, key string, v any) error { +func putJSON(ctx context.Context, kv jetstream.KeyValue, key string, v any) error { data, err := json.Marshal(v) if err != nil { return err } - if _, err := kv.Put(key, data); err != nil { + if _, err := kv.Put(ctx, key, data); err != nil { return fmt.Errorf("put %s: %w", key, err) } return nil @@ -198,14 +199,14 @@ func putJSON(kv nats.KeyValue, key string, v any) error { // to sweep a cluster's whole key subtree (meta/instances/tasks/services/ // assignments) after its tasks are stopped. Best-effort per key; the first // delete error is returned after attempting the rest. -func deleteKeysWithPrefix(kv nats.KeyValue, prefix string) error { - keys, err := keysWithPrefix(kv, prefix) +func deleteKeysWithPrefix(ctx context.Context, kv jetstream.KeyValue, prefix string) error { + keys, err := keysWithPrefix(ctx, kv, prefix) if err != nil { return err } var firstErr error for _, k := range keys { - if derr := kv.Delete(k); derr != nil && firstErr == nil { + if derr := kv.Delete(ctx, k); derr != nil && firstErr == nil { firstErr = derr } } @@ -213,9 +214,9 @@ func deleteKeysWithPrefix(kv nats.KeyValue, prefix string) error { } // keysWithPrefix returns all KV keys under prefix. -func keysWithPrefix(kv nats.KeyValue, prefix string) ([]string, error) { - keys, err := kv.Keys() - if errors.Is(err, nats.ErrNoKeysFound) { +func keysWithPrefix(ctx context.Context, kv jetstream.KeyValue, prefix string) ([]string, error) { + keys, err := kv.Keys(ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { return nil, nil } if err != nil { @@ -235,18 +236,18 @@ func keysWithPrefix(kv nats.KeyValue, prefix string) ([]string, error) { // CreateCluster persists a cluster meta record. Idempotent: re-creating an // existing cluster returns the existing ACTIVE record. -func (s *Service) CreateCluster(_ context.Context, input *ecs.CreateClusterInput, accountID string) (*ecs.CreateClusterOutput, error) { +func (s *Service) CreateCluster(ctx context.Context, input *ecs.CreateClusterInput, accountID string) (*ecs.CreateClusterOutput, error) { name := aws.StringValue(input.ClusterName) if name == "" { name = defaultCluster } - kv, err := s.bucket(accountID) + kv, err := s.bucket(ctx, accountID) if err != nil { return nil, err } var rec ClusterRecord - found, err := getJSON(kv, ClusterMetaKey(name), &rec) + found, err := getJSON(ctx, kv, ClusterMetaKey(name), &rec) if err != nil { return nil, err } @@ -263,7 +264,7 @@ func (s *Service) CreateCluster(_ context.Context, input *ecs.CreateClusterInput rec.Tags[aws.StringValue(t.Key)] = aws.StringValue(t.Value) } } - if err := putJSON(kv, ClusterMetaKey(name), &rec); err != nil { + if err := putJSON(ctx, kv, ClusterMetaKey(name), &rec); err != nil { return nil, err } } @@ -272,8 +273,8 @@ func (s *Service) CreateCluster(_ context.Context, input *ecs.CreateClusterInput // DescribeClusters returns meta for the named clusters; unknown names are // silently skipped (AWS returns them under "failures", omitted here in v1). -func (s *Service) DescribeClusters(_ context.Context, input *ecs.DescribeClustersInput, accountID string) (*ecs.DescribeClustersOutput, error) { - kv, err := s.bucket(accountID) +func (s *Service) DescribeClusters(ctx context.Context, input *ecs.DescribeClustersInput, accountID string) (*ecs.DescribeClustersOutput, error) { + kv, err := s.bucket(ctx, accountID) if err != nil { return nil, err } @@ -285,7 +286,7 @@ func (s *Service) DescribeClusters(_ context.Context, input *ecs.DescribeCluster for _, name := range names { name = clusterShortName(name) var rec ClusterRecord - found, err := getJSON(kv, ClusterMetaKey(name), &rec) + found, err := getJSON(ctx, kv, ClusterMetaKey(name), &rec) if err != nil { return nil, err } @@ -297,12 +298,12 @@ func (s *Service) DescribeClusters(_ context.Context, input *ecs.DescribeCluster } // ListClusters returns the ARNs of all clusters in the account. -func (s *Service) ListClusters(_ context.Context, _ *ecs.ListClustersInput, accountID string) (*ecs.ListClustersOutput, error) { - kv, err := s.bucket(accountID) +func (s *Service) ListClusters(ctx context.Context, _ *ecs.ListClustersInput, accountID string) (*ecs.ListClustersOutput, error) { + kv, err := s.bucket(ctx, accountID) if err != nil { return nil, err } - keys, err := keysWithPrefix(kv, "clusters/") + keys, err := keysWithPrefix(ctx, kv, "clusters/") if err != nil { return nil, err } @@ -312,7 +313,7 @@ func (s *Service) ListClusters(_ context.Context, _ *ecs.ListClustersInput, acco continue } var rec ClusterRecord - found, err := getJSON(kv, k, &rec) + found, err := getJSON(ctx, kv, k, &rec) if err != nil { return nil, err } @@ -384,12 +385,12 @@ func (s *Service) RegisterTaskDefinition(ctx context.Context, input *ecs.Registe return nil, err } warnUnsupportedLogDrivers(ctx, family, input.ContainerDefinitions) - kv, err := s.bucket(accountID) + kv, err := s.bucket(ctx, accountID) if err != nil { return nil, err } - rev, err := s.nextRevision(kv, family) + rev, err := s.nextRevision(ctx, kv, family) if err != nil { return nil, err } @@ -408,10 +409,10 @@ func (s *Service) RegisterTaskDefinition(ctx context.Context, input *ecs.Registe RegisteredAt: time.Now().UTC(), Containers: containerDefsFromAWS(input.ContainerDefinitions), } - if err := putJSON(kv, TaskDefRevKey(family, rev), &rec); err != nil { + if err := putJSON(ctx, kv, TaskDefRevKey(family, rev), &rec); err != nil { return nil, err } - if err := putJSON(kv, TaskDefLatestRevKey(family), rev); err != nil { + if err := putJSON(ctx, kv, TaskDefLatestRevKey(family), rev); err != nil { return nil, err } return &ecs.RegisterTaskDefinitionOutput{TaskDefinition: rec.toAWS(), Tags: tagsToAWS(rec.Tags)}, nil @@ -447,9 +448,9 @@ func warnUnsupportedLogDrivers(ctx context.Context, family string, defs []*ecs.C } // nextRevision reads the family's latest-rev and returns latest+1 (1 if absent). -func (s *Service) nextRevision(kv nats.KeyValue, family string) (int, error) { +func (s *Service) nextRevision(ctx context.Context, kv jetstream.KeyValue, family string) (int, error) { var latest int - found, err := getJSON(kv, TaskDefLatestRevKey(family), &latest) + found, err := getJSON(ctx, kv, TaskDefLatestRevKey(family), &latest) if err != nil { return 0, err } @@ -462,17 +463,17 @@ func (s *Service) nextRevision(kv nats.KeyValue, family string) (int, error) { // DeregisterTaskDefinition marks a specific task-definition revision INACTIVE. // AWS requires an explicit family:revision (a bare family is rejected); the // revision stays describable, matching AWS. Idempotent. -func (s *Service) DeregisterTaskDefinition(_ context.Context, input *ecs.DeregisterTaskDefinitionInput, accountID string) (*ecs.DeregisterTaskDefinitionOutput, error) { +func (s *Service) DeregisterTaskDefinition(ctx context.Context, input *ecs.DeregisterTaskDefinitionInput, accountID string) (*ecs.DeregisterTaskDefinitionOutput, error) { family, rev := parseTaskDefRef(aws.StringValue(input.TaskDefinition)) if family == "" || rev == 0 { return nil, errors.New(awserrors.ErrorECSInvalidParameter) } - kv, err := s.bucket(accountID) + kv, err := s.bucket(ctx, accountID) if err != nil { return nil, err } var rec TaskDefRecord - found, err := getJSON(kv, TaskDefRevKey(family, rev), &rec) + found, err := getJSON(ctx, kv, TaskDefRevKey(family, rev), &rec) if err != nil { return nil, err } @@ -480,19 +481,19 @@ func (s *Service) DeregisterTaskDefinition(_ context.Context, input *ecs.Deregis return nil, errors.New(awserrors.ErrorECSInvalidParameter) } rec.Status = TaskDefStatusInactive - if err := putJSON(kv, TaskDefRevKey(family, rev), &rec); err != nil { + if err := putJSON(ctx, kv, TaskDefRevKey(family, rev), &rec); err != nil { return nil, err } return &ecs.DeregisterTaskDefinitionOutput{TaskDefinition: rec.toAWS()}, nil } // DescribeTaskDefinition resolves "family", "family:rev" or an ARN to a revision. -func (s *Service) DescribeTaskDefinition(_ context.Context, input *ecs.DescribeTaskDefinitionInput, accountID string) (*ecs.DescribeTaskDefinitionOutput, error) { - kv, err := s.bucket(accountID) +func (s *Service) DescribeTaskDefinition(ctx context.Context, input *ecs.DescribeTaskDefinitionInput, accountID string) (*ecs.DescribeTaskDefinitionOutput, error) { + kv, err := s.bucket(ctx, accountID) if err != nil { return nil, err } - rec, err := s.resolveTaskDef(kv, aws.StringValue(input.TaskDefinition)) + rec, err := s.resolveTaskDef(ctx, kv, aws.StringValue(input.TaskDefinition)) if err != nil { return nil, err } @@ -502,8 +503,8 @@ func (s *Service) DescribeTaskDefinition(_ context.Context, input *ecs.DescribeT // ListTaskDefinitions returns revision ARNs, optionally filtered by family and // by status. Matching AWS, the status defaults to ACTIVE when unset, so // deregistered (INACTIVE) revisions drop off the default listing. -func (s *Service) ListTaskDefinitions(_ context.Context, input *ecs.ListTaskDefinitionsInput, accountID string) (*ecs.ListTaskDefinitionsOutput, error) { - kv, err := s.bucket(accountID) +func (s *Service) ListTaskDefinitions(ctx context.Context, input *ecs.ListTaskDefinitionsInput, accountID string) (*ecs.ListTaskDefinitionsOutput, error) { + kv, err := s.bucket(ctx, accountID) if err != nil { return nil, err } @@ -515,7 +516,7 @@ func (s *Service) ListTaskDefinitions(_ context.Context, input *ecs.ListTaskDefi if fam := aws.StringValue(input.FamilyPrefix); fam != "" { prefix = TaskDefRevsPrefix(fam) } - keys, err := keysWithPrefix(kv, prefix) + keys, err := keysWithPrefix(ctx, kv, prefix) if err != nil { return nil, err } @@ -525,7 +526,7 @@ func (s *Service) ListTaskDefinitions(_ context.Context, input *ecs.ListTaskDefi continue } var rec TaskDefRecord - found, err := getJSON(kv, k, &rec) + found, err := getJSON(ctx, kv, k, &rec) if err != nil { return nil, err } @@ -538,14 +539,14 @@ func (s *Service) ListTaskDefinitions(_ context.Context, input *ecs.ListTaskDefi // resolveTaskDef loads the TaskDefRecord named by ref ("family", "family:rev", // or a task-definition ARN). A bare family resolves to its latest revision. -func (s *Service) resolveTaskDef(kv nats.KeyValue, ref string) (*TaskDefRecord, error) { +func (s *Service) resolveTaskDef(ctx context.Context, kv jetstream.KeyValue, ref string) (*TaskDefRecord, error) { family, rev := parseTaskDefRef(ref) if family == "" { return nil, errors.New(awserrors.ErrorECSInvalidParameter) } if rev == 0 { var latest int - found, err := getJSON(kv, TaskDefLatestRevKey(family), &latest) + found, err := getJSON(ctx, kv, TaskDefLatestRevKey(family), &latest) if err != nil { return nil, err } @@ -555,7 +556,7 @@ func (s *Service) resolveTaskDef(kv nats.KeyValue, ref string) (*TaskDefRecord, rev = latest } var rec TaskDefRecord - found, err := getJSON(kv, TaskDefRevKey(family, rev), &rec) + found, err := getJSON(ctx, kv, TaskDefRevKey(family, rev), &rec) if err != nil { return nil, err } diff --git a/spinifex/handlers/ecs/service_test.go b/spinifex/handlers/ecs/service_test.go index cf7447a53..7f03e01df 100644 --- a/spinifex/handlers/ecs/service_test.go +++ b/spinifex/handlers/ecs/service_test.go @@ -166,7 +166,7 @@ func TestService_DescribeTaskDefinition_Unknown(t *testing.T) { // register path (the way the agent registers in production). func registerInstance(t *testing.T, svc *Service, cluster, id string, cpu, mem int) { t.Helper() - require.NoError(t, svc.recordRegister(&bus.RegisterInstance{ + require.NoError(t, svc.recordRegister(t.Context(), &bus.RegisterInstance{ AccountID: testAccountID, ClusterName: cluster, InstanceID: id, @@ -393,7 +393,7 @@ func TestService_SubmitTaskStateChange_StopsTask(t *testing.T) { func TestService_RecordHeartbeat_UpdatesStatus(t *testing.T) { svc, _ := newTestService(t) registerInstance(t, svc, "web", "i-1", 1024, 2048) - require.NoError(t, svc.recordHeartbeat(&bus.Heartbeat{ + require.NoError(t, svc.recordHeartbeat(t.Context(), &bus.Heartbeat{ AccountID: testAccountID, ClusterName: "web", InstanceID: "i-1", Status: bus.StatusDraining, })) di, err := svc.DescribeContainerInstances(context.Background(), &ecs.DescribeContainerInstancesInput{ @@ -415,13 +415,13 @@ func TestScheduler_ReapBucket_StopsStaleInstanceTasks(t *testing.T) { taskID := containerInstanceShortID(aws.StringValue(out.Tasks[0].TaskArn)) // Backdate LastSeen beyond the heartbeat timeout. - kv, err := svc.bucket(testAccountID) + kv, err := svc.bucket(t.Context(), testAccountID) require.NoError(t, err) var rec InstanceRecord - _, err = getJSON(kv, InstanceKey("web", "i-1"), &rec) + _, err = getJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec) require.NoError(t, err) rec.LastSeen = time.Now().UTC().Add(-2 * heartbeatTimeout) - require.NoError(t, putJSON(kv, InstanceKey("web", "i-1"), &rec)) + require.NoError(t, putJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec)) sc := NewScheduler(nc, svc, "test-holder") sc.reapBucket(context.Background(), kv, testAccountID, time.Now().UTC()) @@ -444,9 +444,9 @@ func TestScheduler_AcquireLease_SingleLeader(t *testing.T) { svc := NewService(nc, testRegion, "") a := NewScheduler(nc, svc, "holder-a") b := NewScheduler(nc, svc, "holder-b") - assert.True(t, a.acquireOrRefresh()) - assert.False(t, b.acquireOrRefresh()) - assert.True(t, a.acquireOrRefresh()) // refresh keeps leadership + assert.True(t, a.acquireOrRefresh(t.Context())) + assert.False(t, b.acquireOrRefresh(t.Context())) + assert.True(t, a.acquireOrRefresh(t.Context())) // refresh keeps leadership } // --- GPU placement dimension (Epic C2) --- @@ -476,7 +476,7 @@ func registerTaskDefGPU(t *testing.T, svc *Service, family string, cpu, mem, gpu // capacity via the bus register path. func registerInstanceGPU(t *testing.T, svc *Service, cluster, id string, cpu, mem, gpu int) { t.Helper() - require.NoError(t, svc.recordRegister(&bus.RegisterInstance{ + require.NoError(t, svc.recordRegister(t.Context(), &bus.RegisterInstance{ AccountID: testAccountID, ClusterName: cluster, InstanceID: id, @@ -526,10 +526,10 @@ func TestService_RunTask_GPU_ReservesOnPlacement(t *testing.T) { assert.Empty(t, registeredGPU.StringSetValue) assert.Empty(t, remainingGPU.StringSetValue) - kv, err := svc.bucket(testAccountID) + kv, err := svc.bucket(t.Context(), testAccountID) require.NoError(t, err) var rec InstanceRecord - found, err := getJSON(kv, InstanceKey("web", "i-1"), &rec) + found, err := getJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec) require.NoError(t, err) require.True(t, found) assert.Equal(t, 2, rec.TotalGPU) @@ -540,7 +540,7 @@ func TestService_RunTask_GPU_ReservesOnPlacement(t *testing.T) { AccountID: testAccountID, ClusterName: "web", InstanceID: "i-1", TaskID: taskID, LastStatus: bus.TaskStatusStopped, Reason: "exited", })) - found, err = getJSON(kv, InstanceKey("web", "i-1"), &rec) + found, err = getJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec) require.NoError(t, err) require.True(t, found) assert.Equal(t, 0, rec.ReservedGPU) @@ -624,7 +624,7 @@ func TestRegisterContainerInstance_GPU_StringSetResource(t *testing.T) { // path (Epic C3 register report-back). func TestRecordRegister_GPU_CarriesDeviceUUIDs(t *testing.T) { svc, _ := newTestService(t) - require.NoError(t, svc.recordRegister(&bus.RegisterInstance{ + require.NoError(t, svc.recordRegister(t.Context(), &bus.RegisterInstance{ AccountID: testAccountID, ClusterName: "web", InstanceID: "i-1", Capacity: bus.InstanceCapacity{CPU: 1024, MemoryMiB: 2048, GPU: 2, GPUIDs: []string{"GPU-aaa", "GPU-bbb"}}, })) diff --git a/spinifex/handlers/ecs/services.go b/spinifex/handlers/ecs/services.go index 80b75ea8c..eb1cbf7f1 100644 --- a/spinifex/handlers/ecs/services.go +++ b/spinifex/handlers/ecs/services.go @@ -16,6 +16,7 @@ import ( handlers_ec2_eip "github.com/mulgadc/spinifex/spinifex/handlers/ec2/eip" handlers_elbv2 "github.com/mulgadc/spinifex/spinifex/handlers/elbv2" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // targetRegistrar registers/deregisters a service task's awsvpc ENI IP with an @@ -136,12 +137,12 @@ func (s *Service) CreateService(ctx context.Context, input *ecs.CreateServiceInp } cluster := clusterShortName(aws.StringValue(input.Cluster)) - kv, err := s.bucket(accountID) + kv, err := s.bucket(ctx, accountID) if err != nil { return nil, err } var clusterRec ClusterRecord - found, err := getJSON(kv, ClusterMetaKey(cluster), &clusterRec) + found, err := getJSON(ctx, kv, ClusterMetaKey(cluster), &clusterRec) if err != nil { return nil, err } @@ -150,13 +151,13 @@ func (s *Service) CreateService(ctx context.Context, input *ecs.CreateServiceInp } var existing ServiceRecord - if ok, gerr := getJSON(kv, ServiceKey(cluster, name), &existing); gerr != nil { + if ok, gerr := getJSON(ctx, kv, ServiceKey(cluster, name), &existing); gerr != nil { return nil, gerr } else if ok && existing.Status == ServiceStatusActive { return &ecs.CreateServiceOutput{Service: s.serviceToAWS(accountID, &existing)}, nil } - taskDef, err := s.resolveTaskDef(kv, aws.StringValue(input.TaskDefinition)) + taskDef, err := s.resolveTaskDef(ctx, kv, aws.StringValue(input.TaskDefinition)) if err != nil { return nil, err } @@ -190,7 +191,7 @@ func (s *Service) CreateService(ctx context.Context, input *ecs.CreateServiceInp applyDeploymentConfig(&rec, input.DeploymentConfiguration) rec.LastGoodTaskDefARN = taskDef.ARN rec.Deployments = []Deployment{newPrimaryDeployment(rec.DeploymentID, taskDef, rec.DesiredCount)} - if err := putJSON(kv, ServiceKey(cluster, name), &rec); err != nil { + if err := putJSON(ctx, kv, ServiceKey(cluster, name), &rec); err != nil { return nil, err } @@ -204,12 +205,12 @@ func (s *Service) CreateService(ctx context.Context, input *ecs.CreateServiceInp func (s *Service) UpdateService(ctx context.Context, input *ecs.UpdateServiceInput, accountID string) (*ecs.UpdateServiceOutput, error) { cluster := clusterShortName(aws.StringValue(input.Cluster)) name := serviceShortName(aws.StringValue(input.Service)) - kv, err := s.bucket(accountID) + kv, err := s.bucket(ctx, accountID) if err != nil { return nil, err } var rec ServiceRecord - found, err := getJSON(kv, ServiceKey(cluster, name), &rec) + found, err := getJSON(ctx, kv, ServiceKey(cluster, name), &rec) if err != nil { return nil, err } @@ -225,7 +226,7 @@ func (s *Service) UpdateService(ctx context.Context, input *ecs.UpdateServiceInp rec.DesiredCount = int(aws.Int64Value(input.DesiredCount)) } if td := aws.StringValue(input.TaskDefinition); td != "" { - taskDef, terr := s.resolveTaskDef(kv, td) + taskDef, terr := s.resolveTaskDef(ctx, kv, td) if terr != nil { return nil, terr } @@ -236,7 +237,7 @@ func (s *Service) UpdateService(ctx context.Context, input *ecs.UpdateServiceInp } } rec.UpdatedAt = time.Now().UTC() - if err := putJSON(kv, ServiceKey(cluster, name), &rec); err != nil { + if err := putJSON(ctx, kv, ServiceKey(cluster, name), &rec); err != nil { return nil, err } if err := s.reconcileService(ctx, kv, accountID, &rec); err != nil { @@ -250,12 +251,12 @@ func (s *Service) UpdateService(ctx context.Context, input *ecs.UpdateServiceInp func (s *Service) DeleteService(ctx context.Context, input *ecs.DeleteServiceInput, accountID string) (*ecs.DeleteServiceOutput, error) { cluster := clusterShortName(aws.StringValue(input.Cluster)) name := serviceShortName(aws.StringValue(input.Service)) - kv, err := s.bucket(accountID) + kv, err := s.bucket(ctx, accountID) if err != nil { return nil, err } var rec ServiceRecord - found, err := getJSON(kv, ServiceKey(cluster, name), &rec) + found, err := getJSON(ctx, kv, ServiceKey(cluster, name), &rec) if err != nil { return nil, err } @@ -264,7 +265,7 @@ func (s *Service) DeleteService(ctx context.Context, input *ecs.DeleteServiceInp } rec.DesiredCount = 0 - tasks, err := s.listServiceTasks(kv, cluster, name) + tasks, err := s.listServiceTasks(ctx, kv, cluster, name) if err != nil { return nil, err } @@ -275,16 +276,16 @@ func (s *Service) DeleteService(ctx context.Context, input *ecs.DeleteServiceInp rec.RunningCount = 0 rec.PendingCount = 0 rec.UpdatedAt = time.Now().UTC() - if err := putJSON(kv, ServiceKey(cluster, name), &rec); err != nil { + if err := putJSON(ctx, kv, ServiceKey(cluster, name), &rec); err != nil { return nil, err } return &ecs.DeleteServiceOutput{Service: s.serviceToAWS(accountID, &rec)}, nil } // DescribeServices returns the named services; unknown names surface as failures. -func (s *Service) DescribeServices(_ context.Context, input *ecs.DescribeServicesInput, accountID string) (*ecs.DescribeServicesOutput, error) { +func (s *Service) DescribeServices(ctx context.Context, input *ecs.DescribeServicesInput, accountID string) (*ecs.DescribeServicesOutput, error) { cluster := clusterShortName(aws.StringValue(input.Cluster)) - kv, err := s.bucket(accountID) + kv, err := s.bucket(ctx, accountID) if err != nil { return nil, err } @@ -292,7 +293,7 @@ func (s *Service) DescribeServices(_ context.Context, input *ecs.DescribeService for _, ref := range awsStringSlice(input.Services) { name := serviceShortName(ref) var rec ServiceRecord - found, gerr := getJSON(kv, ServiceKey(cluster, name), &rec) + found, gerr := getJSON(ctx, kv, ServiceKey(cluster, name), &rec) if gerr != nil { return nil, gerr } @@ -306,13 +307,13 @@ func (s *Service) DescribeServices(_ context.Context, input *ecs.DescribeService } // ListServices returns the ARNs of every service in a cluster. -func (s *Service) ListServices(_ context.Context, input *ecs.ListServicesInput, accountID string) (*ecs.ListServicesOutput, error) { +func (s *Service) ListServices(ctx context.Context, input *ecs.ListServicesInput, accountID string) (*ecs.ListServicesOutput, error) { cluster := clusterShortName(aws.StringValue(input.Cluster)) - kv, err := s.bucket(accountID) + kv, err := s.bucket(ctx, accountID) if err != nil { return nil, err } - recs, err := s.listServiceRecords(kv, cluster) + recs, err := s.listServiceRecords(ctx, kv, cluster) if err != nil { return nil, err } @@ -330,14 +331,14 @@ func (s *Service) ListServices(_ context.Context, input *ecs.ListServicesInput, // and drains superseded (old) tasks while healthy running stays at/above // minimumHealthyPercent, then advances the rollout / circuit-breaker state. Per- // deployment and overall counts are refreshed as a side effect. -func (s *Service) reconcileService(ctx context.Context, kv nats.KeyValue, accountID string, svc *ServiceRecord) error { +func (s *Service) reconcileService(ctx context.Context, kv jetstream.KeyValue, accountID string, svc *ServiceRecord) error { if svc.Status != ServiceStatusActive { return nil } svc.normalizeDeploymentConfig() svc.ensurePrimaryDeployment() - tasks, err := s.listServiceTasks(kv, svc.Cluster, svc.Name) + tasks, err := s.listServiceTasks(ctx, kv, svc.Cluster, svc.Name) if err != nil { return err } @@ -349,7 +350,7 @@ func (s *Service) reconcileService(ctx context.Context, kv nats.KeyValue, accoun // let the next tick roll the replacement in. if tripCircuitBreaker(svc, primary) { svc.RunningCount, svc.PendingCount = running, pending - return putJSON(kv, ServiceKey(svc.Cluster, svc.Name), svc) + return putJSON(ctx, kv, ServiceKey(svc.Cluster, svc.Name), svc) } desired := svc.DesiredCount maxCount := desired * svc.MaximumPercent / 100 @@ -365,43 +366,48 @@ func (s *Service) reconcileService(ctx context.Context, kv nats.KeyValue, accoun s.stopSurplusTasks(ctx, kv, accountID, tasks, primary.ID, desired, running, minCount) // Re-tally after launch/stop so rollout state + persisted counts are current. - if fresh, ferr := s.listServiceTasks(kv, svc.Cluster, svc.Name); ferr == nil { + if fresh, ferr := s.listServiceTasks(ctx, kv, svc.Cluster, svc.Name); ferr == nil { running, pending = tallyDeployments(svc, fresh) } updateRolloutState(svc, primary, desired) } svc.RunningCount, svc.PendingCount = running, pending - return putJSON(kv, ServiceKey(svc.Cluster, svc.Name), svc) + return putJSON(ctx, kv, ServiceKey(svc.Cluster, svc.Name), svc) } // reconcileAllServices is the scheduler-tick fan-out: every ACTIVE service in // every ECS account bucket is reconciled. Runs only on the scheduler leader. -func (s *Service) reconcileAllServices() { - ctx := context.Background() +// Returns an error when the account enumeration could not be completed, so a +// pass that could not see the whole fleet is reported rather than read as "no +// services to reconcile" — every unlisted account stalls below its desired count. +func (s *Service) reconcileAllServices(ctx context.Context) error { js, err := s.js() if err != nil { - return + return err } - for bucket := range js.KeyValueStoreNames() { - accountID, ok := accountIDFromBucket(bucket) - if !ok { + buckets, err := accountBuckets(ctx, s.nc) + if err != nil { + return err + } + for _, bucket := range buckets { + kv, err := js.KeyValue(ctx, bucket.name) + if err != nil { + slog.Error("ECS reconcile: open bucket failed", "bucket", bucket.name, "err", err) continue } - kv, err := js.KeyValue(bucket) - if err != nil { - continue - } - recs, err := s.allServiceRecords(kv) + recs, err := s.allServiceRecords(ctx, kv) if err != nil { + slog.Error("ECS reconcile: list services failed", "bucket", bucket.name, "err", err) continue } for i := range recs { - if err := s.reconcileService(ctx, kv, accountID, &recs[i]); err != nil { + if err := s.reconcileService(ctx, kv, bucket.accountID, &recs[i]); err != nil { slog.Error("ECS reconcile: service failed", "service", recs[i].Name, "err", err) } } } + return nil } // launchDeploymentTasks places n tasks for a specific deployment via the standard @@ -443,7 +449,7 @@ func (s *Service) launchDeploymentTasks(ctx context.Context, accountID string, s // deployment tasks toward zero, and the primary's own surplus on a scale-in. // PENDING surplus is stopped freely; RUNNING surplus is gated so healthy running // never drops below minimumHealthyPercent (minCount). -func (s *Service) stopSurplusTasks(ctx context.Context, kv nats.KeyValue, accountID string, tasks []TaskRecord, primaryID string, desired, runningTotal, minCount int) { +func (s *Service) stopSurplusTasks(ctx context.Context, kv jetstream.KeyValue, accountID string, tasks []TaskRecord, primaryID string, desired, runningTotal, minCount int) { var oldPending, oldRunning, primaryPending, primaryRunning []int for i := range tasks { isPrimary := deploymentIDFromStartedBy(tasks[i].StartedBy) == primaryID @@ -501,9 +507,9 @@ func (s *Service) stopSurplusTasks(ctx context.Context, kv nats.KeyValue, accoun // STOPPED nor already requested to stop (DesiredStatus=STOPPED). A cooperatively // stopped task lingers RUNNING until the agent reaps it, but it is on its way out, // so it must not count toward the service's running total for scaling decisions. -func (s *Service) listServiceTasks(kv nats.KeyValue, cluster, name string) ([]TaskRecord, error) { +func (s *Service) listServiceTasks(ctx context.Context, kv jetstream.KeyValue, cluster, name string) ([]TaskRecord, error) { group := serviceTaskGroup(name) - all, err := s.listTaskRecords(kv, cluster) + all, err := s.listTaskRecords(ctx, kv, cluster) if err != nil { return nil, err } @@ -517,15 +523,15 @@ func (s *Service) listServiceTasks(kv nats.KeyValue, cluster, name string) ([]Ta } // listTaskRecords returns every task record in a cluster. -func (s *Service) listTaskRecords(kv nats.KeyValue, cluster string) ([]TaskRecord, error) { - keys, err := keysWithPrefix(kv, TasksPrefix(cluster)) +func (s *Service) listTaskRecords(ctx context.Context, kv jetstream.KeyValue, cluster string) ([]TaskRecord, error) { + keys, err := keysWithPrefix(ctx, kv, TasksPrefix(cluster)) if err != nil { return nil, err } out := make([]TaskRecord, 0, len(keys)) for _, k := range keys { var rec TaskRecord - found, err := getJSON(kv, k, &rec) + found, err := getJSON(ctx, kv, k, &rec) if err != nil { return nil, err } @@ -537,15 +543,15 @@ func (s *Service) listTaskRecords(kv nats.KeyValue, cluster string) ([]TaskRecor } // listServiceRecords returns every service record in a cluster. -func (s *Service) listServiceRecords(kv nats.KeyValue, cluster string) ([]ServiceRecord, error) { - keys, err := keysWithPrefix(kv, ServicesPrefix(cluster)) +func (s *Service) listServiceRecords(ctx context.Context, kv jetstream.KeyValue, cluster string) ([]ServiceRecord, error) { + keys, err := keysWithPrefix(ctx, kv, ServicesPrefix(cluster)) if err != nil { return nil, err } out := make([]ServiceRecord, 0, len(keys)) for _, k := range keys { var rec ServiceRecord - found, err := getJSON(kv, k, &rec) + found, err := getJSON(ctx, kv, k, &rec) if err != nil { return nil, err } @@ -557,8 +563,8 @@ func (s *Service) listServiceRecords(kv nats.KeyValue, cluster string) ([]Servic } // allServiceRecords returns every service record across all clusters in a bucket. -func (s *Service) allServiceRecords(kv nats.KeyValue) ([]ServiceRecord, error) { - keys, err := keysWithPrefix(kv, "clusters/") +func (s *Service) allServiceRecords(ctx context.Context, kv jetstream.KeyValue) ([]ServiceRecord, error) { + keys, err := keysWithPrefix(ctx, kv, "clusters/") if err != nil { return nil, err } @@ -568,7 +574,7 @@ func (s *Service) allServiceRecords(kv nats.KeyValue) ([]ServiceRecord, error) { continue } var rec ServiceRecord - found, err := getJSON(kv, k, &rec) + found, err := getJSON(ctx, kv, k, &rec) if err != nil { return nil, err } diff --git a/spinifex/handlers/ecs/services_test.go b/spinifex/handlers/ecs/services_test.go index c500c73e6..f00b70b50 100644 --- a/spinifex/handlers/ecs/services_test.go +++ b/spinifex/handlers/ecs/services_test.go @@ -8,7 +8,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ecs" "github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -34,7 +34,7 @@ func (r *stubRegistrar) Deregister(_ context.Context, _, tg, ip string, port int // serviceTestRig wires a Service with a stub registrar and a seeded cluster + // task definition + one fat container instance. -func serviceTestRig(t *testing.T) (*Service, *stubRegistrar, nats.KeyValue) { +func serviceTestRig(t *testing.T) (*Service, *stubRegistrar, jetstream.KeyValue) { t.Helper() svc, _ := newTestService(t) reg := &stubRegistrar{} @@ -43,7 +43,7 @@ func serviceTestRig(t *testing.T) (*Service, *stubRegistrar, nats.KeyValue) { require.NoError(t, err) registerTaskDef(t, svc, "app", 128, 256) registerInstance(t, svc, "web", "i-1", 4096, 8192) - kv, err := svc.bucket(testAccountID) + kv, err := svc.bucket(t.Context(), testAccountID) require.NoError(t, err) return svc, reg, kv } @@ -62,7 +62,7 @@ func TestService_CreateService_LaunchesReplicas(t *testing.T) { assert.Equal(t, ServiceARN(testRegion, testAccountID, "web", "web"), aws.StringValue(out.Service.ServiceArn)) assert.Equal(t, int64(2), aws.Int64Value(out.Service.PendingCount)) - tasks, err := svc.listServiceTasks(kv, "web", "web") + tasks, err := svc.listServiceTasks(t.Context(), kv, "web", "web") require.NoError(t, err) require.Len(t, tasks, 2) for _, task := range tasks { @@ -106,7 +106,7 @@ func TestService_Reconcile_ReplacesStoppedTask(t *testing.T) { }, testAccountID) require.NoError(t, err) - tasks, err := svc.listServiceTasks(kv, "web", "web") + tasks, err := svc.listServiceTasks(t.Context(), kv, "web", "web") require.NoError(t, err) require.Len(t, tasks, 2) @@ -115,13 +115,13 @@ func TestService_Reconcile_ReplacesStoppedTask(t *testing.T) { AccountID: testAccountID, ClusterName: "web", InstanceID: "i-1", TaskID: tasks[0].TaskID, LastStatus: bus.TaskStatusStopped, })) - live, err := svc.listServiceTasks(kv, "web", "web") + live, err := svc.listServiceTasks(t.Context(), kv, "web", "web") require.NoError(t, err) require.Len(t, live, 1) // STOPPED task drops out of the active set // Reconcile launches a replacement back to desired=2. - svc.reconcileAllServices() - live, err = svc.listServiceTasks(kv, "web", "web") + require.NoError(t, svc.reconcileAllServices(t.Context())) + live, err = svc.listServiceTasks(t.Context(), kv, "web", "web") require.NoError(t, err) assert.Len(t, live, 2) } @@ -134,7 +134,7 @@ func TestService_UpdateService_ScalesIn(t *testing.T) { DesiredCount: aws.Int64(3), }, testAccountID) require.NoError(t, err) - live, _ := svc.listServiceTasks(kv, "web", "web") + live, _ := svc.listServiceTasks(t.Context(), kv, "web", "web") require.Len(t, live, 3) out, err := svc.UpdateService(context.Background(), &ecs.UpdateServiceInput{ @@ -143,7 +143,7 @@ func TestService_UpdateService_ScalesIn(t *testing.T) { require.NoError(t, err) assert.Equal(t, int64(1), aws.Int64Value(out.Service.DesiredCount)) - live, err = svc.listServiceTasks(kv, "web", "web") + live, err = svc.listServiceTasks(t.Context(), kv, "web", "web") require.NoError(t, err) assert.Len(t, live, 1) } @@ -162,7 +162,7 @@ func TestService_DeleteService_DrainsAndInactivates(t *testing.T) { require.NoError(t, err) assert.Equal(t, ServiceStatusInactive, aws.StringValue(out.Service.Status)) - live, err := svc.listServiceTasks(kv, "web", "web") + live, err := svc.listServiceTasks(t.Context(), kv, "web", "web") require.NoError(t, err) assert.Empty(t, live) @@ -172,8 +172,8 @@ func TestService_DeleteService_DrainsAndInactivates(t *testing.T) { }, testAccountID) require.NoError(t, err) require.Len(t, desc.Services, 1) - svc.reconcileAllServices() - live, _ = svc.listServiceTasks(kv, "web", "web") + require.NoError(t, svc.reconcileAllServices(t.Context())) + live, _ = svc.listServiceTasks(t.Context(), kv, "web", "web") assert.Empty(t, live) } @@ -214,7 +214,7 @@ func TestService_StartTask_PlacesPerInstance(t *testing.T) { assert.Empty(t, out.Failures) assert.Equal(t, TaskStatusPending, aws.StringValue(out.Tasks[0].LastStatus)) - tasks, err := svc.listTaskRecords(kv, "web") + tasks, err := svc.listTaskRecords(t.Context(), kv, "web") require.NoError(t, err) require.Len(t, tasks, 1) assert.Equal(t, "custom", tasks[0].Group) @@ -263,7 +263,7 @@ func TestService_StopTask_StopsAndDeregisters(t *testing.T) { NetworkMode: NetworkModeAwsvpc, ENIPrivateIP: "10.0.1.9", ReservedCPU: 64, ReservedMemoryMiB: 64, } - require.NoError(t, putJSON(kv, TaskKey("web", "t-9"), task)) + require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "t-9"), task)) // Cooperative stop: the task stays RUNNING with desiredStatus=STOPPED and a // stop directive lands in the instance inbox; nothing is deregistered until the @@ -277,7 +277,7 @@ func TestService_StopTask_StopsAndDeregisters(t *testing.T) { assert.Empty(t, reg.deregistered) var sd bus.StopDirective - found, err := getJSON(kv, StopKey("web", "i-1", "t-9"), &sd) + found, err := getJSON(t.Context(), kv, StopKey("web", "i-1", "t-9"), &sd) require.NoError(t, err) require.True(t, found) assert.Equal(t, "bye", sd.Reason) @@ -288,7 +288,7 @@ func TestService_StopTask_StopsAndDeregisters(t *testing.T) { TaskID: "t-9", LastStatus: bus.TaskStatusStopped, })) assert.Equal(t, []string{key(tgARN, "10.0.1.9", 80)}, reg.deregistered) - found, err = getJSON(kv, StopKey("web", "i-1", "t-9"), &sd) + found, err = getJSON(t.Context(), kv, StopKey("web", "i-1", "t-9"), &sd) require.NoError(t, err) assert.False(t, found) @@ -312,7 +312,7 @@ func TestService_StopTask_UnplacedForceStops(t *testing.T) { TaskID: "t-np", Cluster: "web", LastStatus: TaskStatusPending, DesiredStatus: TaskStatusRunning, } - require.NoError(t, putJSON(kv, TaskKey("web", "t-np"), task)) + require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "t-np"), task)) out, err := svc.StopTask(context.Background(), &ecs.StopTaskInput{ Cluster: aws.String("web"), Task: aws.String("t-np"), Reason: aws.String("cancel"), @@ -344,7 +344,7 @@ func TestService_TargetGroup_RegisterOnRunningDeregisterOnStopped(t *testing.T) ContainerInstanceID: "i-1", LastStatus: TaskStatusPending, NetworkMode: NetworkModeAwsvpc, ENIPrivateIP: "10.0.1.5", } - require.NoError(t, putJSON(kv, TaskKey("web", "t-1"), task)) + require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "t-1"), task)) // RUNNING → register. require.NoError(t, svc.recordTaskState(context.Background(), &bus.TaskState{ diff --git a/spinifex/handlers/ecs/store.go b/spinifex/handlers/ecs/store.go index c6d1d6a8b..bb9a98c39 100644 --- a/spinifex/handlers/ecs/store.go +++ b/spinifex/handlers/ecs/store.go @@ -1,14 +1,16 @@ package handlers_ecs import ( + "context" "errors" "fmt" "log/slog" "time" + "github.com/mulgadc/spinifex/spinifex/kvutil" "github.com/mulgadc/spinifex/spinifex/migrate" - "github.com/mulgadc/spinifex/spinifex/utils" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // JetStream KV bucket and key-path constants for the ECS control plane. @@ -169,16 +171,46 @@ func AccountBucketName(accountID string) string { return KVBucketECSAccountPrefix + accountID } +// accountBucket pairs a per-account KV bucket name with the account it belongs +// to, so the sweeps that walk every account do not each re-derive the ID. +type accountBucket struct { + name string + accountID string +} + +// accountBuckets returns every ECS per-account bucket. It fails rather than +// returning a short list when the enumeration could not be completed: the lister +// behind it closes its channel identically on success and on error, so a sweep +// that ignored the failure would read an unreachable JetStream as an empty fleet +// and silently skip every account it could not see. +func accountBuckets(ctx context.Context, nc *nats.Conn) ([]accountBucket, error) { + js, err := jetstream.New(nc) + if err != nil { + return nil, fmt.Errorf("jetstream: %w", err) + } + names, err := kvutil.BucketNames(ctx, js) + if err != nil { + return nil, err + } + buckets := make([]accountBucket, 0, len(names)) + for _, name := range names { + if accountID, ok := accountIDFromBucket(name); ok { + buckets = append(buckets, accountBucket{name: name, accountID: accountID}) + } + } + return buckets, nil +} + // GetOrCreateAccountBucket returns the per-account KV bucket for accountID, // creating it on first use. Idempotent: subsequent calls with the same accountID // return the existing handle. -func GetOrCreateAccountBucket(js nats.JetStreamContext, accountID string) (nats.KeyValue, error) { +func GetOrCreateAccountBucket(ctx context.Context, js jetstream.JetStream, accountID string) (jetstream.KeyValue, error) { bucket := AccountBucketName(accountID) - kv, err := utils.GetOrCreateKVBucket(js, bucket, KVBucketECSAccountHistory) + kv, err := kvutil.GetOrCreateBucket(ctx, js, bucket, KVBucketECSAccountHistory) if err != nil { return nil, fmt.Errorf("failed to create ECS per-account KV bucket %s: %w", bucket, err) } - if err := migrate.DefaultRegistry.RunKV(bucket, kv, KVBucketECSAccountVersion); err != nil { + if err := migrate.DefaultRegistry.RunKV(ctx, bucket, kv, KVBucketECSAccountVersion); err != nil { return nil, fmt.Errorf("migrate %s: %w", bucket, err) } return kv, nil @@ -187,22 +219,22 @@ func GetOrCreateAccountBucket(js nats.JetStreamContext, accountID string) (nats. // InitLeaderBucket creates (or attaches to) the shared spinifex-ecs-leader bucket // used for per-cluster scheduler leader-lease CAS locks. The bucket is configured // with History=1 and a 60s TTL so stale leases expire on their own when a leader -// dies mid-cycle. utils.GetOrCreateKVBucket doesn't expose a TTL knob, so this +// dies mid-cycle. kvutil.GetOrCreateBucket doesn't expose a TTL knob, so this // takes the direct js.CreateKeyValue path and falls back to js.KeyValue on // already-exists. -func InitLeaderBucket(js nats.JetStreamContext) (nats.KeyValue, error) { - kv, err := js.CreateKeyValue(&nats.KeyValueConfig{ +func InitLeaderBucket(ctx context.Context, js jetstream.JetStream) (jetstream.KeyValue, error) { + kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{ Bucket: KVBucketECSLeader, History: 1, TTL: KVBucketECSLeaderTTL, }) if err != nil { - kv, err = js.KeyValue(KVBucketECSLeader) + kv, err = js.KeyValue(ctx, KVBucketECSLeader) if err != nil { return nil, fmt.Errorf("failed to create or open ECS leader bucket %s: %w", KVBucketECSLeader, err) } } - if err := migrate.DefaultRegistry.RunKV(KVBucketECSLeader, kv, KVBucketECSLeaderVersion); err != nil { + if err := migrate.DefaultRegistry.RunKV(ctx, KVBucketECSLeader, kv, KVBucketECSLeaderVersion); err != nil { return nil, fmt.Errorf("migrate %s: %w", KVBucketECSLeader, err) } slog.Info("ECS leader bucket initialized", "bucket", KVBucketECSLeader, "ttl", KVBucketECSLeaderTTL) diff --git a/spinifex/handlers/ecs/store_test.go b/spinifex/handlers/ecs/store_test.go index 9c28de6ae..de121cfcb 100644 --- a/spinifex/handlers/ecs/store_test.go +++ b/spinifex/handlers/ecs/store_test.go @@ -27,13 +27,14 @@ func TestNewStore_Valid(t *testing.T) { } func TestGetOrCreateAccountBucket_Idempotent(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) - kv1, err := GetOrCreateAccountBucket(js, testAccountID) + kv1, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID) require.NoError(t, err) require.NotNil(t, kv1) - kv2, err := GetOrCreateAccountBucket(js, testAccountID) + kv2, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID) require.NoError(t, err) require.NotNil(t, kv2) @@ -42,14 +43,15 @@ func TestGetOrCreateAccountBucket_Idempotent(t *testing.T) { } func TestInitLeaderBucket_Idempotent(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) - kv1, err := InitLeaderBucket(js) + kv1, err := InitLeaderBucket(t.Context(), js) require.NoError(t, err) require.NotNil(t, kv1) assert.Equal(t, KVBucketECSLeader, kv1.Bucket()) - kv2, err := InitLeaderBucket(js) + kv2, err := InitLeaderBucket(t.Context(), js) require.NoError(t, err) require.NotNil(t, kv2) assert.Equal(t, KVBucketECSLeader, kv2.Bucket()) diff --git a/spinifex/handlers/ecs/tags.go b/spinifex/handlers/ecs/tags.go index 7e8c4dda7..666d35193 100644 --- a/spinifex/handlers/ecs/tags.go +++ b/spinifex/handlers/ecs/tags.go @@ -9,7 +9,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ecs" "github.com/mulgadc/spinifex/spinifex/awserrors" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // ecsResourceKind identifies which stored record kind an ARN refers to. @@ -64,11 +64,11 @@ func parseResourceARN(resourceARN string) (kind ecsResourceKind, cluster, id str } // resourceTags reads the tag map stored on the ARN-identified resource. -func (s *Service) resourceTags(kv nats.KeyValue, kind ecsResourceKind, cluster, id string) (map[string]string, error) { +func (s *Service) resourceTags(ctx context.Context, kv jetstream.KeyValue, kind ecsResourceKind, cluster, id string) (map[string]string, error) { switch kind { case ecsResourceCluster: var rec ClusterRecord - found, err := getJSON(kv, ClusterMetaKey(id), &rec) + found, err := getJSON(ctx, kv, ClusterMetaKey(id), &rec) if err != nil { return nil, err } @@ -82,7 +82,7 @@ func (s *Service) resourceTags(kv nats.KeyValue, kind ecsResourceKind, cluster, return nil, errors.New(awserrors.ErrorECSInvalidParameter) } var rec TaskDefRecord - found, err := getJSON(kv, TaskDefRevKey(family, rev), &rec) + found, err := getJSON(ctx, kv, TaskDefRevKey(family, rev), &rec) if err != nil { return nil, err } @@ -92,7 +92,7 @@ func (s *Service) resourceTags(kv nats.KeyValue, kind ecsResourceKind, cluster, return rec.Tags, nil case ecsResourceService: var rec ServiceRecord - found, err := getJSON(kv, ServiceKey(cluster, id), &rec) + found, err := getJSON(ctx, kv, ServiceKey(cluster, id), &rec) if err != nil { return nil, err } @@ -102,7 +102,7 @@ func (s *Service) resourceTags(kv nats.KeyValue, kind ecsResourceKind, cluster, return rec.Tags, nil case ecsResourceTask: var rec TaskRecord - found, err := getJSON(kv, TaskKey(cluster, id), &rec) + found, err := getJSON(ctx, kv, TaskKey(cluster, id), &rec) if err != nil { return nil, err } @@ -112,7 +112,7 @@ func (s *Service) resourceTags(kv nats.KeyValue, kind ecsResourceKind, cluster, return rec.Tags, nil case ecsResourceContainerInstance: var rec InstanceRecord - found, err := getJSON(kv, InstanceKey(cluster, id), &rec) + found, err := getJSON(ctx, kv, InstanceKey(cluster, id), &rec) if err != nil { return nil, err } @@ -128,11 +128,11 @@ func (s *Service) resourceTags(kv nats.KeyValue, kind ecsResourceKind, cluster, // mutateResourceTags reads the ARN-identified resource, applies mutate to its // tag map, and re-puts the record. Used by TagResource (merge) and // UntagResource (delete). -func (s *Service) mutateResourceTags(kv nats.KeyValue, kind ecsResourceKind, cluster, id string, mutate func(map[string]string) map[string]string) error { +func (s *Service) mutateResourceTags(ctx context.Context, kv jetstream.KeyValue, kind ecsResourceKind, cluster, id string, mutate func(map[string]string) map[string]string) error { switch kind { case ecsResourceCluster: var rec ClusterRecord - found, err := getJSON(kv, ClusterMetaKey(id), &rec) + found, err := getJSON(ctx, kv, ClusterMetaKey(id), &rec) if err != nil { return err } @@ -140,14 +140,14 @@ func (s *Service) mutateResourceTags(kv nats.KeyValue, kind ecsResourceKind, clu return errors.New(awserrors.ErrorECSClusterNotFound) } rec.Tags = mutate(rec.Tags) - return putJSON(kv, ClusterMetaKey(id), &rec) + return putJSON(ctx, kv, ClusterMetaKey(id), &rec) case ecsResourceTaskDefinition: family, rev := parseTaskDefRef(id) if family == "" || rev == 0 { return errors.New(awserrors.ErrorECSInvalidParameter) } var rec TaskDefRecord - found, err := getJSON(kv, TaskDefRevKey(family, rev), &rec) + found, err := getJSON(ctx, kv, TaskDefRevKey(family, rev), &rec) if err != nil { return err } @@ -155,10 +155,10 @@ func (s *Service) mutateResourceTags(kv nats.KeyValue, kind ecsResourceKind, clu return errors.New(awserrors.ErrorECSInvalidParameter) } rec.Tags = mutate(rec.Tags) - return putJSON(kv, TaskDefRevKey(family, rev), &rec) + return putJSON(ctx, kv, TaskDefRevKey(family, rev), &rec) case ecsResourceService: var rec ServiceRecord - found, err := getJSON(kv, ServiceKey(cluster, id), &rec) + found, err := getJSON(ctx, kv, ServiceKey(cluster, id), &rec) if err != nil { return err } @@ -166,10 +166,10 @@ func (s *Service) mutateResourceTags(kv nats.KeyValue, kind ecsResourceKind, clu return errors.New(awserrors.ErrorECSServiceNotFound) } rec.Tags = mutate(rec.Tags) - return putJSON(kv, ServiceKey(cluster, id), &rec) + return putJSON(ctx, kv, ServiceKey(cluster, id), &rec) case ecsResourceTask: var rec TaskRecord - found, err := getJSON(kv, TaskKey(cluster, id), &rec) + found, err := getJSON(ctx, kv, TaskKey(cluster, id), &rec) if err != nil { return err } @@ -177,10 +177,10 @@ func (s *Service) mutateResourceTags(kv nats.KeyValue, kind ecsResourceKind, clu return errors.New(awserrors.ErrorECSInvalidParameter) } rec.Tags = mutate(rec.Tags) - return putJSON(kv, TaskKey(cluster, id), &rec) + return putJSON(ctx, kv, TaskKey(cluster, id), &rec) case ecsResourceContainerInstance: var rec InstanceRecord - found, err := getJSON(kv, InstanceKey(cluster, id), &rec) + found, err := getJSON(ctx, kv, InstanceKey(cluster, id), &rec) if err != nil { return err } @@ -188,14 +188,14 @@ func (s *Service) mutateResourceTags(kv nats.KeyValue, kind ecsResourceKind, clu return errors.New(awserrors.ErrorECSInvalidParameter) } rec.Tags = mutate(rec.Tags) - return putJSON(kv, InstanceKey(cluster, id), &rec) + return putJSON(ctx, kv, InstanceKey(cluster, id), &rec) default: return errors.New(awserrors.ErrorECSInvalidParameter) } } // ListTagsForResource returns the tags stored on the ARN-identified resource. -func (s *Service) ListTagsForResource(_ context.Context, input *ecs.ListTagsForResourceInput, accountID string) (*ecs.ListTagsForResourceOutput, error) { +func (s *Service) ListTagsForResource(ctx context.Context, input *ecs.ListTagsForResourceInput, accountID string) (*ecs.ListTagsForResourceOutput, error) { if input == nil || aws.StringValue(input.ResourceArn) == "" { return nil, errors.New(awserrors.ErrorECSInvalidParameter) } @@ -203,11 +203,11 @@ func (s *Service) ListTagsForResource(_ context.Context, input *ecs.ListTagsForR if err != nil { return nil, err } - kv, err := s.bucket(accountID) + kv, err := s.bucket(ctx, accountID) if err != nil { return nil, err } - tags, err := s.resourceTags(kv, kind, cluster, id) + tags, err := s.resourceTags(ctx, kv, kind, cluster, id) if err != nil { return nil, err } @@ -215,7 +215,7 @@ func (s *Service) ListTagsForResource(_ context.Context, input *ecs.ListTagsForR } // TagResource merges the supplied tags onto the ARN-identified resource. -func (s *Service) TagResource(_ context.Context, input *ecs.TagResourceInput, accountID string) (*ecs.TagResourceOutput, error) { +func (s *Service) TagResource(ctx context.Context, input *ecs.TagResourceInput, accountID string) (*ecs.TagResourceOutput, error) { if input == nil || aws.StringValue(input.ResourceArn) == "" { return nil, errors.New(awserrors.ErrorECSInvalidParameter) } @@ -223,12 +223,12 @@ func (s *Service) TagResource(_ context.Context, input *ecs.TagResourceInput, ac if err != nil { return nil, err } - kv, err := s.bucket(accountID) + kv, err := s.bucket(ctx, accountID) if err != nil { return nil, err } add := tagsToMap(input.Tags) - err = s.mutateResourceTags(kv, kind, cluster, id, func(existing map[string]string) map[string]string { + err = s.mutateResourceTags(ctx, kv, kind, cluster, id, func(existing map[string]string) map[string]string { if existing == nil { existing = map[string]string{} } @@ -242,7 +242,7 @@ func (s *Service) TagResource(_ context.Context, input *ecs.TagResourceInput, ac } // UntagResource deletes the named tag keys from the ARN-identified resource. -func (s *Service) UntagResource(_ context.Context, input *ecs.UntagResourceInput, accountID string) (*ecs.UntagResourceOutput, error) { +func (s *Service) UntagResource(ctx context.Context, input *ecs.UntagResourceInput, accountID string) (*ecs.UntagResourceOutput, error) { if input == nil || aws.StringValue(input.ResourceArn) == "" { return nil, errors.New(awserrors.ErrorECSInvalidParameter) } @@ -250,12 +250,12 @@ func (s *Service) UntagResource(_ context.Context, input *ecs.UntagResourceInput if err != nil { return nil, err } - kv, err := s.bucket(accountID) + kv, err := s.bucket(ctx, accountID) if err != nil { return nil, err } keys := awsStringSlice(input.TagKeys) - err = s.mutateResourceTags(kv, kind, cluster, id, func(existing map[string]string) map[string]string { + err = s.mutateResourceTags(ctx, kv, kind, cluster, id, func(existing map[string]string) map[string]string { for _, k := range keys { delete(existing, k) } diff --git a/spinifex/handlers/ecs/task_stop.go b/spinifex/handlers/ecs/task_stop.go index 5660cbedd..20b85ef59 100644 --- a/spinifex/handlers/ecs/task_stop.go +++ b/spinifex/handlers/ecs/task_stop.go @@ -13,7 +13,7 @@ import ( "github.com/google/uuid" "github.com/mulgadc/spinifex/spinifex/awserrors" "github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // defaultStopReason is recorded when StopTask is called without a reason. @@ -28,12 +28,12 @@ const defaultStopReason = "Task stopped by user" func (s *Service) StopTask(ctx context.Context, input *ecs.StopTaskInput, accountID string) (*ecs.StopTaskOutput, error) { cluster := clusterShortName(aws.StringValue(input.Cluster)) taskID := taskShortID(aws.StringValue(input.Task)) - kv, err := s.bucket(accountID) + kv, err := s.bucket(ctx, accountID) if err != nil { return nil, err } var task TaskRecord - found, err := getJSON(kv, TaskKey(cluster, taskID), &task) + found, err := getJSON(ctx, kv, TaskKey(cluster, taskID), &task) if err != nil { return nil, err } @@ -52,19 +52,19 @@ func (s *Service) StopTask(ctx context.Context, input *ecs.StopTaskInput, accoun // scheduler bin-pack). Mirrors RunTask's reserve → record → ENI → assign flow. func (s *Service) StartTask(ctx context.Context, input *ecs.StartTaskInput, accountID string) (*ecs.StartTaskOutput, error) { cluster := clusterShortName(aws.StringValue(input.Cluster)) - kv, err := s.bucket(accountID) + kv, err := s.bucket(ctx, accountID) if err != nil { return nil, err } var clusterRec ClusterRecord - found, err := getJSON(kv, ClusterMetaKey(cluster), &clusterRec) + found, err := getJSON(ctx, kv, ClusterMetaKey(cluster), &clusterRec) if err != nil { return nil, err } if !found { return nil, errors.New(awserrors.ErrorECSClusterNotFound) } - taskDef, err := s.resolveTaskDef(kv, aws.StringValue(input.TaskDefinition)) + taskDef, err := s.resolveTaskDef(ctx, kv, aws.StringValue(input.TaskDefinition)) if err != nil { return nil, err } @@ -82,7 +82,7 @@ func (s *Service) StartTask(ctx context.Context, input *ecs.StartTaskInput, acco for _, ref := range awsStringSlice(input.ContainerInstances) { instanceID := containerInstanceShortID(ref) taskID := uuid.NewString() - inst, rerr := s.reserveOnInstance(kv, cluster, instanceID, taskID, cpu, mem, gpu) + inst, rerr := s.reserveOnInstance(ctx, kv, cluster, instanceID, taskID, cpu, mem, gpu) if rerr != nil { out.Failures = append(out.Failures, &ecs.Failure{ Arn: aws.String(ref), Reason: aws.String("RESOURCE:placement"), Detail: aws.String(rerr.Error()), @@ -100,10 +100,10 @@ func (s *Service) StartTask(ctx context.Context, input *ecs.StartTaskInput, acco continue } } - if err := putJSON(kv, TaskKey(cluster, taskID), rec); err != nil { + if err := putJSON(ctx, kv, TaskKey(cluster, taskID), rec); err != nil { return nil, err } - if err := s.publishAssign(kv, accountID, cluster, inst.InstanceID, rec, taskDef); err != nil { + if err := s.publishAssign(ctx, kv, accountID, cluster, inst.InstanceID, rec, taskDef); err != nil { slog.ErrorContext(ctx, "ECS StartTask: failed to publish assign", "task", taskID, "instance", inst.InstanceID, "err", err) } out.Tasks = append(out.Tasks, s.taskToAWS(accountID, rec)) @@ -114,10 +114,10 @@ func (s *Service) StartTask(ctx context.Context, input *ecs.StartTaskInput, acco // reserveOnInstance commits a task's capacity reservation onto a specific // container instance under a KV CAS. Unlike reservePlacement it does not bin-pack; // the instance is fixed by the caller (StartTask). -func (s *Service) reserveOnInstance(kv nats.KeyValue, cluster, instanceID, taskID string, cpu, mem, gpu int) (*InstanceRecord, error) { +func (s *Service) reserveOnInstance(ctx context.Context, kv jetstream.KeyValue, cluster, instanceID, taskID string, cpu, mem, gpu int) (*InstanceRecord, error) { for range reservePlacementRetries { - entry, err := kv.Get(InstanceKey(cluster, instanceID)) - if errors.Is(err, nats.ErrKeyNotFound) { + entry, err := kv.Get(ctx, InstanceKey(cluster, instanceID)) + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil, errors.New(awserrors.ErrorECSInvalidParameter) } if err != nil { @@ -138,7 +138,7 @@ func (s *Service) reserveOnInstance(kv nats.KeyValue, cluster, instanceID, taskI if merr != nil { return nil, merr } - if _, uerr := kv.Update(InstanceKey(cluster, instanceID), data, entry.Revision()); uerr == nil { + if _, uerr := kv.Update(ctx, InstanceKey(cluster, instanceID), data, entry.Revision()); uerr == nil { return &live, nil } } @@ -152,7 +152,7 @@ func (s *Service) reserveOnInstance(kv nats.KeyValue, cluster, instanceID, taskI // then performs the single release. No-op once the task is already STOPPED. A task // with no container instance (never placed) is forced-stopped directly since no // agent can report it. -func (s *Service) requestStopTask(ctx context.Context, kv nats.KeyValue, accountID string, task *TaskRecord, reason string) { +func (s *Service) requestStopTask(ctx context.Context, kv jetstream.KeyValue, accountID string, task *TaskRecord, reason string) { if task.LastStatus == TaskStatusStopped { return } @@ -161,12 +161,12 @@ func (s *Service) requestStopTask(ctx context.Context, kv nats.KeyValue, account return } task.DesiredStatus = TaskStatusStopped - if perr := putJSON(kv, TaskKey(task.Cluster, task.TaskID), task); perr != nil { + if perr := putJSON(ctx, kv, TaskKey(task.Cluster, task.TaskID), task); perr != nil { slog.ErrorContext(ctx, "ECS requestStopTask: persist failed", "task", task.TaskID, "err", perr) return } sd := bus.StopDirective{TaskID: task.TaskID, Reason: reason} - if perr := putJSON(kv, StopKey(task.Cluster, task.ContainerInstanceID, task.TaskID), &sd); perr != nil { + if perr := putJSON(ctx, kv, StopKey(task.Cluster, task.ContainerInstanceID, task.TaskID), &sd); perr != nil { slog.ErrorContext(ctx, "ECS requestStopTask: post stop directive failed", "task", task.TaskID, "err", perr) } } @@ -176,7 +176,7 @@ func (s *Service) requestStopTask(ctx context.Context, kv nats.KeyValue, account // its ELBv2 targets. Used where no live agent can reap the container (heartbeat // reaper, cluster/instance teardown). The mutated record is reflected back into // task. -func (s *Service) forceStopTask(ctx context.Context, kv nats.KeyValue, accountID string, task *TaskRecord, reason string) { +func (s *Service) forceStopTask(ctx context.Context, kv jetstream.KeyValue, accountID string, task *TaskRecord, reason string) { if task.LastStatus == TaskStatusStopped { return } @@ -188,15 +188,15 @@ func (s *Service) forceStopTask(ctx context.Context, kv nats.KeyValue, accountID // Release the auto-assigned EIP before persisting so the cleared public IP // lands in this single write. s.releaseTaskPublicIP(ctx, accountID, task) - if perr := putJSON(kv, TaskKey(task.Cluster, task.TaskID), task); perr != nil { + if perr := putJSON(ctx, kv, TaskKey(task.Cluster, task.TaskID), task); perr != nil { slog.ErrorContext(ctx, "ECS forceStopTask: persist failed", "task", task.TaskID, "err", perr) return } s.deregisterServiceTargets(ctx, kv, accountID, task) - s.reclaimAssignInbox(kv, task.Cluster, task.ContainerInstanceID, task.TaskID) - s.reclaimStopInbox(kv, task.Cluster, task.ContainerInstanceID, task.TaskID) + s.reclaimAssignInbox(ctx, kv, task.Cluster, task.ContainerInstanceID, task.TaskID) + s.reclaimStopInbox(ctx, kv, task.Cluster, task.ContainerInstanceID, task.TaskID) s.reclaimTaskENI(ctx, accountID, task) - if rerr := s.releaseReservation(kv, task.Cluster, task.ContainerInstanceID, task.TaskID, task.ReservedCPU, task.ReservedMemoryMiB, task.GPU); rerr != nil { + if rerr := s.releaseReservation(ctx, kv, task.Cluster, task.ContainerInstanceID, task.TaskID, task.ReservedCPU, task.ReservedMemoryMiB, task.GPU); rerr != nil { slog.ErrorContext(ctx, "ECS forceStopTask: release reservation failed", "task", task.TaskID, "err", rerr) } } @@ -216,22 +216,22 @@ func awsvpcConfigFromStartTask(input *ecs.StartTaskInput) awsvpcConfig { // registerServiceTargets registers a task's awsvpc ENI IP with each of its // owning service's target groups. No-op when the task is not a service task, // the service has no load balancers, or the task has no ENI IP (bridge/host). -func (s *Service) registerServiceTargets(ctx context.Context, kv nats.KeyValue, accountID string, task *TaskRecord) { +func (s *Service) registerServiceTargets(ctx context.Context, kv jetstream.KeyValue, accountID string, task *TaskRecord) { s.applyServiceTargets(ctx, kv, accountID, task, true) } // deregisterServiceTargets is the STOPPED-side inverse of registerServiceTargets. -func (s *Service) deregisterServiceTargets(ctx context.Context, kv nats.KeyValue, accountID string, task *TaskRecord) { +func (s *Service) deregisterServiceTargets(ctx context.Context, kv jetstream.KeyValue, accountID string, task *TaskRecord) { s.applyServiceTargets(ctx, kv, accountID, task, false) } -func (s *Service) applyServiceTargets(ctx context.Context, kv nats.KeyValue, accountID string, task *TaskRecord, register bool) { +func (s *Service) applyServiceTargets(ctx context.Context, kv jetstream.KeyValue, accountID string, task *TaskRecord, register bool) { name := serviceNameFromGroup(task.Group) if name == "" || task.ENIPrivateIP == "" || s.targets == nil { return } var svc ServiceRecord - found, err := getJSON(kv, ServiceKey(task.Cluster, name), &svc) + found, err := getJSON(ctx, kv, ServiceKey(task.Cluster, name), &svc) if err != nil || !found || len(svc.LoadBalancers) == 0 { return } @@ -254,7 +254,7 @@ func (s *Service) applyServiceTargets(ctx context.Context, kv nats.KeyValue, acc // public IP onto the task. No-op for non-service or non-awsvpc tasks, or when an // EIP is already assigned. Best-effort: a failure logs and leaves the task with // only its private endpoint. -func (s *Service) assignTaskPublicIP(ctx context.Context, kv nats.KeyValue, accountID string, task *TaskRecord) { +func (s *Service) assignTaskPublicIP(ctx context.Context, kv jetstream.KeyValue, accountID string, task *TaskRecord) { if s.eips == nil || task.ENIID == "" || task.ENIPublicIP != "" { return } @@ -263,7 +263,7 @@ func (s *Service) assignTaskPublicIP(ctx context.Context, kv nats.KeyValue, acco return } var svc ServiceRecord - found, err := getJSON(kv, ServiceKey(task.Cluster, name), &svc) + found, err := getJSON(ctx, kv, ServiceKey(task.Cluster, name), &svc) if err != nil || !found || !strings.EqualFold(svc.AssignPublicIP, "ENABLED") { return } @@ -274,7 +274,7 @@ func (s *Service) assignTaskPublicIP(ctx context.Context, kv nats.KeyValue, acco } task.ENIPublicIP = publicIP task.ENIEIPAllocationID = allocID - if perr := putJSON(kv, TaskKey(task.Cluster, task.TaskID), task); perr != nil { + if perr := putJSON(ctx, kv, TaskKey(task.Cluster, task.TaskID), task); perr != nil { slog.ErrorContext(ctx, "ECS auto-EIP: persist public IP failed", "task", task.TaskID, "err", perr) } } diff --git a/spinifex/handlers/ecs/taskdef_validation_test.go b/spinifex/handlers/ecs/taskdef_validation_test.go index 08cbe3c94..792ecfe5a 100644 --- a/spinifex/handlers/ecs/taskdef_validation_test.go +++ b/spinifex/handlers/ecs/taskdef_validation_test.go @@ -132,10 +132,10 @@ func TestRunTask_AssignCarriesGPU(t *testing.T) { assert.Zero(t, poll.Assignments[0].Containers[1].GPU) taskID := taskShortID(aws.StringValue(out.Tasks[0].TaskArn)) - kv, err := svc.bucket(testAccountID) + kv, err := svc.bucket(t.Context(), testAccountID) require.NoError(t, err) var rec TaskRecord - found, err := getJSON(kv, TaskKey("web", taskID), &rec) + found, err := getJSON(t.Context(), kv, TaskKey("web", taskID), &rec) require.NoError(t, err) require.True(t, found) assert.Equal(t, 1, rec.GPU) diff --git a/spinifex/handlers/ecs/tasks.go b/spinifex/handlers/ecs/tasks.go index b6990d146..7f3e9cd1c 100644 --- a/spinifex/handlers/ecs/tasks.go +++ b/spinifex/handlers/ecs/tasks.go @@ -13,7 +13,7 @@ import ( "github.com/google/uuid" "github.com/mulgadc/spinifex/spinifex/awserrors" "github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // reservePlacementRetries bounds the CAS retry loop when concurrent RunTask calls @@ -26,13 +26,13 @@ const reservePlacementRetries = 5 // are returned as RunTask failures; already-placed tasks in the same call stay. func (s *Service) RunTask(ctx context.Context, input *ecs.RunTaskInput, accountID string) (*ecs.RunTaskOutput, error) { cluster := clusterShortName(aws.StringValue(input.Cluster)) - kv, err := s.bucket(accountID) + kv, err := s.bucket(ctx, accountID) if err != nil { return nil, err } var clusterRec ClusterRecord - found, err := getJSON(kv, ClusterMetaKey(cluster), &clusterRec) + found, err := getJSON(ctx, kv, ClusterMetaKey(cluster), &clusterRec) if err != nil { return nil, err } @@ -40,7 +40,7 @@ func (s *Service) RunTask(ctx context.Context, input *ecs.RunTaskInput, accountI return nil, errors.New(awserrors.ErrorECSClusterNotFound) } - taskDef, err := s.resolveTaskDef(kv, aws.StringValue(input.TaskDefinition)) + taskDef, err := s.resolveTaskDef(ctx, kv, aws.StringValue(input.TaskDefinition)) if err != nil { return nil, err } @@ -61,7 +61,7 @@ func (s *Service) RunTask(ctx context.Context, input *ecs.RunTaskInput, accountI out := &ecs.RunTaskOutput{} for i := 0; i < count; i++ { taskID := uuid.NewString() - inst, err := s.reservePlacement(kv, cluster, taskID, cpu, mem, gpu, strategy) + inst, err := s.reservePlacement(ctx, kv, cluster, taskID, cpu, mem, gpu, strategy) if err != nil { out.Failures = append(out.Failures, &ecs.Failure{ Reason: aws.String("RESOURCE:placement"), @@ -82,10 +82,10 @@ func (s *Service) RunTask(ctx context.Context, input *ecs.RunTaskInput, accountI } } - if err := putJSON(kv, TaskKey(cluster, taskID), rec); err != nil { + if err := putJSON(ctx, kv, TaskKey(cluster, taskID), rec); err != nil { return nil, err } - if err := s.publishAssign(kv, accountID, cluster, inst.InstanceID, rec, taskDef); err != nil { + if err := s.publishAssign(ctx, kv, accountID, cluster, inst.InstanceID, rec, taskDef); err != nil { slog.ErrorContext(ctx, "ECS RunTask: failed to publish assign", "task", taskID, "instance", inst.InstanceID, "err", err) } out.Tasks = append(out.Tasks, s.taskToAWS(accountID, rec)) @@ -97,9 +97,9 @@ func (s *Service) RunTask(ctx context.Context, input *ecs.RunTaskInput, accountI // identity onto rec. On any failure it rolls back (releases a half-allocated ENI // and the placement reservation) and returns a RunTask failure for this task — // the caller skips it without leaking the ENI or the reserved capacity. -func (s *Service) provisionTaskENI(ctx context.Context, kv nats.KeyValue, accountID, cluster string, rec *TaskRecord, netCfg awsvpcConfig) *ecs.Failure { +func (s *Service) provisionTaskENI(ctx context.Context, kv jetstream.KeyValue, accountID, cluster string, rec *TaskRecord, netCfg awsvpcConfig) *ecs.Failure { rollback := func(reason string, err error) *ecs.Failure { - if rerr := s.releaseReservation(kv, cluster, rec.ContainerInstanceID, rec.TaskID, rec.ReservedCPU, rec.ReservedMemoryMiB, rec.GPU); rerr != nil { + if rerr := s.releaseReservation(ctx, kv, cluster, rec.ContainerInstanceID, rec.TaskID, rec.ReservedCPU, rec.ReservedMemoryMiB, rec.GPU); rerr != nil { slog.ErrorContext(ctx, "ECS RunTask: reservation rollback failed", "task", rec.TaskID, "err", rerr) } return &ecs.Failure{Reason: aws.String(reason), Detail: aws.String(err.Error())} @@ -126,9 +126,9 @@ func (s *Service) provisionTaskENI(ctx context.Context, kv nats.KeyValue, accoun // reservePlacement bin-packs a task onto an ACTIVE instance and commits the // reservation under a KV CAS, retrying on contention. -func (s *Service) reservePlacement(kv nats.KeyValue, cluster, taskID string, cpu, mem, gpu int, strategy string) (*InstanceRecord, error) { +func (s *Service) reservePlacement(ctx context.Context, kv jetstream.KeyValue, cluster, taskID string, cpu, mem, gpu int, strategy string) (*InstanceRecord, error) { for range reservePlacementRetries { - instances, err := s.listInstanceRecords(kv, cluster) + instances, err := s.listInstanceRecords(ctx, kv, cluster) if err != nil { return nil, err } @@ -138,7 +138,7 @@ func (s *Service) reservePlacement(kv nats.KeyValue, cluster, taskID string, cpu } // Re-read the chosen instance for its current revision, then CAS-update. - entry, err := kv.Get(InstanceKey(cluster, chosen.InstanceID)) + entry, err := kv.Get(ctx, InstanceKey(cluster, chosen.InstanceID)) if err != nil { continue } @@ -157,7 +157,7 @@ func (s *Service) reservePlacement(kv nats.KeyValue, cluster, taskID string, cpu if merr != nil { return nil, merr } - if _, uerr := kv.Update(InstanceKey(cluster, live.InstanceID), data, entry.Revision()); uerr != nil { + if _, uerr := kv.Update(ctx, InstanceKey(cluster, live.InstanceID), data, entry.Revision()); uerr != nil { continue // lost the CAS race; retry placement } return &live, nil @@ -194,7 +194,7 @@ func (s *Service) newTaskRecord(accountID, cluster, taskID string, td *TaskDefRe // agent drains it by polling the gateway (PollAssignments) rather than // subscribing to NATS, so the bus stays host-internal. Durable + restart-safe: // an unacked assign survives an agent crash and is re-delivered on the next poll. -func (s *Service) publishAssign(kv nats.KeyValue, accountID, cluster, instanceID string, rec *TaskRecord, td *TaskDefRecord) error { +func (s *Service) publishAssign(ctx context.Context, kv jetstream.KeyValue, accountID, cluster, instanceID string, rec *TaskRecord, td *TaskDefRecord) error { msg := bus.Assign{ AccountID: accountID, ClusterName: cluster, @@ -214,13 +214,13 @@ func (s *Service) publishAssign(kv nats.KeyValue, accountID, cluster, instanceID for _, c := range td.Containers { msg.Containers = append(msg.Containers, c.toAssignContainer()) } - return putJSON(kv, AssignmentKey(cluster, instanceID, rec.TaskID), &msg) + return putJSON(ctx, kv, AssignmentKey(cluster, instanceID, rec.TaskID), &msg) } // DescribeTasks returns task records for the named tasks in a cluster. -func (s *Service) DescribeTasks(_ context.Context, input *ecs.DescribeTasksInput, accountID string) (*ecs.DescribeTasksOutput, error) { +func (s *Service) DescribeTasks(ctx context.Context, input *ecs.DescribeTasksInput, accountID string) (*ecs.DescribeTasksOutput, error) { cluster := clusterShortName(aws.StringValue(input.Cluster)) - kv, err := s.bucket(accountID) + kv, err := s.bucket(ctx, accountID) if err != nil { return nil, err } @@ -228,7 +228,7 @@ func (s *Service) DescribeTasks(_ context.Context, input *ecs.DescribeTasksInput for _, ref := range awsStringSlice(input.Tasks) { taskID := containerInstanceShortID(ref) var rec TaskRecord - found, err := getJSON(kv, TaskKey(cluster, taskID), &rec) + found, err := getJSON(ctx, kv, TaskKey(cluster, taskID), &rec) if err != nil { return nil, err } @@ -242,20 +242,20 @@ func (s *Service) DescribeTasks(_ context.Context, input *ecs.DescribeTasksInput } // ListTasks returns the ARNs of all tasks in a cluster. -func (s *Service) ListTasks(_ context.Context, input *ecs.ListTasksInput, accountID string) (*ecs.ListTasksOutput, error) { +func (s *Service) ListTasks(ctx context.Context, input *ecs.ListTasksInput, accountID string) (*ecs.ListTasksOutput, error) { cluster := clusterShortName(aws.StringValue(input.Cluster)) - kv, err := s.bucket(accountID) + kv, err := s.bucket(ctx, accountID) if err != nil { return nil, err } - keys, err := keysWithPrefix(kv, TasksPrefix(cluster)) + keys, err := keysWithPrefix(ctx, kv, TasksPrefix(cluster)) if err != nil { return nil, err } out := &ecs.ListTasksOutput{} for _, k := range keys { var rec TaskRecord - found, err := getJSON(kv, k, &rec) + found, err := getJSON(ctx, kv, k, &rec) if err != nil { return nil, err } diff --git a/spinifex/handlers/ecs/tasks_gc.go b/spinifex/handlers/ecs/tasks_gc.go index ca7968d8e..ff62e31bd 100644 --- a/spinifex/handlers/ecs/tasks_gc.go +++ b/spinifex/handlers/ecs/tasks_gc.go @@ -1,46 +1,52 @@ package handlers_ecs import ( + "context" "log/slog" "strings" "time" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // sweepStoppedTasks prunes stale STOPPED task records across every ECS account // bucket. Leader-only (scheduler is the single KV writer); runs on the sweep -// tick. Mirrors reap()'s bucket walk. -func (sc *Scheduler) sweepStoppedTasks() { - js, err := sc.nc.JetStream() +// tick. Mirrors reap()'s bucket walk. Returns an error when the account +// enumeration could not be completed, so a pass that saw only part of the fleet +// is reported rather than passing for a clean sweep. +func (sc *Scheduler) sweepStoppedTasks(ctx context.Context) error { + js, err := jetstream.New(sc.nc) if err != nil { - return + return err + } + buckets, err := accountBuckets(ctx, sc.nc) + if err != nil { + return err } now := time.Now().UTC() - for bucket := range js.KeyValueStoreNames() { - if _, ok := accountIDFromBucket(bucket); !ok { - continue - } - kv, err := js.KeyValue(bucket) + for _, bucket := range buckets { + kv, err := js.KeyValue(ctx, bucket.name) if err != nil { + slog.Error("ECS sweep: open bucket failed", "bucket", bucket.name, "err", err) continue } - pruned, serr := sc.svc.sweepStoppedBucket(kv, now, stoppedTaskRetention) + pruned, serr := sc.svc.sweepStoppedBucket(ctx, kv, now, stoppedTaskRetention) if serr != nil { - slog.Error("ECS sweep: bucket failed", "bucket", bucket, "err", serr) + slog.Error("ECS sweep: bucket failed", "bucket", bucket.name, "err", serr) continue } if pruned > 0 { - slog.Info("ECS sweep: pruned stale STOPPED tasks", "bucket", bucket, "count", pruned) + slog.Info("ECS sweep: pruned stale STOPPED tasks", "bucket", bucket.name, "count", pruned) } } + return nil } // sweepStoppedBucket deletes task records that have been STOPPED longer than // retention. A task missing its StoppedAt timestamp is never pruned (defensive: // it would otherwise look infinitely old). Returns the number deleted. -func (s *Service) sweepStoppedBucket(kv nats.KeyValue, now time.Time, retention time.Duration) (int, error) { - keys, err := keysWithPrefix(kv, "clusters/") +func (s *Service) sweepStoppedBucket(ctx context.Context, kv jetstream.KeyValue, now time.Time, retention time.Duration) (int, error) { + keys, err := keysWithPrefix(ctx, kv, "clusters/") if err != nil { return 0, err } @@ -50,7 +56,7 @@ func (s *Service) sweepStoppedBucket(kv nats.KeyValue, now time.Time, retention continue } var task TaskRecord - found, gerr := getJSON(kv, k, &task) + found, gerr := getJSON(ctx, kv, k, &task) if gerr != nil || !found { continue } @@ -60,7 +66,7 @@ func (s *Service) sweepStoppedBucket(kv nats.KeyValue, now time.Time, retention if now.Sub(task.StoppedAt) <= retention { continue } - if derr := kv.Delete(k); derr != nil { + if derr := kv.Delete(ctx, k); derr != nil { slog.Warn("ECS sweep: delete failed", "key", k, "err", derr) continue } diff --git a/spinifex/handlers/ecs/tasks_gc_test.go b/spinifex/handlers/ecs/tasks_gc_test.go index 80c78d5ec..3229fcb1c 100644 --- a/spinifex/handlers/ecs/tasks_gc_test.go +++ b/spinifex/handlers/ecs/tasks_gc_test.go @@ -17,7 +17,7 @@ func TestSweepStoppedBucket_PrunesOnlyStale(t *testing.T) { TaskID: id, Cluster: "web", ARN: TaskARN(testRegion, testAccountID, "web", id), LastStatus: status, DesiredStatus: status, StoppedAt: stoppedAt, } - require.NoError(t, putJSON(kv, TaskKey("web", id), &rec)) + require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", id), &rec)) } put("stale", TaskStatusStopped, now.Add(-2*time.Hour)) // older than retention -> prune @@ -25,13 +25,13 @@ func TestSweepStoppedBucket_PrunesOnlyStale(t *testing.T) { put("running", TaskStatusRunning, time.Time{}) // not stopped -> keep put("nostamp", TaskStatusStopped, time.Time{}) // STOPPED but no timestamp -> keep (defensive) - pruned, err := svc.sweepStoppedBucket(kv, now, time.Hour) + pruned, err := svc.sweepStoppedBucket(t.Context(), kv, now, time.Hour) require.NoError(t, err) assert.Equal(t, 1, pruned) exists := func(id string) bool { var rec TaskRecord - found, gerr := getJSON(kv, TaskKey("web", id), &rec) + found, gerr := getJSON(t.Context(), kv, TaskKey("web", id), &rec) require.NoError(t, gerr) return found } diff --git a/spinifex/handlers/eks/access_entry.go b/spinifex/handlers/eks/access_entry.go index 5719bb7fd..b0368fd60 100644 --- a/spinifex/handlers/eks/access_entry.go +++ b/spinifex/handlers/eks/access_entry.go @@ -1,6 +1,7 @@ package handlers_eks import ( + "context" "encoding/json" "errors" "fmt" @@ -10,7 +11,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/eks" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // Access-entry types. v1 supports STANDARD principals only; node types @@ -45,7 +46,7 @@ func AccessEntryARN(region, accountID, cluster, principalARN string) string { } // PutAccessEntryRecord writes the entry unconditionally. -func PutAccessEntryRecord(kv nats.KeyValue, rec *AccessEntryRecord) error { +func PutAccessEntryRecord(ctx context.Context, kv jetstream.KeyValue, rec *AccessEntryRecord) error { if rec == nil { return errors.New("eks: PutAccessEntryRecord nil record") } @@ -57,20 +58,20 @@ func PutAccessEntryRecord(kv nats.KeyValue, rec *AccessEntryRecord) error { return fmt.Errorf("marshal access entry %s: %w", rec.PrincipalARN, err) } key := AccessEntryKey(rec.ClusterName, rec.PrincipalARN) - if _, err := kv.Put(key, data); err != nil { + if _, err := kv.Put(ctx, key, data); err != nil { return fmt.Errorf("kv put %s: %w", key, err) } return nil } // GetAccessEntryRecord reads one entry. Returns ErrAccessEntryNotFound if absent. -func GetAccessEntryRecord(kv nats.KeyValue, cluster, principalARN string) (*AccessEntryRecord, error) { +func GetAccessEntryRecord(ctx context.Context, kv jetstream.KeyValue, cluster, principalARN string) (*AccessEntryRecord, error) { if cluster == "" || principalARN == "" { return nil, errors.New("eks: GetAccessEntryRecord empty cluster or principal ARN") } - entry, err := kv.Get(AccessEntryKey(cluster, principalARN)) + entry, err := kv.Get(ctx, AccessEntryKey(cluster, principalARN)) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil, ErrAccessEntryNotFound } return nil, fmt.Errorf("kv get access entry: %w", err) @@ -83,13 +84,13 @@ func GetAccessEntryRecord(kv nats.KeyValue, cluster, principalARN string) (*Acce } // ListAccessEntryRecords returns all access entries under a cluster, sorted by principal ARN. -func ListAccessEntryRecords(kv nats.KeyValue, cluster string) ([]*AccessEntryRecord, error) { +func ListAccessEntryRecords(ctx context.Context, kv jetstream.KeyValue, cluster string) ([]*AccessEntryRecord, error) { if cluster == "" { return nil, errors.New("eks: ListAccessEntryRecords empty cluster") } - keys, err := kv.Keys() + keys, err := kv.Keys(ctx) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return nil, nil } return nil, fmt.Errorf("kv keys: %w", err) @@ -100,9 +101,9 @@ func ListAccessEntryRecords(kv nats.KeyValue, cluster string) ([]*AccessEntryRec if !strings.HasPrefix(k, prefix) { continue } - entry, err := kv.Get(k) + entry, err := kv.Get(ctx, k) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { continue } return nil, fmt.Errorf("kv get %s: %w", k, err) @@ -118,15 +119,15 @@ func ListAccessEntryRecords(kv nats.KeyValue, cluster string) ([]*AccessEntryRec } // DeleteAccessEntryRecord removes one entry; returns ErrAccessEntryNotFound if absent. -func DeleteAccessEntryRecord(kv nats.KeyValue, cluster, principalARN string) error { +func DeleteAccessEntryRecord(ctx context.Context, kv jetstream.KeyValue, cluster, principalARN string) error { key := AccessEntryKey(cluster, principalARN) - if _, err := kv.Get(key); err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if _, err := kv.Get(ctx, key); err != nil { + if errors.Is(err, jetstream.ErrKeyNotFound) { return ErrAccessEntryNotFound } return fmt.Errorf("kv get %s: %w", key, err) } - if err := kv.Delete(key); err != nil { + if err := kv.Delete(ctx, key); err != nil { return fmt.Errorf("kv delete %s: %w", key, err) } return nil @@ -134,12 +135,12 @@ func DeleteAccessEntryRecord(kv nats.KeyValue, cluster, principalARN string) err // casUpdateAccessEntry does a revision-checked read-modify-write. // mutate returns true when a field changed. Returns ErrAccessEntryNotFound if absent. -func casUpdateAccessEntry(kv nats.KeyValue, cluster, principalARN string, mutate func(*AccessEntryRecord) bool) (*AccessEntryRecord, error) { +func casUpdateAccessEntry(ctx context.Context, kv jetstream.KeyValue, cluster, principalARN string, mutate func(*AccessEntryRecord) bool) (*AccessEntryRecord, error) { key := AccessEntryKey(cluster, principalARN) for range maxClusterStateCASRetries { - entry, err := kv.Get(key) + entry, err := kv.Get(ctx, key) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil, ErrAccessEntryNotFound } return nil, fmt.Errorf("kv get %s: %w", key, err) @@ -155,11 +156,11 @@ func casUpdateAccessEntry(kv nats.KeyValue, cluster, principalARN string, mutate if err != nil { return nil, fmt.Errorf("marshal access entry %s: %w", principalARN, err) } - _, err = kv.Update(key, data, entry.Revision()) + _, err = kv.Update(ctx, key, data, entry.Revision()) if err == nil { return &rec, nil } - if errors.Is(err, nats.ErrKeyExists) { + if errors.Is(err, jetstream.ErrKeyExists) { continue } return nil, fmt.Errorf("kv update %s: %w", key, err) diff --git a/spinifex/handlers/eks/access_entry_test.go b/spinifex/handlers/eks/access_entry_test.go index 882609c28..1d7006dec 100644 --- a/spinifex/handlers/eks/access_entry_test.go +++ b/spinifex/handlers/eks/access_entry_test.go @@ -84,12 +84,12 @@ func TestAssociatedPolicyToAWS_ScopeNamespaces(t *testing.T) { // The record-store guard clauses reject malformed input before touching the KV, // so a nil handle is enough to exercise them. func TestAccessEntryRecordGuards(t *testing.T) { - require.Error(t, PutAccessEntryRecord(nil, nil)) - require.Error(t, PutAccessEntryRecord(nil, &AccessEntryRecord{PrincipalARN: "arn:aws:iam::000000000001:user/admin"})) + require.Error(t, PutAccessEntryRecord(t.Context(), nil, nil)) + require.Error(t, PutAccessEntryRecord(t.Context(), nil, &AccessEntryRecord{PrincipalARN: "arn:aws:iam::000000000001:user/admin"})) - _, err := GetAccessEntryRecord(nil, "", "arn:aws:iam::000000000001:user/admin") + _, err := GetAccessEntryRecord(t.Context(), nil, "", "arn:aws:iam::000000000001:user/admin") require.Error(t, err) - _, err = ListAccessEntryRecords(nil, "") + _, err = ListAccessEntryRecords(t.Context(), nil, "") require.Error(t, err) } diff --git a/spinifex/handlers/eks/addon_installer.go b/spinifex/handlers/eks/addon_installer.go index f48257eef..0063c3067 100644 --- a/spinifex/handlers/eks/addon_installer.go +++ b/spinifex/handlers/eks/addon_installer.go @@ -1,11 +1,13 @@ package handlers_eks import ( + "context" "encoding/json" "errors" "fmt" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // AddonInstaller delivers and removes a managed add-on's Kubernetes manifests. @@ -14,9 +16,9 @@ import ( type AddonInstaller interface { // Install stages the add-on manifest. Does not transition to ACTIVE — that // is gated on the cluster's state report confirming pods are ready. - Install(accountID, cluster string, rec *AddonRecord) error + Install(ctx context.Context, accountID, cluster string, rec *AddonRecord) error // Uninstall removes the add-on's manifests from the cluster. - Uninstall(accountID, cluster, addon string) error + Uninstall(ctx context.Context, accountID, cluster, addon string) error } // StagedAddonManifest is the artifact the VM-side delivery transport consumes. @@ -47,22 +49,22 @@ func newStagingInstaller(nc *nats.Conn, clusterSize int) *stagingInstaller { return &stagingInstaller{nc: nc, clusterSize: clusterSize} } -func (i *stagingInstaller) acctKV(accountID string) (nats.KeyValue, error) { +func (i *stagingInstaller) acctKV(ctx context.Context, accountID string) (jetstream.KeyValue, error) { if i.nc == nil { return nil, errors.New("eks: stagingInstaller nil NATS connection") } - js, err := i.nc.JetStream() + js, err := jetstream.New(i.nc) if err != nil { return nil, fmt.Errorf("jetstream: %w", err) } - return GetOrCreateAccountBucket(js, accountID, max(i.clusterSize, 1)) + return GetOrCreateAccountBucket(ctx, js, accountID, max(i.clusterSize, 1)) } -func (i *stagingInstaller) Install(accountID, cluster string, rec *AddonRecord) error { +func (i *stagingInstaller) Install(ctx context.Context, accountID, cluster string, rec *AddonRecord) error { if rec == nil { return errors.New("eks: stagingInstaller Install nil record") } - kv, err := i.acctKV(accountID) + kv, err := i.acctKV(ctx, accountID) if err != nil { return err } @@ -77,19 +79,19 @@ func (i *stagingInstaller) Install(accountID, cluster string, rec *AddonRecord) return fmt.Errorf("marshal staged manifest %s: %w", rec.AddonName, err) } key := AddonManifestKey(cluster, rec.AddonName) - if _, err := kv.Put(key, data); err != nil { + if _, err := kv.Put(ctx, key, data); err != nil { return fmt.Errorf("kv put %s: %w", key, err) } return nil } -func (i *stagingInstaller) Uninstall(accountID, cluster, addon string) error { - kv, err := i.acctKV(accountID) +func (i *stagingInstaller) Uninstall(ctx context.Context, accountID, cluster, addon string) error { + kv, err := i.acctKV(ctx, accountID) if err != nil { return err } key := AddonManifestKey(cluster, addon) - if err := kv.Delete(key); err != nil && !errors.Is(err, nats.ErrKeyNotFound) { + if err := kv.Delete(ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { return fmt.Errorf("kv delete %s: %w", key, err) } return nil diff --git a/spinifex/handlers/eks/addon_state.go b/spinifex/handlers/eks/addon_state.go index e27f07fc6..26d986b0f 100644 --- a/spinifex/handlers/eks/addon_state.go +++ b/spinifex/handlers/eks/addon_state.go @@ -1,6 +1,7 @@ package handlers_eks import ( + "context" "encoding/json" "errors" "fmt" @@ -8,7 +9,7 @@ import ( "strings" "time" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // AddonStatus mirrors the AWS EKS addon.status enum verbatim. @@ -48,7 +49,7 @@ func AddonARN(region, accountID, cluster, addon string) string { } // PutAddonRecord writes the record unconditionally. -func PutAddonRecord(kv nats.KeyValue, cluster string, rec *AddonRecord) error { +func PutAddonRecord(ctx context.Context, kv jetstream.KeyValue, cluster string, rec *AddonRecord) error { if rec == nil { return errors.New("eks: PutAddonRecord nil record") } @@ -60,20 +61,20 @@ func PutAddonRecord(kv nats.KeyValue, cluster string, rec *AddonRecord) error { return fmt.Errorf("marshal addon %s: %w", rec.AddonName, err) } key := AddonKey(cluster, rec.AddonName) - if _, err := kv.Put(key, data); err != nil { + if _, err := kv.Put(ctx, key, data); err != nil { return fmt.Errorf("kv put %s: %w", key, err) } return nil } // GetAddonRecord reads one record. Returns ErrAddonNotFound if absent. -func GetAddonRecord(kv nats.KeyValue, cluster, addon string) (*AddonRecord, error) { +func GetAddonRecord(ctx context.Context, kv jetstream.KeyValue, cluster, addon string) (*AddonRecord, error) { if cluster == "" || addon == "" { return nil, errors.New("eks: GetAddonRecord empty cluster or addon name") } - entry, err := kv.Get(AddonKey(cluster, addon)) + entry, err := kv.Get(ctx, AddonKey(cluster, addon)) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil, ErrAddonNotFound } return nil, fmt.Errorf("kv get addon: %w", err) @@ -87,13 +88,13 @@ func GetAddonRecord(kv nats.KeyValue, cluster, addon string) (*AddonRecord, erro // ListAddonRecords returns all add-on records under a cluster, sorted by name. // Staged-manifest sub-keys (one extra path segment) are skipped. -func ListAddonRecords(kv nats.KeyValue, cluster string) ([]*AddonRecord, error) { +func ListAddonRecords(ctx context.Context, kv jetstream.KeyValue, cluster string) ([]*AddonRecord, error) { if cluster == "" { return nil, errors.New("eks: ListAddonRecords empty cluster") } - keys, err := kv.Keys() + keys, err := kv.Keys(ctx) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return nil, nil } return nil, fmt.Errorf("kv keys: %w", err) @@ -108,9 +109,9 @@ func ListAddonRecords(kv nats.KeyValue, cluster string) ([]*AddonRecord, error) if strings.Contains(strings.TrimPrefix(k, prefix), "/") { continue } - entry, err := kv.Get(k) + entry, err := kv.Get(ctx, k) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { continue } return nil, fmt.Errorf("kv get %s: %w", k, err) @@ -126,20 +127,20 @@ func ListAddonRecords(kv nats.KeyValue, cluster string) ([]*AddonRecord, error) } // DeleteAddonRecord removes one record and its staged manifest. Returns ErrAddonNotFound if absent. -func DeleteAddonRecord(kv nats.KeyValue, cluster, addon string) error { +func DeleteAddonRecord(ctx context.Context, kv jetstream.KeyValue, cluster, addon string) error { key := AddonKey(cluster, addon) - if _, err := kv.Get(key); err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if _, err := kv.Get(ctx, key); err != nil { + if errors.Is(err, jetstream.ErrKeyNotFound) { return ErrAddonNotFound } return fmt.Errorf("kv get %s: %w", key, err) } - if err := kv.Delete(key); err != nil { + if err := kv.Delete(ctx, key); err != nil { return fmt.Errorf("kv delete %s: %w", key, err) } // Best-effort: drop the staged manifest so a re-create starts clean. manifestKey := AddonManifestKey(cluster, addon) - if err := kv.Delete(manifestKey); err != nil && !errors.Is(err, nats.ErrKeyNotFound) { + if err := kv.Delete(ctx, manifestKey); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { return fmt.Errorf("kv delete %s: %w", manifestKey, err) } return nil @@ -147,12 +148,12 @@ func DeleteAddonRecord(kv nats.KeyValue, cluster, addon string) error { // casUpdateAddon does a revision-checked read-modify-write. // mutate returns true when a field changed. Returns ErrAddonNotFound if absent. -func casUpdateAddon(kv nats.KeyValue, cluster, addon string, mutate func(*AddonRecord) bool) (*AddonRecord, error) { +func casUpdateAddon(ctx context.Context, kv jetstream.KeyValue, cluster, addon string, mutate func(*AddonRecord) bool) (*AddonRecord, error) { key := AddonKey(cluster, addon) for range maxClusterStateCASRetries { - entry, err := kv.Get(key) + entry, err := kv.Get(ctx, key) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil, ErrAddonNotFound } return nil, fmt.Errorf("kv get %s: %w", key, err) @@ -168,11 +169,11 @@ func casUpdateAddon(kv nats.KeyValue, cluster, addon string, mutate func(*AddonR if err != nil { return nil, fmt.Errorf("marshal addon %s: %w", addon, err) } - _, err = kv.Update(key, data, entry.Revision()) + _, err = kv.Update(ctx, key, data, entry.Revision()) if err == nil { return &rec, nil } - if errors.Is(err, nats.ErrKeyExists) { + if errors.Is(err, jetstream.ErrKeyExists) { continue } return nil, fmt.Errorf("kv update %s: %w", key, err) diff --git a/spinifex/handlers/eks/addon_state_test.go b/spinifex/handlers/eks/addon_state_test.go index 709e7a10a..445d400e8 100644 --- a/spinifex/handlers/eks/addon_state_test.go +++ b/spinifex/handlers/eks/addon_state_test.go @@ -4,7 +4,7 @@ import ( "testing" "time" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -22,14 +22,14 @@ func sampleAddonRecord(name string) *AddonRecord { } func TestAddonRecordGuards(t *testing.T) { - require.Error(t, PutAddonRecord(nil, "c1", nil)) - require.Error(t, PutAddonRecord(nil, "", sampleAddonRecord("x"))) - require.Error(t, PutAddonRecord(nil, "c1", &AddonRecord{})) + require.Error(t, PutAddonRecord(t.Context(), nil, "c1", nil)) + require.Error(t, PutAddonRecord(t.Context(), nil, "", sampleAddonRecord("x"))) + require.Error(t, PutAddonRecord(t.Context(), nil, "c1", &AddonRecord{})) - _, err := GetAddonRecord(nil, "", "x") + _, err := GetAddonRecord(t.Context(), nil, "", "x") require.Error(t, err) - _, err = ListAddonRecords(nil, "") + _, err = ListAddonRecords(t.Context(), nil, "") require.Error(t, err) } @@ -38,9 +38,9 @@ func TestAddonRecord_RoundTrip(t *testing.T) { in := sampleAddonRecord("aws-load-balancer-controller") in.Tags = map[string]string{"team": "platform"} - require.NoError(t, PutAddonRecord(kv, "c1", in)) + require.NoError(t, PutAddonRecord(t.Context(), kv, "c1", in)) - got, err := GetAddonRecord(kv, "c1", in.AddonName) + got, err := GetAddonRecord(t.Context(), kv, "c1", in.AddonName) require.NoError(t, err) assert.Equal(t, in.AddonName, got.AddonName) assert.Equal(t, in.AddonVersion, got.AddonVersion) @@ -50,20 +50,20 @@ func TestAddonRecord_RoundTrip(t *testing.T) { func TestGetAddonRecord_NotFound(t *testing.T) { kv := newClusterStateTestKV(t) - _, err := GetAddonRecord(kv, "c1", "ghost") + _, err := GetAddonRecord(t.Context(), kv, "c1", "ghost") assert.ErrorIs(t, err, ErrAddonNotFound) } func TestListAddonRecords_SkipsManifestSubKeys(t *testing.T) { kv := newClusterStateTestKV(t) - require.NoError(t, PutAddonRecord(kv, "c1", sampleAddonRecord("coredns"))) - require.NoError(t, PutAddonRecord(kv, "c1", sampleAddonRecord("aws-load-balancer-controller"))) + require.NoError(t, PutAddonRecord(t.Context(), kv, "c1", sampleAddonRecord("coredns"))) + require.NoError(t, PutAddonRecord(t.Context(), kv, "c1", sampleAddonRecord("aws-load-balancer-controller"))) // A staged manifest sub-key must not be mistaken for a record. - _, err := kv.Put(AddonManifestKey("c1", "coredns"), []byte(`{"addonName":"coredns"}`)) + _, err := kv.Put(t.Context(), AddonManifestKey("c1", "coredns"), []byte(`{"addonName":"coredns"}`)) require.NoError(t, err) - recs, err := ListAddonRecords(kv, "c1") + recs, err := ListAddonRecords(t.Context(), kv, "c1") require.NoError(t, err) require.Len(t, recs, 2) // Sorted by name. @@ -73,36 +73,36 @@ func TestListAddonRecords_SkipsManifestSubKeys(t *testing.T) { func TestDeleteAddonRecord_RemovesRecordAndManifest(t *testing.T) { kv := newClusterStateTestKV(t) - require.NoError(t, PutAddonRecord(kv, "c1", sampleAddonRecord("coredns"))) - _, err := kv.Put(AddonManifestKey("c1", "coredns"), []byte(`{}`)) + require.NoError(t, PutAddonRecord(t.Context(), kv, "c1", sampleAddonRecord("coredns"))) + _, err := kv.Put(t.Context(), AddonManifestKey("c1", "coredns"), []byte(`{}`)) require.NoError(t, err) - require.NoError(t, DeleteAddonRecord(kv, "c1", "coredns")) + require.NoError(t, DeleteAddonRecord(t.Context(), kv, "c1", "coredns")) - _, err = GetAddonRecord(kv, "c1", "coredns") + _, err = GetAddonRecord(t.Context(), kv, "c1", "coredns") assert.ErrorIs(t, err, ErrAddonNotFound) - _, err = kv.Get(AddonManifestKey("c1", "coredns")) - assert.ErrorIs(t, err, nats.ErrKeyNotFound) + _, err = kv.Get(t.Context(), AddonManifestKey("c1", "coredns")) + assert.ErrorIs(t, err, jetstream.ErrKeyNotFound) // Deleting again is a not-found. - assert.ErrorIs(t, DeleteAddonRecord(kv, "c1", "coredns"), ErrAddonNotFound) + assert.ErrorIs(t, DeleteAddonRecord(t.Context(), kv, "c1", "coredns"), ErrAddonNotFound) } func TestCasUpdateAddon(t *testing.T) { kv := newClusterStateTestKV(t) - require.NoError(t, PutAddonRecord(kv, "c1", sampleAddonRecord("coredns"))) + require.NoError(t, PutAddonRecord(t.Context(), kv, "c1", sampleAddonRecord("coredns"))) - updated, err := casUpdateAddon(kv, "c1", "coredns", func(r *AddonRecord) bool { + updated, err := casUpdateAddon(t.Context(), kv, "c1", "coredns", func(r *AddonRecord) bool { r.Status = AddonStatusActive return true }) require.NoError(t, err) assert.Equal(t, AddonStatusActive, updated.Status) - got, err := GetAddonRecord(kv, "c1", "coredns") + got, err := GetAddonRecord(t.Context(), kv, "c1", "coredns") require.NoError(t, err) assert.Equal(t, AddonStatusActive, got.Status) - _, err = casUpdateAddon(kv, "c1", "ghost", func(*AddonRecord) bool { return true }) + _, err = casUpdateAddon(t.Context(), kv, "c1", "ghost", func(*AddonRecord) bool { return true }) assert.ErrorIs(t, err, ErrAddonNotFound) } diff --git a/spinifex/handlers/eks/addon_status_reconciler.go b/spinifex/handlers/eks/addon_status_reconciler.go index 5dc422cba..bb8d44d52 100644 --- a/spinifex/handlers/eks/addon_status_reconciler.go +++ b/spinifex/handlers/eks/addon_status_reconciler.go @@ -1,6 +1,7 @@ package handlers_eks import ( + "context" "errors" "log/slog" "time" @@ -11,12 +12,12 @@ import ( // report names an unknown add-on (DeleteAddon already removed the record) or // when the status would not change, so a steady stream of "applied"/"ready" // reports is cheap and idempotent. -func (r *ClusterReconciler) applyAddonStatusReport(report AddonStatusReport) { +func (r *ClusterReconciler) applyAddonStatusReport(ctx context.Context, report AddonStatusReport) { if report.Addon == "" { return } now := time.Now().UTC() - _, err := casUpdateAddon(r.acctKV, r.clusterName, report.Addon, func(rec *AddonRecord) bool { + _, err := casUpdateAddon(ctx, r.acctKV, r.clusterName, report.Addon, func(rec *AddonRecord) bool { next, changed := nextAddonStatus(rec.Status, report.Phase) // Surface the agent's message as the add-on health on failure, and clear // it once the add-on recovers to ACTIVE. diff --git a/spinifex/handlers/eks/addon_status_reconciler_test.go b/spinifex/handlers/eks/addon_status_reconciler_test.go index a51672d05..d667e8ef2 100644 --- a/spinifex/handlers/eks/addon_status_reconciler_test.go +++ b/spinifex/handlers/eks/addon_status_reconciler_test.go @@ -4,17 +4,17 @@ import ( "testing" "time" - "github.com/nats-io/nats.go" + "github.com/mulgadc/spinifex/spinifex/testutil" + "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) // acctKVForTest opens the per-account KV bucket the test service is backed by. -func acctKVForTest(t *testing.T, svc *EKSServiceImpl) nats.KeyValue { +func acctKVForTest(t *testing.T, svc *EKSServiceImpl) jetstream.KeyValue { t.Helper() - js, err := svc.deps.NATSConn.JetStream() - require.NoError(t, err) - kv, err := GetOrCreateAccountBucket(js, testAccountID, 1) + js := testutil.NewJetStream(t, svc.deps.NATSConn) + kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1) require.NoError(t, err) return kv } @@ -56,28 +56,28 @@ func TestApplyAddonStatusReport(t *testing.T) { seed := func(t *testing.T) { t.Helper() now := time.Now().UTC() - require.NoError(t, PutAddonRecord(acctKV, "c1", &AddonRecord{ + require.NoError(t, PutAddonRecord(t.Context(), acctKV, "c1", &AddonRecord{ AddonName: "coredns", AddonVersion: "1.11.1", Status: AddonStatusCreating, Arn: AddonARN("us-east-1", testAccountID, "c1", "coredns"), CreatedAt: now, ModifiedAt: now, })) } get := func(t *testing.T) *AddonRecord { t.Helper() - rec, err := GetAddonRecord(acctKV, "c1", "coredns") + rec, err := GetAddonRecord(t.Context(), acctKV, "c1", "coredns") require.NoError(t, err) return rec } t.Run("ready flips creating to active and clears health", func(t *testing.T) { seed(t) - r.applyAddonStatusReport(AddonStatusReport{Addon: "coredns", Phase: AddonPhaseReady, Message: "ignored", TS: time.Now().Unix()}) + r.applyAddonStatusReport(t.Context(), AddonStatusReport{Addon: "coredns", Phase: AddonPhaseReady, Message: "ignored", TS: time.Now().Unix()}) rec := get(t) assert.Equal(t, AddonStatusActive, rec.Status) assert.Empty(t, rec.Health) }) t.Run("failed on active goes degraded and records message", func(t *testing.T) { - r.applyAddonStatusReport(AddonStatusReport{Addon: "coredns", Phase: AddonPhaseFailed, Message: "rollout stalled", TS: time.Now().Unix()}) + r.applyAddonStatusReport(t.Context(), AddonStatusReport{Addon: "coredns", Phase: AddonPhaseFailed, Message: "rollout stalled", TS: time.Now().Unix()}) rec := get(t) assert.Equal(t, AddonStatusDegraded, rec.Status) assert.Equal(t, "rollout stalled", rec.Health) @@ -86,17 +86,17 @@ func TestApplyAddonStatusReport(t *testing.T) { t.Run("applied is a no-op", func(t *testing.T) { seed(t) before := get(t) - r.applyAddonStatusReport(AddonStatusReport{Addon: "coredns", Phase: AddonPhaseApplied, TS: time.Now().Unix()}) + r.applyAddonStatusReport(t.Context(), AddonStatusReport{Addon: "coredns", Phase: AddonPhaseApplied, TS: time.Now().Unix()}) after := get(t) assert.Equal(t, AddonStatusCreating, after.Status) assert.Equal(t, before.ModifiedAt, after.ModifiedAt, "no-op must not bump ModifiedAt") }) t.Run("unknown addon is a no-op (no panic)", func(t *testing.T) { - r.applyAddonStatusReport(AddonStatusReport{Addon: "not-installed", Phase: AddonPhaseReady, TS: time.Now().Unix()}) + r.applyAddonStatusReport(t.Context(), AddonStatusReport{Addon: "not-installed", Phase: AddonPhaseReady, TS: time.Now().Unix()}) }) t.Run("empty addon is ignored", func(t *testing.T) { - r.applyAddonStatusReport(AddonStatusReport{Phase: AddonPhaseReady}) + r.applyAddonStatusReport(t.Context(), AddonStatusReport{Phase: AddonPhaseReady}) }) } diff --git a/spinifex/handlers/eks/addons.go b/spinifex/handlers/eks/addons.go index 42d07adc9..b40f75e6b 100644 --- a/spinifex/handlers/eks/addons.go +++ b/spinifex/handlers/eks/addons.go @@ -13,7 +13,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/eks" "github.com/mulgadc/spinifex/spinifex/awserrors" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // ListAddons returns the names of every managed add-on installed on a cluster. @@ -22,11 +22,11 @@ func (s *EKSServiceImpl) ListAddons(ctx context.Context, input *eks.ListAddonsIn return nil, errors.New(awserrors.ErrorInvalidParameterValue) } cluster := aws.StringValue(input.ClusterName) - acctKV, err := s.acctKVForCluster(accountID, cluster) + acctKV, err := s.acctKVForCluster(ctx, accountID, cluster) if err != nil { return nil, err } - recs, err := ListAddonRecords(acctKV, cluster) + recs, err := ListAddonRecords(ctx, acctKV, cluster) if err != nil { return nil, err } @@ -80,11 +80,11 @@ func (s *EKSServiceImpl) CreateAddon(ctx context.Context, input *eks.CreateAddon } else if !spec.supportsVersion(version) { return nil, errors.New(awserrors.ErrorInvalidParameterValue) } - acctKV, err := s.acctKVForCluster(accountID, cluster) + acctKV, err := s.acctKVForCluster(ctx, accountID, cluster) if err != nil { return nil, err } - if _, err := GetAddonRecord(acctKV, cluster, addonName); err == nil { + if _, err := GetAddonRecord(ctx, acctKV, cluster, addonName); err == nil { return nil, errors.New(awserrors.ErrorEKSResourceInUse) } else if !errors.Is(err, ErrAddonNotFound) { return nil, err @@ -101,11 +101,11 @@ func (s *EKSServiceImpl) CreateAddon(ctx context.Context, input *eks.CreateAddon CreatedAt: now, ModifiedAt: now, } - if err := PutAddonRecord(acctKV, cluster, rec); err != nil { + if err := PutAddonRecord(ctx, acctKV, cluster, rec); err != nil { return nil, err } - if err := s.addonInstaller().Install(accountID, cluster, rec); err != nil { - s.markAddonFailed(acctKV, cluster, addonName, err) + if err := s.addonInstaller().Install(ctx, accountID, cluster, rec); err != nil { + s.markAddonFailed(ctx, acctKV, cluster, addonName, err) return nil, err } return &eks.CreateAddonOutput{Addon: addonRecordToAWS(cluster, rec)}, nil @@ -135,13 +135,13 @@ func (s *EKSServiceImpl) ListStagedAddonManifests(ctx context.Context, input *Li if input == nil || input.ClusterName == "" { return nil, errors.New(awserrors.ErrorInvalidParameterValue) } - acctKV, err := s.acctKVForCluster(accountID, input.ClusterName) + acctKV, err := s.acctKVForCluster(ctx, accountID, input.ClusterName) if err != nil { return nil, err } - keys, err := acctKV.Keys() + keys, err := acctKV.Keys(ctx) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return &ListStagedAddonManifestsOutput{Manifests: []StagedAddonManifest{}}, nil } return nil, err @@ -152,9 +152,9 @@ func (s *EKSServiceImpl) ListStagedAddonManifests(ctx context.Context, input *Li if !strings.HasPrefix(k, prefix) || !strings.HasSuffix(k, "/manifest") { continue } - entry, err := acctKV.Get(k) + entry, err := acctKV.Get(ctx, k) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { continue } return nil, err @@ -184,11 +184,11 @@ func (s *EKSServiceImpl) DescribeAddon(ctx context.Context, input *eks.DescribeA } cluster := aws.StringValue(input.ClusterName) addonName := aws.StringValue(input.AddonName) - acctKV, err := s.acctKVForCluster(accountID, cluster) + acctKV, err := s.acctKVForCluster(ctx, accountID, cluster) if err != nil { return nil, err } - rec, err := GetAddonRecord(acctKV, cluster, addonName) + rec, err := GetAddonRecord(ctx, acctKV, cluster, addonName) if err != nil { if errors.Is(err, ErrAddonNotFound) { return nil, errors.New(awserrors.ErrorEKSResourceNotFound) @@ -205,7 +205,7 @@ func (s *EKSServiceImpl) UpdateAddon(ctx context.Context, input *eks.UpdateAddon } cluster := aws.StringValue(input.ClusterName) addonName := aws.StringValue(input.AddonName) - acctKV, err := s.acctKVForCluster(accountID, cluster) + acctKV, err := s.acctKVForCluster(ctx, accountID, cluster) if err != nil { return nil, err } @@ -217,7 +217,7 @@ func (s *EKSServiceImpl) UpdateAddon(ctx context.Context, input *eks.UpdateAddon } } now := time.Now().UTC() - rec, err := casUpdateAddon(acctKV, cluster, addonName, func(r *AddonRecord) bool { + rec, err := casUpdateAddon(ctx, acctKV, cluster, addonName, func(r *AddonRecord) bool { if v := aws.StringValue(input.AddonVersion); v != "" { r.AddonVersion = v } @@ -237,8 +237,8 @@ func (s *EKSServiceImpl) UpdateAddon(ctx context.Context, input *eks.UpdateAddon } return nil, err } - if err := s.addonInstaller().Install(accountID, cluster, rec); err != nil { - s.markAddonFailed(acctKV, cluster, addonName, err) + if err := s.addonInstaller().Install(ctx, accountID, cluster, rec); err != nil { + s.markAddonFailed(ctx, acctKV, cluster, addonName, err) return nil, err } return &eks.UpdateAddonOutput{Update: &eks.Update{ @@ -256,11 +256,11 @@ func (s *EKSServiceImpl) DeleteAddon(ctx context.Context, input *eks.DeleteAddon } cluster := aws.StringValue(input.ClusterName) addonName := aws.StringValue(input.AddonName) - acctKV, err := s.acctKVForCluster(accountID, cluster) + acctKV, err := s.acctKVForCluster(ctx, accountID, cluster) if err != nil { return nil, err } - rec, err := GetAddonRecord(acctKV, cluster, addonName) + rec, err := GetAddonRecord(ctx, acctKV, cluster, addonName) if err != nil { if errors.Is(err, ErrAddonNotFound) { return nil, errors.New(awserrors.ErrorEKSResourceNotFound) @@ -270,10 +270,10 @@ func (s *EKSServiceImpl) DeleteAddon(ctx context.Context, input *eks.DeleteAddon rec.Status = AddonStatusDeleting rec.ModifiedAt = time.Now().UTC() out := &eks.DeleteAddonOutput{Addon: addonRecordToAWS(cluster, rec)} - if err := s.addonInstaller().Uninstall(accountID, cluster, addonName); err != nil { + if err := s.addonInstaller().Uninstall(ctx, accountID, cluster, addonName); err != nil { return nil, err } - if err := DeleteAddonRecord(acctKV, cluster, addonName); err != nil { + if err := DeleteAddonRecord(ctx, acctKV, cluster, addonName); err != nil { if errors.Is(err, ErrAddonNotFound) { return nil, errors.New(awserrors.ErrorEKSResourceNotFound) } @@ -334,9 +334,9 @@ func (s *EKSServiceImpl) addonInstaller() AddonInstaller { } // markAddonFailed best-effort flips a record to CREATE_FAILED with the error reason. -func (s *EKSServiceImpl) markAddonFailed(acctKV nats.KeyValue, cluster, addon string, cause error) { +func (s *EKSServiceImpl) markAddonFailed(ctx context.Context, acctKV jetstream.KeyValue, cluster, addon string, cause error) { now := time.Now().UTC() - if _, err := casUpdateAddon(acctKV, cluster, addon, func(r *AddonRecord) bool { + if _, err := casUpdateAddon(ctx, acctKV, cluster, addon, func(r *AddonRecord) bool { r.Status = AddonStatusCreateFailed r.Health = cause.Error() r.ModifiedAt = now diff --git a/spinifex/handlers/eks/addons_test.go b/spinifex/handlers/eks/addons_test.go index a5f0f315b..0ad681eed 100644 --- a/spinifex/handlers/eks/addons_test.go +++ b/spinifex/handlers/eks/addons_test.go @@ -8,6 +8,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/eks" "github.com/mulgadc/spinifex/spinifex/awserrors" + "github.com/mulgadc/spinifex/spinifex/testutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -22,12 +23,12 @@ type fakeAddonInstaller struct { var _ AddonInstaller = (*fakeAddonInstaller)(nil) -func (f *fakeAddonInstaller) Install(_, _ string, rec *AddonRecord) error { +func (f *fakeAddonInstaller) Install(_ context.Context, _, _ string, rec *AddonRecord) error { f.installs = append(f.installs, rec) return f.installErr } -func (f *fakeAddonInstaller) Uninstall(_, _, addon string) error { +func (f *fakeAddonInstaller) Uninstall(_ context.Context, _, _, addon string) error { f.uninstalls = append(f.uninstalls, addon) return nil } @@ -241,11 +242,10 @@ func TestStagingInstaller_StagesManifest(t *testing.T) { }, testAccountID) require.NoError(t, err) - js, err := svc.deps.NATSConn.JetStream() + js := testutil.NewJetStream(t, svc.deps.NATSConn) + kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1) require.NoError(t, err) - kv, err := GetOrCreateAccountBucket(js, testAccountID, 1) - require.NoError(t, err) - entry, err := kv.Get(AddonManifestKey("c1", albController)) + entry, err := kv.Get(t.Context(), AddonManifestKey("c1", albController)) require.NoError(t, err, "installer must stage a manifest for VM-side delivery") assert.Contains(t, string(entry.Value()), albController) } diff --git a/spinifex/handlers/eks/claim.go b/spinifex/handlers/eks/claim.go index e9468d5ca..76d0e6347 100644 --- a/spinifex/handlers/eks/claim.go +++ b/spinifex/handlers/eks/claim.go @@ -1,10 +1,11 @@ package handlers_eks import ( + "context" "errors" "fmt" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // claimKey atomically creates key with payload. owned is true when this caller @@ -13,15 +14,15 @@ import ( // owned=false, existing=nil so the caller can retry. // This is the shared idempotency primitive: the first caller wins and launches; // duplicates (SDK retries, re-published handlers) lose and never launch. -func claimKey(kv nats.KeyValue, key string, payload []byte) (owned bool, existing []byte, rev uint64, err error) { - if _, cerr := kv.Create(key, payload); cerr == nil { +func claimKey(ctx context.Context, kv jetstream.KeyValue, key string, payload []byte) (owned bool, existing []byte, rev uint64, err error) { + if _, cerr := kv.Create(ctx, key, payload); cerr == nil { return true, nil, 0, nil - } else if !errors.Is(cerr, nats.ErrKeyExists) { + } else if !errors.Is(cerr, jetstream.ErrKeyExists) { return false, nil, 0, fmt.Errorf("kv create %s: %w", key, cerr) } - entry, gerr := kv.Get(key) + entry, gerr := kv.Get(ctx, key) if gerr != nil { - if errors.Is(gerr, nats.ErrKeyNotFound) { + if errors.Is(gerr, jetstream.ErrKeyNotFound) { return false, nil, 0, nil } return false, nil, 0, fmt.Errorf("kv get %s: %w", key, gerr) diff --git a/spinifex/handlers/eks/claim_race_test.go b/spinifex/handlers/eks/claim_race_test.go index 790ed14a4..bdfd301e2 100644 --- a/spinifex/handlers/eks/claim_race_test.go +++ b/spinifex/handlers/eks/claim_race_test.go @@ -52,7 +52,7 @@ func TestCreateCluster_ConcurrentSameNameSingleOwner(t *testing.T) { assert.Equal(t, 1, ok, "exactly one create owns the name") assert.Equal(t, 1, inUse, "the duplicate is rejected ResourceInUse") - meta, err := GetClusterMeta(f.kv, "race") + meta, err := GetClusterMeta(t.Context(), f.kv, "race") require.NoError(t, err) assert.Equal(t, ClusterStatusCreating, meta.Status) } @@ -65,7 +65,7 @@ func TestCreateCluster_ConcurrentReclaimSingleOwner(t *testing.T) { meta := sampleClusterMeta("race") meta.Status = ClusterStatusFailed - require.NoError(t, PutClusterMeta(f.kv, meta)) + require.NoError(t, PutClusterMeta(t.Context(), f.kv, meta)) const n = 2 errs := make([]error, n) @@ -84,7 +84,7 @@ func TestCreateCluster_ConcurrentReclaimSingleOwner(t *testing.T) { assert.Equal(t, 1, ok, "exactly one retry reclaims the FAILED cluster") assert.Equal(t, 1, inUse, "the concurrent retry is rejected") - got, err := GetClusterMeta(f.kv, "race") + got, err := GetClusterMeta(t.Context(), f.kv, "race") require.NoError(t, err) assert.Equal(t, ClusterStatusCreating, got.Status) } diff --git a/spinifex/handlers/eks/cluster_reconciler.go b/spinifex/handlers/eks/cluster_reconciler.go index 2423ffe0d..2f1dbce5c 100644 --- a/spinifex/handlers/eks/cluster_reconciler.go +++ b/spinifex/handlers/eks/cluster_reconciler.go @@ -12,6 +12,7 @@ import ( "time" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) const ( @@ -124,8 +125,8 @@ type HTTPDoer interface { // ClusterReconciler drives one cluster's CREATING → ACTIVE → DELETING lifecycle // under a leader lease. One instance per (accountID, clusterName). type ClusterReconciler struct { - leaderKV nats.KeyValue - acctKV nats.KeyValue + leaderKV jetstream.KeyValue + acctKV jetstream.KeyValue accountID string clusterName string holderID string @@ -312,7 +313,7 @@ func WithEtcdResetRecovery(grace, backoff time.Duration, maxAttempts int) Reconc // NewClusterReconciler constructs a ClusterReconciler. healthURL is the // fully-qualified probe target (e.g. https://nlb-dns:443/healthz) or empty // to skip /healthz probing (CREATING transitions on bootstrap state alone). -func NewClusterReconciler(leaderKV, acctKV nats.KeyValue, accountID, clusterName, holderID, healthURL string, opts ...ReconcilerOption) (*ClusterReconciler, error) { +func NewClusterReconciler(leaderKV, acctKV jetstream.KeyValue, accountID, clusterName, holderID, healthURL string, opts ...ReconcilerOption) (*ClusterReconciler, error) { if leaderKV == nil { return nil, errors.New("eks: NewClusterReconciler nil leaderKV") } @@ -371,16 +372,20 @@ func reconcilerLeaderKey(accountID, clusterName string) string { // AcquireLease attempts a CAS Create on the leader bucket. Returns // (release, true) on success; (nil, false) otherwise. The bucket TTL reaps // stale leases when release does not run. -func (r *ClusterReconciler) AcquireLease() (func(), bool) { +func (r *ClusterReconciler) AcquireLease(ctx context.Context) (func(), bool) { key := reconcilerLeaderKey(r.accountID, r.clusterName) - if _, err := r.leaderKV.Create(key, []byte(r.holderID)); err != nil { + if _, err := r.leaderKV.Create(ctx, key, []byte(r.holderID)); err != nil { slog.Debug("ClusterReconciler: lease held by another holder", "key", key, "holderID", r.holderID, "err", err) return nil, false } slog.Info("ClusterReconciler: lease acquired", "key", key, "holderID", r.holderID) return func() { - if err := r.leaderKV.Delete(key); err != nil { + // The registry cancels the reconciler's context before invoking release, so + // the delete runs on its own context — otherwise every release would fail + // and leave the lease for the TTL to reap. + releaseCtx := context.Background() + if err := r.leaderKV.Delete(releaseCtx, key); err != nil { slog.Warn("ClusterReconciler: lease release failed (TTL will reap)", "key", key, "holderID", r.holderID, "err", err) } @@ -388,9 +393,9 @@ func (r *ClusterReconciler) AcquireLease() (func(), bool) { } // RefreshLease re-asserts ownership via a CAS update. Returns false if another holder won. -func (r *ClusterReconciler) RefreshLease() bool { +func (r *ClusterReconciler) RefreshLease(ctx context.Context) bool { key := reconcilerLeaderKey(r.accountID, r.clusterName) - entry, err := r.leaderKV.Get(key) + entry, err := r.leaderKV.Get(ctx, key) if err != nil { slog.Warn("ClusterReconciler: lease get failed", "key", key, "err", err) return false @@ -400,7 +405,7 @@ func (r *ClusterReconciler) RefreshLease() bool { "key", key, "holderID", r.holderID, "got", string(entry.Value())) return false } - if _, err := r.leaderKV.Update(key, []byte(r.holderID), entry.Revision()); err != nil { + if _, err := r.leaderKV.Update(ctx, key, []byte(r.holderID), entry.Revision()); err != nil { slog.Warn("ClusterReconciler: lease CAS update failed", "key", key, "holderID", r.holderID, "err", err) return false @@ -435,7 +440,7 @@ func (r *ClusterReconciler) Run(ctx context.Context) error { "cluster", r.clusterName, "subject", m.Subject, "err", perr) return } - r.applyAddonStatusReport(report) + r.applyAddonStatusReport(ctx, report) }) if err != nil { return fmt.Errorf("subscribe addon status %s: %w", r.addonStatusSubject, err) @@ -461,7 +466,7 @@ func (r *ClusterReconciler) Run(ctx context.Context) error { case <-ctx.Done(): return ctx.Err() case <-refreshT.C: - if !r.RefreshLease() { + if !r.RefreshLease(ctx) { return ErrReconcilerLeaseLost } case <-reconcileT.C: @@ -483,13 +488,13 @@ func terminalReconcileErr(err error) bool { } func (r *ClusterReconciler) reconcileOnce(ctx context.Context) error { - meta, err := GetClusterMeta(r.acctKV, r.clusterName) + meta, err := GetClusterMeta(ctx, r.acctKV, r.clusterName) if err != nil { return err } switch meta.Status { case ClusterStatusCreating: - ready, reason := r.bootstrapReady(meta) + ready, reason := r.bootstrapReady(ctx, meta) if !ready { // Info, not Debug: this is the per-tick reason a CREATING cluster has // not yet flipped ACTIVE. At Debug it is invisible at the default @@ -497,18 +502,18 @@ func (r *ClusterReconciler) reconcileOnce(ctx context.Context) error { // no diagnosable cause. Logged only during the CREATING window. slog.Info("ClusterReconciler: bootstrap not ready", "cluster", r.clusterName, "reason", reason) - return r.failIfCreateTimedOut(meta, "bootstrap not ready: "+reason) + return r.failIfCreateTimedOut(ctx, meta, "bootstrap not ready: "+reason) } issue, nodeCount, nodegroupReady := r.observe(ctx) if issue != "" { slog.Info("ClusterReconciler: health not ready", "cluster", r.clusterName, "issue", issue) - return r.failIfCreateTimedOut(meta, "healthz not ready: "+issue) + return r.failIfCreateTimedOut(ctx, meta, "healthz not ready: "+issue) } - if err := SetClusterStatus(r.acctKV, r.clusterName, ClusterStatusActive); err != nil { + if err := SetClusterStatus(ctx, r.acctKV, r.clusterName, ClusterStatusActive); err != nil { return err } - if err := SetClusterHealthState(r.acctKV, r.clusterName, "", nodeCount, nodegroupReady); err != nil { + if err := SetClusterHealthState(ctx, r.acctKV, r.clusterName, "", nodeCount, nodegroupReady); err != nil { if errors.Is(err, ErrClusterNotFound) { return ErrClusterNotFound } @@ -522,7 +527,7 @@ func (r *ClusterReconciler) reconcileOnce(ctx context.Context) error { slog.Warn("ClusterReconciler: health failing for ACTIVE cluster", "cluster", r.clusterName, "issue", issue) } - if err := SetClusterHealthState(r.acctKV, r.clusterName, issue, nodeCount, nodegroupReady); err != nil { + if err := SetClusterHealthState(ctx, r.acctKV, r.clusterName, issue, nodeCount, nodegroupReady); err != nil { if errors.Is(err, ErrClusterNotFound) { return ErrClusterNotFound } @@ -627,7 +632,7 @@ func (r *ClusterReconciler) maybeReformEtcdQuorum(ctx context.Context, meta *Clu // re-apply; resetIssued gates it so a steady-healthy cluster never re-bumps // the directive epoch each tick. if r.resetIssued { - r.clearRecoveryDirectives(meta) + r.clearRecoveryDirectives(ctx, meta) r.resetIssued = false } r.resetSince = time.Time{} @@ -684,7 +689,7 @@ func (r *ClusterReconciler) maybeReformEtcdQuorum(ctx context.Context, meta *Clu // consuming an attempt (stopping without the cluster-reset directive would just // reproduce the wedge); a follower failure logs and continues (that member stays // wedged, retried next attempt). - if _, err := StoreRecoveryDirective(r.acctKV, r.clusterName, seed.InstanceID, RecoveryActionClusterReset, "", false); err != nil { + if _, err := StoreRecoveryDirective(ctx, r.acctKV, r.clusterName, seed.InstanceID, RecoveryActionClusterReset, "", false); err != nil { slog.Warn("ClusterReconciler: set cluster-reset directive failed", "cluster", r.clusterName, "seed", seed.InstanceID, "err", err) return @@ -692,7 +697,7 @@ func (r *ClusterReconciler) maybeReformEtcdQuorum(ctx context.Context, meta *Clu r.resetAttempts++ r.resetIssued = true for _, n := range members[1:] { - if _, err := StoreRecoveryDirective(r.acctKV, r.clusterName, n.InstanceID, RecoveryActionWipeRejoin, "", false); err != nil { + if _, err := StoreRecoveryDirective(ctx, r.acctKV, r.clusterName, n.InstanceID, RecoveryActionWipeRejoin, "", false); err != nil { slog.Warn("ClusterReconciler: set wipe-rejoin directive failed", "cluster", r.clusterName, "instanceId", n.InstanceID, "err", err) } @@ -725,9 +730,9 @@ func (r *ClusterReconciler) maybeReformEtcdQuorum(ctx context.Context, meta *Clu // epoch) once a cluster has recovered, so a stale cluster-reset cannot re-apply on // a later reboot. Best-effort: a failure logs and the epoch guard on the guest // still prevents re-application of the already-applied directive. -func (r *ClusterReconciler) clearRecoveryDirectives(meta *ClusterMeta) { +func (r *ClusterReconciler) clearRecoveryDirectives(ctx context.Context, meta *ClusterMeta) { for _, n := range cpMemberNodes(meta) { - if _, err := StoreRecoveryDirective(r.acctKV, r.clusterName, n.InstanceID, RecoveryActionNone, "", false); err != nil { + if _, err := StoreRecoveryDirective(ctx, r.acctKV, r.clusterName, n.InstanceID, RecoveryActionNone, "", false); err != nil { slog.Warn("ClusterReconciler: clear recovery directive failed", "cluster", r.clusterName, "instanceId", n.InstanceID, "err", err) } @@ -925,7 +930,7 @@ func (r *ClusterReconciler) maybeReplaceControlPlaneMember(ctx context.Context, "cluster", r.clusterName, "lost", deadID, "attempt", r.replaceAttempts, "err", err) return } - if err := SwapControlPlaneMember(r.acctKV, r.clusterName, deadID, newNode); err != nil { + if err := SwapControlPlaneMember(ctx, r.acctKV, r.clusterName, deadID, newNode); err != nil { slog.Error("ClusterReconciler: CP replacement launched but meta swap failed (leak risk)", "cluster", r.clusterName, "lost", deadID, "newInstanceId", newNode.InstanceID, "err", err) return @@ -989,7 +994,7 @@ func (r *ClusterReconciler) observe(ctx context.Context) (issue string, nodeCoun // failIfCreateTimedOut marks the cluster FAILED once createTimeout has elapsed // since meta.CreatedAt. Returns nil while still within the window so the next // tick retries. A zero CreatedAt disables the deadline. -func (r *ClusterReconciler) failIfCreateTimedOut(meta *ClusterMeta, reason string) error { +func (r *ClusterReconciler) failIfCreateTimedOut(ctx context.Context, meta *ClusterMeta, reason string) error { if r.createTimeout <= 0 || meta.CreatedAt.IsZero() { return nil } @@ -997,7 +1002,7 @@ func (r *ClusterReconciler) failIfCreateTimedOut(meta *ClusterMeta, reason strin return nil } failReason := fmt.Sprintf("cluster did not become ACTIVE within %s: %s", r.createTimeout, reason) - if err := MarkClusterFailed(r.acctKV, r.clusterName, failReason); err != nil { + if err := MarkClusterFailed(ctx, r.acctKV, r.clusterName, failReason); err != nil { if errors.Is(err, ErrClusterNotFound) { return ErrClusterNotFound } @@ -1011,7 +1016,7 @@ func (r *ClusterReconciler) failIfCreateTimedOut(meta *ClusterMeta, reason strin // bootstrapReady returns true once the four bootstrap KV artifacts are present: // node token, admin kubeconfig, OIDC JWKS verified-marker, and CA on meta. // Uses the verified-marker (not OIDCJWKSKey) so ACTIVE always implies a verified key. -func (r *ClusterReconciler) bootstrapReady(meta *ClusterMeta) (bool, string) { +func (r *ClusterReconciler) bootstrapReady(ctx context.Context, meta *ClusterMeta) (bool, string) { if meta.CertificateAuthorityB64 == "" { return false, "CA absent on meta" } @@ -1021,7 +1026,7 @@ func (r *ClusterReconciler) bootstrapReady(meta *ClusterMeta) (bool, string) { OIDCJWKSVerifiedKey(r.clusterName), } for _, k := range keys { - if _, err := r.acctKV.Get(k); err != nil { + if _, err := r.acctKV.Get(ctx, k); err != nil { return false, fmt.Sprintf("%s missing: %s", k, err) } } diff --git a/spinifex/handlers/eks/cluster_reconciler_recovery_test.go b/spinifex/handlers/eks/cluster_reconciler_recovery_test.go index 53149e7f5..58d070940 100644 --- a/spinifex/handlers/eks/cluster_reconciler_recovery_test.go +++ b/spinifex/handlers/eks/cluster_reconciler_recovery_test.go @@ -181,8 +181,8 @@ func TestMaybeRecoverCP_StateQueryErrorSkipsRestart(t *testing.T) { func TestActiveBranchRestartsWedgedCP(t *testing.T) { cp := &fakeCPControl{state: "stopped"} r, _, acctKV := newStateReconcilerHarness(t, WithCPInstanceControl(cp)) - require.NoError(t, SetClusterStatus(acctKV, "alpha", ClusterStatusActive)) - require.NoError(t, casUpdateMeta(acctKV, "alpha", func(m *ClusterMeta) bool { + require.NoError(t, SetClusterStatus(t.Context(), acctKV, "alpha", ClusterStatusActive)) + require.NoError(t, casUpdateMeta(t.Context(), acctKV, "alpha", func(m *ClusterMeta) bool { m.ControlPlaneInstanceID = "i-cp" return true })) @@ -191,7 +191,7 @@ func TestActiveBranchRestartsWedgedCP(t *testing.T) { require.NoError(t, r.reconcileOnce(context.Background())) - meta, err := GetClusterMeta(acctKV, "alpha") + meta, err := GetClusterMeta(t.Context(), acctKV, "alpha") require.NoError(t, err) assert.Equal(t, ClusterStatusActive, meta.Status, "status stays AWS-faithful ACTIVE, not a new DEGRADED enum") assert.Contains(t, meta.HealthIssue, "stale", "degradation is reflected in the health field") diff --git a/spinifex/handlers/eks/cluster_reconciler_replace_test.go b/spinifex/handlers/eks/cluster_reconciler_replace_test.go index 468556a0b..99e1c5b34 100644 --- a/spinifex/handlers/eks/cluster_reconciler_replace_test.go +++ b/spinifex/handlers/eks/cluster_reconciler_replace_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -84,10 +84,10 @@ func haMeta(nodes []ControlPlaneNode) *ClusterMeta { // newReplaceReconciler wires a reconciler with the given CP control + provisioner // and stores meta in the account KV so SwapControlPlaneMember can CAS it. -func newReplaceReconciler(t *testing.T, cp CPInstanceControl, prov CPProvisioner, meta *ClusterMeta) (*ClusterReconciler, nats.KeyValue) { +func newReplaceReconciler(t *testing.T, cp CPInstanceControl, prov CPProvisioner, meta *ClusterMeta) (*ClusterReconciler, jetstream.KeyValue) { t.Helper() r, _, acctKV := newStateReconcilerHarness(t, WithCPInstanceControl(cp), WithCPProvisioner(prov)) - require.NoError(t, PutClusterMeta(acctKV, meta)) + require.NoError(t, PutClusterMeta(t.Context(), acctKV, meta)) return r, acctKV } @@ -107,7 +107,7 @@ func TestMaybeReplaceCP_ReplacesLostMemberWithHealthyQuorum(t *testing.T) { assert.Equal(t, "10.0.0.12", prov.lastReq.DeadPeerIP, "replacement is asked to prune the dead member's etcd peer") assert.ElementsMatch(t, []string{"node-a", "node-b"}, prov.lastReq.ExcludeHosts, "excludes live-member hosts from placement") - got, err := GetClusterMeta(acctKV, "alpha") + got, err := GetClusterMeta(t.Context(), acctKV, "alpha") require.NoError(t, err) ids := cpMemberInstanceIDs(got) assert.Contains(t, ids, "i-cp-new", "replacement swapped into meta") @@ -213,7 +213,7 @@ func TestMaybeReplaceCP_ProvisionFailureLeavesDegraded(t *testing.T) { r.maybeReplaceControlPlaneMember(context.Background(), meta, "") assert.Equal(t, 1, prov.calls) - got, err := GetClusterMeta(acctKV, "alpha") + got, err := GetClusterMeta(t.Context(), acctKV, "alpha") require.NoError(t, err) assert.Contains(t, cpMemberInstanceIDs(got), "i-cp2", "meta unchanged when provision fails") } @@ -251,7 +251,7 @@ func TestMaybeReplaceCP_NilProvisionerIsNoop(t *testing.T) { cp := &mapCPControl{states: map[string]string{"i-cp0": "running", "i-cp1": "running", "i-cp2": "terminated"}} meta := haMeta(nodes) r, _, acctKV := newStateReconcilerHarness(t, WithCPInstanceControl(cp)) // no provisioner - require.NoError(t, PutClusterMeta(acctKV, meta)) + require.NoError(t, PutClusterMeta(t.Context(), acctKV, meta)) r.replacingSince = time.Now().Add(-10 * time.Minute) require.NotPanics(t, func() { @@ -329,7 +329,7 @@ func TestActiveBranchReplacesLostCPMember(t *testing.T) { require.NoError(t, r.reconcileOnce(context.Background())) require.Equal(t, 1, prov.calls, "wedged member replaced through the ACTIVE arm") - got, err := GetClusterMeta(acctKV, "alpha") + got, err := GetClusterMeta(t.Context(), acctKV, "alpha") require.NoError(t, err) assert.Equal(t, ClusterStatusActive, got.Status, "status stays AWS-faithful ACTIVE") assert.Contains(t, cpMemberInstanceIDs(got), "i-cp-new") diff --git a/spinifex/handlers/eks/cluster_reconciler_reset_test.go b/spinifex/handlers/eks/cluster_reconciler_reset_test.go index 19dc102eb..bfc7ba94a 100644 --- a/spinifex/handlers/eks/cluster_reconciler_reset_test.go +++ b/spinifex/handlers/eks/cluster_reconciler_reset_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -13,7 +13,7 @@ import ( // newResetReconciler wires a reconciler with the given CP control plus the etcd // quorum-reformation escalation enabled at its defaults (5m grace/backoff, 2 // attempts). The acctKV is real so recovery directives round-trip through KV. -func newResetReconciler(t *testing.T, cp CPInstanceControl) (*ClusterReconciler, nats.KeyValue) { +func newResetReconciler(t *testing.T, cp CPInstanceControl) (*ClusterReconciler, jetstream.KeyValue) { t.Helper() r, _, acctKV := newStateReconcilerHarness(t, WithCPInstanceControl(cp), WithEtcdResetRecovery(0, 0, 0)) return r, acctKV @@ -89,12 +89,12 @@ func TestReformEtcd_EscalatesAfterGrace(t *testing.T) { assert.True(t, r.resetIssued) assert.False(t, r.lastResetAt.IsZero()) - seed, err := LoadRecoveryDirective(acctKV, "alpha", "i-seed") + seed, err := LoadRecoveryDirective(t.Context(), acctKV, "alpha", "i-seed") require.NoError(t, err) assert.Equal(t, RecoveryActionClusterReset, seed.Action, "primary reseeds the etcd quorum") assert.Equal(t, int64(1), seed.Epoch) - follower, err := LoadRecoveryDirective(acctKV, "alpha", "i-1") + follower, err := LoadRecoveryDirective(t.Context(), acctKV, "alpha", "i-1") require.NoError(t, err) assert.Equal(t, RecoveryActionWipeRejoin, follower.Action, "followers wipe and rejoin") assert.Equal(t, int64(1), follower.Epoch) @@ -137,7 +137,7 @@ func TestReformEtcd_RecoveryClearsDirectivesAndClock(t *testing.T) { assert.Zero(t, r.resetAttempts, "recovery resets the attempt count") assert.False(t, r.resetIssued, "one-shot clear consumed") - seed, err := LoadRecoveryDirective(acctKV, "alpha", "i-seed") + seed, err := LoadRecoveryDirective(t.Context(), acctKV, "alpha", "i-seed") require.NoError(t, err) assert.Equal(t, RecoveryActionNone, seed.Action, "stale cluster-reset is cleared on recovery") assert.Equal(t, int64(2), seed.Epoch, "clear advances the epoch past the applied directive") @@ -152,7 +152,7 @@ func TestReformEtcd_SteadyHealthyDoesNotBumpEpoch(t *testing.T) { r.maybeReformEtcdQuorum(context.Background(), meta, "") r.maybeReformEtcdQuorum(context.Background(), meta, "") - d, err := LoadRecoveryDirective(acctKV, "alpha", "i-0") + d, err := LoadRecoveryDirective(t.Context(), acctKV, "alpha", "i-0") require.NoError(t, err) assert.Equal(t, int64(0), d.Epoch, "no directive is written for a never-wedged cluster") assert.Equal(t, RecoveryActionNone, d.Action) diff --git a/spinifex/handlers/eks/cluster_reconciler_state_test.go b/spinifex/handlers/eks/cluster_reconciler_state_test.go index fe85480d9..94f613367 100644 --- a/spinifex/handlers/eks/cluster_reconciler_state_test.go +++ b/spinifex/handlers/eks/cluster_reconciler_state_test.go @@ -8,6 +8,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/testutil" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -16,14 +17,15 @@ import ( // state source (so observe() reads r.latest) and returns the conn + account KV. // It does NOT start a subscription — tests drive r.latest directly for // determinism; the integration test below exercises the real Subscribe path. -func newStateReconcilerHarness(t *testing.T, opts ...ReconcilerOption) (*ClusterReconciler, *nats.Conn, nats.KeyValue) { +func newStateReconcilerHarness(t *testing.T, opts ...ReconcilerOption) (*ClusterReconciler, *nats.Conn, jetstream.KeyValue) { t.Helper() - _, nc, js := testutil.StartTestJetStream(t) - leaderKV, err := InitLeaderBucket(js, 1) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + leaderKV, err := InitLeaderBucket(t.Context(), js, 1) require.NoError(t, err) - acctKV, err := GetOrCreateAccountBucket(js, testAccountID, 1) + acctKV, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1) require.NoError(t, err) - require.NoError(t, PutClusterMeta(acctKV, sampleClusterMeta("alpha"))) + require.NoError(t, PutClusterMeta(t.Context(), acctKV, sampleClusterMeta("alpha"))) base := []ReconcilerOption{WithStateSource(nc, StateSubject(testAccountID, "alpha"))} r, err := NewClusterReconciler(leaderKV, acctKV, testAccountID, "alpha", "holder-1", "", append(base, opts...)...) @@ -43,7 +45,7 @@ func TestClusterReconciler_CreatingTransitionsToActiveOnStateReport(t *testing.T require.NoError(t, r.reconcileOnce(context.Background())) - meta, err := GetClusterMeta(acctKV, "alpha") + meta, err := GetClusterMeta(t.Context(), acctKV, "alpha") require.NoError(t, err) assert.Equal(t, ClusterStatusActive, meta.Status) assert.Equal(t, 3, meta.NodeCount) @@ -58,7 +60,7 @@ func TestClusterReconciler_CreatingStaysWithoutStateReport(t *testing.T) { require.NoError(t, r.reconcileOnce(context.Background())) - meta, err := GetClusterMeta(acctKV, "alpha") + meta, err := GetClusterMeta(t.Context(), acctKV, "alpha") require.NoError(t, err) assert.Equal(t, ClusterStatusCreating, meta.Status, "no state report yet → stays CREATING") } @@ -71,20 +73,20 @@ func TestClusterReconciler_CreatingStaysOnUnhealthyReport(t *testing.T) { require.NoError(t, r.reconcileOnce(context.Background())) - meta, err := GetClusterMeta(acctKV, "alpha") + meta, err := GetClusterMeta(t.Context(), acctKV, "alpha") require.NoError(t, err) assert.Equal(t, ClusterStatusCreating, meta.Status, "unhealthy apiserver → stays CREATING") } func TestClusterReconciler_ActiveRecordsNodeCountAndClearsIssue(t *testing.T) { r, _, acctKV := newStateReconcilerHarness(t) - require.NoError(t, SetClusterStatus(acctKV, "alpha", ClusterStatusActive)) - require.NoError(t, SetClusterHealthState(acctKV, "alpha", "stale", 0, nil)) + require.NoError(t, SetClusterStatus(t.Context(), acctKV, "alpha", ClusterStatusActive)) + require.NoError(t, SetClusterHealthState(t.Context(), acctKV, "alpha", "stale", 0, nil)) r.latest.Store(freshReport("ok", 4)) require.NoError(t, r.reconcileOnce(context.Background())) - meta, err := GetClusterMeta(acctKV, "alpha") + meta, err := GetClusterMeta(t.Context(), acctKV, "alpha") require.NoError(t, err) assert.Equal(t, ClusterStatusActive, meta.Status) assert.Equal(t, 4, meta.NodeCount) @@ -93,12 +95,12 @@ func TestClusterReconciler_ActiveRecordsNodeCountAndClearsIssue(t *testing.T) { func TestClusterReconciler_ActiveFlagsStaleReport(t *testing.T) { r, _, acctKV := newStateReconcilerHarness(t, WithStateStaleAfter(90*time.Second)) - require.NoError(t, SetClusterStatus(acctKV, "alpha", ClusterStatusActive)) + require.NoError(t, SetClusterStatus(t.Context(), acctKV, "alpha", ClusterStatusActive)) r.latest.Store(&ServerStateReport{Healthz: "ok", NodeCount: 2, TS: time.Now().Add(-5 * time.Minute).Unix()}) require.NoError(t, r.reconcileOnce(context.Background())) - meta, err := GetClusterMeta(acctKV, "alpha") + meta, err := GetClusterMeta(t.Context(), acctKV, "alpha") require.NoError(t, err) assert.Contains(t, meta.HealthIssue, "stale", "report older than stale window flags an issue") assert.Equal(t, 2, meta.NodeCount, "last known node count still recorded") @@ -106,7 +108,7 @@ func TestClusterReconciler_ActiveFlagsStaleReport(t *testing.T) { func TestClusterReconciler_ActiveUnhealthyReportSurfacesReason(t *testing.T) { r, _, acctKV := newStateReconcilerHarness(t) - require.NoError(t, SetClusterStatus(acctKV, "alpha", ClusterStatusActive)) + require.NoError(t, SetClusterStatus(t.Context(), acctKV, "alpha", ClusterStatusActive)) r.latest.Store(&ServerStateReport{ Healthz: "fail", NodeCount: 0, TS: time.Now().Unix(), Reason: "readyz:[etcd]; etcd:unreachable; disk:ok", @@ -114,7 +116,7 @@ func TestClusterReconciler_ActiveUnhealthyReportSurfacesReason(t *testing.T) { require.NoError(t, r.reconcileOnce(context.Background())) - meta, err := GetClusterMeta(acctKV, "alpha") + meta, err := GetClusterMeta(t.Context(), acctKV, "alpha") require.NoError(t, err) assert.Equal(t, ClusterStatusActive, meta.Status) assert.Contains(t, meta.HealthIssue, `apiserver healthz="fail"`) @@ -123,12 +125,12 @@ func TestClusterReconciler_ActiveUnhealthyReportSurfacesReason(t *testing.T) { func TestClusterReconciler_ActiveUnhealthyReportWithoutReasonStaysTerse(t *testing.T) { r, _, acctKV := newStateReconcilerHarness(t) - require.NoError(t, SetClusterStatus(acctKV, "alpha", ClusterStatusActive)) + require.NoError(t, SetClusterStatus(t.Context(), acctKV, "alpha", ClusterStatusActive)) r.latest.Store(freshReport("fail", 0)) require.NoError(t, r.reconcileOnce(context.Background())) - meta, err := GetClusterMeta(acctKV, "alpha") + meta, err := GetClusterMeta(t.Context(), acctKV, "alpha") require.NoError(t, err) assert.Equal(t, `apiserver healthz="fail"`, meta.HealthIssue, "no reason → terse issue, back-compatible with older AMI") } @@ -141,7 +143,7 @@ func TestClusterReconciler_StateReportSubscriptionDrivesActive(t *testing.T) { freshenClusterCreatedAt(t, acctKV) seedBootstrapState(t, acctKV) - release, ok := r.AcquireLease() + release, ok := r.AcquireLease(t.Context()) require.True(t, ok) defer release() @@ -169,7 +171,7 @@ func TestClusterReconciler_StateReportSubscriptionDrivesActive(t *testing.T) { }() require.Eventually(t, func() bool { - meta, err := GetClusterMeta(acctKV, "alpha") + meta, err := GetClusterMeta(t.Context(), acctKV, "alpha") return err == nil && meta.Status == ClusterStatusActive && meta.NodeCount == 3 }, 1500*time.Millisecond, 10*time.Millisecond, "state reports should drive CREATING → ACTIVE") diff --git a/spinifex/handlers/eks/cluster_reconciler_test.go b/spinifex/handlers/eks/cluster_reconciler_test.go index bc2b397a4..7cf701a73 100644 --- a/spinifex/handlers/eks/cluster_reconciler_test.go +++ b/spinifex/handlers/eks/cluster_reconciler_test.go @@ -12,7 +12,7 @@ import ( "time" "github.com/mulgadc/spinifex/spinifex/testutil" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -45,14 +45,15 @@ func (s *stubHTTPDoer) setResponse(status int, err error) { s.err = err } -func newReconcilerHarness(t *testing.T, healthURL string, opts ...ReconcilerOption) (*ClusterReconciler, nats.KeyValue, nats.KeyValue) { +func newReconcilerHarness(t *testing.T, healthURL string, opts ...ReconcilerOption) (*ClusterReconciler, jetstream.KeyValue, jetstream.KeyValue) { t.Helper() - _, _, js := testutil.StartTestJetStream(t) - leaderKV, err := InitLeaderBucket(js, 1) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + leaderKV, err := InitLeaderBucket(t.Context(), js, 1) require.NoError(t, err) - acctKV, err := GetOrCreateAccountBucket(js, testAccountID, 1) + acctKV, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1) require.NoError(t, err) - require.NoError(t, PutClusterMeta(acctKV, sampleClusterMeta("alpha"))) + require.NoError(t, PutClusterMeta(t.Context(), acctKV, sampleClusterMeta("alpha"))) r, err := NewClusterReconciler(leaderKV, acctKV, testAccountID, "alpha", "holder-1", healthURL, opts...) require.NoError(t, err) @@ -60,10 +61,11 @@ func newReconcilerHarness(t *testing.T, healthURL string, opts ...ReconcilerOpti } func TestNewClusterReconciler_EmptyInputsRejected(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) - leaderKV, err := InitLeaderBucket(js, 1) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + leaderKV, err := InitLeaderBucket(t.Context(), js, 1) require.NoError(t, err) - acctKV, err := GetOrCreateAccountBucket(js, testAccountID, 1) + acctKV, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1) require.NoError(t, err) _, err = NewClusterReconciler(nil, acctKV, testAccountID, "alpha", "h", "") @@ -81,13 +83,13 @@ func TestNewClusterReconciler_EmptyInputsRejected(t *testing.T) { func TestClusterReconciler_AcquireLeaseFirstHolderWins(t *testing.T) { r, _, _ := newReconcilerHarness(t, "") - release, ok := r.AcquireLease() + release, ok := r.AcquireLease(t.Context()) require.True(t, ok) require.NotNil(t, release) defer release() // Second AcquireLease from same holder must fail (Create returns KeyExists). - release2, ok2 := r.AcquireLease() + release2, ok2 := r.AcquireLease(t.Context()) assert.False(t, ok2) assert.Nil(t, release2) } @@ -95,13 +97,13 @@ func TestClusterReconciler_AcquireLeaseFirstHolderWins(t *testing.T) { func TestClusterReconciler_AcquireLeaseSecondHolderLoses(t *testing.T) { r1, leaderKV, acctKV := newReconcilerHarness(t, "") - release, ok := r1.AcquireLease() + release, ok := r1.AcquireLease(t.Context()) require.True(t, ok) defer release() r2, err := NewClusterReconciler(leaderKV, acctKV, testAccountID, "alpha", "holder-2", "") require.NoError(t, err) - release2, ok2 := r2.AcquireLease() + release2, ok2 := r2.AcquireLease(t.Context()) assert.False(t, ok2) assert.Nil(t, release2) } @@ -109,54 +111,54 @@ func TestClusterReconciler_AcquireLeaseSecondHolderLoses(t *testing.T) { func TestClusterReconciler_RefreshLeaseFailsAfterStolen(t *testing.T) { r, leaderKV, _ := newReconcilerHarness(t, "") - release, ok := r.AcquireLease() + release, ok := r.AcquireLease(t.Context()) require.True(t, ok) defer release() - assert.True(t, r.RefreshLease(), "refresh should succeed while we own the key") + assert.True(t, r.RefreshLease(t.Context()), "refresh should succeed while we own the key") // Steal the key by overwriting its value via Put. - _, err := leaderKV.Put(reconcilerLeaderKey(testAccountID, "alpha"), []byte("holder-2")) + _, err := leaderKV.Put(t.Context(), reconcilerLeaderKey(testAccountID, "alpha"), []byte("holder-2")) require.NoError(t, err) - assert.False(t, r.RefreshLease(), "refresh should fail after another holder stole the key") + assert.False(t, r.RefreshLease(t.Context()), "refresh should fail after another holder stole the key") } func TestClusterReconciler_RefreshLeaseFailsWhenKeyDeleted(t *testing.T) { r, leaderKV, _ := newReconcilerHarness(t, "") - release, ok := r.AcquireLease() + release, ok := r.AcquireLease(t.Context()) require.True(t, ok) defer release() - require.NoError(t, leaderKV.Delete(reconcilerLeaderKey(testAccountID, "alpha"))) - assert.False(t, r.RefreshLease()) + require.NoError(t, leaderKV.Delete(t.Context(), reconcilerLeaderKey(testAccountID, "alpha"))) + assert.False(t, r.RefreshLease(t.Context())) } // freshenClusterCreatedAt re-stamps the cluster meta's CreatedAt to now so the // reconciler's CREATING deadline is far from expiry. sampleClusterMeta uses a // fixed past date, which would otherwise trip the create-timeout in tests that // assert the cluster stays CREATING. -func freshenClusterCreatedAt(t *testing.T, kv nats.KeyValue) { +func freshenClusterCreatedAt(t *testing.T, kv jetstream.KeyValue) { t.Helper() - meta, err := GetClusterMeta(kv, "alpha") + meta, err := GetClusterMeta(t.Context(), kv, "alpha") require.NoError(t, err) meta.CreatedAt = time.Now().UTC() - require.NoError(t, PutClusterMeta(kv, meta)) + require.NoError(t, PutClusterMeta(t.Context(), kv, meta)) } -func seedBootstrapState(t *testing.T, kv nats.KeyValue) { +func seedBootstrapState(t *testing.T, kv jetstream.KeyValue) { t.Helper() - _, err := kv.Put(NodeTokenKey("alpha"), []byte("enc-token")) + _, err := kv.Put(t.Context(), NodeTokenKey("alpha"), []byte("enc-token")) require.NoError(t, err) - _, err = kv.Put(AdminKubeconfigKey("alpha"), []byte("enc-kc")) + _, err = kv.Put(t.Context(), AdminKubeconfigKey("alpha"), []byte("enc-kc")) require.NoError(t, err) // Pre-seeded JWKS plus the verified-marker persistJWKS writes on a passing // cross-check — bootstrapReady gates ACTIVE on the marker, not the seed. - _, err = kv.Put(OIDCJWKSKey("alpha"), []byte(`{"keys":[]}`)) + _, err = kv.Put(t.Context(), OIDCJWKSKey("alpha"), []byte(`{"keys":[]}`)) require.NoError(t, err) - _, err = kv.Put(OIDCJWKSVerifiedKey("alpha"), []byte("verified")) + _, err = kv.Put(t.Context(), OIDCJWKSVerifiedKey("alpha"), []byte("verified")) require.NoError(t, err) - require.NoError(t, SetClusterCertificateAuthority(kv, "alpha", "ca-blob-b64")) + require.NoError(t, SetClusterCertificateAuthority(t.Context(), kv, "alpha", "ca-blob-b64")) } func TestClusterReconciler_CreatingTransitionsToActiveOnReadyAndHealthz(t *testing.T) { @@ -167,7 +169,7 @@ func TestClusterReconciler_CreatingTransitionsToActiveOnReadyAndHealthz(t *testi WithReconcileInterval(10*time.Millisecond), WithLeaseRefresh(10*time.Second), ) - release, ok := r.AcquireLease() + release, ok := r.AcquireLease(t.Context()) require.True(t, ok) defer release() @@ -180,7 +182,7 @@ func TestClusterReconciler_CreatingTransitionsToActiveOnReadyAndHealthz(t *testi go func() { runErr <- r.Run(ctx) }() require.Eventually(t, func() bool { - meta, err := GetClusterMeta(acctKV, "alpha") + meta, err := GetClusterMeta(t.Context(), acctKV, "alpha") if err != nil { return false } @@ -205,23 +207,23 @@ func TestClusterReconciler_CreatingStaysWhenBootstrapMissing(t *testing.T) { WithReconcileInterval(10*time.Millisecond), WithLeaseRefresh(10*time.Second), ) - release, ok := r.AcquireLease() + release, ok := r.AcquireLease(t.Context()) require.True(t, ok) defer release() freshenClusterCreatedAt(t, acctKV) // Only seed two of the three KV keys (no CA, no JWKS). - _, err := acctKV.Put(NodeTokenKey("alpha"), []byte("enc-token")) + _, err := acctKV.Put(t.Context(), NodeTokenKey("alpha"), []byte("enc-token")) require.NoError(t, err) - _, err = acctKV.Put(AdminKubeconfigKey("alpha"), []byte("enc-kc")) + _, err = acctKV.Put(t.Context(), AdminKubeconfigKey("alpha"), []byte("enc-kc")) require.NoError(t, err) ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond) defer cancel() _ = r.Run(ctx) - meta, err := GetClusterMeta(acctKV, "alpha") + meta, err := GetClusterMeta(t.Context(), acctKV, "alpha") require.NoError(t, err) assert.Equal(t, ClusterStatusCreating, meta.Status, "must remain CREATING when bootstrap incomplete") } @@ -239,26 +241,26 @@ func TestClusterReconciler_CreatingStaysWhenJWKSUnverified(t *testing.T) { WithReconcileInterval(10*time.Millisecond), WithLeaseRefresh(10*time.Second), ) - release, ok := r.AcquireLease() + release, ok := r.AcquireLease(t.Context()) require.True(t, ok) defer release() freshenClusterCreatedAt(t, acctKV) - _, err := acctKV.Put(NodeTokenKey("alpha"), []byte("enc-token")) + _, err := acctKV.Put(t.Context(), NodeTokenKey("alpha"), []byte("enc-token")) require.NoError(t, err) - _, err = acctKV.Put(AdminKubeconfigKey("alpha"), []byte("enc-kc")) + _, err = acctKV.Put(t.Context(), AdminKubeconfigKey("alpha"), []byte("enc-kc")) require.NoError(t, err) - require.NoError(t, SetClusterCertificateAuthority(acctKV, "alpha", "ca-blob-b64")) + require.NoError(t, SetClusterCertificateAuthority(t.Context(), acctKV, "alpha", "ca-blob-b64")) // Pre-seeded JWKS present, but NO verified-marker. - _, err = acctKV.Put(OIDCJWKSKey("alpha"), []byte(`{"keys":[]}`)) + _, err = acctKV.Put(t.Context(), OIDCJWKSKey("alpha"), []byte(`{"keys":[]}`)) require.NoError(t, err) ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond) defer cancel() _ = r.Run(ctx) - meta, err := GetClusterMeta(acctKV, "alpha") + meta, err := GetClusterMeta(t.Context(), acctKV, "alpha") require.NoError(t, err) assert.Equal(t, ClusterStatusCreating, meta.Status, "must remain CREATING until the JWKS cross-check positively passes") @@ -272,7 +274,7 @@ func TestClusterReconciler_CreatingStaysWhenHealthzFails(t *testing.T) { WithReconcileInterval(10*time.Millisecond), WithLeaseRefresh(10*time.Second), ) - release, ok := r.AcquireLease() + release, ok := r.AcquireLease(t.Context()) require.True(t, ok) defer release() @@ -283,7 +285,7 @@ func TestClusterReconciler_CreatingStaysWhenHealthzFails(t *testing.T) { defer cancel() _ = r.Run(ctx) - meta, err := GetClusterMeta(acctKV, "alpha") + meta, err := GetClusterMeta(t.Context(), acctKV, "alpha") require.NoError(t, err) assert.Equal(t, ClusterStatusCreating, meta.Status, "must remain CREATING when /healthz fails") } @@ -297,16 +299,16 @@ func TestClusterReconciler_CreatingTimesOutToFailed(t *testing.T) { WithLeaseRefresh(10*time.Second), WithCreateTimeout(1*time.Millisecond), ) - release, ok := r.AcquireLease() + release, ok := r.AcquireLease(t.Context()) require.True(t, ok) defer release() // Stamp CreatedAt in the past so the 1ms deadline is already exceeded; the // cluster never becomes ready, so the reconciler must mark it FAILED. - meta, err := GetClusterMeta(acctKV, "alpha") + meta, err := GetClusterMeta(t.Context(), acctKV, "alpha") require.NoError(t, err) meta.CreatedAt = time.Now().Add(-time.Hour).UTC() - require.NoError(t, PutClusterMeta(acctKV, meta)) + require.NoError(t, PutClusterMeta(t.Context(), acctKV, meta)) ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() @@ -314,7 +316,7 @@ func TestClusterReconciler_CreatingTimesOutToFailed(t *testing.T) { err = r.Run(ctx) assert.ErrorIs(t, err, ErrReconcilerClusterFailed) - got, err := GetClusterMeta(acctKV, "alpha") + got, err := GetClusterMeta(t.Context(), acctKV, "alpha") require.NoError(t, err) assert.Equal(t, ClusterStatusFailed, got.Status) assert.Contains(t, got.StatusReason, "did not become ACTIVE") @@ -329,7 +331,7 @@ func TestClusterReconciler_CreatingWithinDeadlineStaysCreating(t *testing.T) { WithLeaseRefresh(10*time.Second), WithCreateTimeout(time.Hour), ) - release, ok := r.AcquireLease() + release, ok := r.AcquireLease(t.Context()) require.True(t, ok) defer release() @@ -339,7 +341,7 @@ func TestClusterReconciler_CreatingWithinDeadlineStaysCreating(t *testing.T) { defer cancel() _ = r.Run(ctx) - meta, err := GetClusterMeta(acctKV, "alpha") + meta, err := GetClusterMeta(t.Context(), acctKV, "alpha") require.NoError(t, err) assert.Equal(t, ClusterStatusCreating, meta.Status, "within deadline must stay CREATING") } @@ -352,11 +354,11 @@ func TestClusterReconciler_DeletingExitsLoop(t *testing.T) { WithReconcileInterval(10*time.Millisecond), WithLeaseRefresh(10*time.Second), ) - release, ok := r.AcquireLease() + release, ok := r.AcquireLease(t.Context()) require.True(t, ok) defer release() - require.NoError(t, SetClusterStatus(acctKV, "alpha", ClusterStatusDeleting)) + require.NoError(t, SetClusterStatus(t.Context(), acctKV, "alpha", ClusterStatusDeleting)) ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() @@ -373,7 +375,7 @@ func TestClusterReconciler_LostLeaseExitsLoop(t *testing.T) { WithReconcileInterval(10*time.Second), WithLeaseRefresh(20*time.Millisecond), ) - release, ok := r.AcquireLease() + release, ok := r.AcquireLease(t.Context()) require.True(t, ok) defer release() @@ -388,7 +390,7 @@ func TestClusterReconciler_LostLeaseExitsLoop(t *testing.T) { // Steal the lease in another holder's name. The next refresh tick should // notice and return ErrReconcilerLeaseLost. time.Sleep(40 * time.Millisecond) - _, err := leaderKV.Put(reconcilerLeaderKey(testAccountID, "alpha"), []byte("holder-2")) + _, err := leaderKV.Put(t.Context(), reconcilerLeaderKey(testAccountID, "alpha"), []byte("holder-2")) require.NoError(t, err) select { @@ -407,18 +409,18 @@ func TestClusterReconciler_ActiveProbesAndWarnsOnHealthzFail(t *testing.T) { WithReconcileInterval(10*time.Millisecond), WithLeaseRefresh(10*time.Second), ) - release, ok := r.AcquireLease() + release, ok := r.AcquireLease(t.Context()) require.True(t, ok) defer release() - require.NoError(t, SetClusterStatus(acctKV, "alpha", ClusterStatusActive)) + require.NoError(t, SetClusterStatus(t.Context(), acctKV, "alpha", ClusterStatusActive)) stub.setResponse(http.StatusServiceUnavailable, nil) ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) defer cancel() _ = r.Run(ctx) - meta, err := GetClusterMeta(acctKV, "alpha") + meta, err := GetClusterMeta(t.Context(), acctKV, "alpha") require.NoError(t, err) assert.Equal(t, ClusterStatusActive, meta.Status, "ACTIVE must not flip on healthz fail in this PR") assert.Positive(t, stub.calls.Load()) @@ -434,23 +436,23 @@ func TestClusterReconciler_ActiveHealthRecoversClearsIssue(t *testing.T) { WithReconcileInterval(10*time.Millisecond), WithLeaseRefresh(10*time.Second), ) - release, ok := r.AcquireLease() + release, ok := r.AcquireLease(t.Context()) require.True(t, ok) defer release() - require.NoError(t, SetClusterStatus(acctKV, "alpha", ClusterStatusActive)) + require.NoError(t, SetClusterStatus(t.Context(), acctKV, "alpha", ClusterStatusActive)) // First probe fails and records the issue. require.Error(t, r.probeHealthz(context.Background())) require.NoError(t, r.reconcileOnce(context.Background())) - meta, err := GetClusterMeta(acctKV, "alpha") + meta, err := GetClusterMeta(t.Context(), acctKV, "alpha") require.NoError(t, err) require.NotEmpty(t, meta.HealthIssue) // Probe recovers: the issue must be cleared. stub.setResponse(http.StatusOK, nil) require.NoError(t, r.reconcileOnce(context.Background())) - meta, err = GetClusterMeta(acctKV, "alpha") + meta, err = GetClusterMeta(t.Context(), acctKV, "alpha") require.NoError(t, err) assert.Empty(t, meta.HealthIssue, "recovered /healthz must clear the recorded issue") } diff --git a/spinifex/handlers/eks/cluster_state.go b/spinifex/handlers/eks/cluster_state.go index 35d5d6444..e63326583 100644 --- a/spinifex/handlers/eks/cluster_state.go +++ b/spinifex/handlers/eks/cluster_state.go @@ -1,6 +1,7 @@ package handlers_eks import ( + "context" "encoding/json" "errors" "fmt" @@ -8,7 +9,7 @@ import ( "strings" "time" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // ClusterStatus is the persisted cluster lifecycle state; values match the AWS EKS status enum. @@ -152,7 +153,7 @@ var ErrClusterNotFound = errors.New("eks: cluster not found") const maxClusterStateCASRetries = 5 // PutClusterMeta writes the meta record unconditionally. -func PutClusterMeta(kv nats.KeyValue, meta *ClusterMeta) error { +func PutClusterMeta(ctx context.Context, kv jetstream.KeyValue, meta *ClusterMeta) error { if meta == nil { return errors.New("eks: PutClusterMeta nil meta") } @@ -163,7 +164,7 @@ func PutClusterMeta(kv nats.KeyValue, meta *ClusterMeta) error { if err != nil { return fmt.Errorf("marshal cluster meta %s: %w", meta.Name, err) } - if _, err := kv.Put(ClusterMetaKey(meta.Name), data); err != nil { + if _, err := kv.Put(ctx, ClusterMetaKey(meta.Name), data); err != nil { return fmt.Errorf("kv put %s: %w", ClusterMetaKey(meta.Name), err) } return nil @@ -171,13 +172,13 @@ func PutClusterMeta(kv nats.KeyValue, meta *ClusterMeta) error { // GetClusterMeta reads the meta record. Returns ErrClusterNotFound if the // key is absent. -func GetClusterMeta(kv nats.KeyValue, name string) (*ClusterMeta, error) { +func GetClusterMeta(ctx context.Context, kv jetstream.KeyValue, name string) (*ClusterMeta, error) { if name == "" { return nil, errors.New("eks: GetClusterMeta empty name") } - entry, err := kv.Get(ClusterMetaKey(name)) + entry, err := kv.Get(ctx, ClusterMetaKey(name)) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil, ErrClusterNotFound } return nil, fmt.Errorf("kv get %s: %w", ClusterMetaKey(name), err) @@ -192,14 +193,14 @@ func GetClusterMeta(kv nats.KeyValue, name string) (*ClusterMeta, error) { // casUpdateMeta does a revision-checked read-modify-write of the cluster meta. // mutate returns true when a field changed. Retries on CAS conflict up to // maxClusterStateCASRetries. Returns ErrClusterNotFound if deleted concurrently. -func casUpdateMeta(kv nats.KeyValue, name string, mutate func(*ClusterMeta) bool) error { +func casUpdateMeta(ctx context.Context, kv jetstream.KeyValue, name string, mutate func(*ClusterMeta) bool) error { if name == "" { return errors.New("eks: casUpdateMeta empty name") } for range maxClusterStateCASRetries { - entry, err := kv.Get(ClusterMetaKey(name)) + entry, err := kv.Get(ctx, ClusterMetaKey(name)) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return ErrClusterNotFound } return fmt.Errorf("kv get %s: %w", ClusterMetaKey(name), err) @@ -215,11 +216,11 @@ func casUpdateMeta(kv nats.KeyValue, name string, mutate func(*ClusterMeta) bool if err != nil { return fmt.Errorf("marshal cluster meta %s: %w", name, err) } - _, err = kv.Update(ClusterMetaKey(name), data, entry.Revision()) + _, err = kv.Update(ctx, ClusterMetaKey(name), data, entry.Revision()) if err == nil { return nil } - if errors.Is(err, nats.ErrKeyExists) { + if errors.Is(err, jetstream.ErrKeyExists) { continue } return fmt.Errorf("kv update %s: %w", ClusterMetaKey(name), err) @@ -228,11 +229,11 @@ func casUpdateMeta(kv nats.KeyValue, name string, mutate func(*ClusterMeta) bool } // SetClusterStatus does a CAS update of meta.Status. Returns ErrClusterNotFound if deleted concurrently. -func SetClusterStatus(kv nats.KeyValue, name string, status ClusterStatus) error { +func SetClusterStatus(ctx context.Context, kv jetstream.KeyValue, name string, status ClusterStatus) error { if name == "" { return errors.New("eks: SetClusterStatus empty name") } - return casUpdateMeta(kv, name, func(m *ClusterMeta) bool { + return casUpdateMeta(ctx, kv, name, func(m *ClusterMeta) bool { if m.Status == status { return false } @@ -246,11 +247,11 @@ func SetClusterStatus(kv nats.KeyValue, name string, status ClusterStatus) error // MarkClusterFailed transitions the cluster to FAILED with a reason, but only // from CREATING — so a late error cannot clobber a concurrent delete or an active cluster. -func MarkClusterFailed(kv nats.KeyValue, name, reason string) error { +func MarkClusterFailed(ctx context.Context, kv jetstream.KeyValue, name, reason string) error { if name == "" { return errors.New("eks: MarkClusterFailed empty name") } - return casUpdateMeta(kv, name, func(m *ClusterMeta) bool { + return casUpdateMeta(ctx, kv, name, func(m *ClusterMeta) bool { if m.Status != ClusterStatusCreating { return false } @@ -262,14 +263,14 @@ func MarkClusterFailed(kv nats.KeyValue, name, reason string) error { // SetClusterCertificateAuthority does a CAS update of meta.CertificateAuthorityB64. // Called by the bootstrap subscriber when the K3s VM publishes its CA. -func SetClusterCertificateAuthority(kv nats.KeyValue, name, caB64 string) error { +func SetClusterCertificateAuthority(ctx context.Context, kv jetstream.KeyValue, name, caB64 string) error { if name == "" { return errors.New("eks: SetClusterCertificateAuthority empty name") } if caB64 == "" { return errors.New("eks: SetClusterCertificateAuthority empty CA") } - return casUpdateMeta(kv, name, func(m *ClusterMeta) bool { + return casUpdateMeta(ctx, kv, name, func(m *ClusterMeta) bool { if m.CertificateAuthorityB64 == caB64 { return false } @@ -280,11 +281,11 @@ func SetClusterCertificateAuthority(kv nats.KeyValue, name, caB64 string) error // SetClusterHealth records the latest health outcome; issue="" means healthy. // Only writes on a state change, stamping LastHealthProbe at the transition. -func SetClusterHealth(kv nats.KeyValue, name, issue string) error { +func SetClusterHealth(ctx context.Context, kv jetstream.KeyValue, name, issue string) error { if name == "" { return errors.New("eks: SetClusterHealth empty name") } - return casUpdateMeta(kv, name, func(m *ClusterMeta) bool { + return casUpdateMeta(ctx, kv, name, func(m *ClusterMeta) bool { if m.HealthIssue == issue { return false } @@ -299,11 +300,11 @@ func SetClusterHealth(kv nats.KeyValue, name, issue string) error { // change; no KV write when nothing changed. nodegroupReady may be nil (older // AMI / no report yet), in which case the persisted per-nodegroup counts are // left untouched. -func SetClusterHealthState(kv nats.KeyValue, name, issue string, nodeCount int, nodegroupReady map[string]int) error { +func SetClusterHealthState(ctx context.Context, kv jetstream.KeyValue, name, issue string, nodeCount int, nodegroupReady map[string]int) error { if name == "" { return errors.New("eks: SetClusterHealthState empty name") } - return casUpdateMeta(kv, name, func(m *ClusterMeta) bool { + return casUpdateMeta(ctx, kv, name, func(m *ClusterMeta) bool { changed := m.HealthIssue != issue || m.NodeCount != nodeCount if nodegroupReady != nil && !maps.Equal(m.NodegroupNodeCounts, nodegroupReady) { changed = true @@ -328,11 +329,11 @@ func SetClusterHealthState(kv nats.KeyValue, name, issue string, nodeCount int, // never retries EC2/OVN calls against a stale VpcId, which is what turns a // single DependencyViolation into a permanent teardown loop. No-op if already // cleared. Returns ErrClusterNotFound if the cluster was deleted concurrently. -func ClearClusterManagedCPVPC(kv nats.KeyValue, name string) error { +func ClearClusterManagedCPVPC(ctx context.Context, kv jetstream.KeyValue, name string) error { if name == "" { return errors.New("eks: ClearClusterManagedCPVPC empty name") } - return casUpdateMeta(kv, name, func(m *ClusterMeta) bool { + return casUpdateMeta(ctx, kv, name, func(m *ClusterMeta) bool { if m.ManagedCPVPC == nil { return false } @@ -346,14 +347,14 @@ func ClearClusterManagedCPVPC(kv nats.KeyValue, name string) error { // refreshes the scalar [0] mirrors if the primary changed. A no-op (no error) // when the dead member is already gone from the list — a concurrent swap already // won. Returns ErrClusterNotFound if the cluster was deleted concurrently. -func SwapControlPlaneMember(kv nats.KeyValue, name, deadInstanceID string, replacement ControlPlaneNode) error { +func SwapControlPlaneMember(ctx context.Context, kv jetstream.KeyValue, name, deadInstanceID string, replacement ControlPlaneNode) error { if name == "" { return errors.New("eks: SwapControlPlaneMember empty name") } if replacement.InstanceID == "" { return errors.New("eks: SwapControlPlaneMember empty replacement instance id") } - return casUpdateMeta(kv, name, func(m *ClusterMeta) bool { + return casUpdateMeta(ctx, kv, name, func(m *ClusterMeta) bool { idx := -1 for i, n := range m.ControlPlaneNodes { if n.InstanceID == deadInstanceID { @@ -376,14 +377,14 @@ func SwapControlPlaneMember(kv nats.KeyValue, name, deadInstanceID string, repla // DeleteClusterPrefix removes every KV key under clusters/{name}/. // Returns the first error but continues sweeping. -func DeleteClusterPrefix(kv nats.KeyValue, name string) error { +func DeleteClusterPrefix(ctx context.Context, kv jetstream.KeyValue, name string) error { if name == "" { return errors.New("eks: DeleteClusterPrefix empty name") } prefix := fmt.Sprintf("clusters/%s/", name) - keys, err := kv.Keys() + keys, err := kv.Keys(ctx) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return nil } return fmt.Errorf("kv keys: %w", err) @@ -393,7 +394,7 @@ func DeleteClusterPrefix(kv nats.KeyValue, name string) error { if !strings.HasPrefix(k, prefix) { continue } - if err := kv.Delete(k); err != nil && firstErr == nil { + if err := kv.Delete(ctx, k); err != nil && firstErr == nil { firstErr = fmt.Errorf("kv delete %s: %w", k, err) } } diff --git a/spinifex/handlers/eks/cluster_state_test.go b/spinifex/handlers/eks/cluster_state_test.go index afaf8644a..c351c8ebd 100644 --- a/spinifex/handlers/eks/cluster_state_test.go +++ b/spinifex/handlers/eks/cluster_state_test.go @@ -6,15 +6,16 @@ import ( "time" "github.com/mulgadc/spinifex/spinifex/testutil" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func newClusterStateTestKV(t *testing.T) nats.KeyValue { +func newClusterStateTestKV(t *testing.T) jetstream.KeyValue { t.Helper() - _, _, js := testutil.StartTestJetStream(t) - kv, err := GetOrCreateAccountBucket(js, testAccountID, 1) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1) require.NoError(t, err) return kv } @@ -37,17 +38,17 @@ func sampleClusterMeta(name string) *ClusterMeta { func TestPutClusterMeta_NilOrEmptyNameRejected(t *testing.T) { kv := newClusterStateTestKV(t) - require.Error(t, PutClusterMeta(kv, nil)) - require.Error(t, PutClusterMeta(kv, &ClusterMeta{})) + require.Error(t, PutClusterMeta(t.Context(), kv, nil)) + require.Error(t, PutClusterMeta(t.Context(), kv, &ClusterMeta{})) } func TestPutClusterMeta_RoundTrip(t *testing.T) { kv := newClusterStateTestKV(t) in := sampleClusterMeta("alpha") - require.NoError(t, PutClusterMeta(kv, in)) + require.NoError(t, PutClusterMeta(t.Context(), kv, in)) - got, err := GetClusterMeta(kv, "alpha") + got, err := GetClusterMeta(t.Context(), kv, "alpha") require.NoError(t, err) assert.Equal(t, in.Name, got.Name) assert.Equal(t, in.Arn, got.Arn) @@ -62,14 +63,14 @@ func TestPutClusterMeta_RoundTrip(t *testing.T) { func TestGetClusterMeta_MissingReturnsErrClusterNotFound(t *testing.T) { kv := newClusterStateTestKV(t) - _, err := GetClusterMeta(kv, "missing") + _, err := GetClusterMeta(t.Context(), kv, "missing") require.ErrorIs(t, err, ErrClusterNotFound) } func TestGetClusterMeta_EmptyNameRejected(t *testing.T) { kv := newClusterStateTestKV(t) - _, err := GetClusterMeta(kv, "") + _, err := GetClusterMeta(t.Context(), kv, "") require.Error(t, err) assert.NotErrorIs(t, err, ErrClusterNotFound) } @@ -77,25 +78,25 @@ func TestGetClusterMeta_EmptyNameRejected(t *testing.T) { func TestGetClusterMeta_CorruptBlobRejected(t *testing.T) { kv := newClusterStateTestKV(t) - _, err := kv.Put(ClusterMetaKey("corrupt"), []byte("{not json")) + _, err := kv.Put(t.Context(), ClusterMetaKey("corrupt"), []byte("{not json")) require.NoError(t, err) - _, err = GetClusterMeta(kv, "corrupt") + _, err = GetClusterMeta(t.Context(), kv, "corrupt") require.Error(t, err) assert.NotErrorIs(t, err, ErrClusterNotFound) } func TestSetClusterStatus_TransitionsAndIsIdempotent(t *testing.T) { kv := newClusterStateTestKV(t) - require.NoError(t, PutClusterMeta(kv, sampleClusterMeta("alpha"))) + require.NoError(t, PutClusterMeta(t.Context(), kv, sampleClusterMeta("alpha"))) - require.NoError(t, SetClusterStatus(kv, "alpha", ClusterStatusActive)) - got, err := GetClusterMeta(kv, "alpha") + require.NoError(t, SetClusterStatus(t.Context(), kv, "alpha", ClusterStatusActive)) + got, err := GetClusterMeta(t.Context(), kv, "alpha") require.NoError(t, err) assert.Equal(t, ClusterStatusActive, got.Status) - require.NoError(t, SetClusterStatus(kv, "alpha", ClusterStatusActive)) - got, err = GetClusterMeta(kv, "alpha") + require.NoError(t, SetClusterStatus(t.Context(), kv, "alpha", ClusterStatusActive)) + got, err = GetClusterMeta(t.Context(), kv, "alpha") require.NoError(t, err) assert.Equal(t, ClusterStatusActive, got.Status) } @@ -106,9 +107,9 @@ func TestSetClusterStatus_DeletingAlwaysAllowedFromAnyState(t *testing.T) { for _, from := range []ClusterStatus{ClusterStatusCreating, ClusterStatusActive, ClusterStatusFailed} { meta := sampleClusterMeta("alpha") meta.Status = from - require.NoError(t, PutClusterMeta(kv, meta)) - require.NoError(t, SetClusterStatus(kv, "alpha", ClusterStatusDeleting)) - got, err := GetClusterMeta(kv, "alpha") + require.NoError(t, PutClusterMeta(t.Context(), kv, meta)) + require.NoError(t, SetClusterStatus(t.Context(), kv, "alpha", ClusterStatusDeleting)) + got, err := GetClusterMeta(t.Context(), kv, "alpha") require.NoError(t, err) assert.Equal(t, ClusterStatusDeleting, got.Status, "from=%s", from) } @@ -117,30 +118,30 @@ func TestSetClusterStatus_DeletingAlwaysAllowedFromAnyState(t *testing.T) { func TestSetClusterStatus_MissingReturnsErrClusterNotFound(t *testing.T) { kv := newClusterStateTestKV(t) - err := SetClusterStatus(kv, "missing", ClusterStatusActive) + err := SetClusterStatus(t.Context(), kv, "missing", ClusterStatusActive) require.ErrorIs(t, err, ErrClusterNotFound) } func TestSetClusterStatus_RecoversFromConcurrentRevisionBump(t *testing.T) { kv := newClusterStateTestKV(t) - require.NoError(t, PutClusterMeta(kv, sampleClusterMeta("alpha"))) + require.NoError(t, PutClusterMeta(t.Context(), kv, sampleClusterMeta("alpha"))) // Read once to get a stale revision, then bump the record from another // goroutine via Put to force one CAS conflict on the next Update. - entry, err := kv.Get(ClusterMetaKey("alpha")) + entry, err := kv.Get(t.Context(), ClusterMetaKey("alpha")) require.NoError(t, err) bumped := sampleClusterMeta("alpha") bumped.Version = "1.32-bumped" data, err := json.Marshal(bumped) require.NoError(t, err) - _, err = kv.Update(ClusterMetaKey("alpha"), data, entry.Revision()) + _, err = kv.Update(t.Context(), ClusterMetaKey("alpha"), data, entry.Revision()) require.NoError(t, err) // SetClusterStatus must succeed even though there has been an intervening // revision bump — its internal loop re-reads + retries. - require.NoError(t, SetClusterStatus(kv, "alpha", ClusterStatusActive)) - got, err := GetClusterMeta(kv, "alpha") + require.NoError(t, SetClusterStatus(t.Context(), kv, "alpha", ClusterStatusActive)) + got, err := GetClusterMeta(t.Context(), kv, "alpha") require.NoError(t, err) assert.Equal(t, ClusterStatusActive, got.Status) assert.Equal(t, "1.32-bumped", got.Version) @@ -148,10 +149,10 @@ func TestSetClusterStatus_RecoversFromConcurrentRevisionBump(t *testing.T) { func TestMarkClusterFailed_FromCreatingSetsStatusAndReason(t *testing.T) { kv := newClusterStateTestKV(t) - require.NoError(t, PutClusterMeta(kv, sampleClusterMeta("alpha"))) + require.NoError(t, PutClusterMeta(t.Context(), kv, sampleClusterMeta("alpha"))) - require.NoError(t, MarkClusterFailed(kv, "alpha", "bootstrap failed: boom")) - got, err := GetClusterMeta(kv, "alpha") + require.NoError(t, MarkClusterFailed(t.Context(), kv, "alpha", "bootstrap failed: boom")) + got, err := GetClusterMeta(t.Context(), kv, "alpha") require.NoError(t, err) assert.Equal(t, ClusterStatusFailed, got.Status) assert.Equal(t, "bootstrap failed: boom", got.StatusReason) @@ -163,10 +164,10 @@ func TestMarkClusterFailed_NoopFromNonCreating(t *testing.T) { for _, from := range []ClusterStatus{ClusterStatusActive, ClusterStatusDeleting, ClusterStatusFailed} { meta := sampleClusterMeta("alpha") meta.Status = from - require.NoError(t, PutClusterMeta(kv, meta)) + require.NoError(t, PutClusterMeta(t.Context(), kv, meta)) - require.NoError(t, MarkClusterFailed(kv, "alpha", "late bootstrap error")) - got, err := GetClusterMeta(kv, "alpha") + require.NoError(t, MarkClusterFailed(t.Context(), kv, "alpha", "late bootstrap error")) + got, err := GetClusterMeta(t.Context(), kv, "alpha") require.NoError(t, err) assert.Equal(t, from, got.Status, "from=%s must be untouched", from) assert.Empty(t, got.StatusReason, "from=%s reason must not be set", from) @@ -176,21 +177,21 @@ func TestMarkClusterFailed_NoopFromNonCreating(t *testing.T) { func TestMarkClusterFailed_MissingReturnsErrClusterNotFound(t *testing.T) { kv := newClusterStateTestKV(t) - err := MarkClusterFailed(kv, "ghost", "boom") + err := MarkClusterFailed(t.Context(), kv, "ghost", "boom") require.ErrorIs(t, err, ErrClusterNotFound) } func TestSetClusterCertificateAuthority_WritesAndIsIdempotent(t *testing.T) { kv := newClusterStateTestKV(t) - require.NoError(t, PutClusterMeta(kv, sampleClusterMeta("alpha"))) + require.NoError(t, PutClusterMeta(t.Context(), kv, sampleClusterMeta("alpha"))) - require.NoError(t, SetClusterCertificateAuthority(kv, "alpha", "ca-blob-b64")) - got, err := GetClusterMeta(kv, "alpha") + require.NoError(t, SetClusterCertificateAuthority(t.Context(), kv, "alpha", "ca-blob-b64")) + got, err := GetClusterMeta(t.Context(), kv, "alpha") require.NoError(t, err) assert.Equal(t, "ca-blob-b64", got.CertificateAuthorityB64) - require.NoError(t, SetClusterCertificateAuthority(kv, "alpha", "ca-blob-b64")) - got, err = GetClusterMeta(kv, "alpha") + require.NoError(t, SetClusterCertificateAuthority(t.Context(), kv, "alpha", "ca-blob-b64")) + got, err = GetClusterMeta(t.Context(), kv, "alpha") require.NoError(t, err) assert.Equal(t, "ca-blob-b64", got.CertificateAuthorityB64) } @@ -198,33 +199,33 @@ func TestSetClusterCertificateAuthority_WritesAndIsIdempotent(t *testing.T) { func TestSetClusterCertificateAuthority_MissingReturnsErrClusterNotFound(t *testing.T) { kv := newClusterStateTestKV(t) - err := SetClusterCertificateAuthority(kv, "ghost", "ca-blob") + err := SetClusterCertificateAuthority(t.Context(), kv, "ghost", "ca-blob") require.ErrorIs(t, err, ErrClusterNotFound) } func TestSetClusterCertificateAuthority_EmptyInputsRejected(t *testing.T) { kv := newClusterStateTestKV(t) - require.Error(t, SetClusterCertificateAuthority(kv, "", "ca-blob")) - require.Error(t, SetClusterCertificateAuthority(kv, "alpha", "")) + require.Error(t, SetClusterCertificateAuthority(t.Context(), kv, "", "ca-blob")) + require.Error(t, SetClusterCertificateAuthority(t.Context(), kv, "alpha", "")) } func TestDeleteClusterPrefix_SweepsEveryKey(t *testing.T) { kv := newClusterStateTestKV(t) - require.NoError(t, PutClusterMeta(kv, sampleClusterMeta("alpha"))) - _, err := kv.Put(NodegroupKey("alpha", "ng-1"), []byte(`{"name":"ng-1"}`)) + require.NoError(t, PutClusterMeta(t.Context(), kv, sampleClusterMeta("alpha"))) + _, err := kv.Put(t.Context(), NodegroupKey("alpha", "ng-1"), []byte(`{"name":"ng-1"}`)) require.NoError(t, err) - _, err = kv.Put(OIDCSigningKeyKey("alpha"), []byte("enc-blob")) + _, err = kv.Put(t.Context(), OIDCSigningKeyKey("alpha"), []byte("enc-blob")) require.NoError(t, err) - _, err = kv.Put(OIDCJWKSKey("alpha"), []byte(`{"keys":[]}`)) + _, err = kv.Put(t.Context(), OIDCJWKSKey("alpha"), []byte(`{"keys":[]}`)) require.NoError(t, err) - _, err = kv.Put(EventKey("alpha", "1700000000"), []byte(`{"ts":"1700000000"}`)) + _, err = kv.Put(t.Context(), EventKey("alpha", "1700000000"), []byte(`{"ts":"1700000000"}`)) require.NoError(t, err) // Sibling cluster must survive the sweep. - require.NoError(t, PutClusterMeta(kv, sampleClusterMeta("beta"))) + require.NoError(t, PutClusterMeta(t.Context(), kv, sampleClusterMeta("beta"))) - require.NoError(t, DeleteClusterPrefix(kv, "alpha")) + require.NoError(t, DeleteClusterPrefix(t.Context(), kv, "alpha")) for _, k := range []string{ ClusterMetaKey("alpha"), @@ -233,11 +234,11 @@ func TestDeleteClusterPrefix_SweepsEveryKey(t *testing.T) { OIDCJWKSKey("alpha"), EventKey("alpha", "1700000000"), } { - _, err := kv.Get(k) - assert.ErrorIs(t, err, nats.ErrKeyNotFound, "key %s should be gone, got err=%v", k, err) + _, err := kv.Get(t.Context(), k) + assert.ErrorIs(t, err, jetstream.ErrKeyNotFound, "key %s should be gone, got err=%v", k, err) } - got, err := GetClusterMeta(kv, "beta") + got, err := GetClusterMeta(t.Context(), kv, "beta") require.NoError(t, err) assert.Equal(t, "beta", got.Name) } @@ -245,11 +246,11 @@ func TestDeleteClusterPrefix_SweepsEveryKey(t *testing.T) { func TestDeleteClusterPrefix_EmptyBucketIsNoop(t *testing.T) { kv := newClusterStateTestKV(t) - require.NoError(t, DeleteClusterPrefix(kv, "ghost")) + require.NoError(t, DeleteClusterPrefix(t.Context(), kv, "ghost")) } func TestDeleteClusterPrefix_EmptyNameRejected(t *testing.T) { kv := newClusterStateTestKV(t) - require.Error(t, DeleteClusterPrefix(kv, "")) + require.Error(t, DeleteClusterPrefix(t.Context(), kv, "")) } diff --git a/spinifex/handlers/eks/create_cluster_test.go b/spinifex/handlers/eks/create_cluster_test.go index dc8bc1614..ff84dc2f3 100644 --- a/spinifex/handlers/eks/create_cluster_test.go +++ b/spinifex/handlers/eks/create_cluster_test.go @@ -42,7 +42,7 @@ func TestCreateCluster_NLBArnPersistedBeforeLaterFailure(t *testing.T) { require.NoError(t, err) f.svc.WaitLaunches() - meta, getErr := GetClusterMeta(f.kv, "alpha") + meta, getErr := GetClusterMeta(t.Context(), f.kv, "alpha") require.NoError(t, getErr, "meta must remain so the leaked NLB has an owning record") assert.Equal(t, ClusterStatusFailed, meta.Status) assert.NotEmpty(t, meta.NLBArn, "NLB ARN must be persisted before the VM-launch step") @@ -66,7 +66,7 @@ func TestCreateCluster_FailedCreateThenDeleteReclaimsNLB(t *testing.T) { require.NoError(t, err) assert.NotEmpty(t, f.nlb.deleteLBCalls, "DeleteCluster must tear down the NLB recorded by the failed create") - _, getErr := GetClusterMeta(f.kv, "alpha") + _, getErr := GetClusterMeta(t.Context(), f.kv, "alpha") assert.ErrorIs(t, getErr, ErrClusterNotFound) } @@ -85,7 +85,7 @@ func TestCreateCluster_PostPlacementFailureLeavesControlPlaneRecorded(t *testing f.svc.WaitLaunches() require.NotEmpty(t, f.inst.launchCalls, "control-plane VM was launched before the register-targets step") - meta, getErr := GetClusterMeta(f.kv, "alpha") + meta, getErr := GetClusterMeta(t.Context(), f.kv, "alpha") require.NoError(t, getErr, "meta must remain so the leaked CP VM has an owning record") assert.Equal(t, ClusterStatusFailed, meta.Status) assert.NotEmpty(t, meta.ControlPlaneInstanceID, "CP instance ID must be persisted before the register-targets step") @@ -108,7 +108,7 @@ func TestCreateCluster_PostPlacementFailedThenDeleteTerminatesControlPlane(t *te require.NoError(t, err) assert.NotEmpty(t, f.inst.terminateCalls, "DeleteCluster must terminate the CP VM recorded by the failed create") - _, getErr := GetClusterMeta(f.kv, "alpha") + _, getErr := GetClusterMeta(t.Context(), f.kv, "alpha") assert.ErrorIs(t, getErr, ErrClusterNotFound) } @@ -130,7 +130,7 @@ func TestCreateCluster_FailedLaunchEagerlyPurgesInfra(t *testing.T) { // No DeleteCluster here: the failure handler itself must reap the NLB/LB VM. assert.NotEmpty(t, f.nlb.deleteLBCalls, "failed launch must eagerly tear down the NLB it provisioned") - meta, getErr := GetClusterMeta(f.kv, "alpha") + meta, getErr := GetClusterMeta(t.Context(), f.kv, "alpha") require.NoError(t, getErr, "FAILED meta is retained for observability") assert.Equal(t, ClusterStatusFailed, meta.Status) } @@ -142,7 +142,7 @@ func TestCreateCluster_ExistingClusterReturnsResourceInUse(t *testing.T) { meta := sampleClusterMeta("alpha") meta.Status = ClusterStatusActive - require.NoError(t, PutClusterMeta(f.kv, meta)) + require.NoError(t, PutClusterMeta(t.Context(), f.kv, meta)) _, err := f.svc.CreateCluster(context.Background(), createInput("alpha"), testAccountID, "") require.EqualError(t, err, awserrors.ErrorEKSResourceInUse) @@ -158,7 +158,7 @@ func TestCreateCluster_FailedClusterIsReclaimedOnRetry(t *testing.T) { _, err := f.svc.CreateCluster(context.Background(), createInput("alpha"), testAccountID, "") require.NoError(t, err) f.svc.WaitLaunches() - failed, err := GetClusterMeta(f.kv, "alpha") + failed, err := GetClusterMeta(t.Context(), f.kv, "alpha") require.NoError(t, err) require.Equal(t, ClusterStatusFailed, failed.Status) @@ -170,7 +170,7 @@ func TestCreateCluster_FailedClusterIsReclaimedOnRetry(t *testing.T) { require.NoError(t, err) f.svc.WaitLaunches() - got, err := GetClusterMeta(f.kv, "alpha") + got, err := GetClusterMeta(t.Context(), f.kv, "alpha") require.NoError(t, err) assert.Equal(t, ClusterStatusCreating, got.Status) assert.NotEmpty(t, f.nlb.deleteLBCalls, "reclaim must tear down the failed attempt's NLB") @@ -184,7 +184,7 @@ func TestCreateCluster_FailedClusterVisibleInDescribeAndList(t *testing.T) { meta := sampleClusterMeta("alpha") meta.Status = ClusterStatusFailed meta.StatusReason = "bootstrap failed: boom" - require.NoError(t, PutClusterMeta(f.kv, meta)) + require.NoError(t, PutClusterMeta(t.Context(), f.kv, meta)) desc, err := f.svc.DescribeCluster(context.Background(), &eks.DescribeClusterInput{Name: aws.String("alpha")}, testAccountID) require.NoError(t, err) @@ -209,8 +209,8 @@ func TestDescribeCluster_SurfacesHealthIssueForActiveCluster(t *testing.T) { meta := sampleClusterMeta("alpha") meta.Status = ClusterStatusActive - require.NoError(t, PutClusterMeta(f.kv, meta)) - require.NoError(t, SetClusterHealth(f.kv, "alpha", "healthz request: connection refused")) + require.NoError(t, PutClusterMeta(t.Context(), f.kv, meta)) + require.NoError(t, SetClusterHealth(t.Context(), f.kv, "alpha", "healthz request: connection refused")) desc, err := f.svc.DescribeCluster(context.Background(), &eks.DescribeClusterInput{Name: aws.String("alpha")}, testAccountID) require.NoError(t, err) @@ -227,7 +227,7 @@ func TestDescribeCluster_HealthyActiveClusterHasNoIssues(t *testing.T) { meta := sampleClusterMeta("alpha") meta.Status = ClusterStatusActive - require.NoError(t, PutClusterMeta(f.kv, meta)) + require.NoError(t, PutClusterMeta(t.Context(), f.kv, meta)) desc, err := f.svc.DescribeCluster(context.Background(), &eks.DescribeClusterInput{Name: aws.String("alpha")}, testAccountID) require.NoError(t, err) @@ -245,7 +245,7 @@ func TestCreateCluster_MissingAMIReturnsServiceUnavailable(t *testing.T) { require.NoError(t, err) f.svc.WaitLaunches() - meta, getErr := GetClusterMeta(f.kv, "alpha") + meta, getErr := GetClusterMeta(t.Context(), f.kv, "alpha") require.NoError(t, getErr) assert.Equal(t, ClusterStatusFailed, meta.Status) } @@ -258,7 +258,7 @@ func TestCreateCluster_HappyPathPersistsActiveCreatingMeta(t *testing.T) { require.NotNil(t, out) f.svc.WaitLaunches() - meta, err := GetClusterMeta(f.kv, "alpha") + meta, err := GetClusterMeta(t.Context(), f.kv, "alpha") require.NoError(t, err) assert.Equal(t, ClusterStatusCreating, meta.Status) assert.NotEmpty(t, meta.NLBArn) @@ -277,7 +277,7 @@ func TestCreateCluster_SeedsCreatorAdminAccessEntry(t *testing.T) { require.NoError(t, err) f.svc.WaitLaunches() - rec, err := GetAccessEntryRecord(f.kv, "alpha", caller) + rec, err := GetAccessEntryRecord(t.Context(), f.kv, "alpha", caller) require.NoError(t, err) assert.Equal(t, []string{"system:masters"}, rec.KubernetesGroups) assert.Equal(t, caller, rec.KubernetesUsername) @@ -297,7 +297,7 @@ func TestCreateCluster_SkipsCreatorAdminWhenDisabled(t *testing.T) { require.NoError(t, err) f.svc.WaitLaunches() - _, err = GetAccessEntryRecord(f.kv, "alpha", caller) + _, err = GetAccessEntryRecord(t.Context(), f.kv, "alpha", caller) assert.ErrorIs(t, err, ErrAccessEntryNotFound) } @@ -313,7 +313,7 @@ func TestCreateCluster_BuildsManagedCPVPCUnderSystemAccount(t *testing.T) { require.NoError(t, err) f.svc.WaitLaunches() - meta, err := GetClusterMeta(f.kv, "setb") + meta, err := GetClusterMeta(t.Context(), f.kv, "setb") require.NoError(t, err) require.NotNil(t, meta.ManagedCPVPC, "managed CP VPC refs must be persisted") assert.NotEmpty(t, meta.ManagedCPVPC.VpcId) diff --git a/spinifex/handlers/eks/delete_cluster_test.go b/spinifex/handlers/eks/delete_cluster_test.go index dc0c2e408..378619b63 100644 --- a/spinifex/handlers/eks/delete_cluster_test.go +++ b/spinifex/handlers/eks/delete_cluster_test.go @@ -14,6 +14,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/awserrors" "github.com/mulgadc/spinifex/spinifex/testutil" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -41,7 +42,7 @@ func deleteInput(name string) *eks.DeleteClusterInput { // failures at specific lifecycle steps. type eksServiceFixture struct { svc *EKSServiceImpl - kv nats.KeyValue + kv jetstream.KeyValue nlb *fakeNLBProvisioner inst *fakeK3sInst vpc *fakeK3sVPC @@ -57,8 +58,9 @@ type eksServiceFixture struct { func newEKSServiceFixture(t *testing.T) *eksServiceFixture { t.Helper() - _, nc, js := testutil.StartTestJetStream(t) - kv, err := GetOrCreateAccountBucket(js, testAccountID, 1) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1) require.NoError(t, err) nlb := newFakeNLBProvisioner() @@ -122,7 +124,7 @@ func newEKSServiceFixture(t *testing.T) *eksServiceFixture { // DeleteCluster exercises every teardown branch. type deleteClusterFixture struct { svc *EKSServiceImpl - kv nats.KeyValue + kv jetstream.KeyValue nlb *fakeNLBProvisioner inst *fakeK3sInst vpc *fakeK3sVPC @@ -144,10 +146,10 @@ func newDeleteClusterFixture(t *testing.T, clusterName string) *deleteClusterFix meta.ResourcesVpcConfig.VpcId = "vpc-aaa" meta.EgressEIPPublicIP = "203.0.113.50" meta.EgressEIPAllocationID = "eipalloc-fake01" - require.NoError(t, PutClusterMeta(kv, meta)) + require.NoError(t, PutClusterMeta(t.Context(), kv, meta)) // Real OIDC key so ZeroizeClusterOIDCKey has material to wipe. - _, _, err := GenerateClusterOIDCKeypair(kv, clusterName, bootstrapTestMasterKey) + _, _, err := GenerateClusterOIDCKeypair(t.Context(), kv, clusterName, bootstrapTestMasterKey) require.NoError(t, err) return &deleteClusterFixture{svc: f.svc, kv: kv, nlb: f.nlb, inst: f.inst, vpc: f.vpc, eip: f.eip, sg: f.sg} @@ -173,7 +175,7 @@ func TestDeleteCluster_LeakedSGKeepsClusterDeleting(t *testing.T) { _, err := f.svc.DeleteCluster(context.Background(), deleteInput("alpha"), testAccountID) require.Error(t, err, "a leaked customer-VPC SG must surface, not be swallowed") - meta, getErr := GetClusterMeta(f.kv, "alpha") + meta, getErr := GetClusterMeta(t.Context(), f.kv, "alpha") require.NoError(t, getErr, "the KV sweep must NOT run while an SG is still leaked — meta must survive for retry") assert.Equal(t, ClusterStatusDeleting, meta.Status, "a cluster with a leaked SG must stay DELETING") } @@ -246,7 +248,7 @@ func TestDeleteCluster_AllTeardownSucceedsSweepsKV(t *testing.T) { require.NoError(t, err) require.NotNil(t, out) - _, err = GetClusterMeta(f.kv, "alpha") + _, err = GetClusterMeta(t.Context(), f.kv, "alpha") assert.ErrorIs(t, err, ErrClusterNotFound, "meta must be swept after successful teardown") assert.Len(t, f.inst.terminateCalls, 1) assert.Len(t, f.vpc.deleteCalls, 1) @@ -262,10 +264,10 @@ func TestDeleteCluster_AllTeardownSucceedsSweepsKV(t *testing.T) { func TestDeleteCluster_DetachesPrivateEndpointENIBeforeDelete(t *testing.T) { f := newDeleteClusterFixture(t, "alpha") - meta, err := GetClusterMeta(f.kv, "alpha") + meta, err := GetClusterMeta(t.Context(), f.kv, "alpha") require.NoError(t, err) meta.PrivateEndpointENIID = "eni-pe-001" - require.NoError(t, PutClusterMeta(f.kv, meta)) + require.NoError(t, PutClusterMeta(t.Context(), f.kv, meta)) f.vpc.inUseUntilDetached = map[string]bool{"eni-pe-001": true} out, err := f.svc.DeleteCluster(context.Background(), deleteInput("alpha"), testAccountID) @@ -273,7 +275,7 @@ func TestDeleteCluster_DetachesPrivateEndpointENIBeforeDelete(t *testing.T) { require.NotNil(t, out) assert.Contains(t, f.vpc.detachCalls, "eni-pe-001", "private-endpoint ENI must be detached before delete") - _, getErr := GetClusterMeta(f.kv, "alpha") + _, getErr := GetClusterMeta(t.Context(), f.kv, "alpha") assert.ErrorIs(t, getErr, ErrClusterNotFound, "successful teardown must sweep the meta") } @@ -291,7 +293,7 @@ func TestDeleteCluster_NLBFailureLeavesMetaAndDELETING(t *testing.T) { _, err := f.svc.DeleteCluster(context.Background(), deleteInput("alpha"), testAccountID) require.Error(t, err, "teardown failure must surface, not be swallowed") - meta, getErr := GetClusterMeta(f.kv, "alpha") + meta, getErr := GetClusterMeta(t.Context(), f.kv, "alpha") require.NoError(t, getErr, "meta must survive so a retry can find the resource ARNs") assert.Equal(t, ClusterStatusDeleting, meta.Status) assert.NotEmpty(t, meta.NLBArn, "NLB ARN must remain recorded for retry") @@ -304,7 +306,7 @@ func TestDeleteCluster_VMTerminateFailureLeavesMeta(t *testing.T) { _, err := f.svc.DeleteCluster(context.Background(), deleteInput("alpha"), testAccountID) require.Error(t, err) - meta, getErr := GetClusterMeta(f.kv, "alpha") + meta, getErr := GetClusterMeta(t.Context(), f.kv, "alpha") require.NoError(t, getErr) assert.Equal(t, ClusterStatusDeleting, meta.Status) } @@ -321,7 +323,7 @@ func TestDeleteCluster_EgressEIPAlreadyReleasedSweepsKV(t *testing.T) { require.NotNil(t, out) require.Len(t, f.eip.releaseCalls, 1, "release must still be attempted") - _, getErr := GetClusterMeta(f.kv, "alpha") + _, getErr := GetClusterMeta(t.Context(), f.kv, "alpha") assert.ErrorIs(t, getErr, ErrClusterNotFound, "NotFound release must not block the KV sweep") } @@ -334,7 +336,7 @@ func TestDeleteCluster_EgressEIPReleaseRealErrorLeavesMeta(t *testing.T) { _, err := f.svc.DeleteCluster(context.Background(), deleteInput("alpha"), testAccountID) require.Error(t, err) - meta, getErr := GetClusterMeta(f.kv, "alpha") + meta, getErr := GetClusterMeta(t.Context(), f.kv, "alpha") require.NoError(t, getErr) assert.Equal(t, ClusterStatusDeleting, meta.Status) assert.NotEmpty(t, meta.EgressEIPAllocationID, "alloc ID must remain for retry") @@ -349,7 +351,7 @@ func TestDeleteClusterSingleFlightLoserSkipsTeardown(t *testing.T) { f := newDeleteClusterFixture(t, "alpha") // Stand in for the winning handler: hold the teardown lease. - _, err := f.svc.leaderKV.Create(teardownLeaderKey(testAccountID, "alpha"), []byte("node-2")) + _, err := f.svc.leaderKV.Create(t.Context(), teardownLeaderKey(testAccountID, "alpha"), []byte("node-2")) require.NoError(t, err) out, err := f.svc.DeleteCluster(context.Background(), deleteInput("alpha"), testAccountID) @@ -361,7 +363,7 @@ func TestDeleteClusterSingleFlightLoserSkipsTeardown(t *testing.T) { assert.Empty(t, f.inst.terminateCalls, "the loser must not terminate the CP VM — the lease holder owns the teardown") assert.Empty(t, f.eip.releaseCalls, "the loser must not release the billable EIP") - meta, getErr := GetClusterMeta(f.kv, "alpha") + meta, getErr := GetClusterMeta(t.Context(), f.kv, "alpha") require.NoError(t, getErr, "the loser must leave the meta for the lease holder to sweep") assert.Equal(t, ClusterStatusCreating, meta.Status, "the loser must not mutate KV state; the winner owns the DELETING transition") } @@ -375,8 +377,21 @@ func TestDeleteClusterWinnerReleasesTeardownLease(t *testing.T) { _, err := f.svc.DeleteCluster(context.Background(), deleteInput("alpha"), testAccountID) require.NoError(t, err) - _, getErr := f.svc.leaderKV.Get(teardownLeaderKey(testAccountID, "alpha")) - assert.ErrorIs(t, getErr, nats.ErrKeyNotFound, "the teardown lease must be released after a successful delete") + _, getErr := f.svc.leaderKV.Get(t.Context(), teardownLeaderKey(testAccountID, "alpha")) + assert.ErrorIs(t, getErr, jetstream.ErrKeyNotFound, "the teardown lease must be released after a successful delete") +} + +func TestTeardownLeaseReleaseSurvivesCallerCancellation(t *testing.T) { + f := newDeleteClusterFixture(t, "alpha") + ctx, cancel := context.WithCancel(t.Context()) + + release, acquired := f.svc.acquireTeardownLease(ctx, testAccountID, "alpha") + require.True(t, acquired) + cancel() + release() + + _, err := f.svc.leaderKV.Get(t.Context(), teardownLeaderKey(testAccountID, "alpha")) + require.ErrorIs(t, err, jetstream.ErrKeyNotFound) } // TestDeleteCluster_ManagedCPVPCReclaimsCustomerVPCSGs guards the contract: in @@ -388,10 +403,10 @@ func TestDeleteClusterWinnerReleasesTeardownLease(t *testing.T) { func TestDeleteCluster_ManagedCPVPCReclaimsCustomerVPCSGs(t *testing.T) { f := newDeleteClusterFixture(t, "alpha") - meta, err := GetClusterMeta(f.kv, "alpha") + meta, err := GetClusterMeta(t.Context(), f.kv, "alpha") require.NoError(t, err) meta.ManagedCPVPC = &ManagedCPVPC{VpcId: "vpc-cp"} - require.NoError(t, PutClusterMeta(f.kv, meta)) + require.NoError(t, PutClusterMeta(t.Context(), f.kv, meta)) // Worker-side SGs created in the customer VPC (vpc-aaa) by launchNodegroupInfra. f.sg.existing["eks-cluster-alpha-control-plane-sg|vpc-aaa"] = "sg-cust-cp" @@ -421,10 +436,10 @@ func TestDeleteCluster_ManagedCPVPCReclaimsCustomerVPCSGs(t *testing.T) { func TestDeleteCluster_ManagedCPVPCAlreadyGoneClearsStateRecord(t *testing.T) { f := newDeleteClusterFixture(t, "alpha") - meta, err := GetClusterMeta(f.kv, "alpha") + meta, err := GetClusterMeta(t.Context(), f.kv, "alpha") require.NoError(t, err) meta.ManagedCPVPC = &ManagedCPVPC{VpcId: "vpc-cp-gone", PublicSubnetId: "subnet-cp-pub"} - require.NoError(t, PutClusterMeta(f.kv, meta)) + require.NoError(t, PutClusterMeta(t.Context(), f.kv, meta)) // The CP VPC's own EC2 record is already gone: f.vpcMgr has nothing tagged // for "alpha", so DeleteClusterCPVPC's describe-by-tag finds zero VPCs and @@ -439,7 +454,7 @@ func TestDeleteCluster_ManagedCPVPCAlreadyGoneClearsStateRecord(t *testing.T) { _, err = f.svc.DeleteCluster(context.Background(), deleteInput("alpha"), testAccountID) require.Error(t, err, "the unrelated NLB failure must still surface") - stuck, getErr := GetClusterMeta(f.kv, "alpha") + stuck, getErr := GetClusterMeta(t.Context(), f.kv, "alpha") require.NoError(t, getErr, "meta must survive for retry") assert.Equal(t, ClusterStatusDeleting, stuck.Status) assert.Nil(t, stuck.ManagedCPVPC, @@ -452,7 +467,7 @@ func TestDeleteCluster_ManagedCPVPCAlreadyGoneClearsStateRecord(t *testing.T) { _, err = f.svc.DeleteCluster(context.Background(), deleteInput("alpha"), testAccountID) require.NoError(t, err, "the re-drive must converge to fully deleted") - _, getErr = GetClusterMeta(f.kv, "alpha") + _, getErr = GetClusterMeta(t.Context(), f.kv, "alpha") assert.ErrorIs(t, getErr, ErrClusterNotFound, "meta must be swept once teardown converges") } diff --git a/spinifex/handlers/eks/dns_reconcile_test.go b/spinifex/handlers/eks/dns_reconcile_test.go index 08d4e1943..621b8de8c 100644 --- a/spinifex/handlers/eks/dns_reconcile_test.go +++ b/spinifex/handlers/eks/dns_reconcile_test.go @@ -5,6 +5,7 @@ import ( "testing" handlers_dns "github.com/mulgadc/spinifex/spinifex/handlers/dns" + "github.com/mulgadc/spinifex/spinifex/testutil" "github.com/nats-io/nats.go" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -25,7 +26,7 @@ func TestCreateCluster_EndpointDNSNameUsesOwnerAccount(t *testing.T) { require.NoError(t, err) fixture.svc.WaitLaunches() - meta, err := GetClusterMeta(fixture.kv, "alpha") + meta, err := GetClusterMeta(t.Context(), fixture.kv, "alpha") require.NoError(t, err) assert.Equal(t, "alpha.111122223333.us-east-1.eks.spx3.net", meta.EndpointDNSName) assert.Contains(t, meta.Endpoint, meta.EndpointDNSName) @@ -39,19 +40,19 @@ func TestDesiredDNSChanges_IncludesEndpointReadyCreatingClusters(t *testing.T) { creating.Status = ClusterStatusCreating creating.EndpointDNSName = "creating.111122223333.us-east-1.eks.spx3.net" creating.EndpointIP = "203.0.113.10" - require.NoError(t, PutClusterMeta(fixture.kv, creating)) + require.NoError(t, PutClusterMeta(t.Context(), fixture.kv, creating)) active := sampleClusterMeta("active") active.Status = ClusterStatusActive active.EndpointDNSName = "active.111122223333.us-east-1.eks.spx3.net" active.EndpointIP = "203.0.113.11" - require.NoError(t, PutClusterMeta(fixture.kv, active)) + require.NoError(t, PutClusterMeta(t.Context(), fixture.kv, active)) failed := sampleClusterMeta("failed") failed.Status = ClusterStatusFailed failed.EndpointDNSName = "failed.111122223333.us-east-1.eks.spx3.net" failed.EndpointIP = "203.0.113.12" - require.NoError(t, PutClusterMeta(fixture.kv, failed)) + require.NoError(t, PutClusterMeta(t.Context(), fixture.kv, failed)) changes, authoritative := fixture.svc.DesiredDNSChanges() require.True(t, authoritative) @@ -67,13 +68,12 @@ func TestDesiredDNSChanges_MetadataReadFailureIsNotAuthoritative(t *testing.T) { active.Status = ClusterStatusActive active.EndpointDNSName = "healthy.111122223333.us-east-1.eks.spx3.net" active.EndpointIP = "203.0.113.10" - require.NoError(t, PutClusterMeta(fixture.kv, active)) + require.NoError(t, PutClusterMeta(t.Context(), fixture.kv, active)) - js, err := fixture.svc.deps.NATSConn.JetStream() + js := testutil.NewJetStream(t, fixture.svc.deps.NATSConn) + corruptKV, err := GetOrCreateAccountBucket(t.Context(), js, "444455556666", 1) require.NoError(t, err) - corruptKV, err := GetOrCreateAccountBucket(js, "444455556666", 1) - require.NoError(t, err) - _, err = corruptKV.Put(ClusterMetaKey("unreadable"), []byte("{not json")) + _, err = corruptKV.Put(t.Context(), ClusterMetaKey("unreadable"), []byte("{not json")) require.NoError(t, err) changes, authoritative := fixture.svc.DesiredDNSChanges() @@ -89,7 +89,7 @@ func TestDesiredDNSChanges_EnumerationFailureIsNotAuthoritative(t *testing.T) { active.Status = ClusterStatusActive active.EndpointDNSName = "healthy.111122223333.us-east-1.eks.spx3.net" active.EndpointIP = "203.0.113.10" - require.NoError(t, PutClusterMeta(fixture.kv, active)) + require.NoError(t, PutClusterMeta(t.Context(), fixture.kv, active)) // A failed bucket enumeration must never be mistaken for "no buckets": closing the // connection makes the stream-names request error, and the reconcile must suppress diff --git a/spinifex/handlers/eks/eks_billable_reaper.go b/spinifex/handlers/eks/eks_billable_reaper.go index a3bdc771d..20f432b5e 100644 --- a/spinifex/handlers/eks/eks_billable_reaper.go +++ b/spinifex/handlers/eks/eks_billable_reaper.go @@ -10,6 +10,7 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" "github.com/mulgadc/spinifex/spinifex/tags" "github.com/mulgadc/spinifex/spinifex/vm" + "github.com/nats-io/nats.go/jetstream" ) // EKSBillableReaper is ADR-0006 §5's meta-independent billable cleanup, built on @@ -47,7 +48,7 @@ func (r *EKSBillableReaper) Sweep(ctx context.Context) (int, error) { return 0, fmt.Errorf("list running VMs: %w", err) } - js, err := r.svc.deps.NATSConn.JetStream() + js, err := jetstream.New(r.svc.deps.NATSConn) if err != nil { return 0, fmt.Errorf("jetstream: %w", err) } @@ -68,12 +69,12 @@ func (r *EKSBillableReaper) Sweep(ctx context.Context) (int, error) { continue // ENI gone or untagged: cannot confirm orphan-hood, never reap } - acctKV, err := GetOrCreateAccountBucket(js, clusterAccount, max(r.svc.deps.ClusterSize, 1)) + acctKV, err := GetOrCreateAccountBucket(ctx, js, clusterAccount, max(r.svc.deps.ClusterSize, 1)) if err != nil { slog.WarnContext(ctx, "eks-billable: account bucket lookup failed", "account", clusterAccount, "err", err) continue } - if _, err := GetClusterMeta(acctKV, clusterName); !errors.Is(err, ErrClusterNotFound) { + if _, err := GetClusterMeta(ctx, acctKV, clusterName); !errors.Is(err, ErrClusterNotFound) { // Meta present (live/CREATING/DELETING) or unreadable: not a definitive // orphan — leave it to the cluster's own teardown/retry. continue diff --git a/spinifex/handlers/eks/eks_billable_reaper_test.go b/spinifex/handlers/eks/eks_billable_reaper_test.go index 65e6ad07f..8a46ee79f 100644 --- a/spinifex/handlers/eks/eks_billable_reaper_test.go +++ b/spinifex/handlers/eks/eks_billable_reaper_test.go @@ -87,7 +87,7 @@ func TestRLC5_EKSBillableReaperTerminatesOrphanCPVM(t *testing.T) { // untagged is never reaped (orphan-hood cannot be confirmed). func TestRLC5_EKSBillableReaperSpareLiveAndUncertain(t *testing.T) { f := newEKSServiceFixture(t) - require.NoError(t, PutClusterMeta(f.kv, sampleClusterMeta("alive"))) + require.NoError(t, PutClusterMeta(t.Context(), f.kv, sampleClusterMeta("alive"))) f.vpc.describeByENI = map[string]*ec2.NetworkInterface{ "eni-live": cpENI("eni-live", "alive", testAccountID), diff --git a/spinifex/handlers/eks/eks_deleting_reaper.go b/spinifex/handlers/eks/eks_deleting_reaper.go index 509657ef4..eb8272d2d 100644 --- a/spinifex/handlers/eks/eks_deleting_reaper.go +++ b/spinifex/handlers/eks/eks_deleting_reaper.go @@ -8,7 +8,7 @@ import ( "time" "github.com/mulgadc/spinifex/spinifex/vm" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // deletingReapMinAge is how long a cluster must sit in DELETING before the @@ -47,32 +47,35 @@ func (r *EKSDeletingReaper) Sweep(ctx context.Context) (int, error) { if !r.svc.depsReadyForOrchestration() { return 0, nil } - js, err := r.svc.deps.NATSConn.JetStream() + js, err := jetstream.New(r.svc.deps.NATSConn) if err != nil { return 0, fmt.Errorf("jetstream: %w", err) } + // A truncated bucket listing would silently leave a wedged teardown in place + // for another sweep interval, so an incomplete enumeration fails the sweep. + names, err := accountBucketNames(ctx, r.svc.deps.NATSConn) + if err != nil { + return 0, fmt.Errorf("enumerate account buckets: %w", err) + } reaped := 0 - for name := range js.KeyValueStoreNames() { - if !strings.HasPrefix(name, KVBucketEKSAccountPrefix) { - continue - } + for _, name := range names { select { case <-ctx.Done(): return reaped, ctx.Err() default: } accountID := strings.TrimPrefix(name, KVBucketEKSAccountPrefix) - acctKV, err := js.KeyValue(name) + acctKV, err := js.KeyValue(ctx, name) if err != nil { continue } - clusters, err := listClusterNames(acctKV) + clusters, err := listClusterNames(ctx, acctKV) if err != nil { continue } for _, cluster := range clusters { - n, err := r.reapCluster(accountID, acctKV, cluster) + n, err := r.reapCluster(ctx, accountID, acctKV, cluster) if err != nil { slog.Warn("eks-deleting: re-drive teardown failed", "cluster", cluster, "err", err) } @@ -85,8 +88,8 @@ func (r *EKSDeletingReaper) Sweep(ctx context.Context) (int, error) { // reapCluster re-drives one cluster's teardown if it is wedged in DELETING past // minAge and its leader lease can be acquired. Returns 1 when it completed a // teardown, 0 otherwise. -func (r *EKSDeletingReaper) reapCluster(accountID string, acctKV nats.KeyValue, cluster string) (int, error) { - meta, err := GetClusterMeta(acctKV, cluster) +func (r *EKSDeletingReaper) reapCluster(ctx context.Context, accountID string, acctKV jetstream.KeyValue, cluster string) (int, error) { + meta, err := GetClusterMeta(ctx, acctKV, cluster) if err != nil { return 0, nil // gone or unreadable: nothing to re-drive } @@ -97,7 +100,7 @@ func (r *EKSDeletingReaper) reapCluster(accountID string, acctKV nats.KeyValue, return 0, nil // still within the in-flight synchronous-delete window } - release, ok := r.svc.acquireTeardownLease(accountID, cluster) + release, ok := r.svc.acquireTeardownLease(ctx, accountID, cluster) if !ok { return 0, nil // a synchronous delete or another node's reaper owns this teardown } diff --git a/spinifex/handlers/eks/eks_deleting_reaper_test.go b/spinifex/handlers/eks/eks_deleting_reaper_test.go index 1b3c2ee32..86829fa29 100644 --- a/spinifex/handlers/eks/eks_deleting_reaper_test.go +++ b/spinifex/handlers/eks/eks_deleting_reaper_test.go @@ -13,11 +13,11 @@ import ( // DeletingSince so the reaper treats it as wedged (past min-age). func markDeleting(t *testing.T, f *deleteClusterFixture, name string, age time.Duration) { t.Helper() - require.NoError(t, SetClusterStatus(f.kv, name, ClusterStatusDeleting)) - meta, err := GetClusterMeta(f.kv, name) + require.NoError(t, SetClusterStatus(t.Context(), f.kv, name, ClusterStatusDeleting)) + meta, err := GetClusterMeta(t.Context(), f.kv, name) require.NoError(t, err) meta.DeletingSince = time.Now().UTC().Add(-age) - require.NoError(t, PutClusterMeta(f.kv, meta)) + require.NoError(t, PutClusterMeta(t.Context(), f.kv, meta)) } // TestRLC4_DeletingReaperReDrivesWedgedTeardown locks the contract: a cluster @@ -33,7 +33,7 @@ func TestRLC4_DeletingReaperReDrivesWedgedTeardown(t *testing.T) { require.NoError(t, err) assert.Equal(t, 1, n, "the wedged DELETING cluster must be re-driven") - _, getErr := GetClusterMeta(f.kv, "alpha") + _, getErr := GetClusterMeta(t.Context(), f.kv, "alpha") assert.ErrorIs(t, getErr, ErrClusterNotFound, "meta must be swept after the backstop completes teardown") assert.GreaterOrEqual(t, len(f.inst.terminateCalls), 1, "CP VM must be terminated") assert.Len(t, f.eip.releaseCalls, 1, "the billable egress EIP must be released, not stranded") @@ -50,12 +50,30 @@ func TestDeletingReaperSkipsFreshDelete(t *testing.T) { require.NoError(t, err) assert.Equal(t, 0, n, "a freshly-DELETING cluster must be left to its in-flight delete") - meta, getErr := GetClusterMeta(f.kv, "beta") + meta, getErr := GetClusterMeta(t.Context(), f.kv, "beta") require.NoError(t, getErr, "meta must survive — the reaper did not re-drive") assert.Equal(t, ClusterStatusDeleting, meta.Status) assert.Empty(t, f.inst.terminateCalls, "no teardown must run within the min-age window") } +// TestDeletingReaper_EnumerationFailureIsReported: the KV bucket lister closes +// its channel the same way whether the listing completed or failed, so a sweep +// that ignored the terminal error would report "nothing wedged" for an +// unreachable JetStream — leaving a stuck teardown holding its billable EIP with +// nothing to say the sweep never looked. +func TestDeletingReaper_EnumerationFailureIsReported(t *testing.T) { + f := newDeleteClusterFixture(t, "delta") + markDeleting(t, f, "delta", 10*time.Minute) + + // Closing the connection fails the stream-names request behind the listing. + f.svc.deps.NATSConn.Close() + + reaper := f.svc.NewDeletingReaper() + n, err := reaper.Sweep(context.Background()) + require.Error(t, err, "a failed bucket enumeration must not report a completed sweep") + assert.Zero(t, n) +} + // TestDeletingReaperSkipsNonDeleting: a CREATING/ACTIVE cluster is never touched. func TestDeletingReaperSkipsNonDeleting(t *testing.T) { f := newDeleteClusterFixture(t, "gamma") // stays CREATING @@ -65,7 +83,7 @@ func TestDeletingReaperSkipsNonDeleting(t *testing.T) { require.NoError(t, err) assert.Equal(t, 0, n) - meta, getErr := GetClusterMeta(f.kv, "gamma") + meta, getErr := GetClusterMeta(t.Context(), f.kv, "gamma") require.NoError(t, getErr) assert.Equal(t, ClusterStatusCreating, meta.Status) assert.Empty(t, f.inst.terminateCalls) diff --git a/spinifex/handlers/eks/lifecycle_invariants_test.go b/spinifex/handlers/eks/lifecycle_invariants_test.go index 061167995..57bb7460f 100644 --- a/spinifex/handlers/eks/lifecycle_invariants_test.go +++ b/spinifex/handlers/eks/lifecycle_invariants_test.go @@ -40,7 +40,7 @@ func TestRLC2_EKSBillableTeardownBeforeKVSweep(t *testing.T) { require.Error(t, err, "ADR-0006 §6: a failed billable teardown must surface, not be swallowed") require.Len(t, f.inst.terminateCalls, 1, "the VM teardown must have been attempted") - meta, getErr := GetClusterMeta(f.kv, "alpha") + meta, getErr := GetClusterMeta(t.Context(), f.kv, "alpha") require.NoErrorf(t, getErr, "ADR-0006 §6 billable-before-sweep: the KV sweep must NOT run while a billable teardown is failing — the meta must survive for retry") assert.Equal(t, ClusterStatusDeleting, meta.Status, "a cluster with failed teardown must stay DELETING") } @@ -67,7 +67,7 @@ func TestRLC3_EKSNLBNoOrphanTargetGroupAfterDelete(t *testing.T) { _, err := f.svc.DeleteCluster(context.Background(), deleteInput("alpha"), testAccountID) require.NoError(t, err) - _, getErr := GetClusterMeta(f.kv, "alpha") + _, getErr := GetClusterMeta(t.Context(), f.kv, "alpha") require.ErrorIs(t, getErr, ErrClusterNotFound, "a fully torn-down cluster must be swept") assert.NotContainsf(t, f.nlb.tgByName, tgName, "ADR-0006 §6 NLB no-orphan: the eks-{cluster}-cp target group must not survive DeleteCluster (rides ADR-0002/172)") diff --git a/spinifex/handlers/eks/list_clusters_test.go b/spinifex/handlers/eks/list_clusters_test.go index ab52f130b..47dc98892 100644 --- a/spinifex/handlers/eks/list_clusters_test.go +++ b/spinifex/handlers/eks/list_clusters_test.go @@ -20,7 +20,7 @@ import ( func TestEKSServiceImpl_ListClustersPagination(t *testing.T) { f := newEKSServiceFixture(t) for _, n := range []string{"c01", "c02", "c03", "c04", "c05"} { - require.NoError(t, PutClusterMeta(f.kv, sampleClusterMeta(n))) + require.NoError(t, PutClusterMeta(t.Context(), f.kv, sampleClusterMeta(n))) } p1, err := f.svc.ListClusters(context.Background(), &eks.ListClustersInput{MaxResults: aws.Int64(2)}, testAccountID) @@ -48,7 +48,7 @@ func TestEKSServiceImpl_ListClustersMaxResultsClamped(t *testing.T) { f := newEKSServiceFixture(t) want := []string{"c01", "c02", "c03"} for _, n := range want { - require.NoError(t, PutClusterMeta(f.kv, sampleClusterMeta(n))) + require.NoError(t, PutClusterMeta(t.Context(), f.kv, sampleClusterMeta(n))) } for _, mr := range []int64{0, -5, 250} { diff --git a/spinifex/handlers/eks/nats_bootstrap.go b/spinifex/handlers/eks/nats_bootstrap.go index 31c8185b2..877a03a84 100644 --- a/spinifex/handlers/eks/nats_bootstrap.go +++ b/spinifex/handlers/eks/nats_bootstrap.go @@ -12,6 +12,7 @@ import ( handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // errBootstrapPermanent marks failures that retrying won't fix (malformed envelope, @@ -74,7 +75,7 @@ type BootstrapEnvelope struct { // completion via the four KV keys + meta.CertificateAuthorityB64. type NATSBootstrap struct { nc *nats.Conn - kv nats.KeyValue + kv jetstream.KeyValue masterKey []byte accountID string clusterName string @@ -82,7 +83,7 @@ type NATSBootstrap struct { // NewNATSBootstrap validates inputs and returns a ready subscriber. Run must // be called to actually subscribe; constructor is side-effect-free. -func NewNATSBootstrap(nc *nats.Conn, kv nats.KeyValue, masterKey []byte, accountID, clusterName string) (*NATSBootstrap, error) { +func NewNATSBootstrap(nc *nats.Conn, kv jetstream.KeyValue, masterKey []byte, accountID, clusterName string) (*NATSBootstrap, error) { if nc == nil { return nil, errors.New("eks: NewNATSBootstrap nil nats conn") } @@ -111,15 +112,15 @@ func NewNATSBootstrap(nc *nats.Conn, kv nats.KeyValue, masterKey []byte, account // a daemon-restart resumes only on pending subjects (K3s publishes each once). // JWKS is tracked via OIDCJWKSVerifiedKey (set after cross-check), not OIDCJWKSKey // (pre-seeded by the controller), so ACTIVE always implies a verified keypair. -func BootstrapPendingKinds(kv nats.KeyValue, clusterName string, meta *ClusterMeta) []string { +func BootstrapPendingKinds(ctx context.Context, kv jetstream.KeyValue, clusterName string, meta *ClusterMeta) []string { var pending []string - if _, err := kv.Get(NodeTokenKey(clusterName)); err != nil { + if _, err := kv.Get(ctx, NodeTokenKey(clusterName)); err != nil { pending = append(pending, BootstrapSubjectToken) } - if _, err := kv.Get(AdminKubeconfigKey(clusterName)); err != nil { + if _, err := kv.Get(ctx, AdminKubeconfigKey(clusterName)); err != nil { pending = append(pending, BootstrapSubjectKubeconfig) } - if _, err := kv.Get(OIDCJWKSVerifiedKey(clusterName)); err != nil { + if _, err := kv.Get(ctx, OIDCJWKSVerifiedKey(clusterName)); err != nil { pending = append(pending, BootstrapSubjectJWKS) } if meta == nil || meta.CertificateAuthorityB64 == "" { @@ -131,14 +132,14 @@ func BootstrapPendingKinds(kv nats.KeyValue, clusterName string, meta *ClusterMe type bootstrapSubjectHandler struct { subject string kind string - handler func([]byte) error + handler func(context.Context, []byte) error } // subjectHandlers returns the subject/handler entry for each requested kind. // An unknown kind is an error — callers pass kinds from BootstrapSubject* // constants. A nil/empty kinds slice means all four. func (b *NATSBootstrap) subjectHandlers(kinds []string) ([]bootstrapSubjectHandler, error) { - all := map[string]func([]byte) error{ + all := map[string]func(context.Context, []byte) error{ BootstrapSubjectToken: b.persistToken, BootstrapSubjectKubeconfig: b.persistKubeconfig, BootstrapSubjectJWKS: b.persistJWKS, @@ -200,7 +201,7 @@ func (b *NATSBootstrap) RunForKinds(ctx context.Context, kinds []string) error { return } mu.Unlock() - if err := persistWithRetry(func() error { return s.handler(m.Data) }); err != nil { + if err := persistWithRetry(func() error { return s.handler(ctx, m.Data) }); err != nil { errCh <- fmt.Errorf("persist %s: %w", s.kind, err) return } @@ -235,7 +236,7 @@ func (b *NATSBootstrap) RunForKinds(ctx context.Context, kinds []string) error { } } -func (b *NATSBootstrap) persistToken(data []byte) error { +func (b *NATSBootstrap) persistToken(ctx context.Context, data []byte) error { env, err := unmarshalBootstrapEnvelope(data) if err != nil { return err @@ -247,13 +248,13 @@ func (b *NATSBootstrap) persistToken(data []byte) error { if err != nil { return fmt.Errorf("encrypt token: %w", err) } - if _, err := b.kv.Put(NodeTokenKey(b.clusterName), []byte(ct)); err != nil { + if _, err := b.kv.Put(ctx, NodeTokenKey(b.clusterName), []byte(ct)); err != nil { return fmt.Errorf("kv put %s: %w", NodeTokenKey(b.clusterName), err) } return nil } -func (b *NATSBootstrap) persistKubeconfig(data []byte) error { +func (b *NATSBootstrap) persistKubeconfig(ctx context.Context, data []byte) error { env, err := unmarshalBootstrapEnvelope(data) if err != nil { return err @@ -265,13 +266,13 @@ func (b *NATSBootstrap) persistKubeconfig(data []byte) error { if err != nil { return fmt.Errorf("encrypt kubeconfig: %w", err) } - if _, err := b.kv.Put(AdminKubeconfigKey(b.clusterName), []byte(ct)); err != nil { + if _, err := b.kv.Put(ctx, AdminKubeconfigKey(b.clusterName), []byte(ct)); err != nil { return fmt.Errorf("kv put %s: %w", AdminKubeconfigKey(b.clusterName), err) } return nil } -func (b *NATSBootstrap) persistJWKS(data []byte) error { +func (b *NATSBootstrap) persistJWKS(ctx context.Context, data []byte) error { env, err := unmarshalBootstrapEnvelope(data) if err != nil { return err @@ -279,7 +280,7 @@ func (b *NATSBootstrap) persistJWKS(data []byte) error { if env.JWKS == "" { return fmt.Errorf("%w: jwks envelope empty", errBootstrapPermanent) } - existing, err := b.kv.Get(OIDCJWKSKey(b.clusterName)) + existing, err := b.kv.Get(ctx, OIDCJWKSKey(b.clusterName)) if err != nil { return fmt.Errorf("kv get %s: %w", OIDCJWKSKey(b.clusterName), err) } @@ -287,13 +288,13 @@ func (b *NATSBootstrap) persistJWKS(data []byte) error { return err } // Reconciler gates ACTIVE on this verified-marker, not OIDCJWKSKey (pre-seeded by controller). - if _, err := b.kv.Put(OIDCJWKSVerifiedKey(b.clusterName), []byte("verified")); err != nil { + if _, err := b.kv.Put(ctx, OIDCJWKSVerifiedKey(b.clusterName), []byte("verified")); err != nil { return fmt.Errorf("kv put %s: %w", OIDCJWKSVerifiedKey(b.clusterName), err) } return nil } -func (b *NATSBootstrap) persistCA(data []byte) error { +func (b *NATSBootstrap) persistCA(ctx context.Context, data []byte) error { env, err := unmarshalBootstrapEnvelope(data) if err != nil { return err @@ -304,7 +305,7 @@ func (b *NATSBootstrap) persistCA(data []byte) error { if _, err := base64.StdEncoding.DecodeString(env.CertificateAuthority); err != nil { return fmt.Errorf("%w: CA not base64: %w", errBootstrapPermanent, err) } - return SetClusterCertificateAuthority(b.kv, b.clusterName, env.CertificateAuthority) + return SetClusterCertificateAuthority(ctx, b.kv, b.clusterName, env.CertificateAuthority) } func unmarshalBootstrapEnvelope(data []byte) (BootstrapEnvelope, error) { diff --git a/spinifex/handlers/eks/nats_bootstrap_test.go b/spinifex/handlers/eks/nats_bootstrap_test.go index 060f13c15..9c7ca411f 100644 --- a/spinifex/handlers/eks/nats_bootstrap_test.go +++ b/spinifex/handlers/eks/nats_bootstrap_test.go @@ -13,6 +13,7 @@ import ( handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" "github.com/mulgadc/spinifex/spinifex/testutil" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -29,22 +30,23 @@ var bootstrapTestMasterKey = []byte("0123456789abcdef0123456789abcdef") type natsBootstrapHarness struct { t *testing.T nc *nats.Conn - kv nats.KeyValue + kv jetstream.KeyValue subscriber *NATSBootstrap } func newBootstrapHarness(t *testing.T) *natsBootstrapHarness { t.Helper() - _, nc, js := testutil.StartTestJetStream(t) - kv, err := GetOrCreateAccountBucket(js, testAccountID, 1) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1) require.NoError(t, err) - require.NoError(t, PutClusterMeta(kv, sampleClusterMeta(bootstrapTestCluster))) + require.NoError(t, PutClusterMeta(t.Context(), kv, sampleClusterMeta(bootstrapTestCluster))) // Lay down the controller-generated JWKS so persistJWKS has something // to cross-check against. - _, _, err = GenerateClusterOIDCKeypair(kv, bootstrapTestCluster, bootstrapTestMasterKey) + _, _, err = GenerateClusterOIDCKeypair(t.Context(), kv, bootstrapTestCluster, bootstrapTestMasterKey) require.NoError(t, err) sub, err := NewNATSBootstrap(nc, kv, bootstrapTestMasterKey, testAccountID, bootstrapTestCluster) @@ -64,7 +66,7 @@ func (h *natsBootstrapHarness) publish(kind string, env BootstrapEnvelope) { func (h *natsBootstrapHarness) loadControllerJWKS() []byte { h.t.Helper() - entry, err := h.kv.Get(OIDCJWKSKey(bootstrapTestCluster)) + entry, err := h.kv.Get(h.t.Context(), OIDCJWKSKey(bootstrapTestCluster)) require.NoError(h.t, err) return entry.Value() } @@ -81,8 +83,9 @@ func (h *natsBootstrapHarness) waitSubsBound(base, want int) { } func TestNewNATSBootstrap_RejectsBadInputs(t *testing.T) { - _, nc, js := testutil.StartTestJetStream(t) - kv, err := GetOrCreateAccountBucket(js, testAccountID, 1) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1) require.NoError(t, err) _, err = NewNATSBootstrap(nil, kv, bootstrapTestMasterKey, testAccountID, "alpha") @@ -122,25 +125,25 @@ func TestNATSBootstrap_HappyPath(t *testing.T) { t.Fatal("Run did not return after all four publications") } - tokEntry, err := h.kv.Get(NodeTokenKey(bootstrapTestCluster)) + tokEntry, err := h.kv.Get(h.t.Context(), NodeTokenKey(bootstrapTestCluster)) require.NoError(t, err) dec, err := handlers_iam.DecryptSecret(string(tokEntry.Value()), bootstrapTestMasterKey) require.NoError(t, err) assert.Equal(t, "k3s::server-node-token::v1", dec) - kcEntry, err := h.kv.Get(AdminKubeconfigKey(bootstrapTestCluster)) + kcEntry, err := h.kv.Get(h.t.Context(), AdminKubeconfigKey(bootstrapTestCluster)) require.NoError(t, err) dec, err = handlers_iam.DecryptSecret(string(kcEntry.Value()), bootstrapTestMasterKey) require.NoError(t, err) assert.Equal(t, "apiVersion: v1\nkind: Config\n", dec) - meta, err := GetClusterMeta(h.kv, bootstrapTestCluster) + meta, err := GetClusterMeta(t.Context(), h.kv, bootstrapTestCluster) require.NoError(t, err) assert.Equal(t, caB64, meta.CertificateAuthorityB64) // The passing JWKS cross-check must record the verified-marker the // reconciler gates ACTIVE on. - _, err = h.kv.Get(OIDCJWKSVerifiedKey(bootstrapTestCluster)) + _, err = h.kv.Get(h.t.Context(), OIDCJWKSVerifiedKey(bootstrapTestCluster)) require.NoError(t, err, "persistJWKS must write the verified-marker on a passing cross-check") // Run's deferred cleanup must tear down every subscription it bound, so the @@ -247,7 +250,7 @@ func TestNATSBootstrap_OneShotIgnoresSubsequentMessages(t *testing.T) { // Wait until the first token is persisted before racing in a second, so the // one-shot drop is what's under test rather than publish ordering. require.Eventually(t, func() bool { - _, err := h.kv.Get(NodeTokenKey(bootstrapTestCluster)) + _, err := h.kv.Get(h.t.Context(), NodeTokenKey(bootstrapTestCluster)) return err == nil }, 2*time.Second, 5*time.Millisecond, "first token never persisted") // Second token publication has a different value; persistToken should not @@ -265,7 +268,7 @@ func TestNATSBootstrap_OneShotIgnoresSubsequentMessages(t *testing.T) { t.Fatal("Run did not return") } - entry, err := h.kv.Get(NodeTokenKey(bootstrapTestCluster)) + entry, err := h.kv.Get(h.t.Context(), NodeTokenKey(bootstrapTestCluster)) require.NoError(t, err) dec, err := handlers_iam.DecryptSecret(string(entry.Value()), bootstrapTestMasterKey) require.NoError(t, err) @@ -335,37 +338,37 @@ func TestPersistWithRetry(t *testing.T) { func TestBootstrapPendingKinds(t *testing.T) { h := newBootstrapHarness(t) - meta, err := GetClusterMeta(h.kv, bootstrapTestCluster) + meta, err := GetClusterMeta(t.Context(), h.kv, bootstrapTestCluster) require.NoError(t, err) // Fresh cluster: all four pending. The JWKS verified-marker is absent even // though the controller pre-seeded OIDCJWKSKey, so JWKS is still pending. assert.Equal(t, []string{BootstrapSubjectToken, BootstrapSubjectKubeconfig, BootstrapSubjectJWKS, BootstrapSubjectCA}, - BootstrapPendingKinds(h.kv, bootstrapTestCluster, meta)) + BootstrapPendingKinds(t.Context(), h.kv, bootstrapTestCluster, meta)) // Persist token only -> token drops out of pending. ct, err := handlers_iam.EncryptSecret("tok", bootstrapTestMasterKey) require.NoError(t, err) - _, err = h.kv.Put(NodeTokenKey(bootstrapTestCluster), []byte(ct)) + _, err = h.kv.Put(t.Context(), NodeTokenKey(bootstrapTestCluster), []byte(ct)) require.NoError(t, err) assert.Equal(t, []string{BootstrapSubjectKubeconfig, BootstrapSubjectJWKS, BootstrapSubjectCA}, - BootstrapPendingKinds(h.kv, bootstrapTestCluster, meta)) + BootstrapPendingKinds(t.Context(), h.kv, bootstrapTestCluster, meta)) // Persist kubeconfig + CA-on-meta -> only JWKS (unverified) still pending. - _, err = h.kv.Put(AdminKubeconfigKey(bootstrapTestCluster), []byte(ct)) + _, err = h.kv.Put(t.Context(), AdminKubeconfigKey(bootstrapTestCluster), []byte(ct)) require.NoError(t, err) meta.CertificateAuthorityB64 = "ca-b64" assert.Equal(t, []string{BootstrapSubjectJWKS}, - BootstrapPendingKinds(h.kv, bootstrapTestCluster, meta)) + BootstrapPendingKinds(t.Context(), h.kv, bootstrapTestCluster, meta)) // Write the JWKS verified-marker (persistJWKS does this on a passing // cross-check) -> nothing pending. - _, err = h.kv.Put(OIDCJWKSVerifiedKey(bootstrapTestCluster), []byte("verified")) + _, err = h.kv.Put(t.Context(), OIDCJWKSVerifiedKey(bootstrapTestCluster), []byte("verified")) require.NoError(t, err) - assert.Empty(t, BootstrapPendingKinds(h.kv, bootstrapTestCluster, meta)) + assert.Empty(t, BootstrapPendingKinds(t.Context(), h.kv, bootstrapTestCluster, meta)) } func TestNATSBootstrap_RunForKindsWaitsOnlyForSubset(t *testing.T) { @@ -391,7 +394,7 @@ func TestNATSBootstrap_RunForKindsWaitsOnlyForSubset(t *testing.T) { t.Fatal("RunForKinds did not return after CA-only publication") } - meta, err := GetClusterMeta(h.kv, bootstrapTestCluster) + meta, err := GetClusterMeta(t.Context(), h.kv, bootstrapTestCluster) require.NoError(t, err) assert.Equal(t, caB64, meta.CertificateAuthorityB64) } diff --git a/spinifex/handlers/eks/nodegroup.go b/spinifex/handlers/eks/nodegroup.go index d8057183b..0120ed0c0 100644 --- a/spinifex/handlers/eks/nodegroup.go +++ b/spinifex/handlers/eks/nodegroup.go @@ -23,7 +23,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/awserrors" handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" "github.com/mulgadc/spinifex/spinifex/instancetypes" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // defaultNodegroupInstanceType mirrors the AWS EKS managed-nodegroup default @@ -86,7 +86,7 @@ func NodegroupARN(region, accountID, cluster, ng, id string) string { } // PutNodegroupRecord writes the record unconditionally. -func PutNodegroupRecord(kv nats.KeyValue, rec *NodegroupRecord) error { +func PutNodegroupRecord(ctx context.Context, kv jetstream.KeyValue, rec *NodegroupRecord) error { if rec == nil { return errors.New("eks: PutNodegroupRecord nil record") } @@ -98,7 +98,7 @@ func PutNodegroupRecord(kv nats.KeyValue, rec *NodegroupRecord) error { return fmt.Errorf("marshal nodegroup %s: %w", rec.Name, err) } key := NodegroupKey(rec.ClusterName, rec.Name) - if _, err := kv.Put(key, data); err != nil { + if _, err := kv.Put(ctx, key, data); err != nil { return fmt.Errorf("kv put %s: %w", key, err) } return nil @@ -108,7 +108,7 @@ func PutNodegroupRecord(kv nats.KeyValue, rec *NodegroupRecord) error { // owned=false when a record already exists. This is the idempotency barrier for // CreateNodegroup — a duplicate request loses the claim and never launches // workers. Owner updates after the claim use PutNodegroupRecord. -func ClaimNodegroupRecord(kv nats.KeyValue, rec *NodegroupRecord) (bool, error) { +func ClaimNodegroupRecord(ctx context.Context, kv jetstream.KeyValue, rec *NodegroupRecord) (bool, error) { if rec == nil { return false, errors.New("eks: ClaimNodegroupRecord nil record") } @@ -119,18 +119,18 @@ func ClaimNodegroupRecord(kv nats.KeyValue, rec *NodegroupRecord) (bool, error) if err != nil { return false, fmt.Errorf("marshal nodegroup %s: %w", rec.Name, err) } - owned, _, _, err := claimKey(kv, NodegroupKey(rec.ClusterName, rec.Name), data) + owned, _, _, err := claimKey(ctx, kv, NodegroupKey(rec.ClusterName, rec.Name), data) return owned, err } // GetNodegroupRecord reads one record. Returns ErrNodegroupNotFound if absent. -func GetNodegroupRecord(kv nats.KeyValue, cluster, ng string) (*NodegroupRecord, error) { +func GetNodegroupRecord(ctx context.Context, kv jetstream.KeyValue, cluster, ng string) (*NodegroupRecord, error) { if cluster == "" || ng == "" { return nil, errors.New("eks: GetNodegroupRecord empty cluster or name") } - entry, err := kv.Get(NodegroupKey(cluster, ng)) + entry, err := kv.Get(ctx, NodegroupKey(cluster, ng)) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil, ErrNodegroupNotFound } return nil, fmt.Errorf("kv get nodegroup: %w", err) @@ -144,13 +144,13 @@ func GetNodegroupRecord(kv nats.KeyValue, cluster, ng string) (*NodegroupRecord, // ListNodegroupRecords returns every nodegroup record under a cluster, sorted // by name for stable output. -func ListNodegroupRecords(kv nats.KeyValue, cluster string) ([]*NodegroupRecord, error) { +func ListNodegroupRecords(ctx context.Context, kv jetstream.KeyValue, cluster string) ([]*NodegroupRecord, error) { if cluster == "" { return nil, errors.New("eks: ListNodegroupRecords empty cluster") } - keys, err := kv.Keys() + keys, err := kv.Keys(ctx) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return nil, nil } return nil, fmt.Errorf("kv keys: %w", err) @@ -161,9 +161,9 @@ func ListNodegroupRecords(kv nats.KeyValue, cluster string) ([]*NodegroupRecord, if !strings.HasPrefix(k, prefix) { continue } - entry, err := kv.Get(k) + entry, err := kv.Get(ctx, k) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { continue } return nil, fmt.Errorf("kv get %s: %w", k, err) @@ -180,28 +180,28 @@ func ListNodegroupRecords(kv nats.KeyValue, cluster string) ([]*NodegroupRecord, // DeleteNodegroupRecord removes one record. A missing key is a no-op so // DeleteNodegroup stays idempotent. -func DeleteNodegroupRecord(kv nats.KeyValue, cluster, ng string) error { +func DeleteNodegroupRecord(ctx context.Context, kv jetstream.KeyValue, cluster, ng string) error { key := NodegroupKey(cluster, ng) - if err := kv.Delete(key); err != nil && !errors.Is(err, nats.ErrKeyNotFound) { + if err := kv.Delete(ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { return fmt.Errorf("kv delete %s: %w", key, err) } return nil } // nodegroupAcctKV opens the per-account bucket for nodegroup handlers. -func (s *EKSServiceImpl) nodegroupAcctKV(accountID string) (nats.KeyValue, error) { - js, err := s.deps.NATSConn.JetStream() +func (s *EKSServiceImpl) nodegroupAcctKV(ctx context.Context, accountID string) (jetstream.KeyValue, error) { + js, err := jetstream.New(s.deps.NATSConn) if err != nil { return nil, fmt.Errorf("jetstream: %w", err) } - acctKV, err := GetOrCreateAccountBucket(js, accountID, max(s.deps.ClusterSize, 1)) + acctKV, err := GetOrCreateAccountBucket(ctx, js, accountID, max(s.deps.ClusterSize, 1)) if err != nil { return nil, fmt.Errorf("get account bucket: %w", err) } return acctKV, nil } -func (s *EKSServiceImpl) createNodegroup(ctx context.Context, acctKV nats.KeyValue, input *eks.CreateNodegroupInput, accountID string) (*eks.CreateNodegroupOutput, error) { +func (s *EKSServiceImpl) createNodegroup(ctx context.Context, acctKV jetstream.KeyValue, input *eks.CreateNodegroupInput, accountID string) (*eks.CreateNodegroupOutput, error) { if input == nil { return nil, errors.New(awserrors.ErrorInvalidParameterValue) } @@ -215,7 +215,7 @@ func (s *EKSServiceImpl) createNodegroup(ctx context.Context, acctKV nats.KeyVal return nil, errors.New(awserrors.ErrorInvalidParameterValue) } - meta, err := GetClusterMeta(acctKV, cluster) + meta, err := GetClusterMeta(ctx, acctKV, cluster) if err != nil { if errors.Is(err, ErrClusterNotFound) { return nil, errors.New(awserrors.ErrorEKSResourceNotFound) @@ -239,7 +239,7 @@ func (s *EKSServiceImpl) createNodegroup(ctx context.Context, acctKV nats.KeyVal return nil, errors.New(awserrors.ErrorInvalidRequest) } - if _, err := GetNodegroupRecord(acctKV, cluster, ng); err == nil { + if _, err := GetNodegroupRecord(ctx, acctKV, cluster, ng); err == nil { return nil, errors.New(awserrors.ErrorEKSResourceInUse) } else if !errors.Is(err, ErrNodegroupNotFound) { return nil, err @@ -300,7 +300,7 @@ func (s *EKSServiceImpl) createNodegroup(ctx context.Context, acctKV nats.KeyVal } // Atomically claim the record; concurrent/duplicate CreateNodegroup requests lose here. - owned, err := ClaimNodegroupRecord(acctKV, rec) + owned, err := ClaimNodegroupRecord(ctx, acctKV, rec) if err != nil { return nil, err } @@ -312,13 +312,16 @@ func (s *EKSServiceImpl) createNodegroup(ctx context.Context, acctKV nats.KeyVal // Failures surface as CREATE_FAILED, which DeleteNodegroup reclaims. out := &eks.CreateNodegroupOutput{Nodegroup: nodegroupRecordToAWS(rec)} + // The launch outlives the request, so it runs on its own background context + // rather than the caller's, which is cancelled once this reply is written. + launchCtx := context.Background() s.launchWG.Go(func() { defer func() { if r := recover(); r != nil { - s.markNodegroupFailed(acctKV, cluster, ng, fmt.Sprintf("launch panic: %v", r)) + s.markNodegroupFailed(launchCtx, acctKV, cluster, ng, fmt.Sprintf("launch panic: %v", r)) } }() - s.launchNodegroupInfra(context.Background(), nodegroupLaunchCtx{ + s.launchNodegroupInfra(launchCtx, nodegroupLaunchCtx{ accountID: accountID, cluster: cluster, ng: ng, @@ -380,7 +383,7 @@ type nodegroupLaunchCtx struct { meta *ClusterMeta rec *NodegroupRecord desired int - acctKV nats.KeyValue + acctKV jetstream.KeyValue } // launchNodegroupInfra runs the slow provisioning tail of createNodegroup on a @@ -393,23 +396,23 @@ func (s *EKSServiceImpl) launchNodegroupInfra(ctx context.Context, lc nodegroupL cpSGID, ngSGID, err := EnsureClusterSGs(ctx, s.deps.VPCSG, accountID, cluster, meta.ResourcesVpcConfig.VpcId) if err != nil { - s.markNodegroupFailed(acctKV, cluster, ng, "ensure cluster SGs: "+err.Error()) + s.markNodegroupFailed(ctx, acctKV, cluster, ng, "ensure cluster SGs: "+err.Error()) return } if err := EnsureNodegroupSGRules(ctx, s.deps.VPCSG, accountID, cluster, cpSGID, ngSGID); err != nil { - s.markNodegroupFailed(acctKV, cluster, ng, "ensure nodegroup SG rules: "+err.Error()) + s.markNodegroupFailed(ctx, acctKV, cluster, ng, "ensure nodegroup SG rules: "+err.Error()) return } - token, err := s.decryptNodeToken(acctKV, cluster) + token, err := s.decryptNodeToken(ctx, acctKV, cluster) if err != nil { - s.markNodegroupFailed(acctKV, cluster, ng, "decrypt node token: "+err.Error()) + s.markNodegroupFailed(ctx, acctKV, cluster, ng, "decrypt node token: "+err.Error()) return } amiID, err := resolveWorkerAMI(ctx, s.deps.Image, accountID, rec) if err != nil { - s.markNodegroupFailed(acctKV, cluster, ng, "resolve eks-node AMI: "+err.Error()) + s.markNodegroupFailed(ctx, acctKV, cluster, ng, "resolve eks-node AMI: "+err.Error()) return } @@ -423,7 +426,7 @@ func (s *EKSServiceImpl) launchNodegroupInfra(ctx context.Context, lc nodegroupL rec.Status = eks.NodegroupStatusCreateFailed rec.StatusReason = "launch workers: " + err.Error() rec.ModifiedAt = time.Now().UTC() - if perr := PutNodegroupRecord(acctKV, rec); perr != nil { + if perr := PutNodegroupRecord(ctx, acctKV, rec); perr != nil { slog.ErrorContext(ctx, "createNodegroup: persist CREATE_FAILED record", "cluster", cluster, "nodegroup", ng, "err", perr) } return @@ -450,11 +453,11 @@ func (s *EKSServiceImpl) launchNodegroupInfra(ctx context.Context, lc nodegroupL // inflate the count and satisfy the gate early. They go NotReady within // about a minute of termination, whereas baselining on them fails every // replacement permanently. - if err := s.waitWorkersReady(acctKV, cluster, ng, 0, lc.desired); err != nil { + if err := s.waitWorkersReady(ctx, acctKV, cluster, ng, 0, lc.desired); err != nil { rec.Status = eks.NodegroupStatusCreateFailed rec.StatusReason = "workers did not become Ready: " + err.Error() rec.ModifiedAt = time.Now().UTC() - if perr := PutNodegroupRecord(acctKV, rec); perr != nil { + if perr := PutNodegroupRecord(ctx, acctKV, rec); perr != nil { slog.ErrorContext(ctx, "createNodegroup: persist CREATE_FAILED record", "cluster", cluster, "nodegroup", ng, "err", perr) } return @@ -462,7 +465,7 @@ func (s *EKSServiceImpl) launchNodegroupInfra(ctx context.Context, lc nodegroupL rec.Status = eks.NodegroupStatusActive rec.ModifiedAt = time.Now().UTC() - if err := PutNodegroupRecord(acctKV, rec); err != nil { + if err := PutNodegroupRecord(ctx, acctKV, rec); err != nil { slog.ErrorContext(ctx, "createNodegroup: persist ACTIVE record", "cluster", cluster, "nodegroup", ng, "err", err) } } @@ -476,11 +479,11 @@ func (s *EKSServiceImpl) launchNodegroupInfra(ctx context.Context, lc nodegroupL // cluster-wide NodeCount total) is deliberate: in a multi-nodegroup cluster, // another nodegroup's Ready workers must never let this one falsely reach // ACTIVE while its own workers never joined. -func (s *EKSServiceImpl) waitWorkersReady(acctKV nats.KeyValue, cluster, ng string, baseline, want int) error { +func (s *EKSServiceImpl) waitWorkersReady(ctx context.Context, acctKV jetstream.KeyValue, cluster, ng string, baseline, want int) error { target := baseline + want deadline := time.Now().Add(s.nodegroupReadyTimeout) for { - meta, err := GetClusterMeta(acctKV, cluster) + meta, err := GetClusterMeta(ctx, acctKV, cluster) if err != nil { return err } @@ -504,7 +507,7 @@ func (s *EKSServiceImpl) waitWorkersReady(acctKV nats.KeyValue, cluster, ng stri // so each gets a distinct node name + node label in its user-data) and returns // the launched instance IDs. On a partial failure it returns the IDs that did // launch plus the error so the caller can persist them for teardown. -func (s *EKSServiceImpl) launchWorkers(ctx context.Context, acctKV nats.KeyValue, accountID string, rec *NodegroupRecord, meta *ClusterMeta, ngSGID, amiID, token string, count int) ([]string, error) { +func (s *EKSServiceImpl) launchWorkers(ctx context.Context, acctKV jetstream.KeyValue, accountID string, rec *NodegroupRecord, meta *ClusterMeta, ngSGID, amiID, token string, count int) ([]string, error) { // base is the worker set already on the record (non-empty on a scale-up). // Each incremental persist below writes base+newly-launched so the durable // record always reflects every live worker, never just this call's additions. @@ -522,7 +525,7 @@ func (s *EKSServiceImpl) launchWorkers(ctx context.Context, acctKV nats.KeyValue // its claim-time InstanceIDs until the loop's terminal write, which strands // the already-launched workers if the daemon dies in between. rec.InstanceIDs = append(append([]string(nil), base...), ids...) - if perr := PutNodegroupRecord(acctKV, rec); perr != nil { + if perr := PutNodegroupRecord(ctx, acctKV, rec); perr != nil { return ids, fmt.Errorf("persist launched workers %d/%d: %w", i+1, count, perr) } } @@ -536,7 +539,7 @@ func (s *EKSServiceImpl) launchWorkers(ctx context.Context, acctKV nats.KeyValue // already-launched workers are never reissued — until the desired count is // reached, workerLaunchRetryTimeout elapses, or bgCtx is cancelled. Every retry // is logged so a stuck launch is diagnosable without live-tailing the daemon. -func (s *EKSServiceImpl) launchWorkersUntilDesired(ctx context.Context, acctKV nats.KeyValue, accountID string, rec *NodegroupRecord, meta *ClusterMeta, ngSGID, amiID, token string, count int) ([]string, error) { +func (s *EKSServiceImpl) launchWorkersUntilDesired(ctx context.Context, acctKV jetstream.KeyValue, accountID string, rec *NodegroupRecord, meta *ClusterMeta, ngSGID, amiID, token string, count int) ([]string, error) { deadline := time.Now().Add(s.workerLaunchRetryTimeout) all := make([]string, 0, count) remaining := count @@ -772,13 +775,13 @@ func roleNameFromARN(arn string) string { return "" } -func (s *EKSServiceImpl) describeNodegroup(acctKV nats.KeyValue, input *eks.DescribeNodegroupInput) (*eks.DescribeNodegroupOutput, error) { +func (s *EKSServiceImpl) describeNodegroup(ctx context.Context, acctKV jetstream.KeyValue, input *eks.DescribeNodegroupInput) (*eks.DescribeNodegroupOutput, error) { cluster := aws.StringValue(input.ClusterName) ng := aws.StringValue(input.NodegroupName) if cluster == "" || ng == "" { return nil, errors.New(awserrors.ErrorInvalidParameterValue) } - rec, err := GetNodegroupRecord(acctKV, cluster, ng) + rec, err := GetNodegroupRecord(ctx, acctKV, cluster, ng) if err != nil { if errors.Is(err, ErrNodegroupNotFound) { return nil, errors.New(awserrors.ErrorEKSResourceNotFound) @@ -788,18 +791,18 @@ func (s *EKSServiceImpl) describeNodegroup(acctKV nats.KeyValue, input *eks.Desc return &eks.DescribeNodegroupOutput{Nodegroup: nodegroupRecordToAWS(rec)}, nil } -func (s *EKSServiceImpl) listNodegroups(acctKV nats.KeyValue, input *eks.ListNodegroupsInput) (*eks.ListNodegroupsOutput, error) { +func (s *EKSServiceImpl) listNodegroups(ctx context.Context, acctKV jetstream.KeyValue, input *eks.ListNodegroupsInput) (*eks.ListNodegroupsOutput, error) { cluster := aws.StringValue(input.ClusterName) if cluster == "" { return nil, errors.New(awserrors.ErrorInvalidParameterValue) } - if _, err := GetClusterMeta(acctKV, cluster); err != nil { + if _, err := GetClusterMeta(ctx, acctKV, cluster); err != nil { if errors.Is(err, ErrClusterNotFound) { return nil, errors.New(awserrors.ErrorEKSResourceNotFound) } return nil, err } - recs, err := ListNodegroupRecords(acctKV, cluster) + recs, err := ListNodegroupRecords(ctx, acctKV, cluster) if err != nil { return nil, err } @@ -810,7 +813,7 @@ func (s *EKSServiceImpl) listNodegroups(acctKV nats.KeyValue, input *eks.ListNod return &eks.ListNodegroupsOutput{Nodegroups: aws.StringSlice(names)}, nil } -func (s *EKSServiceImpl) updateNodegroupConfig(ctx context.Context, acctKV nats.KeyValue, input *eks.UpdateNodegroupConfigInput, accountID string) (*eks.UpdateNodegroupConfigOutput, error) { +func (s *EKSServiceImpl) updateNodegroupConfig(ctx context.Context, acctKV jetstream.KeyValue, input *eks.UpdateNodegroupConfigInput, accountID string) (*eks.UpdateNodegroupConfigOutput, error) { if input == nil { return nil, errors.New(awserrors.ErrorInvalidParameterValue) } @@ -819,7 +822,7 @@ func (s *EKSServiceImpl) updateNodegroupConfig(ctx context.Context, acctKV nats. if cluster == "" || ng == "" { return nil, errors.New(awserrors.ErrorInvalidParameterValue) } - meta, err := GetClusterMeta(acctKV, cluster) + meta, err := GetClusterMeta(ctx, acctKV, cluster) if err != nil { if errors.Is(err, ErrClusterNotFound) { return nil, errors.New(awserrors.ErrorEKSResourceNotFound) @@ -845,9 +848,9 @@ func (s *EKSServiceImpl) updateNodegroupConfig(ctx context.Context, acctKV nats. // CAS on the record revision, so two overlapping UpdateNodegroupConfig calls can // never both launch the same delta — the lost-update that scaled 1 worker to 5 // (two reconciles each reading current=1 and each launching desired-current=2). -func (s *EKSServiceImpl) reconcileNodegroup(ctx context.Context, acctKV nats.KeyValue, accountID, cluster, ng string, meta *ClusterMeta, scaling *eks.NodegroupScalingConfig, labels *eks.UpdateLabelsPayload) (*NodegroupRecord, error) { +func (s *EKSServiceImpl) reconcileNodegroup(ctx context.Context, acctKV jetstream.KeyValue, accountID, cluster, ng string, meta *ClusterMeta, scaling *eks.NodegroupScalingConfig, labels *eks.UpdateLabelsPayload) (*NodegroupRecord, error) { for range ngCASMaxRetries { - rec, rev, err := getNodegroupEntry(acctKV, cluster, ng) + rec, rev, err := getNodegroupEntry(ctx, acctKV, cluster, ng) if err != nil { if errors.Is(err, ErrNodegroupNotFound) { return nil, errors.New(awserrors.ErrorEKSResourceNotFound) @@ -874,7 +877,7 @@ func (s *EKSServiceImpl) reconcileNodegroup(ctx context.Context, acctKV nats.Key return nil, fmt.Errorf("terminate surplus workers: %w", err) } rec.InstanceIDs = rec.InstanceIDs[:desired] - if ok, err := s.casPutNodegroup(acctKV, rec, rev); err != nil { + if ok, err := s.casPutNodegroup(ctx, acctKV, rec, rev); err != nil { return nil, err } else if !ok { continue @@ -884,7 +887,7 @@ func (s *EKSServiceImpl) reconcileNodegroup(ctx context.Context, acctKV nats.Key case desired > current: // Commit the deltas + new desired first, then launch the gap one worker // at a time with a per-VM CAS-append (launchWorkersCAS). - if ok, err := s.casPutNodegroup(acctKV, rec, rev); err != nil { + if ok, err := s.casPutNodegroup(ctx, acctKV, rec, rev); err != nil { return nil, err } else if !ok { continue @@ -892,7 +895,7 @@ func (s *EKSServiceImpl) reconcileNodegroup(ctx context.Context, acctKV nats.Key return s.launchWorkersCAS(ctx, acctKV, accountID, cluster, ng, meta, desired) default: - if ok, err := s.casPutNodegroup(acctKV, rec, rev); err != nil { + if ok, err := s.casPutNodegroup(ctx, acctKV, rec, rev); err != nil { return nil, err } else if !ok { continue @@ -907,12 +910,12 @@ func (s *EKSServiceImpl) reconcileNodegroup(ctx context.Context, acctKV nats.Key // worker at a time and CAS-appending its ID only while len(InstanceIDs) < // desired. A worker launched into an already-full record (a concurrent reconcile // got there first) is terminated, so the count never overshoots desired. -func (s *EKSServiceImpl) launchWorkersCAS(ctx context.Context, acctKV nats.KeyValue, accountID, cluster, ng string, meta *ClusterMeta, desired int) (*NodegroupRecord, error) { - token, err := s.decryptNodeToken(acctKV, cluster) +func (s *EKSServiceImpl) launchWorkersCAS(ctx context.Context, acctKV jetstream.KeyValue, accountID, cluster, ng string, meta *ClusterMeta, desired int) (*NodegroupRecord, error) { + token, err := s.decryptNodeToken(ctx, acctKV, cluster) if err != nil { return nil, fmt.Errorf("decrypt node token: %w", err) } - recForAMI, _, err := getNodegroupEntry(acctKV, cluster, ng) + recForAMI, _, err := getNodegroupEntry(ctx, acctKV, cluster, ng) if err != nil { return nil, err } @@ -932,7 +935,7 @@ func (s *EKSServiceImpl) launchWorkersCAS(ctx context.Context, acctKV nats.KeyVa // The target is re-read from the record each pass (not the entry-time desired), // so a concurrent rescale is honored rather than overrun. for range desired + ngCASMaxRetries { - rec, _, err := getNodegroupEntry(acctKV, cluster, ng) + rec, _, err := getNodegroupEntry(ctx, acctKV, cluster, ng) if err != nil { return nil, err } @@ -960,9 +963,9 @@ func (s *EKSServiceImpl) launchWorkersCAS(ctx context.Context, acctKV nats.KeyVa // recordLaunchedWorker CAS-appends id while the record is below its live // ScalingDesired, or terminates the worker when a concurrent reconcile already // filled the gap (or lowered the target). Returns true when id was recorded. -func (s *EKSServiceImpl) recordLaunchedWorker(ctx context.Context, kv nats.KeyValue, accountID, cluster, ng, id string) (bool, error) { +func (s *EKSServiceImpl) recordLaunchedWorker(ctx context.Context, kv jetstream.KeyValue, accountID, cluster, ng, id string) (bool, error) { for range ngCASMaxRetries { - rec, rev, err := getNodegroupEntry(kv, cluster, ng) + rec, rev, err := getNodegroupEntry(ctx, kv, cluster, ng) if err != nil { return false, err } @@ -977,7 +980,7 @@ func (s *EKSServiceImpl) recordLaunchedWorker(ctx context.Context, kv nats.KeyVa } rec.InstanceIDs = append(rec.InstanceIDs, id) rec.ModifiedAt = time.Now().UTC() - if ok, err := s.casPutNodegroup(kv, rec, rev); err != nil { + if ok, err := s.casPutNodegroup(ctx, kv, rec, rev); err != nil { return false, err } else if ok { return true, nil @@ -1008,13 +1011,13 @@ func applyScalingUpdate(rec *NodegroupRecord, scaling *eks.NodegroupScalingConfi } // getNodegroupEntry reads one record with its KV revision for CAS updates. -func getNodegroupEntry(kv nats.KeyValue, cluster, ng string) (*NodegroupRecord, uint64, error) { +func getNodegroupEntry(ctx context.Context, kv jetstream.KeyValue, cluster, ng string) (*NodegroupRecord, uint64, error) { if cluster == "" || ng == "" { return nil, 0, errors.New("eks: getNodegroupEntry empty cluster or name") } - entry, err := kv.Get(NodegroupKey(cluster, ng)) + entry, err := kv.Get(ctx, NodegroupKey(cluster, ng)) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil, 0, ErrNodegroupNotFound } return nil, 0, fmt.Errorf("kv get nodegroup: %w", err) @@ -1028,13 +1031,13 @@ func getNodegroupEntry(kv nats.KeyValue, cluster, ng string) (*NodegroupRecord, // casPutNodegroup writes rec only if the durable revision still matches rev. // ok=false signals a concurrent writer won; the caller re-reads and retries. -func (s *EKSServiceImpl) casPutNodegroup(kv nats.KeyValue, rec *NodegroupRecord, rev uint64) (bool, error) { +func (s *EKSServiceImpl) casPutNodegroup(ctx context.Context, kv jetstream.KeyValue, rec *NodegroupRecord, rev uint64) (bool, error) { data, err := json.Marshal(rec) if err != nil { return false, fmt.Errorf("marshal nodegroup %s: %w", rec.Name, err) } - if _, err := kv.Update(NodegroupKey(rec.ClusterName, rec.Name), data, rev); err != nil { - if errors.Is(err, nats.ErrKeyExists) { + if _, err := kv.Update(ctx, NodegroupKey(rec.ClusterName, rec.Name), data, rev); err != nil { + if errors.Is(err, jetstream.ErrKeyExists) { return false, nil } return false, fmt.Errorf("kv update nodegroup %s: %w", rec.Name, err) @@ -1051,7 +1054,7 @@ func (s *EKSServiceImpl) updateNodegroupVersion(input *eks.UpdateNodegroupVersio return nil, notImpl() } -func (s *EKSServiceImpl) deleteNodegroup(ctx context.Context, acctKV nats.KeyValue, input *eks.DeleteNodegroupInput, accountID string) (*eks.DeleteNodegroupOutput, error) { +func (s *EKSServiceImpl) deleteNodegroup(ctx context.Context, acctKV jetstream.KeyValue, input *eks.DeleteNodegroupInput, accountID string) (*eks.DeleteNodegroupOutput, error) { if input == nil { return nil, errors.New(awserrors.ErrorInvalidParameterValue) } @@ -1060,7 +1063,7 @@ func (s *EKSServiceImpl) deleteNodegroup(ctx context.Context, acctKV nats.KeyVal if cluster == "" || ng == "" { return nil, errors.New(awserrors.ErrorInvalidParameterValue) } - rec, err := GetNodegroupRecord(acctKV, cluster, ng) + rec, err := GetNodegroupRecord(ctx, acctKV, cluster, ng) if err != nil { if errors.Is(err, ErrNodegroupNotFound) { return nil, errors.New(awserrors.ErrorEKSResourceNotFound) @@ -1073,7 +1076,7 @@ func (s *EKSServiceImpl) deleteNodegroup(ctx context.Context, acctKV nats.KeyVal return nil, fmt.Errorf("terminate workers: %w", err) } } - if err := DeleteNodegroupRecord(acctKV, cluster, ng); err != nil { + if err := DeleteNodegroupRecord(ctx, acctKV, cluster, ng); err != nil { return nil, err } @@ -1082,13 +1085,13 @@ func (s *EKSServiceImpl) deleteNodegroup(ctx context.Context, acctKV nats.KeyVal } // decryptNodeToken reads + decrypts the cluster's K3s join token from KV. -func (s *EKSServiceImpl) decryptNodeToken(kv nats.KeyValue, cluster string) (string, error) { +func (s *EKSServiceImpl) decryptNodeToken(ctx context.Context, kv jetstream.KeyValue, cluster string) (string, error) { if kv == nil { return "", errors.New("eks: nil KV for node token") } - entry, err := kv.Get(NodeTokenKey(cluster)) + entry, err := kv.Get(ctx, NodeTokenKey(cluster)) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return "", errors.New("eks: cluster join token not yet provisioned") } return "", fmt.Errorf("kv get node token: %w", err) @@ -1111,8 +1114,8 @@ func (s *EKSServiceImpl) decryptNodeToken(kv nats.KeyValue, cluster string) (str // state. Such records — and any CREATE_FAILED record still holding worker IDs — // have their recorded workers terminated and the record settled to CREATE_FAILED // with InstanceIDs cleared, which is idempotent on the next boot. -func (s *EKSServiceImpl) reclaimOrphanedNodegroups(ctx context.Context, accountID string, acctKV nats.KeyValue, cluster string) { - recs, err := ListNodegroupRecords(acctKV, cluster) +func (s *EKSServiceImpl) reclaimOrphanedNodegroups(ctx context.Context, accountID string, acctKV jetstream.KeyValue, cluster string) { + recs, err := ListNodegroupRecords(ctx, acctKV, cluster) if err != nil { slog.WarnContext(ctx, "reclaimOrphanedNodegroups: list records failed", "cluster", cluster, "err", err) return @@ -1142,7 +1145,7 @@ func (s *EKSServiceImpl) reclaimOrphanedNodegroups(ctx context.Context, accountI rec.StatusReason = reason rec.InstanceIDs = nil rec.ModifiedAt = time.Now().UTC() - if err := PutNodegroupRecord(acctKV, rec); err != nil { + if err := PutNodegroupRecord(ctx, acctKV, rec); err != nil { slog.WarnContext(ctx, "reclaimOrphanedNodegroups: persist settled record failed", "cluster", cluster, "nodegroup", rec.Name, "err", err) continue @@ -1152,15 +1155,15 @@ func (s *EKSServiceImpl) reclaimOrphanedNodegroups(ctx context.Context, accountI } } -func (s *EKSServiceImpl) markNodegroupFailed(kv nats.KeyValue, cluster, ng, reason string) { - rec, err := GetNodegroupRecord(kv, cluster, ng) +func (s *EKSServiceImpl) markNodegroupFailed(ctx context.Context, kv jetstream.KeyValue, cluster, ng, reason string) { + rec, err := GetNodegroupRecord(ctx, kv, cluster, ng) if err != nil { return } rec.Status = eks.NodegroupStatusCreateFailed rec.StatusReason = reason rec.ModifiedAt = time.Now().UTC() - if err := PutNodegroupRecord(kv, rec); err != nil { + if err := PutNodegroupRecord(ctx, kv, rec); err != nil { slog.Warn("markNodegroupFailed: persist failed", "cluster", cluster, "nodegroup", ng, "err", err) } } diff --git a/spinifex/handlers/eks/nodegroup_recreate_test.go b/spinifex/handlers/eks/nodegroup_recreate_test.go index 1309cbbba..706ff3023 100644 --- a/spinifex/handlers/eks/nodegroup_recreate_test.go +++ b/spinifex/handlers/eks/nodegroup_recreate_test.go @@ -29,7 +29,7 @@ func TestCreateNodegroup_RecreateDoesNotBaselineOnPriorWorkers(t *testing.T) { markWorkersReady(t, f, "c1", "ng", 2) f.svc.WaitLaunches() - rec, err := GetNodegroupRecord(f.kv, "c1", "ng") + rec, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng") require.NoError(t, err) require.Equal(t, eks.NodegroupStatusActive, rec.Status) @@ -37,7 +37,7 @@ func TestCreateNodegroup_RecreateDoesNotBaselineOnPriorWorkers(t *testing.T) { // The CP's Ready report lags the delete, so the previous incarnation's two // workers are still counted Ready when the create begins — exactly the state // that produced "want >= 4 (baseline 2 + 2 workers)" in the field. - require.NoError(t, DeleteNodegroupRecord(f.kv, "c1", "ng")) + require.NoError(t, DeleteNodegroupRecord(t.Context(), f.kv, "c1", "ng")) markWorkersReady(t, f, "c1", "ng", 2) _, err = f.svc.CreateNodegroup(context.Background(), createNGInput("c1", "ng", 2), testAccountID) @@ -49,7 +49,7 @@ func TestCreateNodegroup_RecreateDoesNotBaselineOnPriorWorkers(t *testing.T) { markWorkersReady(t, f, "c1", "ng", 2) f.svc.WaitLaunches() - rec, err = GetNodegroupRecord(f.kv, "c1", "ng") + rec, err = GetNodegroupRecord(t.Context(), f.kv, "c1", "ng") require.NoError(t, err) assert.Equal(t, eks.NodegroupStatusActive, rec.Status, "a re-created nodegroup whose own workers are Ready must reach ACTIVE, not CREATE_FAILED") diff --git a/spinifex/handlers/eks/nodegroup_test.go b/spinifex/handlers/eks/nodegroup_test.go index 077581438..221a3e3a0 100644 --- a/spinifex/handlers/eks/nodegroup_test.go +++ b/spinifex/handlers/eks/nodegroup_test.go @@ -159,11 +159,11 @@ func seedActiveClusterWithToken(t *testing.T, f *eksServiceFixture, cluster stri VpcId: "vpc-aaa", }, } - require.NoError(t, PutClusterMeta(f.kv, meta)) + require.NoError(t, PutClusterMeta(t.Context(), f.kv, meta)) ct, err := handlers_iam.EncryptSecret("K10node-join-token::server:abc", bootstrapTestMasterKey) require.NoError(t, err) - _, err = f.kv.Put(NodeTokenKey(cluster), []byte(ct)) + _, err = f.kv.Put(t.Context(), NodeTokenKey(cluster), []byte(ct)) require.NoError(t, err) } @@ -174,7 +174,7 @@ func seedActiveClusterWithToken(t *testing.T, f *eksServiceFixture, cluster stri // baseline (0) plus the worker count the test expects Ready. func markWorkersReady(t *testing.T, f *eksServiceFixture, cluster, ng string, n int) { t.Helper() - require.NoError(t, SetClusterHealthState(f.kv, cluster, "", n, map[string]int{ng: n})) + require.NoError(t, SetClusterHealthState(t.Context(), f.kv, cluster, "", n, map[string]int{ng: n})) } func createNGInput(cluster, ng string, desired int64) *eks.CreateNodegroupInput { @@ -197,19 +197,19 @@ func TestReclaimOrphanedNodegroups_TerminatesStrandedWorkers(t *testing.T) { seedActiveClusterWithToken(t, f, cluster) // CREATING left behind by a crashed launcher, workers already recorded. - require.NoError(t, PutNodegroupRecord(f.kv, &NodegroupRecord{ + require.NoError(t, PutNodegroupRecord(t.Context(), f.kv, &NodegroupRecord{ ClusterName: cluster, Name: "ng-stuck", Status: eks.NodegroupStatusCreating, InstanceIDs: []string{"i-worker1", "i-worker2"}, })) // CREATE_FAILED that still holds a partially-launched worker. - require.NoError(t, PutNodegroupRecord(f.kv, &NodegroupRecord{ + require.NoError(t, PutNodegroupRecord(t.Context(), f.kv, &NodegroupRecord{ ClusterName: cluster, Name: "ng-failed", Status: eks.NodegroupStatusCreateFailed, InstanceIDs: []string{"i-worker3"}, })) // ACTIVE must be left untouched. - require.NoError(t, PutNodegroupRecord(f.kv, &NodegroupRecord{ + require.NoError(t, PutNodegroupRecord(t.Context(), f.kv, &NodegroupRecord{ ClusterName: cluster, Name: "ng-active", Status: eks.NodegroupStatusActive, InstanceIDs: []string{"i-worker9"}, @@ -224,21 +224,22 @@ func TestReclaimOrphanedNodegroups_TerminatesStrandedWorkers(t *testing.T) { assert.ElementsMatch(t, []string{"i-worker1", "i-worker2", "i-worker3"}, terminated) for _, ng := range []string{"ng-stuck", "ng-failed"} { - got, err := GetNodegroupRecord(f.kv, cluster, ng) + got, err := GetNodegroupRecord(t.Context(), f.kv, cluster, ng) require.NoError(t, err) assert.Equal(t, eks.NodegroupStatusCreateFailed, got.Status, ng) assert.Empty(t, got.InstanceIDs, ng) } - got, err := GetNodegroupRecord(f.kv, cluster, "ng-active") + got, err := GetNodegroupRecord(t.Context(), f.kv, cluster, "ng-active") require.NoError(t, err) assert.Equal(t, eks.NodegroupStatusActive, got.Status) assert.Equal(t, []string{"i-worker9"}, got.InstanceIDs) } func TestNodegroupRecord_CRUDRoundTrip(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) - kv, err := GetOrCreateAccountBucket(js, testAccountID, 1) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1) require.NoError(t, err) rec := &NodegroupRecord{ @@ -253,24 +254,24 @@ func TestNodegroupRecord_CRUDRoundTrip(t *testing.T) { ScalingDesired: 2, InstanceIDs: []string{"i-1", "i-2"}, } - require.NoError(t, PutNodegroupRecord(kv, rec)) + require.NoError(t, PutNodegroupRecord(t.Context(), kv, rec)) - got, err := GetNodegroupRecord(kv, "c1", "ng1") + got, err := GetNodegroupRecord(t.Context(), kv, "c1", "ng1") require.NoError(t, err) assert.Equal(t, rec.InstanceIDs, got.InstanceIDs) assert.Equal(t, int64(2), got.ScalingDesired) - list, err := ListNodegroupRecords(kv, "c1") + list, err := ListNodegroupRecords(t.Context(), kv, "c1") require.NoError(t, err) require.Len(t, list, 1) assert.Equal(t, "ng1", list[0].Name) - require.NoError(t, DeleteNodegroupRecord(kv, "c1", "ng1")) - _, err = GetNodegroupRecord(kv, "c1", "ng1") + require.NoError(t, DeleteNodegroupRecord(t.Context(), kv, "c1", "ng1")) + _, err = GetNodegroupRecord(t.Context(), kv, "c1", "ng1") assert.ErrorIs(t, err, ErrNodegroupNotFound) // Delete of an absent record is idempotent. - require.NoError(t, DeleteNodegroupRecord(kv, "c1", "ng1")) + require.NoError(t, DeleteNodegroupRecord(t.Context(), kv, "c1", "ng1")) } func TestBuildAgentUserData_Shape(t *testing.T) { @@ -345,13 +346,13 @@ func TestCreateNodegroup_HappyPath(t *testing.T) { f.svc.WaitLaunches() // The async launch transitions the record to ACTIVE once workers run. - active, err := GetNodegroupRecord(f.kv, "c1", "ng1") + active, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1") require.NoError(t, err) assert.Equal(t, eks.NodegroupStatusActive, active.Status) // Two workers launched and tracked. assert.Len(t, f.worker.runCalls, 2) - rec, err := GetNodegroupRecord(f.kv, "c1", "ng1") + rec, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1") require.NoError(t, err) assert.Len(t, rec.InstanceIDs, 2) @@ -385,7 +386,7 @@ func TestCreateNodegroup_OmittedDiskSizeUsesAWSDefault(t *testing.T) { _, err := f.svc.CreateNodegroup(context.Background(), createNGInput("c1", "ng1", 1), testAccountID) require.NoError(t, err) - rec, err := GetNodegroupRecord(f.kv, "c1", "ng1") + rec, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1") require.NoError(t, err) assert.Equal(t, int64(defaultNodegroupDiskSizeGiB), rec.DiskSize) @@ -412,7 +413,7 @@ func TestCreateNodegroup_GPUInstanceTypeSetsGPUFields(t *testing.T) { _, err := f.svc.CreateNodegroup(context.Background(), in, testAccountID) require.NoError(t, err) - rec, err := GetNodegroupRecord(f.kv, "c1", "ng-gpu") + rec, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng-gpu") require.NoError(t, err) assert.True(t, rec.GPUEnabled) assert.Equal(t, "nvidia", rec.GPUVendor) @@ -436,7 +437,7 @@ func TestCreateNodegroup_GPUNodegroupStagesDevicePluginAddon(t *testing.T) { markWorkersReady(t, f, "c1", "ng-gpu", 1) f.svc.WaitLaunches() - rec, err := GetAddonRecord(f.kv, "c1", nvidiaDevicePluginAddonName) + rec, err := GetAddonRecord(t.Context(), f.kv, "c1", nvidiaDevicePluginAddonName) require.NoError(t, err, "nvidia-device-plugin must be staged for a GPU nodegroup") assert.Equal(t, "0.17.4", rec.AddonVersion) } @@ -453,7 +454,7 @@ func TestCreateNodegroup_NonGPUNodegroupDoesNotStageDevicePluginAddon(t *testing markWorkersReady(t, f, "c1", "ng1", 1) f.svc.WaitLaunches() - _, err = GetAddonRecord(f.kv, "c1", nvidiaDevicePluginAddonName) + _, err = GetAddonRecord(t.Context(), f.kv, "c1", nvidiaDevicePluginAddonName) require.ErrorIs(t, err, ErrAddonNotFound) } @@ -466,7 +467,7 @@ func TestCreateNodegroup_NonGPUInstanceTypeClearsGPUFields(t *testing.T) { _, err := f.svc.CreateNodegroup(context.Background(), createNGInput("c1", "ng1", 1), testAccountID) require.NoError(t, err) - rec, err := GetNodegroupRecord(f.kv, "c1", "ng1") + rec, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1") require.NoError(t, err) assert.False(t, rec.GPUEnabled) assert.Empty(t, rec.GPUVendor) @@ -543,7 +544,7 @@ func TestCreateNodegroup_GPUWorkerLaunchUsesGPUAMI(t *testing.T) { markWorkersReady(t, f, "c1", "ng-gpu", 1) f.svc.WaitLaunches() - rec, err := GetNodegroupRecord(f.kv, "c1", "ng-gpu") + rec, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng-gpu") require.NoError(t, err) assert.Equal(t, eks.NodegroupStatusActive, rec.Status) @@ -570,7 +571,7 @@ func TestCreateNodegroup_GPUWorkerLaunchNoGPUAMINoFallback(t *testing.T) { f.svc.WaitLaunches() - rec, err := GetNodegroupRecord(f.kv, "c1", "ng-gpu") + rec, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng-gpu") require.NoError(t, err) assert.Equal(t, eks.NodegroupStatusCreateFailed, rec.Status) assert.Contains(t, rec.StatusReason, "resolve eks-node AMI") @@ -590,7 +591,7 @@ func TestCreateNodegroup_WorkersNeverReady_CreateFailed(t *testing.T) { // No markWorkersReady → NodeCount stays at the baseline; the Ready-gate times out. f.svc.WaitLaunches() - rec, err := GetNodegroupRecord(f.kv, "c1", "ng1") + rec, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1") require.NoError(t, err) assert.Equal(t, eks.NodegroupStatusCreateFailed, rec.Status) assert.Contains(t, rec.StatusReason, "did not become Ready") @@ -618,11 +619,11 @@ func TestCreateNodegroup_ReadyGateScopedPerNodegroupNotGlobal(t *testing.T) { markWorkersReady(t, f, "c1", "ng-a", 1) f.svc.WaitLaunches() - ngA, err := GetNodegroupRecord(f.kv, "c1", "ng-a") + ngA, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng-a") require.NoError(t, err) assert.Equal(t, eks.NodegroupStatusActive, ngA.Status, "ng-a's own Ready worker must activate it") - ngB, err := GetNodegroupRecord(f.kv, "c1", "ng-b") + ngB, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng-b") require.NoError(t, err) assert.Equal(t, eks.NodegroupStatusCreateFailed, ngB.Status, "ng-b must not be marked ACTIVE by ng-a's Ready nodes — its own workers never registered") @@ -647,7 +648,7 @@ func TestCreateNodegroup_RelaunchesOnWorkerLaunchFailure(t *testing.T) { markWorkersReady(t, f, "c1", "ng1", 1) f.svc.WaitLaunches() - rec, err := GetNodegroupRecord(f.kv, "c1", "ng1") + rec, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1") require.NoError(t, err) assert.Equal(t, eks.NodegroupStatusActive, rec.Status, "the retried launch must converge to ACTIVE, not CREATE_FAILED") @@ -668,7 +669,7 @@ func TestCreateNodegroup_WorkerLaunchFailureExhaustsRetryBudget(t *testing.T) { require.NoError(t, err) f.svc.WaitLaunches() - rec, err := GetNodegroupRecord(f.kv, "c1", "ng1") + rec, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1") require.NoError(t, err) assert.Equal(t, eks.NodegroupStatusCreateFailed, rec.Status) assert.Contains(t, rec.StatusReason, "worker launch shortfall after") @@ -696,7 +697,7 @@ func TestCreateNodegroup_DiskSizePropagatesToBlockDeviceMapping(t *testing.T) { func TestCreateNodegroup_ClusterNotActive(t *testing.T) { f := newEKSServiceFixture(t) - require.NoError(t, PutClusterMeta(f.kv, &ClusterMeta{ + require.NoError(t, PutClusterMeta(t.Context(), f.kv, &ClusterMeta{ Name: "c1", Status: ClusterStatusCreating, ControlPlaneENIIP: "10.0.1.42", @@ -756,7 +757,7 @@ func TestUpdateNodegroupConfig_ScaleUpAndDown(t *testing.T) { require.NoError(t, err) assert.Equal(t, eks.UpdateStatusSuccessful, aws.StringValue(upOut.Update.Status)) assert.Len(t, f.worker.runCalls, 3) - rec, err := GetNodegroupRecord(f.kv, "c1", "ng1") + rec, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1") require.NoError(t, err) assert.Len(t, rec.InstanceIDs, 3) @@ -769,7 +770,7 @@ func TestUpdateNodegroupConfig_ScaleUpAndDown(t *testing.T) { require.NoError(t, err) require.Len(t, f.worker.terminateCalls, 1) assert.Len(t, f.worker.terminateCalls[0], 2) - rec, err = GetNodegroupRecord(f.kv, "c1", "ng1") + rec, err = GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1") require.NoError(t, err) assert.Len(t, rec.InstanceIDs, 1) } @@ -809,7 +810,7 @@ func TestUpdateNodegroupConfig_ConcurrentScaleUpConverges(t *testing.T) { require.NoError(t, e) } - rec, err := GetNodegroupRecord(f.kv, "c1", "ng1") + rec, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1") require.NoError(t, err) assert.Len(t, rec.InstanceIDs, 3, "record must converge to desired=3, not overshoot") @@ -879,7 +880,7 @@ func TestDeleteNodegroup_TerminatesAndIsIdempotent(t *testing.T) { require.Len(t, f.worker.terminateCalls, 1) assert.Len(t, f.worker.terminateCalls[0], 2) - _, err = GetNodegroupRecord(f.kv, "c1", "ng1") + _, err = GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1") assert.ErrorIs(t, err, ErrNodegroupNotFound) // Second delete → ResourceNotFoundException (record already gone). diff --git a/spinifex/handlers/eks/oidc_keypair.go b/spinifex/handlers/eks/oidc_keypair.go index 67f919436..e9e6abb73 100644 --- a/spinifex/handlers/eks/oidc_keypair.go +++ b/spinifex/handlers/eks/oidc_keypair.go @@ -1,6 +1,7 @@ package handlers_eks import ( + "context" "crypto" "crypto/ecdsa" "crypto/elliptic" @@ -14,7 +15,7 @@ import ( "fmt" handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // PublicKeyPEMFromPrivate parses a PKCS#8 private-key PEM and returns the PKIX public-key PEM. @@ -59,7 +60,7 @@ const p256CoordLen = 32 // GenerateClusterOIDCKeypair creates an ECDSA-P256 keypair, encrypts the private key PEM // into KV, and writes the JWKS document. Returns plaintext PEM + JWKS for immediate use. -func GenerateClusterOIDCKeypair(kv nats.KeyValue, clusterName string, masterKey []byte) (privPEM string, jwksBytes []byte, err error) { +func GenerateClusterOIDCKeypair(ctx context.Context, kv jetstream.KeyValue, clusterName string, masterKey []byte) (privPEM string, jwksBytes []byte, err error) { if clusterName == "" { return "", nil, errors.New("eks: GenerateClusterOIDCKeypair empty cluster name") } @@ -82,7 +83,7 @@ func GenerateClusterOIDCKeypair(kv nats.KeyValue, clusterName string, masterKey if err != nil { return "", nil, fmt.Errorf("encrypt OIDC private key: %w", err) } - if _, err := kv.Put(OIDCSigningKeyKey(clusterName), []byte(ciphertext)); err != nil { + if _, err := kv.Put(ctx, OIDCSigningKeyKey(clusterName), []byte(ciphertext)); err != nil { return "", nil, fmt.Errorf("kv put %s: %w", OIDCSigningKeyKey(clusterName), err) } @@ -90,7 +91,7 @@ func GenerateClusterOIDCKeypair(kv nats.KeyValue, clusterName string, masterKey if err != nil { return "", nil, err } - if _, err := kv.Put(OIDCJWKSKey(clusterName), jwksBytes); err != nil { + if _, err := kv.Put(ctx, OIDCJWKSKey(clusterName), jwksBytes); err != nil { return "", nil, fmt.Errorf("kv put %s: %w", OIDCJWKSKey(clusterName), err) } return string(pemBlock), jwksBytes, nil @@ -98,16 +99,16 @@ func GenerateClusterOIDCKeypair(kv nats.KeyValue, clusterName string, masterKey // LoadClusterOIDCPrivateKey decrypts and parses the cluster OIDC private key from KV. // Returns ErrClusterNotFound if absent (supports idempotent DeleteCluster). -func LoadClusterOIDCPrivateKey(kv nats.KeyValue, clusterName string, masterKey []byte) (*ecdsa.PrivateKey, error) { +func LoadClusterOIDCPrivateKey(ctx context.Context, kv jetstream.KeyValue, clusterName string, masterKey []byte) (*ecdsa.PrivateKey, error) { if clusterName == "" { return nil, errors.New("eks: LoadClusterOIDCPrivateKey empty cluster name") } if len(masterKey) == 0 { return nil, errors.New("eks: LoadClusterOIDCPrivateKey empty master key") } - entry, err := kv.Get(OIDCSigningKeyKey(clusterName)) + entry, err := kv.Get(ctx, OIDCSigningKeyKey(clusterName)) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil, ErrClusterNotFound } return nil, fmt.Errorf("kv get %s: %w", OIDCSigningKeyKey(clusterName), err) @@ -137,23 +138,23 @@ func LoadClusterOIDCPrivateKey(kv nats.KeyValue, clusterName string, masterKey [ // ZeroizeClusterOIDCKey overwrites the encrypted key blob with zeros then purges it. // Purge (not Delete) drops all revision history, leaving no recoverable key material. // Absent key is a no-op for idempotent DeleteCluster retries. -func ZeroizeClusterOIDCKey(kv nats.KeyValue, clusterName string) error { +func ZeroizeClusterOIDCKey(ctx context.Context, kv jetstream.KeyValue, clusterName string) error { if clusterName == "" { return errors.New("eks: ZeroizeClusterOIDCKey empty cluster name") } key := OIDCSigningKeyKey(clusterName) - entry, err := kv.Get(key) + entry, err := kv.Get(ctx, key) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil } return fmt.Errorf("kv get %s: %w", key, err) } zero := make([]byte, len(entry.Value())) - if _, err := kv.Put(key, zero); err != nil { + if _, err := kv.Put(ctx, key, zero); err != nil { return fmt.Errorf("kv put zero %s: %w", key, err) } - if err := kv.Purge(key); err != nil { + if err := kv.Purge(ctx, key); err != nil { return fmt.Errorf("kv purge %s: %w", key, err) } return nil diff --git a/spinifex/handlers/eks/oidc_keypair_test.go b/spinifex/handlers/eks/oidc_keypair_test.go index 4e6130e73..19207d8cc 100644 --- a/spinifex/handlers/eks/oidc_keypair_test.go +++ b/spinifex/handlers/eks/oidc_keypair_test.go @@ -11,7 +11,7 @@ import ( "encoding/pem" "testing" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -27,15 +27,15 @@ var testMasterKey = func() []byte { func TestGenerateClusterOIDCKeypair_PersistsJWKSAndEncryptedPrivateKey(t *testing.T) { kv := newClusterStateTestKV(t) - _, jwksBytes, err := GenerateClusterOIDCKeypair(kv, "alpha", testMasterKey) + _, jwksBytes, err := GenerateClusterOIDCKeypair(t.Context(), kv, "alpha", testMasterKey) require.NoError(t, err) require.NotEmpty(t, jwksBytes) - jwksEntry, err := kv.Get(OIDCJWKSKey("alpha")) + jwksEntry, err := kv.Get(t.Context(), OIDCJWKSKey("alpha")) require.NoError(t, err) assert.Equal(t, jwksBytes, jwksEntry.Value()) - keyEntry, err := kv.Get(OIDCSigningKeyKey("alpha")) + keyEntry, err := kv.Get(t.Context(), OIDCSigningKeyKey("alpha")) require.NoError(t, err) assert.NotEmpty(t, keyEntry.Value()) assert.NotContains(t, string(keyEntry.Value()), "BEGIN PRIVATE KEY", @@ -47,7 +47,7 @@ func TestGenerateClusterOIDCKeypair_PersistsJWKSAndEncryptedPrivateKey(t *testin // public key and crash-loops on a private-key PEM. func TestPublicKeyPEMFromPrivate_RoundTrips(t *testing.T) { kv := newClusterStateTestKV(t) - privPEM, _, err := GenerateClusterOIDCKeypair(kv, "alpha", testMasterKey) + privPEM, _, err := GenerateClusterOIDCKeypair(t.Context(), kv, "alpha", testMasterKey) require.NoError(t, err) pubPEM, err := PublicKeyPEMFromPrivate(privPEM) @@ -62,7 +62,7 @@ func TestPublicKeyPEMFromPrivate_RoundTrips(t *testing.T) { pubEC, ok := pub.(*ecdsa.PublicKey) require.True(t, ok) - stored, err := LoadClusterOIDCPrivateKey(kv, "alpha", testMasterKey) + stored, err := LoadClusterOIDCPrivateKey(t.Context(), kv, "alpha", testMasterKey) require.NoError(t, err) assert.True(t, stored.PublicKey.Equal(pubEC), "derived public key must match the private key") } @@ -80,7 +80,7 @@ func TestPublicKeyPEMFromPrivate_RejectsGarbage(t *testing.T) { func TestGenerateClusterOIDCKeypair_ReturnsPrivateKeyPEMMatchingStored(t *testing.T) { kv := newClusterStateTestKV(t) - privPEM, _, err := GenerateClusterOIDCKeypair(kv, "alpha", testMasterKey) + privPEM, _, err := GenerateClusterOIDCKeypair(t.Context(), kv, "alpha", testMasterKey) require.NoError(t, err) require.Contains(t, privPEM, "BEGIN PRIVATE KEY", "returned PEM must be plaintext") @@ -91,7 +91,7 @@ func TestGenerateClusterOIDCKeypair_ReturnsPrivateKeyPEMMatchingStored(t *testin returnedEC, ok := returned.(*ecdsa.PrivateKey) require.True(t, ok) - stored, err := LoadClusterOIDCPrivateKey(kv, "alpha", testMasterKey) + stored, err := LoadClusterOIDCPrivateKey(t.Context(), kv, "alpha", testMasterKey) require.NoError(t, err) assert.True(t, stored.Equal(returnedEC), "returned PEM must match the persisted key") } @@ -99,7 +99,7 @@ func TestGenerateClusterOIDCKeypair_ReturnsPrivateKeyPEMMatchingStored(t *testin func TestGenerateClusterOIDCKeypair_JWKSShapeIsRFC7517EC_P256(t *testing.T) { kv := newClusterStateTestKV(t) - _, jwksBytes, err := GenerateClusterOIDCKeypair(kv, "alpha", testMasterKey) + _, jwksBytes, err := GenerateClusterOIDCKeypair(t.Context(), kv, "alpha", testMasterKey) require.NoError(t, err) var doc oidcJWKS @@ -124,9 +124,9 @@ func TestGenerateClusterOIDCKeypair_JWKSShapeIsRFC7517EC_P256(t *testing.T) { func TestGenerateClusterOIDCKeypair_TwoCallsProduceDistinctKeys(t *testing.T) { kv := newClusterStateTestKV(t) - _, a, err := GenerateClusterOIDCKeypair(kv, "alpha", testMasterKey) + _, a, err := GenerateClusterOIDCKeypair(t.Context(), kv, "alpha", testMasterKey) require.NoError(t, err) - _, b, err := GenerateClusterOIDCKeypair(kv, "beta", testMasterKey) + _, b, err := GenerateClusterOIDCKeypair(t.Context(), kv, "beta", testMasterKey) require.NoError(t, err) assert.NotEqual(t, a, b, "JWKS for distinct clusters must differ") } @@ -134,19 +134,19 @@ func TestGenerateClusterOIDCKeypair_TwoCallsProduceDistinctKeys(t *testing.T) { func TestGenerateClusterOIDCKeypair_EmptyArgsRejected(t *testing.T) { kv := newClusterStateTestKV(t) - _, _, err := GenerateClusterOIDCKeypair(kv, "", testMasterKey) + _, _, err := GenerateClusterOIDCKeypair(t.Context(), kv, "", testMasterKey) require.Error(t, err) - _, _, err = GenerateClusterOIDCKeypair(kv, "alpha", nil) + _, _, err = GenerateClusterOIDCKeypair(t.Context(), kv, "alpha", nil) require.Error(t, err) } func TestLoadClusterOIDCPrivateKey_RoundTripMatchesJWKSPublic(t *testing.T) { kv := newClusterStateTestKV(t) - _, jwksBytes, err := GenerateClusterOIDCKeypair(kv, "alpha", testMasterKey) + _, jwksBytes, err := GenerateClusterOIDCKeypair(t.Context(), kv, "alpha", testMasterKey) require.NoError(t, err) - priv, err := LoadClusterOIDCPrivateKey(kv, "alpha", testMasterKey) + priv, err := LoadClusterOIDCPrivateKey(t.Context(), kv, "alpha", testMasterKey) require.NoError(t, err) require.NotNil(t, priv) assert.Equal(t, elliptic.P256(), priv.Curve) @@ -168,44 +168,44 @@ func TestLoadClusterOIDCPrivateKey_RoundTripMatchesJWKSPublic(t *testing.T) { func TestLoadClusterOIDCPrivateKey_WrongMasterKeyFails(t *testing.T) { kv := newClusterStateTestKV(t) - _, _, err := GenerateClusterOIDCKeypair(kv, "alpha", testMasterKey) + _, _, err := GenerateClusterOIDCKeypair(t.Context(), kv, "alpha", testMasterKey) require.NoError(t, err) other := make([]byte, 32) other[0] = 0xff - _, err = LoadClusterOIDCPrivateKey(kv, "alpha", other) + _, err = LoadClusterOIDCPrivateKey(t.Context(), kv, "alpha", other) require.Error(t, err) } func TestLoadClusterOIDCPrivateKey_MissingReturnsErrClusterNotFound(t *testing.T) { kv := newClusterStateTestKV(t) - _, err := LoadClusterOIDCPrivateKey(kv, "ghost", testMasterKey) + _, err := LoadClusterOIDCPrivateKey(t.Context(), kv, "ghost", testMasterKey) require.ErrorIs(t, err, ErrClusterNotFound) } func TestZeroizeClusterOIDCKey_DeletesKeyAndLeavesSiblingsIntact(t *testing.T) { kv := newClusterStateTestKV(t) - _, _, err := GenerateClusterOIDCKeypair(kv, "alpha", testMasterKey) + _, _, err := GenerateClusterOIDCKeypair(t.Context(), kv, "alpha", testMasterKey) require.NoError(t, err) - require.NoError(t, ZeroizeClusterOIDCKey(kv, "alpha")) + require.NoError(t, ZeroizeClusterOIDCKey(t.Context(), kv, "alpha")) - _, err = kv.Get(OIDCSigningKeyKey("alpha")) - assert.ErrorIs(t, err, nats.ErrKeyNotFound) + _, err = kv.Get(t.Context(), OIDCSigningKeyKey("alpha")) + assert.ErrorIs(t, err, jetstream.ErrKeyNotFound) - jwks, err := kv.Get(OIDCJWKSKey("alpha")) + jwks, err := kv.Get(t.Context(), OIDCJWKSKey("alpha")) require.NoError(t, err, "JWKS must survive zeroize (it carries no secret material)") assert.NotEmpty(t, jwks.Value()) } func TestZeroizeClusterOIDCKey_MissingIsNoop(t *testing.T) { kv := newClusterStateTestKV(t) - require.NoError(t, ZeroizeClusterOIDCKey(kv, "ghost")) + require.NoError(t, ZeroizeClusterOIDCKey(t.Context(), kv, "ghost")) } func TestZeroizeClusterOIDCKey_EmptyNameRejected(t *testing.T) { kv := newClusterStateTestKV(t) - require.Error(t, ZeroizeClusterOIDCKey(kv, "")) + require.Error(t, ZeroizeClusterOIDCKey(t.Context(), kv, "")) } func TestMarshalJWKS_DeterministicForSameKey(t *testing.T) { diff --git a/spinifex/handlers/eks/reconciler_registry.go b/spinifex/handlers/eks/reconciler_registry.go index 352800971..50303da02 100644 --- a/spinifex/handlers/eks/reconciler_registry.go +++ b/spinifex/handlers/eks/reconciler_registry.go @@ -6,7 +6,7 @@ import ( "log/slog" "sync" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // ErrLeaseHeld signals the leader lease is held by another node. @@ -155,7 +155,7 @@ func registryKey(accountID, clusterName string) string { // acquires the lease, and drives Run in a goroutine. Returns ErrLeaseHeld if another node wins. func RunClusterReconciler( ctx context.Context, - leaderKV, acctKV nats.KeyValue, + leaderKV, acctKV jetstream.KeyValue, accountID, clusterName, holderID, healthURL string, opts ...ReconcilerOption, ) (func(), <-chan struct{}, error) { @@ -163,7 +163,7 @@ func RunClusterReconciler( if err != nil { return nil, nil, err } - release, ok := r.AcquireLease() + release, ok := r.AcquireLease(ctx) if !ok { slog.Info("RunClusterReconciler: lease held elsewhere, skipping spawn", "accountID", accountID, "cluster", clusterName) diff --git a/spinifex/handlers/eks/recovery_directive.go b/spinifex/handlers/eks/recovery_directive.go index 0550b4ebf..c9e6322f0 100644 --- a/spinifex/handlers/eks/recovery_directive.go +++ b/spinifex/handlers/eks/recovery_directive.go @@ -7,7 +7,7 @@ import ( "fmt" "github.com/mulgadc/spinifex/spinifex/awserrors" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // RecoveryAction is the directive a control-plane member applies on its next boot @@ -69,10 +69,10 @@ type SetRecoveryDirectiveOutput struct { // LoadRecoveryDirective reads a member's directive from the account KV, returning // a zero-epoch none directive when the key is absent. -func LoadRecoveryDirective(acctKV nats.KeyValue, cluster, instanceID string) (RecoveryDirective, error) { - entry, err := acctKV.Get(RecoveryDirectiveKey(cluster, instanceID)) +func LoadRecoveryDirective(ctx context.Context, acctKV jetstream.KeyValue, cluster, instanceID string) (RecoveryDirective, error) { + entry, err := acctKV.Get(ctx, RecoveryDirectiveKey(cluster, instanceID)) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return RecoveryDirective{Epoch: 0, Action: RecoveryActionNone}, nil } return RecoveryDirective{}, err @@ -89,8 +89,8 @@ func LoadRecoveryDirective(acctKV nats.KeyValue, cluster, instanceID string) (Re // StoreRecoveryDirective writes a member's directive with epoch = previous+1 and // returns the stored value. Used by the reconciler escalation and the operator CLI. -func StoreRecoveryDirective(acctKV nats.KeyValue, cluster, instanceID string, action RecoveryAction, snapshot string, snapshotRequired bool) (RecoveryDirective, error) { - prev, err := LoadRecoveryDirective(acctKV, cluster, instanceID) +func StoreRecoveryDirective(ctx context.Context, acctKV jetstream.KeyValue, cluster, instanceID string, action RecoveryAction, snapshot string, snapshotRequired bool) (RecoveryDirective, error) { + prev, err := LoadRecoveryDirective(ctx, acctKV, cluster, instanceID) if err != nil { return RecoveryDirective{}, err } @@ -99,7 +99,7 @@ func StoreRecoveryDirective(acctKV nats.KeyValue, cluster, instanceID string, ac if err != nil { return RecoveryDirective{}, fmt.Errorf("marshal recovery directive: %w", err) } - if _, err := acctKV.Put(RecoveryDirectiveKey(cluster, instanceID), data); err != nil { + if _, err := acctKV.Put(ctx, RecoveryDirectiveKey(cluster, instanceID), data); err != nil { return RecoveryDirective{}, fmt.Errorf("kv put recovery directive: %w", err) } return next, nil @@ -107,15 +107,15 @@ func StoreRecoveryDirective(acctKV nats.KeyValue, cluster, instanceID string, ac // GetRecoveryDirective serves a member's current recovery directive to the on-VM // k3s-recovery agent via the internal-recovery gateway route. -func (s *EKSServiceImpl) GetRecoveryDirective(_ context.Context, input *GetRecoveryDirectiveInput, accountID string) (*GetRecoveryDirectiveOutput, error) { +func (s *EKSServiceImpl) GetRecoveryDirective(ctx context.Context, input *GetRecoveryDirectiveInput, accountID string) (*GetRecoveryDirectiveOutput, error) { if input == nil || input.ClusterName == "" || input.InstanceID == "" { return nil, errors.New(awserrors.ErrorInvalidParameterValue) } - acctKV, err := s.acctKVForCluster(accountID, input.ClusterName) + acctKV, err := s.acctKVForCluster(ctx, accountID, input.ClusterName) if err != nil { return nil, err } - d, err := LoadRecoveryDirective(acctKV, input.ClusterName, input.InstanceID) + d, err := LoadRecoveryDirective(ctx, acctKV, input.ClusterName, input.InstanceID) if err != nil { return nil, err } @@ -123,7 +123,7 @@ func (s *EKSServiceImpl) GetRecoveryDirective(_ context.Context, input *GetRecov } // SetRecoveryDirective records a member's recovery directive (operator CLI path). -func (s *EKSServiceImpl) SetRecoveryDirective(_ context.Context, input *SetRecoveryDirectiveInput, accountID string) (*SetRecoveryDirectiveOutput, error) { +func (s *EKSServiceImpl) SetRecoveryDirective(ctx context.Context, input *SetRecoveryDirectiveInput, accountID string) (*SetRecoveryDirectiveOutput, error) { if input == nil || input.ClusterName == "" || input.InstanceID == "" { return nil, errors.New(awserrors.ErrorInvalidParameterValue) } @@ -132,11 +132,11 @@ func (s *EKSServiceImpl) SetRecoveryDirective(_ context.Context, input *SetRecov default: return nil, errors.New(awserrors.ErrorInvalidParameterValue) } - acctKV, err := s.acctKVForCluster(accountID, input.ClusterName) + acctKV, err := s.acctKVForCluster(ctx, accountID, input.ClusterName) if err != nil { return nil, err } - d, err := StoreRecoveryDirective(acctKV, input.ClusterName, input.InstanceID, input.Action, input.Snapshot, input.SnapshotRequired) + d, err := StoreRecoveryDirective(ctx, acctKV, input.ClusterName, input.InstanceID, input.Action, input.Snapshot, input.SnapshotRequired) if err != nil { return nil, err } diff --git a/spinifex/handlers/eks/recovery_directive_test.go b/spinifex/handlers/eks/recovery_directive_test.go index 91d000d01..00236a439 100644 --- a/spinifex/handlers/eks/recovery_directive_test.go +++ b/spinifex/handlers/eks/recovery_directive_test.go @@ -11,30 +11,32 @@ import ( ) func TestLoadRecoveryDirective_AbsentIsZeroNone(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) - acctKV, err := GetOrCreateAccountBucket(js, testAccountID, 1) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + acctKV, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1) require.NoError(t, err) - d, err := LoadRecoveryDirective(acctKV, "alpha", "i-missing") + d, err := LoadRecoveryDirective(t.Context(), acctKV, "alpha", "i-missing") require.NoError(t, err) assert.Equal(t, int64(0), d.Epoch, "absent directive reads as epoch 0") assert.Equal(t, RecoveryActionNone, d.Action, "absent directive is a none action") } func TestStoreRecoveryDirective_EpochIncrements(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) - acctKV, err := GetOrCreateAccountBucket(js, testAccountID, 1) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + acctKV, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1) require.NoError(t, err) - first, err := StoreRecoveryDirective(acctKV, "alpha", "i-0", RecoveryActionClusterReset, "", false) + first, err := StoreRecoveryDirective(t.Context(), acctKV, "alpha", "i-0", RecoveryActionClusterReset, "", false) require.NoError(t, err) assert.Equal(t, int64(1), first.Epoch, "first set starts at epoch 1") - second, err := StoreRecoveryDirective(acctKV, "alpha", "i-0", RecoveryActionWipeRejoin, "snap.snap", false) + second, err := StoreRecoveryDirective(t.Context(), acctKV, "alpha", "i-0", RecoveryActionWipeRejoin, "snap.snap", false) require.NoError(t, err) assert.Equal(t, int64(2), second.Epoch, "each set advances the epoch") - got, err := LoadRecoveryDirective(acctKV, "alpha", "i-0") + got, err := LoadRecoveryDirective(t.Context(), acctKV, "alpha", "i-0") require.NoError(t, err) assert.Equal(t, RecoveryActionWipeRejoin, got.Action) assert.Equal(t, "snap.snap", got.Snapshot) @@ -42,15 +44,16 @@ func TestStoreRecoveryDirective_EpochIncrements(t *testing.T) { } func TestStoreRecoveryDirective_SnapshotRequiredRoundTrips(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) - acctKV, err := GetOrCreateAccountBucket(js, testAccountID, 1) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + acctKV, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1) require.NoError(t, err) - stored, err := StoreRecoveryDirective(acctKV, "alpha", "i-seed", RecoveryActionClusterReset, "etcd-frequent-20260709T010000Z.snap", true) + stored, err := StoreRecoveryDirective(t.Context(), acctKV, "alpha", "i-seed", RecoveryActionClusterReset, "etcd-frequent-20260709T010000Z.snap", true) require.NoError(t, err) assert.True(t, stored.SnapshotRequired, "a required snapshot is recorded so the guest aborts on fetch failure") - got, err := LoadRecoveryDirective(acctKV, "alpha", "i-seed") + got, err := LoadRecoveryDirective(t.Context(), acctKV, "alpha", "i-seed") require.NoError(t, err) assert.True(t, got.SnapshotRequired, "SnapshotRequired survives the KV round-trip") assert.Equal(t, "etcd-frequent-20260709T010000Z.snap", got.Snapshot) diff --git a/spinifex/handlers/eks/restore_snapshot.go b/spinifex/handlers/eks/restore_snapshot.go index 14c47c881..dd6a25f1d 100644 --- a/spinifex/handlers/eks/restore_snapshot.go +++ b/spinifex/handlers/eks/restore_snapshot.go @@ -13,7 +13,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/admin" "github.com/mulgadc/spinifex/spinifex/awserrors" "github.com/mulgadc/spinifex/spinifex/objectstore" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // eksBackupsBucket is the predastore bucket the guest etcd-snapshot cron PUTs @@ -142,8 +142,8 @@ func validateSnapshotExists(ctx context.Context, store objectstore.ObjectStore, // (service_impl.go CreateCluster). restore-snapshot only ever targets a // single-CP cluster (isHAControlPlane guards the caller), so this replaces // wholesale rather than matching a dead instance ID like SwapControlPlaneMember. -func replaceControlPlaneForRestore(kv nats.KeyValue, name string, newNode ControlPlaneNode) error { - return casUpdateMeta(kv, name, func(m *ClusterMeta) bool { +func replaceControlPlaneForRestore(ctx context.Context, kv jetstream.KeyValue, name string, newNode ControlPlaneNode) error { + return casUpdateMeta(ctx, kv, name, func(m *ClusterMeta) bool { m.ControlPlaneNodes = []ControlPlaneNode{newNode} m.ControlPlaneInstanceID = newNode.InstanceID m.ControlPlaneENIID = newNode.ENIID @@ -214,11 +214,11 @@ func (s *EKSServiceImpl) RestoreSnapshot(ctx context.Context, input *RestoreSnap if input == nil || input.ClusterName == "" { return nil, errors.New(awserrors.ErrorInvalidParameterValue) } - acctKV, err := s.acctKVForCluster(accountID, input.ClusterName) + acctKV, err := s.acctKVForCluster(ctx, accountID, input.ClusterName) if err != nil { return nil, err } - meta, err := GetClusterMeta(acctKV, input.ClusterName) + meta, err := GetClusterMeta(ctx, acctKV, input.ClusterName) if err != nil { if errors.Is(err, ErrClusterNotFound) { return nil, errors.New(awserrors.ErrorEKSResourceNotFound) @@ -285,7 +285,7 @@ func (s *EKSServiceImpl) RestoreSnapshot(ctx context.Context, input *RestoreSnap // Persist the new CP into meta BEFORE re-pointing the NLB. Once committed the // new CP is canonical, so a later NLB failure is convergeable by the reconciler // (provisional success) rather than a bare error implying nothing happened. - if perr := replaceControlPlaneForRestore(acctKV, input.ClusterName, newNode); perr != nil { + if perr := replaceControlPlaneForRestore(ctx, acctKV, input.ClusterName, newNode); perr != nil { s.unwindFreshCP(ctx, accountID, input.ClusterName, newNode) return nil, fmt.Errorf("eks: persist replacement control plane: %w", perr) } diff --git a/spinifex/handlers/eks/restore_snapshot_test.go b/spinifex/handlers/eks/restore_snapshot_test.go index 9dc650104..e12be3179 100644 --- a/spinifex/handlers/eks/restore_snapshot_test.go +++ b/spinifex/handlers/eks/restore_snapshot_test.go @@ -118,7 +118,7 @@ func TestRestoreSnapshot_HappyPathLaunchesDirectsAndRepoints(t *testing.T) { seedSnapshot(t, store, testAccountID, "alpha", "etcd-frequent-20260709T010000Z.snap") meta := restoreSnapshotFixtureMeta("alpha") - require.NoError(t, PutClusterMeta(f.kv, meta)) + require.NoError(t, PutClusterMeta(t.Context(), f.kv, meta)) out, err := f.svc.RestoreSnapshot(context.Background(), &RestoreSnapshotInput{ClusterName: "alpha"}, testAccountID) require.NoError(t, err) @@ -147,7 +147,7 @@ func TestRestoreSnapshot_HappyPathLaunchesDirectsAndRepoints(t *testing.T) { } // Cluster meta reflects the replacement. - got, err := GetClusterMeta(f.kv, "alpha") + got, err := GetClusterMeta(t.Context(), f.kv, "alpha") require.NoError(t, err) assert.Equal(t, out.NewInstanceID, got.ControlPlaneInstanceID) require.Len(t, got.ControlPlaneNodes, 1) @@ -166,7 +166,7 @@ func TestRestoreSnapshot_ExplicitSnapshotUsedVerbatim(t *testing.T) { seedSnapshot(t, store, testAccountID, "alpha", "etcd-frequent-20260710T010000Z.snap") seedSnapshot(t, store, testAccountID, "alpha", "etcd-daily-20260601T000000Z.snap") meta := restoreSnapshotFixtureMeta("alpha") - require.NoError(t, PutClusterMeta(f.kv, meta)) + require.NoError(t, PutClusterMeta(t.Context(), f.kv, meta)) out, err := f.svc.RestoreSnapshot(context.Background(), &RestoreSnapshotInput{ClusterName: "alpha", Snapshot: "etcd-daily-20260601T000000Z.snap"}, testAccountID) @@ -190,7 +190,7 @@ func TestRestoreSnapshot_ExplicitSnapshotMissingHardFailsBeforeLaunch(t *testing // Only an unrelated snapshot exists — the requested key is absent. seedSnapshot(t, store, testAccountID, "alpha", "etcd-frequent-20260709T010000Z.snap") meta := restoreSnapshotFixtureMeta("alpha") - require.NoError(t, PutClusterMeta(f.kv, meta)) + require.NoError(t, PutClusterMeta(t.Context(), f.kv, meta)) _, err := f.svc.RestoreSnapshot(context.Background(), &RestoreSnapshotInput{ClusterName: "alpha", Snapshot: "etcd-daily-20260601T000000Z.snap"}, testAccountID) @@ -204,7 +204,7 @@ func TestRestoreSnapshot_ExplicitSnapshotWithNoStoreHardFails(t *testing.T) { f.svc.deps.Scheduler = &fakeHostScheduler{hosts: []string{"node-new"}} f.svc.deps.SnapshotStore = nil meta := restoreSnapshotFixtureMeta("alpha") - require.NoError(t, PutClusterMeta(f.kv, meta)) + require.NoError(t, PutClusterMeta(t.Context(), f.kv, meta)) _, err := f.svc.RestoreSnapshot(context.Background(), &RestoreSnapshotInput{ClusterName: "alpha", Snapshot: "etcd-daily-20260601T000000Z.snap"}, testAccountID) @@ -217,7 +217,7 @@ func TestRestoreSnapshot_HARejected(t *testing.T) { meta := restoreSnapshotFixtureMeta("alpha") meta.ControlPlaneSpreadGroup = "eks-cp-111122223333-alpha" meta.ControlPlaneNodes = append(meta.ControlPlaneNodes, ControlPlaneNode{NodeID: "node-2", InstanceID: "i-cp2"}) - require.NoError(t, PutClusterMeta(f.kv, meta)) + require.NoError(t, PutClusterMeta(t.Context(), f.kv, meta)) _, err := f.svc.RestoreSnapshot(context.Background(), &RestoreSnapshotInput{ClusterName: "alpha", Snapshot: "etcd-daily-20260601T000000Z.snap"}, testAccountID) @@ -230,7 +230,7 @@ func TestRestoreSnapshot_NoTemplateRejected(t *testing.T) { f := newEKSServiceFixture(t) meta := restoreSnapshotFixtureMeta("alpha") meta.ControlPlaneTemplate = nil - require.NoError(t, PutClusterMeta(f.kv, meta)) + require.NoError(t, PutClusterMeta(t.Context(), f.kv, meta)) _, err := f.svc.RestoreSnapshot(context.Background(), &RestoreSnapshotInput{ClusterName: "alpha", Snapshot: "etcd-daily-20260601T000000Z.snap"}, testAccountID) @@ -255,7 +255,7 @@ func TestRestoreSnapshot_NLBFailureIsProvisionalNotError(t *testing.T) { f.nlb.registerErr = errors.New("TargetGroupNotFound") meta := restoreSnapshotFixtureMeta("alpha") - require.NoError(t, PutClusterMeta(f.kv, meta)) + require.NoError(t, PutClusterMeta(t.Context(), f.kv, meta)) out, err := f.svc.RestoreSnapshot(context.Background(), &RestoreSnapshotInput{ClusterName: "alpha"}, testAccountID) require.NoError(t, err, "an NLB re-point failure after meta commit is provisional, not a hard error") @@ -263,7 +263,7 @@ func TestRestoreSnapshot_NLBFailureIsProvisionalNotError(t *testing.T) { // Meta was persisted BEFORE the NLB step, so it already names the new CP and // the reconciler can converge the target group. - got, err := GetClusterMeta(f.kv, "alpha") + got, err := GetClusterMeta(t.Context(), f.kv, "alpha") require.NoError(t, err) assert.Equal(t, out.NewInstanceID, got.ControlPlaneInstanceID, "meta persisted before NLB re-point") @@ -293,7 +293,7 @@ func TestConfirmOldCPTerminated_NoOldNodeSucceeds(t *testing.T) { func TestUnwindFreshCP_TerminatesAndClearsDirective(t *testing.T) { f := newEKSServiceFixture(t) meta := restoreSnapshotFixtureMeta("alpha") - require.NoError(t, PutClusterMeta(f.kv, meta)) + require.NoError(t, PutClusterMeta(t.Context(), f.kv, meta)) // Pretend a directive was already set for the fresh CP (as the real flow does // just before a meta-commit failure), then unwind. diff --git a/spinifex/handlers/eks/service_impl.go b/spinifex/handlers/eks/service_impl.go index fe09e83a7..ced1acf96 100644 --- a/spinifex/handlers/eks/service_impl.go +++ b/spinifex/handlers/eks/service_impl.go @@ -28,6 +28,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/objectstore" "github.com/mulgadc/spinifex/spinifex/utils" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // SubnetVPCResolver resolves a subnet to its VPC ID and CIDR. @@ -228,7 +229,7 @@ type eipProvisioner interface { // EKSServiceImpl is the daemon-side EKSService implementation. type EKSServiceImpl struct { deps EKSServiceDeps - leaderKV nats.KeyValue + leaderKV jetstream.KeyValue registry *ReconcilerRegistry mu sync.Mutex @@ -272,15 +273,18 @@ func NewEKSServiceImpl(deps EKSServiceDeps) (*EKSServiceImpl, error) { if deps.NATSConn == nil { return nil, errors.New("eks: NewEKSServiceImpl nil NATSConn") } - js, err := deps.NATSConn.JetStream() + js, err := jetstream.New(deps.NATSConn) if err != nil { return nil, fmt.Errorf("failed to get JetStream context: %w", err) } - leaderKV, err := InitLeaderBucket(js, max(deps.ClusterSize, 1)) + // The leader bucket outlives every request, so its open is bounded by the + // service's own background context rather than a caller's. + ctx, cancel := context.WithCancel(context.Background()) + leaderKV, err := InitLeaderBucket(ctx, js, max(deps.ClusterSize, 1)) if err != nil { + cancel() return nil, err } - ctx, cancel := context.WithCancel(context.Background()) return &EKSServiceImpl{ deps: deps, leaderKV: leaderKV, @@ -319,47 +323,59 @@ func (s *EKSServiceImpl) SpawnRegisteredReconcilers() error { slog.Debug("SpawnRegisteredReconcilers: deps not ready, skipping") return nil } - js, err := s.deps.NATSConn.JetStream() + js, err := jetstream.New(s.deps.NATSConn) if err != nil { return fmt.Errorf("jetstream: %w", err) } prev := -1 + var scanErr error for attempt := range spawnScanMaxAttempts { - observed := s.spawnRegisteredReconcilersScan(js) - if observed <= prev { - break + observed, err := s.spawnRegisteredReconcilersScan(s.bgCtx, js) + switch { + // A failed enumeration is not a settled one. Reading an unreachable + // JetStream as "no clusters" would leave every cluster without its + // reconciler until the next daemon restart, so the pass is retried and the + // last failure surfaces to the caller. + case err != nil: + slog.Warn("SpawnRegisteredReconcilers: scan failed", "attempt", attempt, "err", err) + case observed <= prev: + return nil + default: + prev = observed } - prev = observed + scanErr = err if attempt < spawnScanMaxAttempts-1 { time.Sleep(s.spawnScanRetryBackoff) } } - return nil + return scanErr } // spawnRegisteredReconcilersScan runs one enumeration pass and resumes reconcilers // for every CREATING/ACTIVE cluster it can see, returning the count observed so the -// caller can detect when a lagging JetStream enumeration has settled. -func (s *EKSServiceImpl) spawnRegisteredReconcilersScan(js nats.JetStreamContext) int { +// caller can detect when a lagging JetStream enumeration has settled. An +// incomplete enumeration is reported rather than counted, so a truncated pass is +// never mistaken for a settled one. +func (s *EKSServiceImpl) spawnRegisteredReconcilersScan(ctx context.Context, js jetstream.JetStream) (int, error) { + names, err := accountBucketNames(ctx, s.deps.NATSConn) + if err != nil { + return 0, fmt.Errorf("enumerate account buckets: %w", err) + } observed := 0 - buckets := js.KeyValueStoreNames() - for name := range buckets { - if !strings.HasPrefix(name, KVBucketEKSAccountPrefix) { - continue - } + for _, name := range names { accountID := strings.TrimPrefix(name, KVBucketEKSAccountPrefix) - acctKV, err := js.KeyValue(name) + acctKV, err := js.KeyValue(ctx, name) if err != nil { slog.Warn("SpawnRegisteredReconcilers: open bucket failed", "bucket", name, "err", err) continue } - clusters, err := listClusterNames(acctKV) + clusters, err := listClusterNames(ctx, acctKV) if err != nil { slog.Warn("SpawnRegisteredReconcilers: list clusters failed", "bucket", name, "err", err) continue } for _, cluster := range clusters { - meta, err := GetClusterMeta(acctKV, cluster) + meta, err := GetClusterMeta(ctx, acctKV, cluster) if err != nil { continue } @@ -371,10 +387,10 @@ func (s *EKSServiceImpl) spawnRegisteredReconcilersScan(js nats.JetStreamContext // was in flight when the prior process died left a CREATING (or // partially-launched CREATE_FAILED) record whose workers nothing else // will ever terminate. - s.reclaimOrphanedNodegroups(context.Background(), accountID, acctKV, cluster) + s.reclaimOrphanedNodegroups(ctx, accountID, acctKV, cluster) // Re-subscribe to missing artifacts; K3s publishes each one-shot message once. if meta.Status == ClusterStatusCreating { - if pending := BootstrapPendingKinds(acctKV, cluster, meta); len(pending) > 0 { + if pending := BootstrapPendingKinds(ctx, acctKV, cluster, meta); len(pending) > 0 { slog.Info("SpawnRegisteredReconcilers: resuming bootstrap", "cluster", cluster, "pending", pending) s.spawnBootstrap(accountID, cluster, acctKV, pending) @@ -383,7 +399,7 @@ func (s *EKSServiceImpl) spawnRegisteredReconcilersScan(js nats.JetStreamContext s.spawnReconciler(accountID, cluster, meta) } } - return observed + return observed, nil } func (s *EKSServiceImpl) depsReadyForOrchestration() bool { @@ -474,11 +490,11 @@ func (s *EKSServiceImpl) CreateCluster(ctx context.Context, input *eks.CreateClu name := aws.StringValue(input.Name) subnetIDs := aws.StringValueSlice(input.ResourcesVpcConfig.SubnetIds) - js, err := s.deps.NATSConn.JetStream() + js, err := jetstream.New(s.deps.NATSConn) if err != nil { return nil, logCreateErr(name, accountID, "jetstream", err) } - acctKV, err := GetOrCreateAccountBucket(js, accountID, max(s.deps.ClusterSize, 1)) + acctKV, err := GetOrCreateAccountBucket(ctx, js, accountID, max(s.deps.ClusterSize, 1)) if err != nil { return nil, logCreateErr(name, accountID, "get account bucket", err) } @@ -521,13 +537,16 @@ func (s *EKSServiceImpl) CreateCluster(ctx context.Context, input *eks.CreateClu // Respond CREATING immediately; run the slow launch on a background goroutine. // The goroutine owns meta from here on — do not touch it after this point. out := &eks.CreateClusterOutput{Cluster: clusterMetaToAWS(meta)} + // The launch outlives the request, so it runs on its own background context + // rather than the caller's, which is cancelled once this reply is written. + launchCtx := context.Background() s.launchWG.Go(func() { defer func() { if r := recover(); r != nil { - s.failClusterLaunch(context.Background(), acctKV, name, accountID, meta, "launch panic", fmt.Errorf("%v", r)) + s.failClusterLaunch(launchCtx, acctKV, name, accountID, meta, "launch panic", fmt.Errorf("%v", r)) } }() - s.launchClusterInfra(context.Background(), clusterLaunchCtx{ + s.launchClusterInfra(launchCtx, clusterLaunchCtx{ accountID: accountID, callerPrincipalARN: callerPrincipalARN, name: name, @@ -585,7 +604,7 @@ type clusterLaunchCtx struct { publicCidrs []string input *eks.CreateClusterInput meta *ClusterMeta - acctKV nats.KeyValue + acctKV jetstream.KeyValue } // failClusterLaunch records an asynchronous control-plane launch failure: it @@ -593,7 +612,7 @@ type clusterLaunchCtx struct { // and the 267.3 reclaim path can recreate cleanly) and logs the stage. The // launch runs after the CREATING response was already sent, so a failure can // only surface as status, never as the CreateCluster return value. -func (s *EKSServiceImpl) failClusterLaunch(ctx context.Context, kv nats.KeyValue, name, accountID string, meta *ClusterMeta, stage string, err error) { +func (s *EKSServiceImpl) failClusterLaunch(ctx context.Context, kv jetstream.KeyValue, name, accountID string, meta *ClusterMeta, stage string, err error) { _ = logCreateErr(name, accountID, stage, err) // Release any billable infra already provisioned this launch (LB VM + its // associated EIP, egress EIP, NLB, CP VMs, OIDC) so a failed create never @@ -608,7 +627,7 @@ func (s *EKSServiceImpl) failClusterLaunch(ctx context.Context, kv nats.KeyValue slog.WarnContext(ctx, "failClusterLaunch: infra purge incomplete", "cluster", name, "stage", stage, "err", perr) } } - if mErr := MarkClusterFailed(kv, name, stage+": "+err.Error()); mErr != nil && !errors.Is(mErr, ErrClusterNotFound) { + if mErr := MarkClusterFailed(ctx, kv, name, stage+": "+err.Error()); mErr != nil && !errors.Is(mErr, ErrClusterNotFound) { slog.WarnContext(ctx, "launchClusterInfra: MarkClusterFailed failed", "cluster", name, "err", mErr) } } @@ -643,7 +662,7 @@ func (s *EKSServiceImpl) launchClusterInfra(ctx context.Context, lc clusterLaunc meta.ManagedCPVPC = managedCPVPCFromRefs(cpRefs) // Persist the CP VPC refs before any further fallible step so teardown can // reclaim the VPC/subnets/IGW/NAT GW even if the launch fails after here. - if err := PutClusterMeta(acctKV, meta); err != nil { + if err := PutClusterMeta(ctx, acctKV, meta); err != nil { s.failClusterLaunch(ctx, acctKV, name, accountID, meta, "persist managed CP VPC", err) return } @@ -687,7 +706,7 @@ func (s *EKSServiceImpl) launchClusterInfra(ctx context.Context, lc clusterLaunc } meta.PrivateEndpointENIID = pe.ENIID meta.PrivateEndpointIP = pe.ENIIP - if err := PutClusterMeta(acctKV, meta); err != nil { + if err := PutClusterMeta(ctx, acctKV, meta); err != nil { s.failClusterLaunch(ctx, acctKV, name, accountID, meta, "persist private endpoint refs", err) return } @@ -739,7 +758,7 @@ func (s *EKSServiceImpl) launchClusterInfra(ctx context.Context, lc clusterLaunc // Persist the endpoint before publishing it so reconcile never observes a DNS // record whose creating cluster lacks its desired endpoint metadata. - if err := PutClusterMeta(acctKV, meta); err != nil { + if err := PutClusterMeta(ctx, acctKV, meta); err != nil { s.failClusterLaunch(ctx, acctKV, name, accountID, meta, "persist endpoint metadata", err) return } @@ -753,7 +772,7 @@ func (s *EKSServiceImpl) launchClusterInfra(ctx context.Context, lc clusterLaunc } meta.OIDCIssuer = oidcIssuer - privPEM, _, err := GenerateClusterOIDCKeypair(acctKV, name, s.deps.MasterKey) + privPEM, _, err := GenerateClusterOIDCKeypair(ctx, acctKV, name, s.deps.MasterKey) if err != nil { s.failClusterLaunch(ctx, acctKV, name, accountID, meta, "generate OIDC keypair", err) return @@ -851,7 +870,7 @@ func (s *EKSServiceImpl) launchClusterInfra(ctx context.Context, lc clusterLaunc // failure between here and the next PutClusterMeta leaves them launched but // unrecorded, so neither DeleteCluster nor the FAILED-cluster reclaim could // reach them and the sys.medium VMs leak. - if err := PutClusterMeta(acctKV, meta); err != nil { + if err := PutClusterMeta(ctx, acctKV, meta); err != nil { s.failClusterLaunch(ctx, acctKV, name, accountID, meta, "persist control-plane ids", err) return } @@ -875,7 +894,7 @@ func (s *EKSServiceImpl) launchClusterInfra(ctx context.Context, lc clusterLaunc return } - if err := PutClusterMeta(acctKV, meta); err != nil { + if err := PutClusterMeta(ctx, acctKV, meta); err != nil { s.failClusterLaunch(ctx, acctKV, name, accountID, meta, "persist final meta", err) return } @@ -884,7 +903,7 @@ func (s *EKSServiceImpl) launchClusterInfra(ctx context.Context, lc clusterLaunc if bootstrapCreatorAdmin(input) && callerPrincipalARN != "" { rec := newAccessEntryRecord(region, accountID, name, callerPrincipalARN, "", []string{"system:masters"}, AccessEntryTypeStandard, nil, time.Now().UTC()) - if err := PutAccessEntryRecord(acctKV, rec); err != nil { + if err := PutAccessEntryRecord(ctx, acctKV, rec); err != nil { s.failClusterLaunch(ctx, acctKV, name, accountID, meta, "seed cluster-creator admin access entry", err) return } @@ -921,15 +940,15 @@ func (s *EKSServiceImpl) DescribeCluster(ctx context.Context, input *eks.Describ if name == "" { return nil, errors.New(awserrors.ErrorInvalidParameterValue) } - js, err := s.deps.NATSConn.JetStream() + js, err := jetstream.New(s.deps.NATSConn) if err != nil { return nil, fmt.Errorf("jetstream: %w", err) } - acctKV, err := GetOrCreateAccountBucket(js, accountID, max(s.deps.ClusterSize, 1)) + acctKV, err := GetOrCreateAccountBucket(ctx, js, accountID, max(s.deps.ClusterSize, 1)) if err != nil { return nil, fmt.Errorf("get account bucket: %w", err) } - meta, err := GetClusterMeta(acctKV, name) + meta, err := GetClusterMeta(ctx, acctKV, name) if err != nil { if errors.Is(err, ErrClusterNotFound) { return nil, errors.New(awserrors.ErrorEKSResourceNotFound) @@ -940,15 +959,15 @@ func (s *EKSServiceImpl) DescribeCluster(ctx context.Context, input *eks.Describ } func (s *EKSServiceImpl) ListClusters(ctx context.Context, input *eks.ListClustersInput, accountID string) (*eks.ListClustersOutput, error) { - js, err := s.deps.NATSConn.JetStream() + js, err := jetstream.New(s.deps.NATSConn) if err != nil { return nil, eksReadUnavailableOr(err, "jetstream") } - acctKV, err := GetOrCreateAccountBucket(js, accountID, max(s.deps.ClusterSize, 1)) + acctKV, err := GetOrCreateAccountBucket(ctx, js, accountID, max(s.deps.ClusterSize, 1)) if err != nil { return nil, eksReadUnavailableOr(err, "get account bucket") } - names, err := listClusterNames(acctKV) + names, err := listClusterNames(ctx, acctKV) if err != nil { return nil, eksReadUnavailableOr(err, "list cluster names") } @@ -987,16 +1006,16 @@ func (s *EKSServiceImpl) DeleteCluster(ctx context.Context, input *eks.DeleteClu if name == "" { return nil, errors.New(awserrors.ErrorInvalidParameterValue) } - js, err := s.deps.NATSConn.JetStream() + js, err := jetstream.New(s.deps.NATSConn) if err != nil { return nil, fmt.Errorf("jetstream: %w", err) } - acctKV, err := GetOrCreateAccountBucket(js, accountID, max(s.deps.ClusterSize, 1)) + acctKV, err := GetOrCreateAccountBucket(ctx, js, accountID, max(s.deps.ClusterSize, 1)) if err != nil { return nil, fmt.Errorf("get account bucket: %w", err) } - meta, err := GetClusterMeta(acctKV, name) + meta, err := GetClusterMeta(ctx, acctKV, name) if err != nil { if errors.Is(err, ErrClusterNotFound) { // Idempotent: a cluster already swept from KV is "deleted". Returning @@ -1013,14 +1032,14 @@ func (s *EKSServiceImpl) DeleteCluster(ctx context.Context, input *eks.DeleteClu // risking a double-release on the billable NAT-GW EIP). The loser returns the // cluster as DELETING — AWS-async delete semantics — and the winner (or the // DELETING backstop reaper) drives the teardown to completion. - release, ok := s.acquireTeardownLease(accountID, name) + release, ok := s.acquireTeardownLease(ctx, accountID, name) if !ok { meta.Status = ClusterStatusDeleting return &eks.DeleteClusterOutput{Cluster: clusterMetaToAWS(meta)}, nil } defer release() - if err := SetClusterStatus(acctKV, name, ClusterStatusDeleting); err != nil { + if err := SetClusterStatus(ctx, acctKV, name, ClusterStatusDeleting); err != nil { return nil, fmt.Errorf("set DELETING: %w", err) } meta.Status = ClusterStatusDeleting @@ -1049,16 +1068,18 @@ func teardownLeaderKey(accountID, clusterName string) string { // DELETING backstop reaper. CAS Create fails when another holder owns it. A nil // leaderKV (single-node/test) skips gating; the bucket TTL reaps a lease whose // release never runs. -func (s *EKSServiceImpl) acquireTeardownLease(accountID, clusterName string) (func(), bool) { +func (s *EKSServiceImpl) acquireTeardownLease(ctx context.Context, accountID, clusterName string) (func(), bool) { if s.leaderKV == nil { return func() {}, true } key := teardownLeaderKey(accountID, clusterName) - if _, err := s.leaderKV.Create(key, []byte(s.deps.HolderID)); err != nil { + if _, err := s.leaderKV.Create(ctx, key, []byte(s.deps.HolderID)); err != nil { return nil, false } return func() { - if err := s.leaderKV.Delete(key); err != nil { + releaseCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := s.leaderKV.Delete(releaseCtx, key); err != nil { slog.Warn("eks: teardown lease release failed (TTL will reap)", "key", key, "err", err) } }, true @@ -1066,23 +1087,23 @@ func (s *EKSServiceImpl) acquireTeardownLease(accountID, clusterName string) (fu // claimClusterName atomically claims the cluster meta key before any launch work. // A prior FAILED attempt is reclaimed via CAS (FAILED→CREATING); live clusters reject with ResourceInUse. -func (s *EKSServiceImpl) claimClusterName(ctx context.Context, accountID string, acctKV nats.KeyValue, meta *ClusterMeta) error { +func (s *EKSServiceImpl) claimClusterName(ctx context.Context, accountID string, acctKV jetstream.KeyValue, meta *ClusterMeta) error { name := meta.Name data, err := json.Marshal(meta) if err != nil { return logCreateErr(name, accountID, "marshal initial meta", err) } - if _, cerr := acctKV.Create(ClusterMetaKey(name), data); cerr == nil { + if _, cerr := acctKV.Create(ctx, ClusterMetaKey(name), data); cerr == nil { return nil // won a fresh claim - } else if !errors.Is(cerr, nats.ErrKeyExists) { + } else if !errors.Is(cerr, jetstream.ErrKeyExists) { return logCreateErr(name, accountID, "claim cluster name", cerr) } // Key already present: reclaim a FAILED attempt via CAS; old refs survive for teardown. var oldRefs *ClusterMeta reclaimed := false - if err := casUpdateMeta(acctKV, name, func(m *ClusterMeta) bool { + if err := casUpdateMeta(ctx, acctKV, name, func(m *ClusterMeta) bool { // casUpdateMeta re-runs this closure on every CAS-conflict retry, so reset // the outputs each pass: a retry that now observes CREATING (a concurrent // reclaimer won) must leave reclaimed=false, or we double-own the cluster. @@ -1115,12 +1136,12 @@ func (s *EKSServiceImpl) claimClusterName(ctx context.Context, accountID string, // Purge the failed attempt's infra but KEEP the meta claim (deleteMeta=false) // — the CREATING record is our lock for the rest of this create. if perr := s.purgeClusterInfra(ctx, accountID, name, oldRefs, acctKV, false); perr != nil { - s.markFailed(acctKV, name) + s.markFailed(ctx, acctKV, name) return logCreateErr(name, accountID, "reclaim failed cluster", perr) } // Infra gone: overwrite with the fresh CREATING meta (clears the old refs). - if err := PutClusterMeta(acctKV, meta); err != nil { - s.markFailed(acctKV, name) + if err := PutClusterMeta(ctx, acctKV, meta); err != nil { + s.markFailed(ctx, acctKV, name) return logCreateErr(name, accountID, "persist reclaimed meta", err) } return nil @@ -1134,7 +1155,7 @@ func (s *EKSServiceImpl) claimClusterName(ctx context.Context, accountID string, // its IDs. The cluster SGs sit in the customer VPC, so a leaked SG pins the VPC // on DependencyViolation — leaving the cluster DELETING for a retry is correct. // Shared by DeleteCluster and the FAILED-cluster reclaim path in CreateCluster. -func (s *EKSServiceImpl) purgeClusterInfra(ctx context.Context, accountID, name string, meta *ClusterMeta, acctKV nats.KeyValue, deleteMeta bool) error { +func (s *EKSServiceImpl) purgeClusterInfra(ctx context.Context, accountID, name string, meta *ClusterMeta, acctKV jetstream.KeyValue, deleteMeta bool) error { s.registry.Stop(accountID, name) // infraAcct owns the NLB, CP VMs, and CP VPC. Clusters built with the managed @@ -1147,7 +1168,7 @@ func (s *EKSServiceImpl) purgeClusterInfra(ctx context.Context, accountID, name var teardownErrs []error - if err := ZeroizeClusterOIDCKey(acctKV, name); err != nil { + if err := ZeroizeClusterOIDCKey(ctx, acctKV, name); err != nil { teardownErrs = append(teardownErrs, fmt.Errorf("zeroize OIDC key: %w", err)) } @@ -1273,7 +1294,7 @@ func (s *EKSServiceImpl) purgeClusterInfra(ctx context.Context, accountID, name // SG/VPC API calls against a stale VpcId forever, turning a single // DependencyViolation into a permanent DELETING loop. meta.ManagedCPVPC = nil - if perr := ClearClusterManagedCPVPC(acctKV, name); perr != nil && !errors.Is(perr, ErrClusterNotFound) { + if perr := ClearClusterManagedCPVPC(ctx, acctKV, name); perr != nil && !errors.Is(perr, ErrClusterNotFound) { teardownErrs = append(teardownErrs, fmt.Errorf("clear managed CP VPC state: %w", perr)) } } @@ -1308,7 +1329,7 @@ func (s *EKSServiceImpl) purgeClusterInfra(ctx context.Context, accountID, name // meta key (status CREATING) as its lock for the recreate, so the key must // survive — only the failed attempt's infra is purged here. if deleteMeta { - if err := DeleteClusterPrefix(acctKV, name); err != nil { + if err := DeleteClusterPrefix(ctx, acctKV, name); err != nil { return fmt.Errorf("delete cluster prefix: %w", err) } } @@ -1329,7 +1350,7 @@ func (s *EKSServiceImpl) CreateNodegroup(ctx context.Context, input *eks.CreateN if err := s.requireOrchestrationDeps("CreateNodegroup"); err != nil { return nil, err } - acctKV, err := s.nodegroupAcctKV(accountID) + acctKV, err := s.nodegroupAcctKV(ctx, accountID) if err != nil { return nil, err } @@ -1340,29 +1361,29 @@ func (s *EKSServiceImpl) DescribeNodegroup(ctx context.Context, input *eks.Descr if input == nil { return nil, errors.New(awserrors.ErrorInvalidParameterValue) } - acctKV, err := s.nodegroupAcctKV(accountID) + acctKV, err := s.nodegroupAcctKV(ctx, accountID) if err != nil { return nil, err } - return s.describeNodegroup(acctKV, input) + return s.describeNodegroup(ctx, acctKV, input) } func (s *EKSServiceImpl) ListNodegroups(ctx context.Context, input *eks.ListNodegroupsInput, accountID string) (*eks.ListNodegroupsOutput, error) { if input == nil { return nil, errors.New(awserrors.ErrorInvalidParameterValue) } - acctKV, err := s.nodegroupAcctKV(accountID) + acctKV, err := s.nodegroupAcctKV(ctx, accountID) if err != nil { return nil, err } - return s.listNodegroups(acctKV, input) + return s.listNodegroups(ctx, acctKV, input) } func (s *EKSServiceImpl) UpdateNodegroupConfig(ctx context.Context, input *eks.UpdateNodegroupConfigInput, accountID string) (*eks.UpdateNodegroupConfigOutput, error) { if err := s.requireOrchestrationDeps("UpdateNodegroupConfig"); err != nil { return nil, err } - acctKV, err := s.nodegroupAcctKV(accountID) + acctKV, err := s.nodegroupAcctKV(ctx, accountID) if err != nil { return nil, err } @@ -1377,7 +1398,7 @@ func (s *EKSServiceImpl) DeleteNodegroup(ctx context.Context, input *eks.DeleteN if err := s.requireOrchestrationDeps("DeleteNodegroup"); err != nil { return nil, err } - acctKV, err := s.nodegroupAcctKV(accountID) + acctKV, err := s.nodegroupAcctKV(ctx, accountID) if err != nil { return nil, err } @@ -1387,19 +1408,19 @@ func (s *EKSServiceImpl) DeleteNodegroup(ctx context.Context, input *eks.DeleteN // --- AccessEntry + AccessPolicy --- // acctKVForCluster opens the per-account bucket and verifies the cluster exists. -func (s *EKSServiceImpl) acctKVForCluster(accountID, cluster string) (nats.KeyValue, error) { +func (s *EKSServiceImpl) acctKVForCluster(ctx context.Context, accountID, cluster string) (jetstream.KeyValue, error) { if cluster == "" { return nil, errors.New(awserrors.ErrorInvalidParameterValue) } - js, err := s.deps.NATSConn.JetStream() + js, err := jetstream.New(s.deps.NATSConn) if err != nil { return nil, fmt.Errorf("jetstream: %w", err) } - acctKV, err := GetOrCreateAccountBucket(js, accountID, max(s.deps.ClusterSize, 1)) + acctKV, err := GetOrCreateAccountBucket(ctx, js, accountID, max(s.deps.ClusterSize, 1)) if err != nil { return nil, fmt.Errorf("get account bucket: %w", err) } - if _, err := GetClusterMeta(acctKV, cluster); err != nil { + if _, err := GetClusterMeta(ctx, acctKV, cluster); err != nil { if errors.Is(err, ErrClusterNotFound) { return nil, errors.New(awserrors.ErrorEKSResourceNotFound) } @@ -1425,11 +1446,11 @@ func (s *EKSServiceImpl) CreateAccessEntry(ctx context.Context, input *eks.Creat // Non-standard types (EC2_LINUX etc.) are not yet implemented. return nil, errors.New(awserrors.ErrorInvalidParameterValue) } - acctKV, err := s.acctKVForCluster(accountID, cluster) + acctKV, err := s.acctKVForCluster(ctx, accountID, cluster) if err != nil { return nil, err } - if _, err := GetAccessEntryRecord(acctKV, cluster, principalARN); err == nil { + if _, err := GetAccessEntryRecord(ctx, acctKV, cluster, principalARN); err == nil { return nil, errors.New(awserrors.ErrorEKSResourceInUse) } else if !errors.Is(err, ErrAccessEntryNotFound) { return nil, err @@ -1437,7 +1458,7 @@ func (s *EKSServiceImpl) CreateAccessEntry(ctx context.Context, input *eks.Creat rec := newAccessEntryRecord(s.deps.Region, accountID, cluster, principalARN, aws.StringValue(input.Username), aws.StringValueSlice(input.KubernetesGroups), entryType, aws.StringValueMap(input.Tags), time.Now().UTC()) - if err := PutAccessEntryRecord(acctKV, rec); err != nil { + if err := PutAccessEntryRecord(ctx, acctKV, rec); err != nil { return nil, err } return &eks.CreateAccessEntryOutput{AccessEntry: accessEntryRecordToAWS(rec)}, nil @@ -1449,11 +1470,11 @@ func (s *EKSServiceImpl) DescribeAccessEntry(ctx context.Context, input *eks.Des } cluster := aws.StringValue(input.ClusterName) principalARN := aws.StringValue(input.PrincipalArn) - acctKV, err := s.acctKVForCluster(accountID, cluster) + acctKV, err := s.acctKVForCluster(ctx, accountID, cluster) if err != nil { return nil, err } - rec, err := GetAccessEntryRecord(acctKV, cluster, principalARN) + rec, err := GetAccessEntryRecord(ctx, acctKV, cluster, principalARN) if err != nil { if errors.Is(err, ErrAccessEntryNotFound) { return nil, errors.New(awserrors.ErrorEKSResourceNotFound) @@ -1468,11 +1489,11 @@ func (s *EKSServiceImpl) ListAccessEntries(ctx context.Context, input *eks.ListA return nil, errors.New(awserrors.ErrorInvalidParameterValue) } cluster := aws.StringValue(input.ClusterName) - acctKV, err := s.acctKVForCluster(accountID, cluster) + acctKV, err := s.acctKVForCluster(ctx, accountID, cluster) if err != nil { return nil, err } - recs, err := ListAccessEntryRecords(acctKV, cluster) + recs, err := ListAccessEntryRecords(ctx, acctKV, cluster) if err != nil { return nil, err } @@ -1493,12 +1514,12 @@ func (s *EKSServiceImpl) UpdateAccessEntry(ctx context.Context, input *eks.Updat } cluster := aws.StringValue(input.ClusterName) principalARN := aws.StringValue(input.PrincipalArn) - acctKV, err := s.acctKVForCluster(accountID, cluster) + acctKV, err := s.acctKVForCluster(ctx, accountID, cluster) if err != nil { return nil, err } now := time.Now().UTC() - rec, err := casUpdateAccessEntry(acctKV, cluster, principalARN, func(r *AccessEntryRecord) bool { + rec, err := casUpdateAccessEntry(ctx, acctKV, cluster, principalARN, func(r *AccessEntryRecord) bool { if input.KubernetesGroups != nil { r.KubernetesGroups = aws.StringValueSlice(input.KubernetesGroups) } @@ -1523,11 +1544,11 @@ func (s *EKSServiceImpl) DeleteAccessEntry(ctx context.Context, input *eks.Delet } cluster := aws.StringValue(input.ClusterName) principalARN := aws.StringValue(input.PrincipalArn) - acctKV, err := s.acctKVForCluster(accountID, cluster) + acctKV, err := s.acctKVForCluster(ctx, accountID, cluster) if err != nil { return nil, err } - if err := DeleteAccessEntryRecord(acctKV, cluster, principalARN); err != nil { + if err := DeleteAccessEntryRecord(ctx, acctKV, cluster, principalARN); err != nil { if errors.Is(err, ErrAccessEntryNotFound) { return nil, errors.New(awserrors.ErrorEKSResourceNotFound) } @@ -1550,13 +1571,13 @@ func (s *EKSServiceImpl) AssociateAccessPolicy(ctx context.Context, input *eks.A if err != nil { return nil, errors.New(awserrors.ErrorInvalidParameterValue) } - acctKV, err := s.acctKVForCluster(accountID, cluster) + acctKV, err := s.acctKVForCluster(ctx, accountID, cluster) if err != nil { return nil, err } now := time.Now().UTC() var assoc AssociatedAccessPolicy - _, err = casUpdateAccessEntry(acctKV, cluster, principalARN, func(r *AccessEntryRecord) bool { + _, err = casUpdateAccessEntry(ctx, acctKV, cluster, principalARN, func(r *AccessEntryRecord) bool { assoc = AssociatedAccessPolicy{PolicyARN: policyARN, AccessScope: scope, AssociatedAt: now, ModifiedAt: now} for i := range r.AssociatedPolicies { if r.AssociatedPolicies[i].PolicyARN == policyARN { @@ -1590,12 +1611,12 @@ func (s *EKSServiceImpl) DisassociateAccessPolicy(ctx context.Context, input *ek cluster := aws.StringValue(input.ClusterName) principalARN := aws.StringValue(input.PrincipalArn) policyARN := aws.StringValue(input.PolicyArn) - acctKV, err := s.acctKVForCluster(accountID, cluster) + acctKV, err := s.acctKVForCluster(ctx, accountID, cluster) if err != nil { return nil, err } now := time.Now().UTC() - _, err = casUpdateAccessEntry(acctKV, cluster, principalARN, func(r *AccessEntryRecord) bool { + _, err = casUpdateAccessEntry(ctx, acctKV, cluster, principalARN, func(r *AccessEntryRecord) bool { for i := range r.AssociatedPolicies { if r.AssociatedPolicies[i].PolicyARN == policyARN { r.AssociatedPolicies = append(r.AssociatedPolicies[:i], r.AssociatedPolicies[i+1:]...) @@ -1620,11 +1641,11 @@ func (s *EKSServiceImpl) ListAssociatedAccessPolicies(ctx context.Context, input } cluster := aws.StringValue(input.ClusterName) principalARN := aws.StringValue(input.PrincipalArn) - acctKV, err := s.acctKVForCluster(accountID, cluster) + acctKV, err := s.acctKVForCluster(ctx, accountID, cluster) if err != nil { return nil, err } - rec, err := GetAccessEntryRecord(acctKV, cluster, principalARN) + rec, err := GetAccessEntryRecord(ctx, acctKV, cluster, principalARN) if err != nil { if errors.Is(err, ErrAccessEntryNotFound) { return nil, errors.New(awserrors.ErrorEKSResourceNotFound) @@ -1706,11 +1727,11 @@ func (s *EKSServiceImpl) TagResource(ctx context.Context, input *eks.TagResource add := aws.StringValueMap(input.Tags) if name, ok := clusterNameFromARN(arn); ok { - acctKV, err := s.accountBucket(accountID) + acctKV, err := s.accountBucket(ctx, accountID) if err != nil { return nil, err } - if err := casUpdateMeta(acctKV, name, func(m *ClusterMeta) bool { + if err := casUpdateMeta(ctx, acctKV, name, func(m *ClusterMeta) bool { if len(add) == 0 { return false } @@ -1726,7 +1747,7 @@ func (s *EKSServiceImpl) TagResource(ctx context.Context, input *eks.TagResource } if cluster, ng, ok := nodegroupRefFromARN(arn); ok { - if err := s.casUpdateNodegroupTags(accountID, cluster, ng, func(tags map[string]string) (map[string]string, bool) { + if err := s.casUpdateNodegroupTags(ctx, accountID, cluster, ng, func(tags map[string]string) (map[string]string, bool) { if len(add) == 0 { return tags, false } @@ -1749,11 +1770,11 @@ func (s *EKSServiceImpl) UntagResource(ctx context.Context, input *eks.UntagReso keys := aws.StringValueSlice(input.TagKeys) if name, ok := clusterNameFromARN(arn); ok { - acctKV, err := s.accountBucket(accountID) + acctKV, err := s.accountBucket(ctx, accountID) if err != nil { return nil, err } - if err := casUpdateMeta(acctKV, name, func(m *ClusterMeta) bool { + if err := casUpdateMeta(ctx, acctKV, name, func(m *ClusterMeta) bool { changed := false for _, k := range keys { if _, ok := m.Tags[k]; ok { @@ -1769,7 +1790,7 @@ func (s *EKSServiceImpl) UntagResource(ctx context.Context, input *eks.UntagReso } if cluster, ng, ok := nodegroupRefFromARN(arn); ok { - if err := s.casUpdateNodegroupTags(accountID, cluster, ng, func(tags map[string]string) (map[string]string, bool) { + if err := s.casUpdateNodegroupTags(ctx, accountID, cluster, ng, func(tags map[string]string) (map[string]string, bool) { changed := false for _, k := range keys { if _, ok := tags[k]; ok { @@ -1791,11 +1812,11 @@ func (s *EKSServiceImpl) ListTagsForResource(ctx context.Context, input *eks.Lis arn := aws.StringValue(input.ResourceArn) if name, ok := clusterNameFromARN(arn); ok { - acctKV, err := s.accountBucket(accountID) + acctKV, err := s.accountBucket(ctx, accountID) if err != nil { return nil, err } - meta, err := GetClusterMeta(acctKV, name) + meta, err := GetClusterMeta(ctx, acctKV, name) if err != nil { return nil, eksTagErr(err) } @@ -1803,11 +1824,11 @@ func (s *EKSServiceImpl) ListTagsForResource(ctx context.Context, input *eks.Lis } if cluster, ng, ok := nodegroupRefFromARN(arn); ok { - acctKV, err := s.accountBucket(accountID) + acctKV, err := s.accountBucket(ctx, accountID) if err != nil { return nil, err } - rec, err := GetNodegroupRecord(acctKV, cluster, ng) + rec, err := GetNodegroupRecord(ctx, acctKV, cluster, ng) if err != nil { if errors.Is(err, ErrNodegroupNotFound) { return nil, errors.New(awserrors.ErrorEKSResourceNotFound) @@ -1823,13 +1844,13 @@ func (s *EKSServiceImpl) ListTagsForResource(ctx context.Context, input *eks.Lis // casUpdateNodegroupTags applies mutate to the nodegroup record's tag map under // compare-and-swap. mutate returns the new tag map and whether it changed; a // no-op skips the write. A missing nodegroup surfaces as ResourceNotFound. -func (s *EKSServiceImpl) casUpdateNodegroupTags(accountID, cluster, ng string, mutate func(map[string]string) (map[string]string, bool)) error { - acctKV, err := s.accountBucket(accountID) +func (s *EKSServiceImpl) casUpdateNodegroupTags(ctx context.Context, accountID, cluster, ng string, mutate func(map[string]string) (map[string]string, bool)) error { + acctKV, err := s.accountBucket(ctx, accountID) if err != nil { return err } for range ngCASMaxRetries { - rec, rev, err := getNodegroupEntry(acctKV, cluster, ng) + rec, rev, err := getNodegroupEntry(ctx, acctKV, cluster, ng) if err != nil { if errors.Is(err, ErrNodegroupNotFound) { return errors.New(awserrors.ErrorEKSResourceNotFound) @@ -1842,7 +1863,7 @@ func (s *EKSServiceImpl) casUpdateNodegroupTags(accountID, cluster, ng string, m } rec.Tags = tags rec.ModifiedAt = time.Now().UTC() - if ok, err := s.casPutNodegroup(acctKV, rec, rev); err != nil { + if ok, err := s.casPutNodegroup(ctx, acctKV, rec, rev); err != nil { return err } else if ok { return nil @@ -1852,12 +1873,12 @@ func (s *EKSServiceImpl) casUpdateNodegroupTags(accountID, cluster, ng string, m } // accountBucket returns the per-account KV bucket for accountID. -func (s *EKSServiceImpl) accountBucket(accountID string) (nats.KeyValue, error) { - js, err := s.deps.NATSConn.JetStream() +func (s *EKSServiceImpl) accountBucket(ctx context.Context, accountID string) (jetstream.KeyValue, error) { + js, err := jetstream.New(s.deps.NATSConn) if err != nil { return nil, fmt.Errorf("jetstream: %w", err) } - return GetOrCreateAccountBucket(js, accountID, max(s.deps.ClusterSize, 1)) + return GetOrCreateAccountBucket(ctx, js, accountID, max(s.deps.ClusterSize, 1)) } // clusterNameFromARN extracts the cluster name from an EKS cluster ARN @@ -2018,29 +2039,26 @@ func (s *EKSServiceImpl) DesiredDNSChanges() (changes []handlers_dns.Change, ok if s == nil || s.baseDomain == "" { return nil, false } - js, err := s.deps.NATSConn.JetStream() + js, err := jetstream.New(s.deps.NATSConn) if err != nil { return nil, false } - names, err := kvBucketNames(s.deps.NATSConn) + names, err := accountBucketNames(s.bgCtx, s.deps.NATSConn) if err != nil { slog.Warn("DesiredDNSChanges: enumerate account buckets", "err", err) return nil, false } for _, name := range names { - if !strings.HasPrefix(name, KVBucketEKSAccountPrefix) { - continue - } - acctKV, err := js.KeyValue(name) + acctKV, err := js.KeyValue(s.bgCtx, name) if err != nil { return nil, false } - clusters, err := listClusterNames(acctKV) + clusters, err := listClusterNames(s.bgCtx, acctKV) if err != nil { return nil, false } for _, cluster := range clusters { - meta, err := GetClusterMeta(acctKV, cluster) + meta, err := GetClusterMeta(s.bgCtx, acctKV, cluster) if err != nil { slog.Warn("DesiredDNSChanges: read cluster metadata", "bucket", name, "cluster", cluster, "err", err) return nil, false @@ -2057,48 +2075,8 @@ func (s *EKSServiceImpl) DesiredDNSChanges() (changes []handlers_dns.Change, ok return changes, true } -// kvBucketNames enumerates KV bucket names via a paginated $JS.API.STREAM.NAMES -// request that surfaces the terminal error, unlike js.KeyValueStoreNames() whose -// channel closes on any API/transport error and cannot signal a truncated listing. -func kvBucketNames(nc *nats.Conn) ([]string, error) { - type namesReq struct { - Offset int `json:"offset"` - Subject string `json:"subject"` - } - type namesResp struct { - Total int `json:"total"` - Streams []string `json:"streams"` - Error *nats.APIError `json:"error"` - } - var names []string - for offset := 0; ; { - body, err := json.Marshal(namesReq{Offset: offset, Subject: "$KV.*.>"}) - if err != nil { - return nil, fmt.Errorf("marshal stream-names request: %w", err) - } - msg, err := nc.Request("$JS.API.STREAM.NAMES", body, 5*time.Second) - if err != nil { - return nil, fmt.Errorf("stream-names request: %w", err) - } - var resp namesResp - if err := json.Unmarshal(msg.Data, &resp); err != nil { - return nil, fmt.Errorf("decode stream-names response: %w", err) - } - if resp.Error != nil { - return nil, fmt.Errorf("stream-names api: %w", resp.Error) - } - for _, stream := range resp.Streams { - names = append(names, strings.TrimPrefix(stream, "KV_")) - } - offset += len(resp.Streams) - if offset >= resp.Total || len(resp.Streams) == 0 { - return names, nil - } - } -} - -func (s *EKSServiceImpl) markFailed(kv nats.KeyValue, name string) { - if err := SetClusterStatus(kv, name, ClusterStatusFailed); err != nil { +func (s *EKSServiceImpl) markFailed(ctx context.Context, kv jetstream.KeyValue, name string) { + if err := SetClusterStatus(ctx, kv, name, ClusterStatusFailed); err != nil { slog.Warn("CreateCluster: SetClusterStatus(FAILED) failed", "cluster", name, "err", err) } } @@ -2106,7 +2084,7 @@ func (s *EKSServiceImpl) markFailed(kv nats.KeyValue, name string) { // spawnBootstrap launches the one-shot NATS bootstrap subscriber for a cluster. // kinds nil means a fresh CreateCluster (wait on all four subjects); a non-nil // subset is a daemon-restart resume that waits only on the missing artifacts. -func (s *EKSServiceImpl) spawnBootstrap(accountID, clusterName string, kv nats.KeyValue, kinds []string) { +func (s *EKSServiceImpl) spawnBootstrap(accountID, clusterName string, kv jetstream.KeyValue, kinds []string) { boot, err := NewNATSBootstrap(s.deps.NATSConn, kv, s.deps.MasterKey, accountID, clusterName) if err != nil { slog.Error("spawnBootstrap: NewNATSBootstrap failed", "cluster", clusterName, "err", err) @@ -2122,7 +2100,7 @@ func (s *EKSServiceImpl) spawnBootstrap(accountID, clusterName string, kv nats.K // DescribeCluster surfaces the cause and DeleteCluster can reclaim the // resources. slog.Warn("NATSBootstrap exited; marking cluster FAILED", "cluster", clusterName, "err", err) - if markErr := MarkClusterFailed(kv, clusterName, "bootstrap failed: "+err.Error()); markErr != nil && + if markErr := MarkClusterFailed(s.bgCtx, kv, clusterName, "bootstrap failed: "+err.Error()); markErr != nil && !errors.Is(markErr, ErrClusterNotFound) { slog.Error("spawnBootstrap: MarkClusterFailed", "cluster", clusterName, "err", markErr) } @@ -2130,12 +2108,12 @@ func (s *EKSServiceImpl) spawnBootstrap(accountID, clusterName string, kv nats.K } func (s *EKSServiceImpl) spawnReconciler(accountID, clusterName string, _ *ClusterMeta) { - js, err := s.deps.NATSConn.JetStream() + js, err := jetstream.New(s.deps.NATSConn) if err != nil { slog.Error("spawnReconciler: jetstream", "err", err) return } - acctKV, err := GetOrCreateAccountBucket(js, accountID, max(s.deps.ClusterSize, 1)) + acctKV, err := GetOrCreateAccountBucket(s.bgCtx, js, accountID, max(s.deps.ClusterSize, 1)) if err != nil { slog.Error("spawnReconciler: account bucket", "err", err) return @@ -2252,10 +2230,10 @@ func eksReadUnavailableOr(err error, op string) error { return fmt.Errorf("%s: %w", op, err) } -func listClusterNames(kv nats.KeyValue) ([]string, error) { - keys, err := kv.Keys() +func listClusterNames(ctx context.Context, kv jetstream.KeyValue) ([]string, error) { + keys, err := kv.Keys(ctx) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return nil, nil } return nil, fmt.Errorf("kv keys: %w", err) diff --git a/spinifex/handlers/eks/service_impl_test.go b/spinifex/handlers/eks/service_impl_test.go index 95e375c62..862544027 100644 --- a/spinifex/handlers/eks/service_impl_test.go +++ b/spinifex/handlers/eks/service_impl_test.go @@ -10,7 +10,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/awserrors" handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" "github.com/mulgadc/spinifex/spinifex/testutil" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -155,10 +155,10 @@ func TestDeleteCluster_ZeroizesOIDCKeyBeforeTeardown(t *testing.T) { _, err := f.svc.DeleteCluster(context.Background(), deleteInput("alpha"), testAccountID) require.Error(t, err, "teardown failure must surface") - _, getErr := f.kv.Get(OIDCSigningKeyKey("alpha")) - assert.ErrorIs(t, getErr, nats.ErrKeyNotFound, "OIDC key must be zeroized before teardown, even when teardown fails") + _, getErr := f.kv.Get(t.Context(), OIDCSigningKeyKey("alpha")) + assert.ErrorIs(t, getErr, jetstream.ErrKeyNotFound, "OIDC key must be zeroized before teardown, even when teardown fails") - meta, metaErr := GetClusterMeta(f.kv, "alpha") + meta, metaErr := GetClusterMeta(t.Context(), f.kv, "alpha") require.NoError(t, metaErr, "meta must survive a failed teardown for retry") assert.Equal(t, ClusterStatusDeleting, meta.Status) } @@ -193,11 +193,10 @@ func TestEKSServiceImpl_NodegroupMethodsShimMode(t *testing.T) { // bucket so the AccessEntry handlers (which gate on cluster existence) can run. func seedTestCluster(t *testing.T, svc *EKSServiceImpl, cluster string) { t.Helper() - js, err := svc.deps.NATSConn.JetStream() + js := testutil.NewJetStream(t, svc.deps.NATSConn) + kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1) require.NoError(t, err) - kv, err := GetOrCreateAccountBucket(js, testAccountID, 1) - require.NoError(t, err) - require.NoError(t, PutClusterMeta(kv, &ClusterMeta{Name: cluster, Status: ClusterStatusActive})) + require.NoError(t, PutClusterMeta(t.Context(), kv, &ClusterMeta{Name: cluster, Status: ClusterStatusActive})) } const testPrincipalARN = "arn:aws:iam::111122223333:role/dev" @@ -372,13 +371,12 @@ func TestEKSServiceImpl_OIDCMethodsReturnNotImplemented(t *testing.T) { func TestEKSServiceImpl_ClusterTagRoundTrip(t *testing.T) { svc := setupTestService(t) - js, err := svc.deps.NATSConn.JetStream() - require.NoError(t, err) - kv, err := GetOrCreateAccountBucket(js, testAccountID, 1) + js := testutil.NewJetStream(t, svc.deps.NATSConn) + kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1) require.NoError(t, err) const arn = "arn:aws:eks:us-east-1:111122223333:cluster/c1" - require.NoError(t, PutClusterMeta(kv, &ClusterMeta{ + require.NoError(t, PutClusterMeta(t.Context(), kv, &ClusterMeta{ Name: "c1", Status: ClusterStatusActive, Tags: map[string]string{"env": "prod"}, @@ -415,13 +413,12 @@ func TestEKSServiceImpl_ClusterTagRoundTrip(t *testing.T) { func TestEKSServiceImpl_NodegroupTagRoundTrip(t *testing.T) { svc := setupTestService(t) - js, err := svc.deps.NATSConn.JetStream() - require.NoError(t, err) - kv, err := GetOrCreateAccountBucket(js, testAccountID, 1) + js := testutil.NewJetStream(t, svc.deps.NATSConn) + kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1) require.NoError(t, err) const arn = "arn:aws:eks:us-east-1:111122223333:nodegroup/c1/ng1/abc123" - require.NoError(t, PutNodegroupRecord(kv, &NodegroupRecord{ + require.NoError(t, PutNodegroupRecord(t.Context(), kv, &NodegroupRecord{ ClusterName: "c1", Name: "ng1", Arn: arn, diff --git a/spinifex/handlers/eks/spawn_registered_reconcilers_test.go b/spinifex/handlers/eks/spawn_registered_reconcilers_test.go index add274292..813b906b8 100644 --- a/spinifex/handlers/eks/spawn_registered_reconcilers_test.go +++ b/spinifex/handlers/eks/spawn_registered_reconcilers_test.go @@ -17,7 +17,7 @@ func TestSpawnRegisteredReconcilers_ResumesNonTerminal(t *testing.T) { active := sampleClusterMeta("alpha") active.Status = ClusterStatusActive - require.NoError(t, PutClusterMeta(f.kv, active)) + require.NoError(t, PutClusterMeta(t.Context(), f.kv, active)) // CREATING resumes both the bootstrap re-subscribe and the reconciler. Stamp a // recent CreatedAt so the resumed reconciler stays CREATING (the shared fixture @@ -26,11 +26,11 @@ func TestSpawnRegisteredReconcilers_ResumesNonTerminal(t *testing.T) { creating := sampleClusterMeta("beta") creating.Status = ClusterStatusCreating creating.CreatedAt = time.Now() - require.NoError(t, PutClusterMeta(f.kv, creating)) + require.NoError(t, PutClusterMeta(t.Context(), f.kv, creating)) failed := sampleClusterMeta("zeta") failed.Status = ClusterStatusFailed - require.NoError(t, PutClusterMeta(f.kv, failed)) + require.NoError(t, PutClusterMeta(t.Context(), f.kv, failed)) require.NoError(t, f.svc.SpawnRegisteredReconcilers()) @@ -39,6 +39,25 @@ func TestSpawnRegisteredReconcilers_ResumesNonTerminal(t *testing.T) { assert.False(t, f.svc.registry.Has(testAccountID, "zeta"), "FAILED cluster must not be resumed") } +// TestSpawnRegisteredReconcilers_EnumerationFailureIsReported: the KV bucket +// lister closes its channel the same way whether the listing completed or +// failed, so a boot scan that ignored the terminal error would read an +// unreachable JetStream as "no accounts" and leave every cluster in the fleet +// without a reconciler until the next daemon restart — with clean boot logs. +func TestSpawnRegisteredReconcilers_EnumerationFailureIsReported(t *testing.T) { + f := newEKSServiceFixture(t) + + active := sampleClusterMeta("alpha") + active.Status = ClusterStatusActive + require.NoError(t, PutClusterMeta(t.Context(), f.kv, active)) + + // Closing the connection fails the stream-names request behind the listing. + f.svc.deps.NATSConn.Close() + + require.Error(t, f.svc.SpawnRegisteredReconcilers(), "a failed bucket enumeration must not report a completed scan") + assert.False(t, f.svc.registry.Has(testAccountID, "alpha"), "a listing that never completed can resume nothing") +} + // TestSpawnRegisteredReconcilers_DepsNotReadyNoops confirms the boot scan is a // safe no-op when orchestration deps are absent (the shim construction path), // so it never panics or registers phantom holders on a half-wired daemon. diff --git a/spinifex/handlers/eks/store.go b/spinifex/handlers/eks/store.go index 292b5c302..a3588bdbf 100644 --- a/spinifex/handlers/eks/store.go +++ b/spinifex/handlers/eks/store.go @@ -1,16 +1,19 @@ package handlers_eks import ( + "context" "crypto/sha256" "encoding/hex" "errors" "fmt" "log/slog" + "strings" "time" + "github.com/mulgadc/spinifex/spinifex/kvutil" "github.com/mulgadc/spinifex/spinifex/migrate" - "github.com/mulgadc/spinifex/spinifex/utils" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // JetStream KV bucket and key-path constants for the EKS control plane. @@ -180,17 +183,41 @@ func AccountBucketName(accountID string) string { return KVBucketEKSAccountPrefix + accountID } +// accountBucketNames returns the name of every EKS per-account KV bucket. It +// fails rather than returning a short list when the enumeration could not be +// completed: the lister behind it closes its channel identically on success and +// on error, so a caller that ignored the failure would read an unreachable +// JetStream as "no accounts" — and prune every tenant's endpoint record on that +// empty view. +func accountBucketNames(ctx context.Context, nc *nats.Conn) ([]string, error) { + js, err := jetstream.New(nc) + if err != nil { + return nil, fmt.Errorf("jetstream: %w", err) + } + all, err := kvutil.BucketNames(ctx, js) + if err != nil { + return nil, err + } + names := make([]string, 0, len(all)) + for _, name := range all { + if strings.HasPrefix(name, KVBucketEKSAccountPrefix) { + names = append(names, name) + } + } + return names, nil +} + // GetOrCreateAccountBucket returns the per-account KV bucket for accountID, // creating it on first use at the given replica count (clamped to a minimum // of 1). Idempotent: subsequent calls with the same accountID return the // existing handle. -func GetOrCreateAccountBucket(js nats.JetStreamContext, accountID string, replicas int) (nats.KeyValue, error) { +func GetOrCreateAccountBucket(ctx context.Context, js jetstream.JetStream, accountID string, replicas int) (jetstream.KeyValue, error) { bucket := AccountBucketName(accountID) - kv, err := utils.GetOrCreateKVBucketWithReplicas(js, bucket, KVBucketEKSAccountHistory, replicas) + kv, err := kvutil.GetOrCreateBucketWithReplicas(ctx, js, bucket, KVBucketEKSAccountHistory, replicas) if err != nil { return nil, fmt.Errorf("failed to create EKS per-account KV bucket %s: %w", bucket, err) } - if err := migrate.DefaultRegistry.RunKV(bucket, kv, KVBucketEKSAccountVersion); err != nil { + if err := migrate.DefaultRegistry.RunKV(ctx, bucket, kv, KVBucketEKSAccountVersion); err != nil { return nil, fmt.Errorf("migrate %s: %w", bucket, err) } return kv, nil @@ -200,23 +227,23 @@ func GetOrCreateAccountBucket(js nats.JetStreamContext, accountID string, replic // bucket used for per-cluster reconciler leader-lease CAS locks, at the given // replica count (clamped to a minimum of 1). The bucket is configured with // History=1 and a 60s TTL so stale leases expire on their own when a leader -// dies mid-cycle. utils.GetOrCreateKVBucketWithReplicas doesn't expose a TTL +// dies mid-cycle. kvutil.GetOrCreateBucketWithReplicas doesn't expose a TTL // knob, so this function sets Replicas directly on its own js.CreateKeyValue // call and falls back to js.KeyValue on already-exists. -func InitLeaderBucket(js nats.JetStreamContext, replicas int) (nats.KeyValue, error) { - kv, err := js.CreateKeyValue(&nats.KeyValueConfig{ +func InitLeaderBucket(ctx context.Context, js jetstream.JetStream, replicas int) (jetstream.KeyValue, error) { + kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{ Bucket: KVBucketEKSLeader, History: 1, TTL: KVBucketEKSLeaderTTL, Replicas: max(replicas, 1), }) if err != nil { - kv, err = js.KeyValue(KVBucketEKSLeader) + kv, err = js.KeyValue(ctx, KVBucketEKSLeader) if err != nil { return nil, fmt.Errorf("failed to create or open EKS leader bucket %s: %w", KVBucketEKSLeader, err) } } - if err := migrate.DefaultRegistry.RunKV(KVBucketEKSLeader, kv, KVBucketEKSLeaderVersion); err != nil { + if err := migrate.DefaultRegistry.RunKV(ctx, KVBucketEKSLeader, kv, KVBucketEKSLeaderVersion); err != nil { return nil, fmt.Errorf("migrate %s: %w", KVBucketEKSLeader, err) } slog.Info("EKS leader bucket initialized", "bucket", KVBucketEKSLeader, "ttl", KVBucketEKSLeaderTTL) diff --git a/spinifex/handlers/eks/store_test.go b/spinifex/handlers/eks/store_test.go index 26eefa652..e7b57fbf2 100644 --- a/spinifex/handlers/eks/store_test.go +++ b/spinifex/handlers/eks/store_test.go @@ -11,13 +11,14 @@ import ( const testAccountID = "111122223333" func TestGetOrCreateAccountBucket_Idempotent(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) - kv1, err := GetOrCreateAccountBucket(js, testAccountID, 1) + kv1, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1) require.NoError(t, err) require.NotNil(t, kv1) - kv2, err := GetOrCreateAccountBucket(js, testAccountID, 1) + kv2, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1) require.NoError(t, err) require.NotNil(t, kv2) @@ -32,26 +33,30 @@ func TestGetOrCreateAccountBucket_Idempotent(t *testing.T) { // path (replicas <= 0 -> 1) is exercisable end-to-end here; multi-node // replica counts are exercised live (see the associated bug doc). func TestGetOrCreateAccountBucket_ReplicasClamped(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) - kv, err := GetOrCreateAccountBucket(js, testAccountID, 0) + kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 0) require.NoError(t, err) require.NotNil(t, kv) - si, err := js.StreamInfo("KV_" + AccountBucketName(testAccountID)) + stream, err := js.Stream(t.Context(), "KV_"+AccountBucketName(testAccountID)) + require.NoError(t, err) + si, err := stream.Info(t.Context()) require.NoError(t, err) assert.Equal(t, 1, si.Config.Replicas) } func TestInitLeaderBucket_Idempotent(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) - kv1, err := InitLeaderBucket(js, 1) + kv1, err := InitLeaderBucket(t.Context(), js, 1) require.NoError(t, err) require.NotNil(t, kv1) assert.Equal(t, KVBucketEKSLeader, kv1.Bucket()) - kv2, err := InitLeaderBucket(js, 1) + kv2, err := InitLeaderBucket(t.Context(), js, 1) require.NoError(t, err) require.NotNil(t, kv2) assert.Equal(t, KVBucketEKSLeader, kv2.Bucket()) @@ -60,13 +65,16 @@ func TestInitLeaderBucket_Idempotent(t *testing.T) { // TestInitLeaderBucket_ReplicasClamped confirms the leader bucket is created // with the requested replica count, clamped to a minimum of 1. func TestInitLeaderBucket_ReplicasClamped(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) - kv, err := InitLeaderBucket(js, -1) + kv, err := InitLeaderBucket(t.Context(), js, -1) require.NoError(t, err) require.NotNil(t, kv) - si, err := js.StreamInfo("KV_" + KVBucketEKSLeader) + stream, err := js.Stream(t.Context(), "KV_"+KVBucketEKSLeader) + require.NoError(t, err) + si, err := stream.Info(t.Context()) require.NoError(t, err) assert.Equal(t, 1, si.Config.Replicas) } diff --git a/spinifex/handlers/eks/token_review.go b/spinifex/handlers/eks/token_review.go index c292ba8af..e56489e4b 100644 --- a/spinifex/handlers/eks/token_review.go +++ b/spinifex/handlers/eks/token_review.go @@ -8,6 +8,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/utils" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // WebhookTokenReviewResult is the gateway's reply to the eks-token-webhook's @@ -71,11 +72,11 @@ func ResolveTokenReview(ctx context.Context, nc *nats.Conn, accountID, clusterNa if nc == nil { return WebhookTokenReviewResult{}, fmt.Errorf("eks: ResolveTokenReview nil nats conn") } - js, err := nc.JetStream() + js, err := jetstream.New(nc) if err != nil { return WebhookTokenReviewResult{}, fmt.Errorf("eks: jetstream: %w", err) } - kv, err := js.KeyValue(AccountBucketName(accountID)) + kv, err := js.KeyValue(ctx, AccountBucketName(accountID)) if err != nil { return WebhookTokenReviewResult{}, fmt.Errorf("eks: open account KV %s: %w", accountID, err) } @@ -87,7 +88,7 @@ func ResolveTokenReview(ctx context.Context, nc *nats.Conn, accountID, clusterNa verifyTimeout, "") } lookup := func(principalARN string) (*AccessEntryRecord, error) { - return GetAccessEntryRecord(kv, clusterName, principalARN) + return GetAccessEntryRecord(ctx, kv, clusterName, principalARN) } return Authenticate(token, verify, lookup), nil } diff --git a/spinifex/handlers/eks/token_review_test.go b/spinifex/handlers/eks/token_review_test.go index e1c608de2..c3a562cac 100644 --- a/spinifex/handlers/eks/token_review_test.go +++ b/spinifex/handlers/eks/token_review_test.go @@ -10,6 +10,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/testutil" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -20,6 +21,19 @@ func validToken(url string) string { const testARN = "arn:aws:iam::111122223333:role/admin" +// seedAccountBucket creates an empty per-account bucket on the jetstream API. +// testutil.SeedKV stays on the legacy handle, which cannot be passed to the +// migrated AccessEntry helpers. +func seedAccountBucket(t *testing.T, js jetstream.JetStream, accountID string) jetstream.KeyValue { + t.Helper() + kv, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{ + Bucket: AccountBucketName(accountID), + History: KVBucketEKSAccountHistory, + }) + require.NoError(t, err) + return kv +} + func okVerify(_ string) (*TokenVerifyResponse, error) { return &TokenVerifyResponse{ AccountID: "111122223333", @@ -107,9 +121,10 @@ func TestResolveTokenReview_MissingBucketErrors(t *testing.T) { } func TestResolveTokenReview_AuthenticatesViaVerifyAndKV(t *testing.T) { - _, nc, js := testutil.StartTestJetStream(t) - kv := testutil.SeedKV(t, js, AccountBucketName("111122223333"), nil) - require.NoError(t, PutAccessEntryRecord(kv, &AccessEntryRecord{ + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + kv := seedAccountBucket(t, js, "111122223333") + require.NoError(t, PutAccessEntryRecord(t.Context(), kv, &AccessEntryRecord{ ClusterName: "alpha", PrincipalARN: testARN, KubernetesUsername: testARN, @@ -141,8 +156,9 @@ func TestResolveTokenReview_AuthenticatesViaVerifyAndKV(t *testing.T) { // A valid IAM principal with no AccessEntry resolves to Authenticated=false // (a clean 401), not an error. func TestResolveTokenReview_NoAccessEntryDenies(t *testing.T) { - _, nc, js := testutil.StartTestJetStream(t) - testutil.SeedKV(t, js, AccountBucketName("111122223333"), nil) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + seedAccountBucket(t, js, "111122223333") sub, err := nc.Subscribe(TokenVerifySubject, func(m *nats.Msg) { resp, _ := json.Marshal(TokenVerifyResponse{AccountID: "111122223333", ARN: testARN}) diff --git a/spinifex/handlers/elbv2/arn_test.go b/spinifex/handlers/elbv2/arn_test.go index 53cd78a2f..3b8fdabc8 100644 --- a/spinifex/handlers/elbv2/arn_test.go +++ b/spinifex/handlers/elbv2/arn_test.go @@ -124,7 +124,7 @@ func setupMicrovmTestService(t *testing.T) (*ELBv2ServiceImpl, *handlers_ec2_vpc _, nc, _ := testutil.StartTestJetStream(t) testutil.StubVpcdSGResponder(t, nc) - vpcSvc, err := handlers_ec2_vpc.NewVPCServiceImplWithNATS(nil, nc) + vpcSvc, err := handlers_ec2_vpc.NewVPCServiceImplWithNATS(t.Context(), nil, nc) require.NoError(t, err) cfg := &config.Config{ diff --git a/spinifex/handlers/elbv2/dns_register_test.go b/spinifex/handlers/elbv2/dns_register_test.go index 81daa1e2c..17ba85395 100644 --- a/spinifex/handlers/elbv2/dns_register_test.go +++ b/spinifex/handlers/elbv2/dns_register_test.go @@ -55,7 +55,7 @@ func TestDesiredDNSChanges_IncludesEndpointReadyProvisioningLoadBalancers(t *tes provisioning.State = StateProvisioning provisioning.DNSName = "provisioning-123.ap-southeast-2.elb.spx3.net" provisioning.VPCIP = "10.0.0.10" - require.NoError(t, svc.store.PutLoadBalancer(provisioning)) + require.NoError(t, svc.store.PutLoadBalancer(t.Context(), provisioning)) provisioningPublic := newTestLB("public12345678", "public") provisioningPublic.State = StateProvisioning @@ -63,19 +63,19 @@ func TestDesiredDNSChanges_IncludesEndpointReadyProvisioningLoadBalancers(t *tes provisioningPublic.DNSName = "public-123.ap-southeast-2.elb.spx3.net" provisioningPublic.VPCIP = "10.0.0.11" provisioningPublic.AvailZones = []AvailZoneInfo{{PublicIP: "203.0.113.11"}} - require.NoError(t, svc.store.PutLoadBalancer(provisioningPublic)) + require.NoError(t, svc.store.PutLoadBalancer(t.Context(), provisioningPublic)) active := newTestLB("active12345678", "active") active.State = StateActive active.DNSName = "active-123.ap-southeast-2.elb.spx3.net" active.VPCIP = "10.0.0.12" - require.NoError(t, svc.store.PutLoadBalancer(active)) + require.NoError(t, svc.store.PutLoadBalancer(t.Context(), active)) failed := newTestLB("failed12345678", "failed") failed.State = StateFailed failed.DNSName = "failed-123.ap-southeast-2.elb.spx3.net" failed.VPCIP = "10.0.0.13" - require.NoError(t, svc.store.PutLoadBalancer(failed)) + require.NoError(t, svc.store.PutLoadBalancer(t.Context(), failed)) changes, authoritative := svc.DesiredDNSChanges() require.True(t, authoritative) diff --git a/spinifex/handlers/elbv2/haproxy_certs_test.go b/spinifex/handlers/elbv2/haproxy_certs_test.go index 48a7a788b..43c502526 100644 --- a/spinifex/handlers/elbv2/haproxy_certs_test.go +++ b/spinifex/handlers/elbv2/haproxy_certs_test.go @@ -93,7 +93,7 @@ func TestConfigCertHash_ChangesOnRotation(t *testing.T) { func putTestCert(t *testing.T, svc *ELBv2ServiceImpl, arn, account, leaf, chain, key string) { t.Helper() - require.NoError(t, svc.acmStore.PutCert(&handlers_acm.CertRecord{ + require.NoError(t, svc.acmStore.PutCert(t.Context(), &handlers_acm.CertRecord{ CertificateArn: arn, AccountID: account, Certificate: leaf, @@ -108,16 +108,16 @@ func TestResolveCertPEM_ConcatOrderAndOwnership(t *testing.T) { require.NotNil(t, svc.acmStore) putTestCert(t, svc, certTestArn, testAccountID, "LEAF", "CHAIN", "KEY") - pem, err := svc.resolveCertPEM(certTestArn, testAccountID) + pem, err := svc.resolveCertPEM(t.Context(), certTestArn, testAccountID) require.NoError(t, err) assert.Equal(t, "LEAF\nCHAIN\nKEY\n", pem) // Wrong account → treated as not found. - _, err = svc.resolveCertPEM(certTestArn, "999999999999") + _, err = svc.resolveCertPEM(t.Context(), certTestArn, "999999999999") require.Error(t, err) // Unknown ARN → not found. - _, err = svc.resolveCertPEM("arn:aws:acm:ap-southeast-2:123456789012:certificate/missing", testAccountID) + _, err = svc.resolveCertPEM(t.Context(), "arn:aws:acm:ap-southeast-2:123456789012:certificate/missing", testAccountID) require.Error(t, err) } @@ -126,22 +126,22 @@ func TestValidateListenerCerts_RejectsUnknownAndWrongAccount(t *testing.T) { putTestCert(t, svc, certTestArn, testAccountID, "LEAF", "", "KEY") // Known, owned cert → accepted. - require.NoError(t, svc.validateListenerCerts( + require.NoError(t, svc.validateListenerCerts(t.Context(), []ListenerCertificate{{CertificateArn: certTestArn}}, testAccountID)) // Unknown ARN → CertificateNotFound, rejected at the API boundary. - err := svc.validateListenerCerts( + err := svc.validateListenerCerts(t.Context(), []ListenerCertificate{{CertificateArn: "arn:aws:acm:ap-southeast-2:123456789012:certificate/missing"}}, testAccountID) require.EqualError(t, err, awserrors.ErrorELBv2CertificateNotFound) // Cert owned by another account → not found. - err = svc.validateListenerCerts( + err = svc.validateListenerCerts(t.Context(), []ListenerCertificate{{CertificateArn: certTestArn}}, "999999999999") require.EqualError(t, err, awserrors.ErrorELBv2CertificateNotFound) // No ACM store (degraded) → validation skipped, never blocks. svc.acmStore = nil - require.NoError(t, svc.validateListenerCerts( + require.NoError(t, svc.validateListenerCerts(t.Context(), []ListenerCertificate{{CertificateArn: certTestArn}}, testAccountID)) } @@ -149,7 +149,7 @@ func TestResolveCertPEM_NoChain(t *testing.T) { svc := setupTestService(t) putTestCert(t, svc, certTestArn, testAccountID, "LEAF", "", "KEY") - pem, err := svc.resolveCertPEM(certTestArn, testAccountID) + pem, err := svc.resolveCertPEM(t.Context(), certTestArn, testAccountID) require.NoError(t, err) assert.Equal(t, "LEAF\nKEY\n", pem) } @@ -159,13 +159,13 @@ func TestResolveListenerCerts_DedupAndResolve(t *testing.T) { putTestCert(t, svc, certTestArn, testAccountID, "LEAF", "", "KEY") // Two listeners share the same cert ARN — resolved once. - got, err := svc.resolveListenerCerts([]*ListenerRecord{httpsListener(), httpsListener()}, testAccountID) + got, err := svc.resolveListenerCerts(t.Context(), []*ListenerRecord{httpsListener(), httpsListener()}, testAccountID) require.NoError(t, err) require.Len(t, got, 1) assert.Equal(t, "LEAF\nKEY\n", got[certTestArn]) // HTTP-only listeners → nil map, no error. - got, err = svc.resolveListenerCerts([]*ListenerRecord{{Protocol: ProtocolHTTP}}, testAccountID) + got, err = svc.resolveListenerCerts(t.Context(), []*ListenerRecord{{Protocol: ProtocolHTTP}}, testAccountID) require.NoError(t, err) assert.Nil(t, got) } @@ -179,7 +179,7 @@ func TestGetLBConfig_DeliversCertFiles(t *testing.T) { ConfigHash: "hash123", CertFiles: map[string]string{"/etc/haproxy/certs/lb-deliver1-lst.pem": "LEAF\nKEY\n"}, } - require.NoError(t, svc.store.PutLoadBalancer(rec)) + require.NoError(t, svc.store.PutLoadBalancer(t.Context(), rec)) out, err := svc.GetLBConfig(context.Background(), &GetLBConfigInput{LBID: aws.String("lb-deliver1")}, testAccountID) require.NoError(t, err) diff --git a/spinifex/handlers/elbv2/health_checker.go b/spinifex/handlers/elbv2/health_checker.go index 05c3b126b..fbc683b43 100644 --- a/spinifex/handlers/elbv2/health_checker.go +++ b/spinifex/handlers/elbv2/health_checker.go @@ -47,9 +47,9 @@ func (hc *healthChecker) handleHealthReportDirect(ctx context.Context, report lb var tgs []*TargetGroupRecord var err error if report.LBID != "" { - tgs, err = hc.store.TargetGroupsForLB(report.LBID) + tgs, err = hc.store.TargetGroupsForLB(ctx, report.LBID) } else { - tgs, err = hc.store.ListTargetGroups() + tgs, err = hc.store.ListTargetGroups(ctx) } if err != nil { slog.WarnContext(ctx, "healthChecker: failed to list target groups", "lbId", report.LBID, "err", err) @@ -115,7 +115,7 @@ func (hc *healthChecker) handleHealthReportDirect(ctx context.Context, report lb hc.mu.Unlock() for _, tg := range changedTGs { - if err := hc.store.PutTargetGroup(tg); err != nil { + if err := hc.store.PutTargetGroup(ctx, tg); err != nil { slog.ErrorContext(ctx, "healthChecker: failed to persist target group", "tgId", tg.TargetGroupID, "err", err) } } diff --git a/spinifex/handlers/elbv2/health_checker_test.go b/spinifex/handlers/elbv2/health_checker_test.go index f421a2492..d191ed6f7 100644 --- a/spinifex/handlers/elbv2/health_checker_test.go +++ b/spinifex/handlers/elbv2/health_checker_test.go @@ -82,7 +82,7 @@ func TestEvaluateHealth_DrainingUnchanged(t *testing.T) { func setupTestNATS(t *testing.T) *Store { t.Helper() _, nc, _ := testutil.StartTestJetStream(t) - store, err := NewStore(nc) + store, err := NewStore(t.Context(), nc) require.NoError(t, err) return store } @@ -92,13 +92,13 @@ func setupTestNATS(t *testing.T) *Store { func setupLBWithTG(t *testing.T, store *Store, lbID string, tg *TargetGroupRecord) { t.Helper() lbArn := "arn:aws:elasticloadbalancing:us-east-1:000:loadbalancer/app/test/" + lbID - require.NoError(t, store.PutLoadBalancer(&LoadBalancerRecord{ + require.NoError(t, store.PutLoadBalancer(t.Context(), &LoadBalancerRecord{ LoadBalancerArn: lbArn, LoadBalancerID: lbID, Name: "test-lb", State: StateActive, })) - require.NoError(t, store.PutListener(&ListenerRecord{ + require.NoError(t, store.PutListener(t.Context(), &ListenerRecord{ ListenerArn: lbArn + "/listener-1", ListenerID: lbID + "-lis", LoadBalancerArn: lbArn, @@ -124,7 +124,7 @@ func TestHandleHealthReport_TransitionsInitialToHealthy(t *testing.T) { {Id: "i-aaa111", Port: 80, HealthState: TargetHealthInitial, PrivateIP: "10.0.1.10"}, }, } - require.NoError(t, store.PutTargetGroup(tg)) + require.NoError(t, store.PutTargetGroup(t.Context(), tg)) setupLBWithTG(t, store, "lb-test1", tg) report := lbagent.HealthReport{ @@ -135,7 +135,7 @@ func TestHandleHealthReport_TransitionsInitialToHealthy(t *testing.T) { } hc.handleHealthReportDirect(context.Background(), report) - stored, err := store.GetTargetGroup("tg-123") + stored, err := store.GetTargetGroup(t.Context(), "tg-123") require.NoError(t, err) assert.Equal(t, TargetHealthHealthy, stored.Targets[0].HealthState) } @@ -157,7 +157,7 @@ func TestHandleHealthReport_UnhealthyAfterThreshold(t *testing.T) { {Id: "i-bbb222", Port: 80, HealthState: TargetHealthInitial, PrivateIP: "10.0.1.11"}, }, } - require.NoError(t, store.PutTargetGroup(tg)) + require.NoError(t, store.PutTargetGroup(t.Context(), tg)) setupLBWithTG(t, store, "lb-test2", tg) srvName := sanitizeName("srv", "i-bbb222") @@ -173,7 +173,7 @@ func TestHandleHealthReport_UnhealthyAfterThreshold(t *testing.T) { hc.handleHealthReportDirect(context.Background(), report) } - stored, err := store.GetTargetGroup("tg-456") + stored, err := store.GetTargetGroup(t.Context(), "tg-456") require.NoError(t, err) assert.Equal(t, TargetHealthUnhealthy, stored.Targets[0].HealthState) } @@ -192,7 +192,7 @@ func TestHandleHealthReport_SkipsDrainingTargets(t *testing.T) { {Id: "i-drain", Port: 80, HealthState: TargetHealthDraining, PrivateIP: "10.0.0.1"}, }, } - require.NoError(t, store.PutTargetGroup(tg)) + require.NoError(t, store.PutTargetGroup(t.Context(), tg)) setupLBWithTG(t, store, "lb-test3", tg) report := lbagent.HealthReport{ @@ -203,7 +203,7 @@ func TestHandleHealthReport_SkipsDrainingTargets(t *testing.T) { } hc.handleHealthReportDirect(context.Background(), report) - stored, err := store.GetTargetGroup("tg-789") + stored, err := store.GetTargetGroup(t.Context(), "tg-789") require.NoError(t, err) assert.Equal(t, TargetHealthDraining, stored.Targets[0].HealthState) } @@ -245,7 +245,7 @@ func TestHandleHealthReport_TargetPortZeroUsesTGPort(t *testing.T) { {Id: "i-port0", Port: 0, HealthState: TargetHealthInitial, PrivateIP: "10.0.0.50"}, }, } - require.NoError(t, store.PutTargetGroup(tg)) + require.NoError(t, store.PutTargetGroup(t.Context(), tg)) setupLBWithTG(t, store, "lb-p0", tg) report := lbagent.HealthReport{ @@ -256,7 +256,7 @@ func TestHandleHealthReport_TargetPortZeroUsesTGPort(t *testing.T) { } hc.handleHealthReportDirect(context.Background(), report) - stored, err := store.GetTargetGroup("tg-p0") + stored, err := store.GetTargetGroup(t.Context(), "tg-p0") require.NoError(t, err) assert.Equal(t, TargetHealthHealthy, stored.Targets[0].HealthState) @@ -280,7 +280,7 @@ func TestHandleHealthReportDirect_TransitionsInitialToHealthy(t *testing.T) { {Id: "i-direct1", Port: 80, HealthState: TargetHealthInitial, PrivateIP: "10.0.1.20"}, }, } - require.NoError(t, store.PutTargetGroup(tg)) + require.NoError(t, store.PutTargetGroup(t.Context(), tg)) setupLBWithTG(t, store, "lb-direct", tg) // Call handleHealthReportDirect with a struct — no JSON round-trip. @@ -291,7 +291,7 @@ func TestHandleHealthReportDirect_TransitionsInitialToHealthy(t *testing.T) { }, }) - stored, err := store.GetTargetGroup("tg-direct") + stored, err := store.GetTargetGroup(t.Context(), "tg-direct") require.NoError(t, err) assert.Equal(t, TargetHealthHealthy, stored.Targets[0].HealthState) } @@ -318,7 +318,7 @@ func TestHandleHealthReport_OnlyProcessesTGsForReportingLB(t *testing.T) { {Id: "i-shared", Port: 80, HealthState: TargetHealthInitial, PrivateIP: "10.0.0.1"}, }, } - require.NoError(t, store.PutTargetGroup(tgA)) + require.NoError(t, store.PutTargetGroup(t.Context(), tgA)) setupLBWithTG(t, store, "lb-A", tgA) // TG attached to lb-B — same target ID, different TG @@ -331,7 +331,7 @@ func TestHandleHealthReport_OnlyProcessesTGsForReportingLB(t *testing.T) { {Id: "i-shared", Port: 80, HealthState: TargetHealthInitial, PrivateIP: "10.0.0.2"}, }, } - require.NoError(t, store.PutTargetGroup(tgB)) + require.NoError(t, store.PutTargetGroup(t.Context(), tgB)) setupLBWithTG(t, store, "lb-B", tgB) // Report from lb-A — only tg-a should be updated. @@ -342,11 +342,11 @@ func TestHandleHealthReport_OnlyProcessesTGsForReportingLB(t *testing.T) { }, }) - storedA, err := store.GetTargetGroup("tg-a") + storedA, err := store.GetTargetGroup(t.Context(), "tg-a") require.NoError(t, err) assert.Equal(t, TargetHealthHealthy, storedA.Targets[0].HealthState, "tg-a should be updated") - storedB, err := store.GetTargetGroup("tg-b") + storedB, err := store.GetTargetGroup(t.Context(), "tg-b") require.NoError(t, err) assert.Equal(t, TargetHealthInitial, storedB.Targets[0].HealthState, "tg-b must NOT be updated by lb-A's report") } @@ -366,7 +366,7 @@ func TestHandleHealthReport_FallbackListTargetGroups(t *testing.T) { {Id: "i-fallback", Port: 80, HealthState: TargetHealthInitial, PrivateIP: "10.0.0.99"}, }, } - require.NoError(t, store.PutTargetGroup(tg)) + require.NoError(t, store.PutTargetGroup(t.Context(), tg)) // Report with empty LBID → should fall back to ListTargetGroups hc.handleHealthReportDirect(context.Background(), lbagent.HealthReport{ @@ -377,7 +377,7 @@ func TestHandleHealthReport_FallbackListTargetGroups(t *testing.T) { }, }) - stored, err := store.GetTargetGroup("tg-fb") + stored, err := store.GetTargetGroup(t.Context(), "tg-fb") require.NoError(t, err) assert.Equal(t, TargetHealthHealthy, stored.Targets[0].HealthState) } diff --git a/spinifex/handlers/elbv2/lifecycle_invariants_test.go b/spinifex/handlers/elbv2/lifecycle_invariants_test.go index 5926370a2..cb144b558 100644 --- a/spinifex/handlers/elbv2/lifecycle_invariants_test.go +++ b/spinifex/handlers/elbv2/lifecycle_invariants_test.go @@ -90,11 +90,11 @@ func TestRLC2_ELBv2NoOrphanAfterDeleteLB(t *testing.T) { _, err = svc.DeleteLoadBalancer(context.Background(), &elbv2.DeleteLoadBalancerInput{LoadBalancerArn: lbArn}, testAccountID) require.NoError(t, err) - listeners, err := svc.store.ListListenersByLB(*lbArn) + listeners, err := svc.store.ListListenersByLB(t.Context(), *lbArn) require.NoError(t, err) require.Emptyf(t, listeners, "ADR-0002 §5 no-orphan completeness: no listener owned by a deleted LB may remain") - rules, err := svc.store.ListRules() + rules, err := svc.store.ListRules(t.Context()) require.NoError(t, err) require.Emptyf(t, rules, "ADR-0002 §5 no-orphan completeness: no rule owned by a deleted LB may remain") } @@ -194,7 +194,7 @@ func TestRLC3_ELBv2TGInUseGuardGatesOnLiveRefsOnly(t *testing.T) { danglingListenerArn := "arn:aws:elasticloadbalancing:us-east-1:" + testAccountID + ":listener/app/gone/0000000000000000/1111111111111111" - require.NoError(t, svc.store.PutRule(&RuleRecord{ + require.NoError(t, svc.store.PutRule(t.Context(), &RuleRecord{ RuleArn: "arn:aws:elasticloadbalancing:us-east-1:" + testAccountID + ":listener-rule/app/gone/0000000000000000/1111111111111111/2222222222222222", RuleID: "rule-orphan00000000", ListenerArn: danglingListenerArn, diff --git a/spinifex/handlers/elbv2/name_claim_test.go b/spinifex/handlers/elbv2/name_claim_test.go index 7ce6899fb..1da346e01 100644 --- a/spinifex/handlers/elbv2/name_claim_test.go +++ b/spinifex/handlers/elbv2/name_claim_test.go @@ -45,7 +45,7 @@ func TestCreateLoadBalancer_ConcurrentSameNameSingleOwner(t *testing.T) { assert.Equal(t, 1, ok, "exactly one create owns the name") assert.Equal(t, 1, dup, "the duplicate is rejected DuplicateLoadBalancerName") - rec, err := svc.store.GetLoadBalancerByName("race-lb", testAccountID) + rec, err := svc.store.GetLoadBalancerByName(t.Context(), "race-lb", testAccountID) require.NoError(t, err) require.NotNil(t, rec) } @@ -58,7 +58,7 @@ func TestCreateLoadBalancer_ReclaimsCrashOrphanNameClaim(t *testing.T) { // Seed a name claim pointing at a non-existent LB (crashed create left no // record). ClaimLBName writes the key with this bogus owner. - ok, dup, err := svc.store.ClaimLBName("orphan-lb", testAccountID, "lb-doesnotexist") + ok, dup, err := svc.store.ClaimLBName(t.Context(), "orphan-lb", testAccountID, "lb-doesnotexist") require.NoError(t, err) require.True(t, ok) require.False(t, dup) diff --git a/spinifex/handlers/elbv2/reaper_test.go b/spinifex/handlers/elbv2/reaper_test.go index 1304ce527..0f3d65039 100644 --- a/spinifex/handlers/elbv2/reaper_test.go +++ b/spinifex/handlers/elbv2/reaper_test.go @@ -27,33 +27,33 @@ func TestReapStuckProvisioningLBs(t *testing.T) { now := time.Now().UTC() stuck := testLBRecord("stuck0000000000", StateProvisioning, now.Add(-10*time.Minute), time.Time{}) - require.NoError(t, svc.store.PutLoadBalancer(stuck)) + require.NoError(t, svc.store.PutLoadBalancer(t.Context(), stuck)) fresh := testLBRecord("fresh0000000000", StateProvisioning, now, time.Time{}) - require.NoError(t, svc.store.PutLoadBalancer(fresh)) + require.NoError(t, svc.store.PutLoadBalancer(t.Context(), fresh)) activeOld := testLBRecord("active000000000", StateActive, now.Add(-10*time.Minute), time.Time{}) - require.NoError(t, svc.store.PutLoadBalancer(activeOld)) + require.NoError(t, svc.store.PutLoadBalancer(t.Context(), activeOld)) heartbeating := testLBRecord("heartbeat0000000", StateProvisioning, now.Add(-10*time.Minute), now.Add(-time.Second)) - require.NoError(t, svc.store.PutLoadBalancer(heartbeating)) + require.NoError(t, svc.store.PutLoadBalancer(t.Context(), heartbeating)) - svc.reapStuckProvisioningLBs(now) + svc.reapStuckProvisioningLBs(t.Context(), now) - got, err := svc.store.GetLoadBalancer(stuck.LoadBalancerID) + got, err := svc.store.GetLoadBalancer(t.Context(), stuck.LoadBalancerID) require.NoError(t, err) assert.Equal(t, StateFailed, got.State) assert.NotEmpty(t, got.StateReason) - got, err = svc.store.GetLoadBalancer(fresh.LoadBalancerID) + got, err = svc.store.GetLoadBalancer(t.Context(), fresh.LoadBalancerID) require.NoError(t, err) assert.Equal(t, StateProvisioning, got.State) - got, err = svc.store.GetLoadBalancer(activeOld.LoadBalancerID) + got, err = svc.store.GetLoadBalancer(t.Context(), activeOld.LoadBalancerID) require.NoError(t, err) assert.Equal(t, StateActive, got.State) - got, err = svc.store.GetLoadBalancer(heartbeating.LoadBalancerID) + got, err = svc.store.GetLoadBalancer(t.Context(), heartbeating.LoadBalancerID) require.NoError(t, err) assert.Equal(t, StateProvisioning, got.State) } @@ -65,14 +65,14 @@ func TestStartLifecycleReaper(t *testing.T) { svc.reaperInterval = 5 * time.Millisecond stuck := testLBRecord("ticker0000000000", StateProvisioning, time.Now().UTC().Add(-10*time.Minute), time.Time{}) - require.NoError(t, svc.store.PutLoadBalancer(stuck)) + require.NoError(t, svc.store.PutLoadBalancer(t.Context(), stuck)) ctx, cancel := context.WithCancel(context.Background()) defer cancel() svc.StartLifecycleReaper(ctx) require.Eventually(t, func() bool { - got, err := svc.store.GetLoadBalancer(stuck.LoadBalancerID) + got, err := svc.store.GetLoadBalancer(t.Context(), stuck.LoadBalancerID) return err == nil && got != nil && got.State == StateFailed }, time.Second, 10*time.Millisecond, "reaper goroutine should mark the stuck LB failed") diff --git a/spinifex/handlers/elbv2/service_impl.go b/spinifex/handlers/elbv2/service_impl.go index f06a54087..0b22aca41 100644 --- a/spinifex/handlers/elbv2/service_impl.go +++ b/spinifex/handlers/elbv2/service_impl.go @@ -144,18 +144,6 @@ func (s *ELBv2ServiceImpl) WaitLaunches() { s.launchWG.Wait() } // NewELBv2ServiceImplWithNATS creates an ELBv2 service backed by JetStream KV. func NewELBv2ServiceImplWithNATS(cfg *config.Config, nc *nats.Conn) (*ELBv2ServiceImpl, error) { - store, err := NewStore(nc) - if err != nil { - return nil, fmt.Errorf("failed to create ELBv2 store: %w", err) - } - - // ACM store shares JetStream KV. Non-fatal: failure only disables HTTPS termination. - acmStore, acmErr := handlers_acm.NewStore(nc) - if acmErr != nil { - slog.Warn("ELBv2: ACM store unavailable, HTTPS listeners cannot resolve certs", "err", acmErr) - acmStore = nil - } - region := "us-east-1" nodeID := "" if cfg != nil { @@ -166,6 +154,23 @@ func NewELBv2ServiceImplWithNATS(cfg *config.Config, nc *nats.Conn) (*ELBv2Servi } ctx, cancel := context.WithCancel(context.Background()) + + // Both KV stores open under the service lifetime context, so the constructor + // has to establish it first. + store, err := NewStore(ctx, nc) + if err != nil { + cancel() + return nil, fmt.Errorf("failed to create ELBv2 store: %w", err) + } + + // ACM store shares JetStream KV; its bucket opens under the service lifetime + // context. Non-fatal: failure only disables HTTPS termination. + acmStore, acmErr := handlers_acm.NewStore(ctx, nc) + if acmErr != nil { + slog.Warn("ELBv2: ACM store unavailable, HTTPS listeners cannot resolve certs", "err", acmErr) + acmStore = nil + } + hc := newHealthChecker(store) return &ELBv2ServiceImpl{ @@ -197,7 +202,7 @@ func (s *ELBv2ServiceImpl) ResetTargetHealthOnStartup(ctx context.Context) error if s == nil || s.store == nil { return nil } - tgs, err := s.store.ListTargetGroups() + tgs, err := s.store.ListTargetGroups(ctx) if err != nil { return fmt.Errorf("list target groups: %w", err) } @@ -223,7 +228,7 @@ func (s *ELBv2ServiceImpl) ResetTargetHealthOnStartup(ctx context.Context) error resetTargets++ } if changed { - if err := s.store.PutTargetGroup(tg); err != nil { + if err := s.store.PutTargetGroup(ctx, tg); err != nil { slog.Error("ResetTargetHealthOnStartup: persist failed", "tgId", tg.TargetGroupID, "err", err) continue @@ -522,13 +527,13 @@ func (s *ELBv2ServiceImpl) launchLBVM(ctx context.Context, lbID, scheme string, // loadRecoveryLBIndex returns the instanceID→LB map, populated lazily. // Errors are not cached so transient JetStream failures don't condemn recovery. -func (s *ELBv2ServiceImpl) loadRecoveryLBIndex() (map[string]*LoadBalancerRecord, error) { +func (s *ELBv2ServiceImpl) loadRecoveryLBIndex(ctx context.Context) (map[string]*LoadBalancerRecord, error) { s.recoveryLBIndexMu.Lock() defer s.recoveryLBIndexMu.Unlock() if s.recoveryLBIndex != nil { return s.recoveryLBIndex, nil } - lbs, err := s.store.ListLoadBalancers() + lbs, err := s.store.ListLoadBalancers(ctx) if err != nil { return nil, fmt.Errorf("list lb records: %w", err) } @@ -544,25 +549,25 @@ func (s *ELBv2ServiceImpl) loadRecoveryLBIndex() (map[string]*LoadBalancerRecord // RebuildSystemInstanceInput reconstructs the launch input for host-reboot recovery. // The instance→LB map is memoised so concurrent recovery candidates share one ListLoadBalancers call. -func (s *ELBv2ServiceImpl) RebuildSystemInstanceInput(ctx RecoveryContext) (*SystemInstanceInput, error) { - index, err := s.loadRecoveryLBIndex() +func (s *ELBv2ServiceImpl) RebuildSystemInstanceInput(rc RecoveryContext) (*SystemInstanceInput, error) { + index, err := s.loadRecoveryLBIndex(s.ctx) if err != nil { return nil, err } - lb := index[ctx.InstanceID] + lb := index[rc.InstanceID] if lb == nil { - return nil, fmt.Errorf("no LB record references instance %s", ctx.InstanceID) + return nil, fmt.Errorf("no LB record references instance %s", rc.InstanceID) } if len(lb.Subnets) == 0 || len(lb.ENIs) == 0 { return nil, fmt.Errorf("lb %s has no subnets/ENIs to rebuild from", lb.LoadBalancerID) } - eniDetails := s.describeENIs(context.Background(), lb.ENIs, lb.AccountID) + eniDetails := s.describeENIs(s.ctx, lb.ENIs, lb.AccountID) // For multi-ENI LBs the VPC store must supply all MAC/IP/subnet details. if len(lb.ENIs) > 1 && len(eniDetails) < len(lb.ENIs) { return nil, fmt.Errorf("describe lb %s ENIs: got %d/%d", lb.LoadBalancerID, len(eniDetails), len(lb.ENIs)) } - primaryMAC := ctx.ENIMac + primaryMAC := rc.ENIMac if primary := eniDetails[lb.ENIs[0]]; primary != nil { if mac := aws.StringValue(primary.MacAddress); mac != "" { primaryMAC = mac @@ -578,10 +583,10 @@ func (s *ELBv2ServiceImpl) RebuildSystemInstanceInput(ctx RecoveryContext) (*Sys nics := s.buildMicrovmNICs(lb.VPCIP, primaryMAC, lb.Subnets[0], lb.ENIs[0], lb.Scheme, extraENIs, lb.AccountID) // Re-inject mgmt NIC MAC/CIDR — buildMicrovmNICs leaves them blank. - if len(nics) > 1 && ctx.MgmtMAC != "" { - nics[1].MAC = ctx.MgmtMAC - if ctx.MgmtIP != "" { - nics[1].CIDR = ctx.MgmtIP + "/24" + if len(nics) > 1 && rc.MgmtMAC != "" { + nics[1].MAC = rc.MgmtMAC + if rc.MgmtIP != "" { + nics[1].CIDR = rc.MgmtIP + "/24" } } @@ -591,7 +596,7 @@ func (s *ELBv2ServiceImpl) RebuildSystemInstanceInput(ctx RecoveryContext) (*Sys profileARN := s.ensureLBInstanceProfile(utils.GlobalAccountID) return &SystemInstanceInput{ - InstanceType: ctx.InstanceType, + InstanceType: rc.InstanceType, SubnetID: lb.Subnets[0], ENIID: lb.ENIs[0], ENIMac: primaryMAC, @@ -634,7 +639,7 @@ func (s *ELBv2ServiceImpl) updateStoredConfig(ctx context.Context, lb *LoadBalan return nil } - listeners, err := s.store.ListListenersByLB(lb.LoadBalancerArn) + listeners, err := s.store.ListListenersByLB(ctx, lb.LoadBalancerArn) if err != nil { slog.ErrorContext(ctx, "updateStoredConfig: failed to list listeners", "lbArn", lb.LoadBalancerArn, "err", err) return fmt.Errorf("list listeners: %w", err) @@ -653,7 +658,7 @@ func (s *ELBv2ServiceImpl) updateStoredConfig(ctx context.Context, lb *LoadBalan if _, ok := tgByArn[tgArn]; ok { return } - tg, tgErr := s.store.GetTargetGroupByArn(tgArn) + tg, tgErr := s.store.GetTargetGroupByArn(ctx, tgArn) if tgErr != nil || tg == nil { slog.DebugContext(ctx, "updateStoredConfig: target group not found", "tgArn", tgArn) return @@ -665,7 +670,7 @@ func (s *ELBv2ServiceImpl) updateStoredConfig(ctx context.Context, lb *LoadBalan for _, a := range l.DefaultActions { loadTG(a.TargetGroupArn) } - rules, rErr := s.store.ListRulesByListener(l.ListenerArn) + rules, rErr := s.store.ListRulesByListener(ctx, l.ListenerArn) if rErr != nil { slog.ErrorContext(ctx, "updateStoredConfig: failed to list rules", "listenerArn", l.ListenerArn, "err", rErr) return fmt.Errorf("list rules: %w", rErr) @@ -680,7 +685,7 @@ func (s *ELBv2ServiceImpl) updateStoredConfig(ctx context.Context, lb *LoadBalan } } - certPEMByArn, err := s.resolveListenerCerts(listeners, lb.AccountID) + certPEMByArn, err := s.resolveListenerCerts(ctx, listeners, lb.AccountID) if err != nil { slog.ErrorContext(ctx, "updateStoredConfig: failed to resolve certs", "lbId", lb.LoadBalancerID, "err", err) return fmt.Errorf("resolve certs: %w", err) @@ -703,7 +708,7 @@ func (s *ELBv2ServiceImpl) updateStoredConfig(ctx context.Context, lb *LoadBalan lb.HealthTargets = nil } - if err := s.store.PutLoadBalancer(lb); err != nil { + if err := s.store.PutLoadBalancer(ctx, lb); err != nil { slog.ErrorContext(ctx, "updateStoredConfig: failed to persist LB", "lbId", lb.LoadBalancerID, "err", err) return fmt.Errorf("persist LB: %w", err) } @@ -715,14 +720,14 @@ func (s *ELBv2ServiceImpl) updateStoredConfig(ctx context.Context, lb *LoadBalan // resolveListenerCerts resolves each distinct certificate ARN to its combined PEM. // Returns nil for HTTP-only listeners. -func (s *ELBv2ServiceImpl) resolveListenerCerts(listeners []*ListenerRecord, accountID string) (map[string]string, error) { +func (s *ELBv2ServiceImpl) resolveListenerCerts(ctx context.Context, listeners []*ListenerRecord, accountID string) (map[string]string, error) { var out map[string]string for _, l := range listeners { for _, c := range l.Certificates { if _, ok := out[c.CertificateArn]; ok { continue } - pem, err := s.resolveCertPEM(c.CertificateArn, accountID) + pem, err := s.resolveCertPEM(ctx, c.CertificateArn, accountID) if err != nil { return nil, fmt.Errorf("cert %s: %w", c.CertificateArn, err) } @@ -737,11 +742,11 @@ func (s *ELBv2ServiceImpl) resolveListenerCerts(listeners []*ListenerRecord, acc // resolveCertPEM loads a certificate from ACM and returns its combined PEM // (leaf + chain + key). Cross-account certs are treated as absent. -func (s *ELBv2ServiceImpl) resolveCertPEM(arn, accountID string) (string, error) { +func (s *ELBv2ServiceImpl) resolveCertPEM(ctx context.Context, arn, accountID string) (string, error) { if s.acmStore == nil { return "", errors.New(awserrors.ErrorELBv2CertificateNotFound) } - rec, err := s.acmStore.GetCert(arn) + rec, err := s.acmStore.GetCert(ctx, arn) if err != nil { return "", fmt.Errorf("get cert: %w", err) } @@ -764,12 +769,12 @@ func (s *ELBv2ServiceImpl) resolveCertPEM(arn, accountID string) (string, error) // validateListenerCerts confirms every certificate ARN resolves in the ACM store // and is owned by the account. Rejects at the API boundary to avoid silently // freezing data-plane convergence at config-render time. -func (s *ELBv2ServiceImpl) validateListenerCerts(certs []ListenerCertificate, accountID string) error { +func (s *ELBv2ServiceImpl) validateListenerCerts(ctx context.Context, certs []ListenerCertificate, accountID string) error { if s.acmStore == nil { return nil } for _, c := range certs { - if _, err := s.resolveCertPEM(c.CertificateArn, accountID); err != nil { + if _, err := s.resolveCertPEM(ctx, c.CertificateArn, accountID); err != nil { return errors.New(awserrors.ErrorELBv2CertificateNotFound) } } @@ -794,7 +799,7 @@ func configCertHash(configContent string, certFiles map[string]string) string { // updateStoredConfigForTargetGroup finds all LBs that reference the given target // group (via listeners) and updates their stored config. func (s *ELBv2ServiceImpl) updateStoredConfigForTargetGroup(ctx context.Context, tgArn string) error { - allListeners, err := s.store.ListListeners() + allListeners, err := s.store.ListListeners(ctx) if err != nil { slog.ErrorContext(ctx, "updateStoredConfigForTargetGroup: failed to list listeners", "err", err) return fmt.Errorf("list listeners: %w", err) @@ -810,7 +815,7 @@ func (s *ELBv2ServiceImpl) updateStoredConfigForTargetGroup(ctx context.Context, } // Rules may forward to a TG even if the listener default action does not. - allRules, rulesErr := s.store.ListRules() + allRules, rulesErr := s.store.ListRules(ctx) if rulesErr != nil { slog.ErrorContext(ctx, "updateStoredConfigForTargetGroup: failed to list rules", "err", rulesErr) return fmt.Errorf("list rules: %w", rulesErr) @@ -831,7 +836,7 @@ func (s *ELBv2ServiceImpl) updateStoredConfigForTargetGroup(ctx context.Context, } for lbArn := range lbArns { - lb, lbErr := s.store.GetLoadBalancerByArn(lbArn) + lb, lbErr := s.store.GetLoadBalancerByArn(ctx, lbArn) if lbErr != nil || lb == nil { continue } @@ -851,7 +856,7 @@ func (s *ELBv2ServiceImpl) LBAgentHeartbeat(ctx context.Context, input *LBAgentH lbID := *input.LBID slog.DebugContext(ctx, "LBAgentHeartbeat received", "lbId", lbID, "accountId", accountID) - lb, err := s.store.GetLoadBalancer(lbID) + lb, err := s.store.GetLoadBalancer(ctx, lbID) if err != nil { slog.ErrorContext(ctx, "LBAgentHeartbeat: failed to get LB", "lbId", lbID, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) @@ -886,7 +891,7 @@ func (s *ELBv2ServiceImpl) LBAgentHeartbeat(ctx context.Context, input *LBAgentH } case heartbeatStale: // Refresh the heartbeat timestamp only; avoid writing the full record on every tick. - if err := s.store.PutLoadBalancer(lb); err != nil { + if err := s.store.PutLoadBalancer(ctx, lb); err != nil { slog.ErrorContext(ctx, "LBAgentHeartbeat: failed to persist LB", "lbId", lbID, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } @@ -910,7 +915,7 @@ func (s *ELBv2ServiceImpl) GetLBConfig(ctx context.Context, input *GetLBConfigIn } lbID := *input.LBID - lb, err := s.store.GetLoadBalancer(lbID) + lb, err := s.store.GetLoadBalancer(ctx, lbID) if err != nil { slog.ErrorContext(ctx, "GetLBConfig: failed to get LB", "lbId", lbID, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) @@ -1115,7 +1120,7 @@ func (s *ELBv2ServiceImpl) createLoadBalancer(ctx context.Context, input *elbv2. name := *input.Name // Check for duplicate name - existing, err := s.store.GetLoadBalancerByName(name, accountID) + existing, err := s.store.GetLoadBalancerByName(ctx, name, accountID) if err != nil { slog.ErrorContext(ctx, "CreateLoadBalancer: failed to check duplicate name", "name", name, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) @@ -1165,7 +1170,7 @@ func (s *ELBv2ServiceImpl) createLoadBalancer(ctx context.Context, input *elbv2. // Atomically claim the name before ENI/VM work. SDK retries lose the claim; // orphaned claims from crashed creates are reclaimed. Every failure releases it. - claimOK, claimDup, claimErr := s.store.ClaimLBName(name, accountID, lbID) + claimOK, claimDup, claimErr := s.store.ClaimLBName(ctx, name, accountID, lbID) if claimErr != nil { slog.ErrorContext(ctx, "CreateLoadBalancer: name claim failed", "name", name, "err", claimErr) return nil, errors.New(awserrors.ErrorServerInternal) @@ -1203,7 +1208,7 @@ func (s *ELBv2ServiceImpl) createLoadBalancer(ctx context.Context, input *elbv2. sgID, sgErr := s.createNLBManagedSG(ctx, lbID, lbArn, subnets[0], accountID) if sgErr != nil { slog.ErrorContext(ctx, "CreateLoadBalancer: failed to create managed NLB SG", "lbId", lbID, "err", sgErr) - s.releaseLBNameClaim(name, accountID) + s.releaseLBNameClaim(ctx, name, accountID) return nil, errors.New(awserrors.ErrorServerInternal) } nlbManagedSGID = sgID @@ -1301,7 +1306,7 @@ func (s *ELBv2ServiceImpl) createLoadBalancer(ctx context.Context, input *elbv2. CreatedAt: time.Now().UTC(), } - if err := s.store.PutLoadBalancer(record); err != nil { + if err := s.store.PutLoadBalancer(ctx, record); err != nil { slog.ErrorContext(ctx, "CreateLoadBalancer: failed to persist record", "lbId", lbID, "err", err) // Rollback so unowned ENIs/SG don't leak. s.rollbackLBInfra(ctx, eniIDs, nlbManagedSGID, name, accountID) @@ -1327,12 +1332,12 @@ func (s *ELBv2ServiceImpl) createLoadBalancer(ctx context.Context, input *elbv2. // no retry can clear. The async path keeps its failed record on // purpose; the caller holds the ARN there and can reclaim it. s.rollbackLBInfra(ctx, eniIDs, nlbManagedSGID, name, accountID) - if delErr := s.store.DeleteLoadBalancer(lbID); delErr != nil { + if delErr := s.store.DeleteLoadBalancer(ctx, lbID); delErr != nil { slog.ErrorContext(ctx, "CreateLoadBalancerSync: rollback failed to delete record", "lbId", lbID, "err", delErr) } return nil, lbLaunchError(err) } - launched, lerr := s.store.GetLoadBalancerByArn(lbArn) + launched, lerr := s.store.GetLoadBalancerByArn(ctx, lbArn) if lerr != nil || launched == nil { slog.ErrorContext(ctx, "CreateLoadBalancerSync: reload after launch failed", "lbArn", lbArn, "err", lerr) return nil, errors.New(awserrors.ErrorServerInternal) @@ -1343,7 +1348,7 @@ func (s *ELBv2ServiceImpl) createLoadBalancer(ctx context.Context, input *elbv2. defer func() { if r := recover(); r != nil { slog.ErrorContext(ctx, "CreateLoadBalancer: async launch panic", "lbId", lc.lbID, "panic", r) - s.markLBFailed(lc.lbArn, fmt.Sprintf("LB VM launch panicked: %v", r)) + s.markLBFailed(ctx, lc.lbArn, fmt.Sprintf("LB VM launch panicked: %v", r)) } }() s.launchLBVMAsync(ctx, lc) @@ -1384,7 +1389,7 @@ type lbLaunchCtx struct { func (s *ELBv2ServiceImpl) provisionLBDataPlane(ctx context.Context, lc lbLaunchCtx) error { launch := s.launchLBVM(ctx, lc.lbID, lc.scheme, lc.eniIDs, lc.subnets, lc.accountID, lc.crossAccountENIs) - record, err := s.store.GetLoadBalancerByArn(lc.lbArn) + record, err := s.store.GetLoadBalancerByArn(ctx, lc.lbArn) if err != nil || record == nil { return fmt.Errorf("reload record for %s: %w", lc.lbArn, err) } @@ -1392,7 +1397,7 @@ func (s *ELBv2ServiceImpl) provisionLBDataPlane(ctx context.Context, lc lbLaunch if launch.failed { record.State = StateFailed record.StateReason = launch.failReason - if putErr := s.store.PutLoadBalancer(record); putErr != nil { + if putErr := s.store.PutLoadBalancer(ctx, record); putErr != nil { return fmt.Errorf("persist failed state for %s: %w", lc.lbArn, putErr) } return fmt.Errorf("lb-vm launch failed for %s: %s", lc.lbArn, launch.failReason) @@ -1404,7 +1409,7 @@ func (s *ELBv2ServiceImpl) provisionLBDataPlane(ctx context.Context, lc lbLaunch if launch.publicIP != "" && len(record.AvailZones) > 0 { record.AvailZones[0].PublicIP = launch.publicIP } - if putErr := s.store.PutLoadBalancer(record); putErr != nil { + if putErr := s.store.PutLoadBalancer(ctx, record); putErr != nil { return fmt.Errorf("persist launch result for %s: %w", lc.lbArn, putErr) } // Register the frontend A record now that the serving IP is allocated. @@ -1438,11 +1443,14 @@ func (s *ELBv2ServiceImpl) publishLBDNS(record *LoadBalancerRecord, action handl // store spans every tenant, so a successful list is a complete cross-account // view; a store error yields ok=false so the reconcile suppresses ELB pruning // rather than delete another tenant's live record on a partial view. +// +// The daemon's reconcile loop calls this without a context, so the read runs on +// the service lifetime context: the sweep should stop once the service closes. func (s *ELBv2ServiceImpl) DesiredDNSChanges() (changes []handlers_dns.Change, ok bool) { if s == nil || s.store == nil || s.dnsBaseDomain == "" { return nil, false } - lbs, err := s.store.ListLoadBalancersStrict() + lbs, err := s.store.ListLoadBalancersStrict(s.ctx) if err != nil { return nil, false } @@ -1505,19 +1513,19 @@ func (s *ELBv2ServiceImpl) rollbackLBInfra(ctx context.Context, eniIDs []string, } } s.deleteNLBManagedSG(ctx, nlbManagedSGID, accountID) - s.releaseLBNameClaim(name, accountID) + s.releaseLBNameClaim(ctx, name, accountID) } // markLBFailed reloads the LB record and flips it to the failed state, // recording reason for State.Reason. -func (s *ELBv2ServiceImpl) markLBFailed(lbArn, reason string) { - record, err := s.store.GetLoadBalancerByArn(lbArn) +func (s *ELBv2ServiceImpl) markLBFailed(ctx context.Context, lbArn, reason string) { + record, err := s.store.GetLoadBalancerByArn(ctx, lbArn) if err != nil || record == nil { return } record.State = StateFailed record.StateReason = reason - if putErr := s.store.PutLoadBalancer(record); putErr != nil { + if putErr := s.store.PutLoadBalancer(ctx, record); putErr != nil { slog.Error("CreateLoadBalancer: failed to persist failed state", "lbArn", lbArn, "err", putErr) } } @@ -1526,8 +1534,8 @@ func (s *ELBv2ServiceImpl) markLBFailed(lbArn, reason string) { // lbProvisioningTimeout without a single lb-agent heartbeat. Such an LB launched // its VM but the agent never reached the daemon, so it would otherwise wedge in // provisioning forever. Synchronous; caller supplies now for testability. -func (s *ELBv2ServiceImpl) reapStuckProvisioningLBs(now time.Time) { - lbs, err := s.store.ListLoadBalancers() +func (s *ELBv2ServiceImpl) reapStuckProvisioningLBs(ctx context.Context, now time.Time) { + lbs, err := s.store.ListLoadBalancers(ctx) if err != nil { slog.Error("ELBv2 lifecycle reaper: failed to list load balancers", "err", err) return @@ -1546,7 +1554,7 @@ func (s *ELBv2ServiceImpl) reapStuckProvisioningLBs(now time.Time) { slog.Warn("ELBv2 lifecycle reaper: LB stuck in provisioning, marking failed", "lbArn", lb.LoadBalancerArn, "lbId", lb.LoadBalancerID, "instanceId", lb.InstanceID, "createdAt", lb.CreatedAt) - s.markLBFailed(lb.LoadBalancerArn, reason) + s.markLBFailed(ctx, lb.LoadBalancerArn, reason) } } @@ -1567,7 +1575,7 @@ func (s *ELBv2ServiceImpl) StartLifecycleReaper(ctx context.Context) { case <-ctx.Done(): return case <-ticker.C: - s.reapStuckProvisioningLBs(time.Now().UTC()) + s.reapStuckProvisioningLBs(ctx, time.Now().UTC()) } } }() @@ -1578,7 +1586,7 @@ func (s *ELBv2ServiceImpl) DeleteLoadBalancer(ctx context.Context, input *elbv2. return nil, errors.New(awserrors.ErrorMissingParameter) } - lb, err := s.store.GetLoadBalancerByArn(*input.LoadBalancerArn) + lb, err := s.store.GetLoadBalancerByArn(ctx, *input.LoadBalancerArn) if err != nil { slog.ErrorContext(ctx, "DeleteLoadBalancer: failed to get LB", "arn", *input.LoadBalancerArn, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) @@ -1589,13 +1597,13 @@ func (s *ELBv2ServiceImpl) DeleteLoadBalancer(ctx context.Context, input *elbv2. } // Cascade-delete all listeners and their rules so no orphan pins a TG as ResourceInUse. - listeners, err := s.store.ListListenersByLB(lb.LoadBalancerArn) + listeners, err := s.store.ListListenersByLB(ctx, lb.LoadBalancerArn) if err != nil { slog.ErrorContext(ctx, "DeleteLoadBalancer: failed to list listeners for cascade", "lbArn", lb.LoadBalancerArn, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } for _, l := range listeners { - if err := s.deleteListenerCascade(l); err != nil { + if err := s.deleteListenerCascade(ctx, l); err != nil { slog.ErrorContext(ctx, "DeleteLoadBalancer: failed to cascade listener delete", "listenerID", l.ListenerID, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } @@ -1630,14 +1638,14 @@ func (s *ELBv2ServiceImpl) DeleteLoadBalancer(ctx context.Context, input *elbv2. s.deleteNLBManagedSG(ctx, lb.NLBManagedSGID, accountID) } - if err := s.store.DeleteLoadBalancer(lb.LoadBalancerID); err != nil { + if err := s.store.DeleteLoadBalancer(ctx, lb.LoadBalancerID); err != nil { slog.ErrorContext(ctx, "DeleteLoadBalancer: failed to delete record", "lbId", lb.LoadBalancerID, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } // Release the name claim so the name is reusable. Idempotent on a missing // key, so a delete that races the record removal still converges. - s.releaseLBNameClaim(lb.Name, accountID) + s.releaseLBNameClaim(ctx, lb.Name, accountID) // Withdraw the frontend A record (best-effort; reconcile repairs a miss). s.publishLBDNS(lb, handlers_dns.ActionDelete) @@ -1649,8 +1657,8 @@ func (s *ELBv2ServiceImpl) DeleteLoadBalancer(ctx context.Context, input *elbv2. // releaseLBNameClaim drops the per-account LB name claim. Failures are logged but // not fatal; leaked claims are reclaimed by the crash-orphan path on next create. -func (s *ELBv2ServiceImpl) releaseLBNameClaim(name, accountID string) { - if err := s.store.ReleaseLBName(name, accountID); err != nil { +func (s *ELBv2ServiceImpl) releaseLBNameClaim(ctx context.Context, name, accountID string) { + if err := s.store.ReleaseLBName(ctx, name, accountID); err != nil { slog.Warn("failed to release LB name claim", "name", name, "accountID", accountID, "err", err) } } @@ -1678,7 +1686,7 @@ func (s *ELBv2ServiceImpl) reapFloatingIPNAT(lb *LoadBalancerRecord) { } func (s *ELBv2ServiceImpl) DescribeLoadBalancers(ctx context.Context, input *elbv2.DescribeLoadBalancersInput, accountID string) (*elbv2.DescribeLoadBalancersOutput, error) { - allLBs, err := s.store.ListLoadBalancers() + allLBs, err := s.store.ListLoadBalancers(ctx) if err != nil { slog.ErrorContext(ctx, "DescribeLoadBalancers: failed to list LBs", "err", err) return nil, errors.New(awserrors.ErrorServerInternal) @@ -1767,7 +1775,7 @@ func (s *ELBv2ServiceImpl) CreateTargetGroup(ctx context.Context, input *elbv2.C } // Check duplicate name within VPC - existing, err := s.store.GetTargetGroupByName(name, vpcID) + existing, err := s.store.GetTargetGroupByName(ctx, name, vpcID) if err != nil { slog.ErrorContext(ctx, "CreateTargetGroup: failed to check duplicate name", "name", name, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) @@ -1847,7 +1855,7 @@ func (s *ELBv2ServiceImpl) CreateTargetGroup(ctx context.Context, input *elbv2.C CreatedAt: time.Now().UTC(), } - if err := s.store.PutTargetGroup(record); err != nil { + if err := s.store.PutTargetGroup(ctx, record); err != nil { slog.ErrorContext(ctx, "CreateTargetGroup: failed to persist record", "tgId", tgID, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } @@ -1864,7 +1872,7 @@ func (s *ELBv2ServiceImpl) ModifyTargetGroup(ctx context.Context, input *elbv2.M return nil, errors.New(awserrors.ErrorMissingParameter) } - tg, err := s.store.GetTargetGroupByArn(*input.TargetGroupArn) + tg, err := s.store.GetTargetGroupByArn(ctx, *input.TargetGroupArn) if err != nil { slog.ErrorContext(ctx, "ModifyTargetGroup: failed to get TG", "arn", *input.TargetGroupArn, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) @@ -1909,7 +1917,7 @@ func (s *ELBv2ServiceImpl) ModifyTargetGroup(ctx context.Context, input *elbv2.M } tg.HealthCheck = hc - if err := s.store.PutTargetGroup(tg); err != nil { + if err := s.store.PutTargetGroup(ctx, tg); err != nil { slog.ErrorContext(ctx, "ModifyTargetGroup: failed to persist record", "arn", tg.TargetGroupArn, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } @@ -1926,7 +1934,7 @@ func (s *ELBv2ServiceImpl) DeleteTargetGroup(ctx context.Context, input *elbv2.D return nil, errors.New(awserrors.ErrorMissingParameter) } - tg, err := s.store.GetTargetGroupByArn(*input.TargetGroupArn) + tg, err := s.store.GetTargetGroupByArn(ctx, *input.TargetGroupArn) if err != nil { slog.ErrorContext(ctx, "DeleteTargetGroup: failed to get TG", "arn", *input.TargetGroupArn, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) @@ -1937,7 +1945,7 @@ func (s *ELBv2ServiceImpl) DeleteTargetGroup(ctx context.Context, input *elbv2.D } // Only live listeners/rules (whose LB still exists) pin the TG as ResourceInUse. - lbs, err := s.store.ListLoadBalancers() + lbs, err := s.store.ListLoadBalancers(ctx) if err != nil { slog.ErrorContext(ctx, "DeleteTargetGroup: failed to list load balancers for in-use check", "err", err) return nil, errors.New(awserrors.ErrorServerInternal) @@ -1948,7 +1956,7 @@ func (s *ELBv2ServiceImpl) DeleteTargetGroup(ctx context.Context, input *elbv2.D } // Check if any live listener references this target group. - listeners, err := s.store.ListListeners() + listeners, err := s.store.ListListeners(ctx) if err != nil { slog.ErrorContext(ctx, "DeleteTargetGroup: failed to list listeners for in-use check", "err", err) return nil, errors.New(awserrors.ErrorServerInternal) @@ -1967,7 +1975,7 @@ func (s *ELBv2ServiceImpl) DeleteTargetGroup(ctx context.Context, input *elbv2.D } // Block deletion when a live rule still forwards to the target group. - allRules, err := s.store.ListRules() + allRules, err := s.store.ListRules(ctx) if err != nil { slog.ErrorContext(ctx, "DeleteTargetGroup: failed to list rules for in-use check", "err", err) return nil, errors.New(awserrors.ErrorServerInternal) @@ -1983,7 +1991,7 @@ func (s *ELBv2ServiceImpl) DeleteTargetGroup(ctx context.Context, input *elbv2.D } } - if err := s.store.DeleteTargetGroup(tg.TargetGroupID); err != nil { + if err := s.store.DeleteTargetGroup(ctx, tg.TargetGroupID); err != nil { slog.ErrorContext(ctx, "DeleteTargetGroup: failed to delete record", "tgId", tg.TargetGroupID, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } @@ -1994,7 +2002,7 @@ func (s *ELBv2ServiceImpl) DeleteTargetGroup(ctx context.Context, input *elbv2.D } func (s *ELBv2ServiceImpl) DescribeTargetGroups(ctx context.Context, input *elbv2.DescribeTargetGroupsInput, accountID string) (*elbv2.DescribeTargetGroupsOutput, error) { - allTGs, err := s.store.ListTargetGroups() + allTGs, err := s.store.ListTargetGroups(ctx) if err != nil { slog.ErrorContext(ctx, "DescribeTargetGroups: failed to list TGs", "err", err) return nil, errors.New(awserrors.ErrorServerInternal) @@ -2027,7 +2035,7 @@ func (s *ELBv2ServiceImpl) DescribeTargetGroups(ctx context.Context, input *elbv // Filter by LB ARN if specified if input.LoadBalancerArn != nil && *input.LoadBalancerArn != "" { // Check if any listener on this LB references this TG - listeners, _ := s.store.ListListenersByLB(*input.LoadBalancerArn) + listeners, _ := s.store.ListListenersByLB(ctx, *input.LoadBalancerArn) found := false for _, l := range listeners { for _, a := range l.DefaultActions { @@ -2055,7 +2063,7 @@ func (s *ELBv2ServiceImpl) RegisterTargets(ctx context.Context, input *elbv2.Reg return nil, errors.New(awserrors.ErrorMissingParameter) } - tg, err := s.store.GetTargetGroupByArn(*input.TargetGroupArn) + tg, err := s.store.GetTargetGroupByArn(ctx, *input.TargetGroupArn) if err != nil { slog.ErrorContext(ctx, "RegisterTargets: failed to get TG", "arn", *input.TargetGroupArn, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) @@ -2099,7 +2107,7 @@ func (s *ELBv2ServiceImpl) RegisterTargets(ctx context.Context, input *elbv2.Reg }) } - if err := s.store.PutTargetGroup(tg); err != nil { + if err := s.store.PutTargetGroup(ctx, tg); err != nil { slog.ErrorContext(ctx, "RegisterTargets: failed to persist TG", "arn", *input.TargetGroupArn, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } @@ -2120,7 +2128,7 @@ func (s *ELBv2ServiceImpl) DeregisterTargets(ctx context.Context, input *elbv2.D return nil, errors.New(awserrors.ErrorMissingParameter) } - tg, err := s.store.GetTargetGroupByArn(*input.TargetGroupArn) + tg, err := s.store.GetTargetGroupByArn(ctx, *input.TargetGroupArn) if err != nil { slog.ErrorContext(ctx, "DeregisterTargets: failed to get TG", "arn", *input.TargetGroupArn, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) @@ -2153,7 +2161,7 @@ func (s *ELBv2ServiceImpl) DeregisterTargets(ctx context.Context, input *elbv2.D } tg.Targets = remaining - if err := s.store.PutTargetGroup(tg); err != nil { + if err := s.store.PutTargetGroup(ctx, tg); err != nil { slog.ErrorContext(ctx, "DeregisterTargets: failed to persist TG", "arn", *input.TargetGroupArn, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } @@ -2174,7 +2182,7 @@ func (s *ELBv2ServiceImpl) DescribeTargetHealth(ctx context.Context, input *elbv return nil, errors.New(awserrors.ErrorMissingParameter) } - tg, err := s.store.GetTargetGroupByArn(*input.TargetGroupArn) + tg, err := s.store.GetTargetGroupByArn(ctx, *input.TargetGroupArn) if err != nil { slog.ErrorContext(ctx, "DescribeTargetHealth: failed to get TG", "arn", *input.TargetGroupArn, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) @@ -2193,7 +2201,7 @@ func (s *ELBv2ServiceImpl) DescribeTargetHealth(ctx context.Context, input *elbv // A target group not forwarded to by any listener serves no traffic; its // targets report "unused" (AWS Target.NotInUse), not "initial". - inUse, err := s.store.TargetGroupInUse(tg.TargetGroupArn) + inUse, err := s.store.TargetGroupInUse(ctx, tg.TargetGroupArn) if err != nil { slog.ErrorContext(ctx, "DescribeTargetHealth: failed to check TG association", "arn", tg.TargetGroupArn, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) @@ -2385,7 +2393,7 @@ func (s *ELBv2ServiceImpl) CreateListener(ctx context.Context, input *elbv2.Crea return nil, errors.New(awserrors.ErrorMissingParameter) } - lb, err := s.store.GetLoadBalancerByArn(*input.LoadBalancerArn) + lb, err := s.store.GetLoadBalancerByArn(ctx, *input.LoadBalancerArn) if err != nil { slog.ErrorContext(ctx, "CreateListener: failed to get LB", "arn", *input.LoadBalancerArn, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) @@ -2423,7 +2431,7 @@ func (s *ELBv2ServiceImpl) CreateListener(ctx context.Context, input *elbv2.Crea } // Check for duplicate listener on same port - existingListeners, err := s.store.ListListenersByLB(lb.LoadBalancerArn) + existingListeners, err := s.store.ListListenersByLB(ctx, lb.LoadBalancerArn) if err != nil { slog.ErrorContext(ctx, "CreateListener: failed to list existing listeners", "lbArn", lb.LoadBalancerArn, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) @@ -2437,7 +2445,7 @@ func (s *ELBv2ServiceImpl) CreateListener(ctx context.Context, input *elbv2.Crea // Validate listener-to-target-group protocol compatibility. for _, a := range input.DefaultActions { if a.Type != nil && *a.Type == ActionTypeForward && a.TargetGroupArn != nil { - tg, tgErr := s.store.GetTargetGroupByArn(*a.TargetGroupArn) + tg, tgErr := s.store.GetTargetGroupByArn(ctx, *a.TargetGroupArn) if tgErr != nil { slog.ErrorContext(ctx, "CreateListener: failed to get target group", "arn", *a.TargetGroupArn, "err", tgErr) return nil, errors.New(awserrors.ErrorServerInternal) @@ -2455,7 +2463,7 @@ func (s *ELBv2ServiceImpl) CreateListener(ctx context.Context, input *elbv2.Crea if err != nil { return nil, err } - if err := s.validateListenerCerts(certs, accountID); err != nil { + if err := s.validateListenerCerts(ctx, certs, accountID); err != nil { return nil, err } @@ -2487,7 +2495,7 @@ func (s *ELBv2ServiceImpl) CreateListener(ctx context.Context, input *elbv2.Crea Tags: tags, } - if err := s.store.PutListener(record); err != nil { + if err := s.store.PutListener(ctx, record); err != nil { slog.ErrorContext(ctx, "CreateListener: failed to persist record", "listenerId", listenerID, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } @@ -2534,7 +2542,7 @@ func (s *ELBv2ServiceImpl) rollbackListener(ctx context.Context, record *Listene slog.ErrorContext(ctx, "CreateListener: rollback failed to revoke listener port", "lbArn", lb.LoadBalancerArn, "port", port, "err", err) } } - if err := s.deleteListenerCascade(record); err != nil { + if err := s.deleteListenerCascade(ctx, record); err != nil { slog.ErrorContext(ctx, "CreateListener: rollback failed to delete listener", "listenerArn", record.ListenerArn, "err", err) } } @@ -2542,17 +2550,17 @@ func (s *ELBv2ServiceImpl) rollbackListener(ctx context.Context, record *Listene // deleteListenerCascade removes a listener and all of its rules. Shared by // DeleteListener and DeleteLoadBalancer so LB teardown never bypasses the rule // cascade and leaves orphan rules that pin a target group as ResourceInUse. -func (s *ELBv2ServiceImpl) deleteListenerCascade(listener *ListenerRecord) error { - rules, err := s.store.ListRulesByListener(listener.ListenerArn) +func (s *ELBv2ServiceImpl) deleteListenerCascade(ctx context.Context, listener *ListenerRecord) error { + rules, err := s.store.ListRulesByListener(ctx, listener.ListenerArn) if err != nil { return fmt.Errorf("list rules for listener %s: %w", listener.ListenerArn, err) } for _, r := range rules { - if err := s.store.DeleteRule(r.RuleID); err != nil { + if err := s.store.DeleteRule(ctx, r.RuleID); err != nil { return fmt.Errorf("delete rule %s: %w", r.RuleID, err) } } - if err := s.store.DeleteListener(listener.ListenerID); err != nil { + if err := s.store.DeleteListener(ctx, listener.ListenerID); err != nil { return fmt.Errorf("delete listener %s: %w", listener.ListenerID, err) } return nil @@ -2563,7 +2571,7 @@ func (s *ELBv2ServiceImpl) DeleteListener(ctx context.Context, input *elbv2.Dele return nil, errors.New(awserrors.ErrorMissingParameter) } - listener, err := s.store.GetListenerByArn(*input.ListenerArn) + listener, err := s.store.GetListenerByArn(ctx, *input.ListenerArn) if err != nil { slog.ErrorContext(ctx, "DeleteListener: failed to get listener", "arn", *input.ListenerArn, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) @@ -2574,13 +2582,13 @@ func (s *ELBv2ServiceImpl) DeleteListener(ctx context.Context, input *elbv2.Dele } // Cascade-delete rules so a recreated listener doesn't inherit orphans. - if err := s.deleteListenerCascade(listener); err != nil { + if err := s.deleteListenerCascade(ctx, listener); err != nil { slog.ErrorContext(ctx, "DeleteListener: failed to cascade-delete listener", "listenerArn", listener.ListenerArn, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } // Reload or stop HAProxy after listener removal - lb, lbErr := s.store.GetLoadBalancerByArn(listener.LoadBalancerArn) + lb, lbErr := s.store.GetLoadBalancerByArn(ctx, listener.LoadBalancerArn) if lbErr == nil && lb != nil { // Close the listener port on the NLB's managed front-end SG. if lb.Type == LoadBalancerTypeNetwork && lb.NLBManagedSGID != "" && s.VPCService != nil { @@ -2608,7 +2616,7 @@ func (s *ELBv2ServiceImpl) ModifyListener(ctx context.Context, input *elbv2.Modi return nil, errors.New(awserrors.ErrorMissingParameter) } - listener, err := s.store.GetListenerByArn(*input.ListenerArn) + listener, err := s.store.GetListenerByArn(ctx, *input.ListenerArn) if err != nil { slog.ErrorContext(ctx, "ModifyListener: failed to get listener", "arn", *input.ListenerArn, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) @@ -2617,7 +2625,7 @@ func (s *ELBv2ServiceImpl) ModifyListener(ctx context.Context, input *elbv2.Modi return nil, errors.New(awserrors.ErrorELBv2ListenerNotFound) } - lb, err := s.store.GetLoadBalancerByArn(listener.LoadBalancerArn) + lb, err := s.store.GetLoadBalancerByArn(ctx, listener.LoadBalancerArn) if err != nil { slog.ErrorContext(ctx, "ModifyListener: failed to get LB", "arn", listener.LoadBalancerArn, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) @@ -2650,7 +2658,7 @@ func (s *ELBv2ServiceImpl) ModifyListener(ctx context.Context, input *elbv2.Modi if input.Port != nil { newPort := *input.Port if newPort != updated.Port { - existingListeners, listErr := s.store.ListListenersByLB(lb.LoadBalancerArn) + existingListeners, listErr := s.store.ListListenersByLB(ctx, lb.LoadBalancerArn) if listErr != nil { slog.ErrorContext(ctx, "ModifyListener: failed to list existing listeners", "lbArn", lb.LoadBalancerArn, "err", listErr) return nil, errors.New(awserrors.ErrorServerInternal) @@ -2675,7 +2683,7 @@ func (s *ELBv2ServiceImpl) ModifyListener(ctx context.Context, input *elbv2.Modi return nil, err } if action.Type == ActionTypeForward && action.TargetGroupArn != "" { - tg, tgErr := s.store.GetTargetGroupByArn(action.TargetGroupArn) + tg, tgErr := s.store.GetTargetGroupByArn(ctx, action.TargetGroupArn) if tgErr != nil { slog.ErrorContext(ctx, "ModifyListener: failed to get target group", "arn", action.TargetGroupArn, "err", tgErr) return nil, errors.New(awserrors.ErrorServerInternal) @@ -2695,7 +2703,7 @@ func (s *ELBv2ServiceImpl) ModifyListener(ctx context.Context, input *elbv2.Modi if a.Type != ActionTypeForward || a.TargetGroupArn == "" { continue } - tg, tgErr := s.store.GetTargetGroupByArn(a.TargetGroupArn) + tg, tgErr := s.store.GetTargetGroupByArn(ctx, a.TargetGroupArn) if tgErr != nil { slog.ErrorContext(ctx, "ModifyListener: failed to get target group", "arn", a.TargetGroupArn, "err", tgErr) return nil, errors.New(awserrors.ErrorServerInternal) @@ -2734,14 +2742,14 @@ func (s *ELBv2ServiceImpl) ModifyListener(ctx context.Context, input *elbv2.Modi if certErr != nil { return nil, certErr } - if certErr := s.validateListenerCerts(certs, accountID); certErr != nil { + if certErr := s.validateListenerCerts(ctx, certs, accountID); certErr != nil { return nil, certErr } updated.Certificates = certs updated.SslPolicy = policy } - if err := s.store.PutListener(&updated); err != nil { + if err := s.store.PutListener(ctx, &updated); err != nil { slog.ErrorContext(ctx, "ModifyListener: failed to persist record", "listenerId", updated.ListenerID, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } @@ -2763,9 +2771,9 @@ func (s *ELBv2ServiceImpl) DescribeListeners(ctx context.Context, input *elbv2.D var err error if input.LoadBalancerArn != nil && *input.LoadBalancerArn != "" { - listeners, err = s.store.ListListenersByLB(*input.LoadBalancerArn) + listeners, err = s.store.ListListenersByLB(ctx, *input.LoadBalancerArn) } else { - listeners, err = s.store.ListListeners() + listeners, err = s.store.ListListeners(ctx) } if err != nil { slog.ErrorContext(ctx, "DescribeListeners: failed to list listeners", "err", err) @@ -2826,7 +2834,7 @@ func (s *ELBv2ServiceImpl) DescribeTags(ctx context.Context, input *elbv2.Descri switch resourceType { case elbv2ResourceLoadBalancer: notFoundError = awserrors.ErrorELBv2LoadBalancerNotFound - lb, lbErr := s.store.GetLoadBalancerByArn(arn) + lb, lbErr := s.store.GetLoadBalancerByArn(ctx, arn) if lbErr != nil { slog.ErrorContext(ctx, "DescribeTags: failed to get LB", "arn", arn, "err", lbErr) return nil, errors.New(awserrors.ErrorServerInternal) @@ -2838,7 +2846,7 @@ func (s *ELBv2ServiceImpl) DescribeTags(ctx context.Context, input *elbv2.Descri } case elbv2ResourceTargetGroup: notFoundError = awserrors.ErrorELBv2TargetGroupNotFound - tg, tgErr := s.store.GetTargetGroupByArn(arn) + tg, tgErr := s.store.GetTargetGroupByArn(ctx, arn) if tgErr != nil { slog.ErrorContext(ctx, "DescribeTags: failed to get target group", "arn", arn, "err", tgErr) return nil, errors.New(awserrors.ErrorServerInternal) @@ -2850,7 +2858,7 @@ func (s *ELBv2ServiceImpl) DescribeTags(ctx context.Context, input *elbv2.Descri } case elbv2ResourceListener: notFoundError = awserrors.ErrorELBv2ListenerNotFound - l, lErr := s.store.GetListenerByArn(arn) + l, lErr := s.store.GetListenerByArn(ctx, arn) if lErr != nil { slog.ErrorContext(ctx, "DescribeTags: failed to get listener", "arn", arn, "err", lErr) return nil, errors.New(awserrors.ErrorServerInternal) @@ -2862,7 +2870,7 @@ func (s *ELBv2ServiceImpl) DescribeTags(ctx context.Context, input *elbv2.Descri } case elbv2ResourceListenerRule: notFoundError = awserrors.ErrorELBv2RuleNotFound - r, rErr := s.store.GetRuleByArn(arn) + r, rErr := s.store.GetRuleByArn(ctx, arn) if rErr != nil { slog.ErrorContext(ctx, "DescribeTags: failed to get rule", "arn", arn, "err", rErr) return nil, errors.New(awserrors.ErrorServerInternal) @@ -2875,7 +2883,7 @@ func (s *ELBv2ServiceImpl) DescribeTags(ctx context.Context, input *elbv2.Descri // Synthetic default rule: not stored, carries no tags. Resolve via // its parent listener so a controller's post-create rule-tag sync // gets an empty TagDescription instead of an error. - l, lErr := s.store.GetListenerByArn(lArn) + l, lErr := s.store.GetListenerByArn(ctx, lArn) if lErr != nil { slog.ErrorContext(ctx, "DescribeTags: failed to get listener", "arn", lArn, "err", lErr) return nil, errors.New(awserrors.ErrorServerInternal) diff --git a/spinifex/handlers/elbv2/service_impl_attributes.go b/spinifex/handlers/elbv2/service_impl_attributes.go index 2ccc60970..f18e13bab 100644 --- a/spinifex/handlers/elbv2/service_impl_attributes.go +++ b/spinifex/handlers/elbv2/service_impl_attributes.go @@ -158,14 +158,16 @@ func sdkAttrsOut[A any](pairs []attrPair, mk func(key, value string) A) []A { return out } -func (s *ELBv2ServiceImpl) tgAttrResource(arn, accountID, opName string) attrResource[TargetGroupRecord] { +func (s *ELBv2ServiceImpl) tgAttrResource(ctx context.Context, arn, accountID, opName string) attrResource[TargetGroupRecord] { return attrResource[TargetGroupRecord]{ - arn: arn, - accountID: accountID, - opName: opName, - notFound: awserrors.ErrorELBv2TargetGroupNotFound, - fetch: s.store.GetTargetGroupByArn, - save: s.store.PutTargetGroup, + arn: arn, + accountID: accountID, + opName: opName, + notFound: awserrors.ErrorELBv2TargetGroupNotFound, + // fetch/save close over ctx: attrResource is consumed by the generic + // attribute helpers, which have no context of their own to pass down. + fetch: func(arn string) (*TargetGroupRecord, error) { return s.store.GetTargetGroupByArn(ctx, arn) }, + save: func(tg *TargetGroupRecord) error { return s.store.PutTargetGroup(ctx, tg) }, accountIDOf: func(r *TargetGroupRecord) string { return r.AccountID }, defaults: func(*TargetGroupRecord) map[string]string { return DefaultTargetGroupAttributes() }, attrsOf: func(r *TargetGroupRecord) map[string]string { return r.Attributes }, @@ -173,14 +175,15 @@ func (s *ELBv2ServiceImpl) tgAttrResource(arn, accountID, opName string) attrRes } } -func (s *ELBv2ServiceImpl) lbAttrResource(arn, accountID, opName string) attrResource[LoadBalancerRecord] { +func (s *ELBv2ServiceImpl) lbAttrResource(ctx context.Context, arn, accountID, opName string) attrResource[LoadBalancerRecord] { return attrResource[LoadBalancerRecord]{ - arn: arn, - accountID: accountID, - opName: opName, - notFound: awserrors.ErrorELBv2LoadBalancerNotFound, - fetch: s.store.GetLoadBalancerByArn, - save: s.store.PutLoadBalancer, + arn: arn, + accountID: accountID, + opName: opName, + notFound: awserrors.ErrorELBv2LoadBalancerNotFound, + // See tgAttrResource: the generic helpers carry no context. + fetch: func(arn string) (*LoadBalancerRecord, error) { return s.store.GetLoadBalancerByArn(ctx, arn) }, + save: func(lb *LoadBalancerRecord) error { return s.store.PutLoadBalancer(ctx, lb) }, accountIDOf: func(r *LoadBalancerRecord) string { return r.AccountID }, defaults: func(r *LoadBalancerRecord) map[string]string { return DefaultLoadBalancerAttributes(r.Type) }, attrsOf: func(r *LoadBalancerRecord) map[string]string { return r.Attributes }, @@ -217,7 +220,7 @@ func (s *ELBv2ServiceImpl) ModifyTargetGroupAttributes(ctx context.Context, inpu const op = "ModifyTargetGroupAttributes" arn := aws.StringValue(input.TargetGroupArn) pairs, raw := sdkAttrsIn(input.Attributes, tgAttrKeyVal, arn, op) - submitted, err := modifyResourceAttributes(s.tgAttrResource(arn, accountID, op), pairs, raw) + submitted, err := modifyResourceAttributes(s.tgAttrResource(ctx, arn, accountID, op), pairs, raw) if err != nil { return nil, err } @@ -230,7 +233,7 @@ func (s *ELBv2ServiceImpl) DescribeTargetGroupAttributes(ctx context.Context, in } const op = "DescribeTargetGroupAttributes" arn := aws.StringValue(input.TargetGroupArn) - pairs, err := describeResourceAttributes(s.tgAttrResource(arn, accountID, op)) + pairs, err := describeResourceAttributes(s.tgAttrResource(ctx, arn, accountID, op)) if err != nil { return nil, err } @@ -244,7 +247,7 @@ func (s *ELBv2ServiceImpl) ModifyLoadBalancerAttributes(ctx context.Context, inp const op = "ModifyLoadBalancerAttributes" arn := aws.StringValue(input.LoadBalancerArn) pairs, raw := sdkAttrsIn(input.Attributes, lbAttrKeyVal, arn, op) - submitted, err := modifyResourceAttributes(s.lbAttrResource(arn, accountID, op), pairs, raw) + submitted, err := modifyResourceAttributes(s.lbAttrResource(ctx, arn, accountID, op), pairs, raw) if err != nil { return nil, err } @@ -257,7 +260,7 @@ func (s *ELBv2ServiceImpl) DescribeLoadBalancerAttributes(ctx context.Context, i } const op = "DescribeLoadBalancerAttributes" arn := aws.StringValue(input.LoadBalancerArn) - pairs, err := describeResourceAttributes(s.lbAttrResource(arn, accountID, op)) + pairs, err := describeResourceAttributes(s.lbAttrResource(ctx, arn, accountID, op)) if err != nil { return nil, err } diff --git a/spinifex/handlers/elbv2/service_impl_health_test.go b/spinifex/handlers/elbv2/service_impl_health_test.go index 0c31f7a23..c7bf78f9a 100644 --- a/spinifex/handlers/elbv2/service_impl_health_test.go +++ b/spinifex/handlers/elbv2/service_impl_health_test.go @@ -20,7 +20,7 @@ func setupTestServiceWithInstance(t *testing.T, instanceID, instanceIP string) ( _, nc, _ := testutil.StartTestJetStream(t) testutil.StubVpcdSGResponder(t, nc) - vpcSvc, err := handlers_ec2_vpc.NewVPCServiceImplWithNATS(nil, nc) + vpcSvc, err := handlers_ec2_vpc.NewVPCServiceImplWithNATS(t.Context(), nil, nc) require.NoError(t, err) elbv2Svc, err := NewELBv2ServiceImplWithNATS(nil, nc) @@ -81,7 +81,7 @@ func TestRegisterTargets_ResolvesPrivateIP(t *testing.T) { require.Len(t, health.TargetHealthDescriptions, 1) // The target should have been registered with the resolved IP in the store - tg, err := svc.store.GetTargetGroupByArn(*tgArn) + tg, err := svc.store.GetTargetGroupByArn(t.Context(), *tgArn) require.NoError(t, err) require.Len(t, tg.Targets, 1) assert.Equal(t, "10.0.1.50", tg.Targets[0].PrivateIP) @@ -103,7 +103,7 @@ func TestRegisterTargets_UnresolvableIP(t *testing.T) { }, testAccountID) require.NoError(t, err) - tg, _ := svc.store.GetTargetGroupByArn(*tgArn) + tg, _ := svc.store.GetTargetGroupByArn(t.Context(), *tgArn) require.Len(t, tg.Targets, 1) assert.Empty(t, tg.Targets[0].PrivateIP) } @@ -124,7 +124,7 @@ func TestRegisterTargets_WithoutVPCService(t *testing.T) { }, testAccountID) require.NoError(t, err) - tg, _ := svc.store.GetTargetGroupByArn(*tgArn) + tg, _ := svc.store.GetTargetGroupByArn(t.Context(), *tgArn) assert.Empty(t, tg.Targets[0].PrivateIP) } @@ -156,7 +156,7 @@ func TestRegisterTargets_MultipleInstances(t *testing.T) { }, testAccountID) require.NoError(t, err) - tg, _ := svc.store.GetTargetGroupByArn(*tgArn) + tg, _ := svc.store.GetTargetGroupByArn(t.Context(), *tgArn) require.Len(t, tg.Targets, 2) assert.Equal(t, "10.0.1.50", tg.Targets[0].PrivateIP) assert.Equal(t, "10.0.1.51", tg.Targets[1].PrivateIP) @@ -181,11 +181,11 @@ func TestResetTargetHealthOnStartup_TransitionsAllNonDrainingToInitial(t *testin {Id: "i-i", HealthState: TargetHealthInitial, PrivateIP: "10.0.0.4"}, }, } - require.NoError(t, store.PutTargetGroup(tg)) + require.NoError(t, store.PutTargetGroup(t.Context(), tg)) require.NoError(t, svc.ResetTargetHealthOnStartup(t.Context())) - got, err := store.GetTargetGroup("tg-reset") + got, err := store.GetTargetGroup(t.Context(), "tg-reset") require.NoError(t, err) require.NotNil(t, got) require.Len(t, got.Targets, 4) diff --git a/spinifex/handlers/elbv2/service_impl_lb_network.go b/spinifex/handlers/elbv2/service_impl_lb_network.go index 88fd0c638..9f5ff6067 100644 --- a/spinifex/handlers/elbv2/service_impl_lb_network.go +++ b/spinifex/handlers/elbv2/service_impl_lb_network.go @@ -24,7 +24,7 @@ func (s *ELBv2ServiceImpl) SetIpAddressType(ctx context.Context, input *elbv2.Se return nil, errors.New(awserrors.ErrorELBv2InvalidConfigurationRequest) } - lb, err := s.store.GetLoadBalancerByArn(*input.LoadBalancerArn) + lb, err := s.store.GetLoadBalancerByArn(ctx, *input.LoadBalancerArn) if err != nil { slog.ErrorContext(ctx, "SetIpAddressType: failed to get LB", "arn", *input.LoadBalancerArn, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) @@ -35,7 +35,7 @@ func (s *ELBv2ServiceImpl) SetIpAddressType(ctx context.Context, input *elbv2.Se if lb.IPAddressType != IPAddressTypeIPv4 { lb.IPAddressType = IPAddressTypeIPv4 - if err := s.store.PutLoadBalancer(lb); err != nil { + if err := s.store.PutLoadBalancer(ctx, lb); err != nil { slog.ErrorContext(ctx, "SetIpAddressType: failed to persist LB", "arn", *input.LoadBalancerArn, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } @@ -56,7 +56,7 @@ func (s *ELBv2ServiceImpl) SetSecurityGroups(ctx context.Context, input *elbv2.S return nil, errors.New(awserrors.ErrorMissingParameter) } - lb, err := s.store.GetLoadBalancerByArn(*input.LoadBalancerArn) + lb, err := s.store.GetLoadBalancerByArn(ctx, *input.LoadBalancerArn) if err != nil { slog.ErrorContext(ctx, "SetSecurityGroups: failed to get LB", "arn", *input.LoadBalancerArn, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) @@ -98,7 +98,7 @@ func (s *ELBv2ServiceImpl) SetSecurityGroups(ctx context.Context, input *elbv2.S } lb.SecurityGroups = sgs - if err := s.store.PutLoadBalancer(lb); err != nil { + if err := s.store.PutLoadBalancer(ctx, lb); err != nil { slog.ErrorContext(ctx, "SetSecurityGroups: failed to persist LB", "arn", *input.LoadBalancerArn, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } @@ -120,7 +120,7 @@ func (s *ELBv2ServiceImpl) SetSubnets(ctx context.Context, input *elbv2.SetSubne return nil, errors.New(awserrors.ErrorMissingParameter) } - lb, err := s.store.GetLoadBalancerByArn(*input.LoadBalancerArn) + lb, err := s.store.GetLoadBalancerByArn(ctx, *input.LoadBalancerArn) if err != nil { slog.ErrorContext(ctx, "SetSubnets: failed to get LB", "arn", *input.LoadBalancerArn, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) @@ -155,7 +155,7 @@ func (s *ELBv2ServiceImpl) SetSubnets(ctx context.Context, input *elbv2.SetSubne if s.VPCService == nil { lb.Subnets = desired lb.AvailZones = rebuildAvailZones(desired, lb.AvailZones, nil) - if err := s.store.PutLoadBalancer(lb); err != nil { + if err := s.store.PutLoadBalancer(ctx, lb); err != nil { slog.ErrorContext(ctx, "SetSubnets: failed to persist LB", "arn", *input.LoadBalancerArn, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } @@ -235,7 +235,7 @@ func (s *ELBv2ServiceImpl) SetSubnets(ctx context.Context, input *elbv2.SetSubne lb.HostPorts = launch.hostPorts lb.State, lb.StateReason = s.lbStateAfterLaunch(launch, lb.Scheme) - if err := s.store.PutLoadBalancer(lb); err != nil { + if err := s.store.PutLoadBalancer(ctx, lb); err != nil { slog.ErrorContext(ctx, "SetSubnets: failed to persist LB", "arn", *input.LoadBalancerArn, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } diff --git a/spinifex/handlers/elbv2/service_impl_lb_network_test.go b/spinifex/handlers/elbv2/service_impl_lb_network_test.go index 2f7e1edac..2b70ee761 100644 --- a/spinifex/handlers/elbv2/service_impl_lb_network_test.go +++ b/spinifex/handlers/elbv2/service_impl_lb_network_test.go @@ -144,7 +144,7 @@ func TestSetSecurityGroups_NLBWithSGs_Replaces(t *testing.T) { require.NoError(t, err) assert.Equal(t, []string{"sg-new1", "sg-new2"}, aws.StringValueSlice(res.SecurityGroupIds)) - rec, err := svc.store.GetLoadBalancerByArn(arn) + rec, err := svc.store.GetLoadBalancerByArn(t.Context(), arn) require.NoError(t, err) assert.Equal(t, []string{"sg-new1", "sg-new2"}, rec.SecurityGroups) } diff --git a/spinifex/handlers/elbv2/service_impl_listener_certs.go b/spinifex/handlers/elbv2/service_impl_listener_certs.go index 9eb84e885..ae27f37c3 100644 --- a/spinifex/handlers/elbv2/service_impl_listener_certs.go +++ b/spinifex/handlers/elbv2/service_impl_listener_certs.go @@ -32,7 +32,7 @@ func (s *ELBv2ServiceImpl) AddListenerCertificates(ctx context.Context, input *e return nil, errors.New(awserrors.ErrorMissingParameter) } - listener, err := s.store.GetListenerByArn(*input.ListenerArn) + listener, err := s.store.GetListenerByArn(ctx, *input.ListenerArn) if err != nil { slog.ErrorContext(ctx, "AddListenerCertificates: failed to get listener", "arn", *input.ListenerArn, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) @@ -62,11 +62,11 @@ func (s *ELBv2ServiceImpl) AddListenerCertificates(ctx context.Context, input *e } updated.Certificates = certs - if err := s.validateListenerCerts(certs, accountID); err != nil { + if err := s.validateListenerCerts(ctx, certs, accountID); err != nil { return nil, err } - if err := s.store.PutListener(&updated); err != nil { + if err := s.store.PutListener(ctx, &updated); err != nil { slog.ErrorContext(ctx, "AddListenerCertificates: failed to persist record", "listenerId", updated.ListenerID, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } @@ -87,7 +87,7 @@ func (s *ELBv2ServiceImpl) RemoveListenerCertificates(ctx context.Context, input return nil, errors.New(awserrors.ErrorMissingParameter) } - listener, err := s.store.GetListenerByArn(*input.ListenerArn) + listener, err := s.store.GetListenerByArn(ctx, *input.ListenerArn) if err != nil { slog.ErrorContext(ctx, "RemoveListenerCertificates: failed to get listener", "arn", *input.ListenerArn, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) @@ -118,7 +118,7 @@ func (s *ELBv2ServiceImpl) RemoveListenerCertificates(ctx context.Context, input } updated.Certificates = kept - if err := s.store.PutListener(&updated); err != nil { + if err := s.store.PutListener(ctx, &updated); err != nil { slog.ErrorContext(ctx, "RemoveListenerCertificates: failed to persist record", "listenerId", updated.ListenerID, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } @@ -132,7 +132,7 @@ func (s *ELBv2ServiceImpl) DescribeListenerCertificates(ctx context.Context, inp return nil, errors.New(awserrors.ErrorMissingParameter) } - listener, err := s.store.GetListenerByArn(*input.ListenerArn) + listener, err := s.store.GetListenerByArn(ctx, *input.ListenerArn) if err != nil { slog.ErrorContext(ctx, "DescribeListenerCertificates: failed to get listener", "arn", *input.ListenerArn, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) diff --git a/spinifex/handlers/elbv2/service_impl_nlb_sg.go b/spinifex/handlers/elbv2/service_impl_nlb_sg.go index 949d3097e..5404aac3f 100644 --- a/spinifex/handlers/elbv2/service_impl_nlb_sg.go +++ b/spinifex/handlers/elbv2/service_impl_nlb_sg.go @@ -192,11 +192,14 @@ func (s *ELBv2ServiceImpl) revokeNLBListenerPort(ctx context.Context, lb *LoadBa // SetLoadBalancerIngressCIDRs overrides the client CIDRs for an NLB's listener ports, // revoking old rules and authorizing new ones. Passing nil/empty reverts to the // scheme-based default (0.0.0.0/0 internet-facing, VPC CIDR internal). +// +// EKS drives this through a narrow interface that predates context threading, so +// the KV reads run on the service lifetime context rather than the caller's. func (s *ELBv2ServiceImpl) SetLoadBalancerIngressCIDRs(lbArn string, cidrs []string, accountID string) error { if lbArn == "" { return errors.New(awserrors.ErrorMissingParameter) } - lb, err := s.store.GetLoadBalancerByArn(lbArn) + lb, err := s.store.GetLoadBalancerByArn(s.ctx, lbArn) if err != nil { slog.Error("SetLoadBalancerIngressCIDRs: failed to get LB", "arn", lbArn, "err", err) return errors.New(awserrors.ErrorServerInternal) @@ -218,7 +221,7 @@ func (s *ELBv2ServiceImpl) SetLoadBalancerIngressCIDRs(lbArn string, cidrs []str slog.Error("SetLoadBalancerIngressCIDRs: resolve old CIDRs failed", "arn", lbArn, "err", err) return errors.New(awserrors.ErrorServerInternal) } - listeners, err := s.store.ListListenersByLB(lb.LoadBalancerArn) + listeners, err := s.store.ListListenersByLB(s.ctx, lb.LoadBalancerArn) if err != nil { slog.Error("SetLoadBalancerIngressCIDRs: list listeners failed", "arn", lbArn, "err", err) return errors.New(awserrors.ErrorServerInternal) @@ -242,7 +245,7 @@ func (s *ELBv2ServiceImpl) SetLoadBalancerIngressCIDRs(lbArn string, cidrs []str } } - if err := s.store.PutLoadBalancer(lb); err != nil { + if err := s.store.PutLoadBalancer(s.ctx, lb); err != nil { slog.Error("SetLoadBalancerIngressCIDRs: persist failed", "arn", lbArn, "err", err) return errors.New(awserrors.ErrorServerInternal) } diff --git a/spinifex/handlers/elbv2/service_impl_nlb_sg_test.go b/spinifex/handlers/elbv2/service_impl_nlb_sg_test.go index b80aa30d4..2047be507 100644 --- a/spinifex/handlers/elbv2/service_impl_nlb_sg_test.go +++ b/spinifex/handlers/elbv2/service_impl_nlb_sg_test.go @@ -137,7 +137,7 @@ func TestCreateNLB_MintsManagedSGAttachedToENI(t *testing.T) { lb := createNLB(t, svc, "nlb-sg", "internal", subnetID) assert.Empty(t, lb.SecurityGroups, "NLB must not surface the managed SG as a customer SG") - rec, err := svc.store.GetLoadBalancerByArn(*lb.LoadBalancerArn) + rec, err := svc.store.GetLoadBalancerByArn(t.Context(), *lb.LoadBalancerArn) require.NoError(t, err) require.NotEmpty(t, rec.NLBManagedSGID, "NLB record must carry the managed SG id") assert.NotEqual(t, defaultSGID, rec.NLBManagedSGID, "managed SG must be distinct from the VPC default SG") @@ -186,7 +186,7 @@ func TestCreateNLB_WithCustomerSGs_AttachesThemNoManagedSG(t *testing.T) { }, testAccountID) require.NoError(t, err) - rec, err := svc.store.GetLoadBalancerByArn(*out.LoadBalancers[0].LoadBalancerArn) + rec, err := svc.store.GetLoadBalancerByArn(t.Context(), *out.LoadBalancers[0].LoadBalancerArn) require.NoError(t, err) assert.Equal(t, []string{sgID}, rec.SecurityGroups) assert.Empty(t, rec.NLBManagedSGID, "NLB with customer SGs must not mint a managed SG") @@ -209,7 +209,7 @@ func TestCreateALB_NoManagedSG(t *testing.T) { }, testAccountID) require.NoError(t, err) - rec, err := svc.store.GetLoadBalancerByArn(*out.LoadBalancers[0].LoadBalancerArn) + rec, err := svc.store.GetLoadBalancerByArn(t.Context(), *out.LoadBalancers[0].LoadBalancerArn) require.NoError(t, err) assert.Empty(t, rec.NLBManagedSGID, "ALB must not mint a managed NLB SG") } @@ -223,7 +223,7 @@ func TestCreateListener_Internal_OpensPortFromVPCCIDR(t *testing.T) { lb := createNLB(t, svc, "nlb-int", "internal", subnetID) createTCPListener(t, svc, lb.LoadBalancerArn, 443) - rec, err := svc.store.GetLoadBalancerByArn(*lb.LoadBalancerArn) + rec, err := svc.store.GetLoadBalancerByArn(t.Context(), *lb.LoadBalancerArn) require.NoError(t, err) sg := describeSG(t, vpcSvc, rec.NLBManagedSGID) assert.True(t, sgHasRule(sg, "tcp", 443, "10.0.0.0/16"), @@ -240,7 +240,7 @@ func TestCreateListener_InternetFacing_OpensPortFromAnywhere(t *testing.T) { lb := createNLB(t, svc, "nlb-pub", "internet-facing", subnetID) createTCPListener(t, svc, lb.LoadBalancerArn, 443) - rec, err := svc.store.GetLoadBalancerByArn(*lb.LoadBalancerArn) + rec, err := svc.store.GetLoadBalancerByArn(t.Context(), *lb.LoadBalancerArn) require.NoError(t, err) sg := describeSG(t, vpcSvc, rec.NLBManagedSGID) assert.True(t, sgHasRule(sg, "tcp", 443, "0.0.0.0/0"), @@ -268,7 +268,7 @@ func TestCreateListener_TCPUDP_OpensBothProtocols(t *testing.T) { }, testAccountID) require.NoError(t, err) - rec, err := svc.store.GetLoadBalancerByArn(*lb.LoadBalancerArn) + rec, err := svc.store.GetLoadBalancerByArn(t.Context(), *lb.LoadBalancerArn) require.NoError(t, err) sg := describeSG(t, vpcSvc, rec.NLBManagedSGID) assert.True(t, sgHasRule(sg, "tcp", 53, "0.0.0.0/0")) @@ -285,7 +285,7 @@ func TestSetLoadBalancerIngressCIDRs_RewritesListenerRules(t *testing.T) { lb := createNLB(t, svc, "nlb-cidr", "internet-facing", subnetID) createTCPListener(t, svc, lb.LoadBalancerArn, 443) - rec, err := svc.store.GetLoadBalancerByArn(*lb.LoadBalancerArn) + rec, err := svc.store.GetLoadBalancerByArn(t.Context(), *lb.LoadBalancerArn) require.NoError(t, err) sgID := rec.NLBManagedSGID @@ -304,7 +304,7 @@ func TestSetLoadBalancerIngressCIDRs_RewritesListenerRules(t *testing.T) { assert.True(t, sgHasRule(sg, "tcp", 443, "203.0.113.0/24")) // Persisted override drives the resolved CIDRs. - rec, err = svc.store.GetLoadBalancerByArn(*lb.LoadBalancerArn) + rec, err = svc.store.GetLoadBalancerByArn(t.Context(), *lb.LoadBalancerArn) require.NoError(t, err) assert.Equal(t, []string{"203.0.113.0/24"}, rec.NLBIngressCIDRs) } @@ -340,7 +340,7 @@ func TestDeleteListener_RevokesPort(t *testing.T) { require.NoError(t, err) require.Len(t, lst.Listeners, 1) - rec, err := svc.store.GetLoadBalancerByArn(*lb.LoadBalancerArn) + rec, err := svc.store.GetLoadBalancerByArn(t.Context(), *lb.LoadBalancerArn) require.NoError(t, err) require.True(t, sgHasRule(describeSG(t, vpcSvc, rec.NLBManagedSGID), "tcp", 443, "0.0.0.0/0")) @@ -358,7 +358,7 @@ func TestDeleteNLB_DeletesManagedSG(t *testing.T) { subnetID, _ := firstSubnet(t, vpcSvc) lb := createNLB(t, svc, "nlb-del", "internal", subnetID) - rec, err := svc.store.GetLoadBalancerByArn(*lb.LoadBalancerArn) + rec, err := svc.store.GetLoadBalancerByArn(t.Context(), *lb.LoadBalancerArn) require.NoError(t, err) sgID := rec.NLBManagedSGID require.NotEmpty(t, sgID) @@ -389,7 +389,7 @@ func TestCreateLoadBalancerSync_LaunchFailure_RollsBackEverything(t *testing.T) _, err := svc.CreateLoadBalancerSync(nlbSyncInput("nlb-sync-fail", subnetID), testAccountID) require.Error(t, err) - lbs, err := svc.store.ListLoadBalancers() + lbs, err := svc.store.ListLoadBalancers(t.Context()) require.NoError(t, err) assert.Empty(t, lbs, "failed sync create must leave no LB record") assert.Equal(t, 0, countManagedENIs(t, vpcSvc), "failed sync create must leave no managed ENI") @@ -438,7 +438,7 @@ func TestCreateLoadBalancer_AsyncLaunchFailure_KeepsFailedRecord(t *testing.T) { arn := *out.LoadBalancers[0].LoadBalancerArn svc.WaitLaunches() - rec, err := svc.store.GetLoadBalancerByArn(arn) + rec, err := svc.store.GetLoadBalancerByArn(t.Context(), arn) require.NoError(t, err) require.NotNil(t, rec, "async failed record must remain for the caller to reclaim") assert.Equal(t, StateFailed, rec.State) diff --git a/spinifex/handlers/elbv2/service_impl_rollback_test.go b/spinifex/handlers/elbv2/service_impl_rollback_test.go index ebbaad867..352aa78fe 100644 --- a/spinifex/handlers/elbv2/service_impl_rollback_test.go +++ b/spinifex/handlers/elbv2/service_impl_rollback_test.go @@ -34,7 +34,7 @@ func TestRollbackLBInfra_RemovesAllInfra(t *testing.T) { subnetID, _ := firstSubnet(t, vpcSvc) lb := createNLB(t, svc, "nlb-rollback", "internal", subnetID) - rec, err := svc.store.GetLoadBalancerByArn(*lb.LoadBalancerArn) + rec, err := svc.store.GetLoadBalancerByArn(t.Context(), *lb.LoadBalancerArn) require.NoError(t, err) sgID := rec.NLBManagedSGID require.NotEmpty(t, sgID) @@ -59,7 +59,7 @@ func TestRollbackLBInfra_RemovesAllInfra(t *testing.T) { } // Name claim released → reclaimable by a fresh owner. - ok, dup, err := svc.store.ClaimLBName("nlb-rollback", testAccountID, "lb-fresh") + ok, dup, err := svc.store.ClaimLBName(t.Context(), "nlb-rollback", testAccountID, "lb-fresh") require.NoError(t, err) assert.True(t, ok, "rollback must release the name claim") assert.False(t, dup) @@ -75,14 +75,14 @@ func TestRollbackListener_RevokesPortAndDeletesRecord(t *testing.T) { lb := createNLB(t, svc, "nlb-lst-rb", "internet-facing", subnetID) createTCPListener(t, svc, lb.LoadBalancerArn, 443) - rec, err := svc.store.GetLoadBalancerByArn(*lb.LoadBalancerArn) + rec, err := svc.store.GetLoadBalancerByArn(t.Context(), *lb.LoadBalancerArn) require.NoError(t, err) require.True(t, sgHasRule(describeSG(t, vpcSvc, rec.NLBManagedSGID), "tcp", 443, "0.0.0.0/0")) lst, err := svc.DescribeListeners(context.Background(), &elbv2.DescribeListenersInput{LoadBalancerArn: lb.LoadBalancerArn}, testAccountID) require.NoError(t, err) require.Len(t, lst.Listeners, 1) - listenerRec, err := svc.store.GetListenerByArn(*lst.Listeners[0].ListenerArn) + listenerRec, err := svc.store.GetListenerByArn(t.Context(), *lst.Listeners[0].ListenerArn) require.NoError(t, err) // authorizedCIDRs non-empty ⇒ the config-failure path that already opened the port. @@ -106,13 +106,13 @@ func TestRollbackListener_NilCIDRsSkipsRevoke(t *testing.T) { lb := createNLB(t, svc, "nlb-lst-nil", "internet-facing", subnetID) createTCPListener(t, svc, lb.LoadBalancerArn, 443) - rec, err := svc.store.GetLoadBalancerByArn(*lb.LoadBalancerArn) + rec, err := svc.store.GetLoadBalancerByArn(t.Context(), *lb.LoadBalancerArn) require.NoError(t, err) lst, err := svc.DescribeListeners(context.Background(), &elbv2.DescribeListenersInput{LoadBalancerArn: lb.LoadBalancerArn}, testAccountID) require.NoError(t, err) require.Len(t, lst.Listeners, 1) - listenerRec, err := svc.store.GetListenerByArn(*lst.Listeners[0].ListenerArn) + listenerRec, err := svc.store.GetListenerByArn(t.Context(), *lst.Listeners[0].ListenerArn) require.NoError(t, err) svc.rollbackListener(context.Background(), listenerRec, rec, "tcp", 443, nil, testAccountID) diff --git a/spinifex/handlers/elbv2/service_impl_rules.go b/spinifex/handlers/elbv2/service_impl_rules.go index f12ed07d3..763bd3e29 100644 --- a/spinifex/handlers/elbv2/service_impl_rules.go +++ b/spinifex/handlers/elbv2/service_impl_rules.go @@ -40,7 +40,7 @@ func (s *ELBv2ServiceImpl) CreateRule(ctx context.Context, input *elbv2.CreateRu return nil, errors.New(awserrors.ErrorMissingParameter) } - listener, err := s.store.GetListenerByArn(*input.ListenerArn) + listener, err := s.store.GetListenerByArn(ctx, *input.ListenerArn) if err != nil { slog.ErrorContext(ctx, "CreateRule: failed to get listener", "arn", *input.ListenerArn, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) @@ -59,12 +59,12 @@ func (s *ELBv2ServiceImpl) CreateRule(ctx context.Context, input *elbv2.CreateRu return nil, err } - actions, err := s.validateAndConvertRuleActions(input.Actions, listener.Protocol) + actions, err := s.validateAndConvertRuleActions(ctx, input.Actions, listener.Protocol) if err != nil { return nil, err } - existing, err := s.store.ListRulesByListener(listener.ListenerArn) + existing, err := s.store.ListRulesByListener(ctx, listener.ListenerArn) if err != nil { slog.ErrorContext(ctx, "CreateRule: failed to list rules", "listenerArn", listener.ListenerArn, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) @@ -99,7 +99,7 @@ func (s *ELBv2ServiceImpl) CreateRule(ctx context.Context, input *elbv2.CreateRu Tags: tags, } - if err := s.store.PutRule(record); err != nil { + if err := s.store.PutRule(ctx, record); err != nil { slog.ErrorContext(ctx, "CreateRule: failed to persist", "ruleId", ruleID, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } @@ -120,7 +120,7 @@ func (s *ELBv2ServiceImpl) ModifyRule(ctx context.Context, input *elbv2.ModifyRu return nil, errors.New(awserrors.ErrorMissingParameter) } - rule, err := s.store.GetRuleByArn(*input.RuleArn) + rule, err := s.store.GetRuleByArn(ctx, *input.RuleArn) if err != nil { slog.ErrorContext(ctx, "ModifyRule: failed to get rule", "arn", *input.RuleArn, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) @@ -129,7 +129,7 @@ func (s *ELBv2ServiceImpl) ModifyRule(ctx context.Context, input *elbv2.ModifyRu return nil, errors.New(awserrors.ErrorELBv2RuleNotFound) } - listener, err := s.store.GetListenerByArn(rule.ListenerArn) + listener, err := s.store.GetListenerByArn(ctx, rule.ListenerArn) if err != nil || listener == nil { slog.ErrorContext(ctx, "ModifyRule: failed to get listener", "arn", rule.ListenerArn, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) @@ -144,14 +144,14 @@ func (s *ELBv2ServiceImpl) ModifyRule(ctx context.Context, input *elbv2.ModifyRu updated.Conditions = conditions } if len(input.Actions) > 0 { - actions, err := s.validateAndConvertRuleActions(input.Actions, listener.Protocol) + actions, err := s.validateAndConvertRuleActions(ctx, input.Actions, listener.Protocol) if err != nil { return nil, err } updated.Actions = actions } - if err := s.store.PutRule(&updated); err != nil { + if err := s.store.PutRule(ctx, &updated); err != nil { slog.ErrorContext(ctx, "ModifyRule: failed to persist", "ruleId", updated.RuleID, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } @@ -172,7 +172,7 @@ func (s *ELBv2ServiceImpl) DeleteRule(ctx context.Context, input *elbv2.DeleteRu return nil, errors.New(awserrors.ErrorMissingParameter) } - rule, err := s.store.GetRuleByArn(*input.RuleArn) + rule, err := s.store.GetRuleByArn(ctx, *input.RuleArn) if err != nil { slog.ErrorContext(ctx, "DeleteRule: failed to get rule", "arn", *input.RuleArn, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) @@ -183,13 +183,13 @@ func (s *ELBv2ServiceImpl) DeleteRule(ctx context.Context, input *elbv2.DeleteRu return &elbv2.DeleteRuleOutput{}, nil } - listener, err := s.store.GetListenerByArn(rule.ListenerArn) + listener, err := s.store.GetListenerByArn(ctx, rule.ListenerArn) if err != nil { slog.ErrorContext(ctx, "DeleteRule: failed to get listener for reload", "listenerArn", rule.ListenerArn, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } - if err := s.store.DeleteRule(rule.RuleID); err != nil { + if err := s.store.DeleteRule(ctx, rule.RuleID); err != nil { slog.ErrorContext(ctx, "DeleteRule: failed to delete", "ruleId", rule.RuleID, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } @@ -214,7 +214,7 @@ func (s *ELBv2ServiceImpl) DescribeRules(ctx context.Context, input *elbv2.Descr switch { case input.ListenerArn != nil && *input.ListenerArn != "": - listener, err := s.store.GetListenerByArn(*input.ListenerArn) + listener, err := s.store.GetListenerByArn(ctx, *input.ListenerArn) if err != nil { slog.ErrorContext(ctx, "DescribeRules: failed to get listener", "arn", *input.ListenerArn, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) @@ -222,7 +222,7 @@ func (s *ELBv2ServiceImpl) DescribeRules(ctx context.Context, input *elbv2.Descr if listener == nil || listener.AccountID != accountID { return nil, errors.New(awserrors.ErrorELBv2ListenerNotFound) } - rules, err := s.store.ListRulesByListener(listener.ListenerArn) + rules, err := s.store.ListRulesByListener(ctx, listener.ListenerArn) if err != nil { slog.ErrorContext(ctx, "DescribeRules: failed to list", "listenerArn", listener.ListenerArn, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) @@ -238,7 +238,7 @@ func (s *ELBv2ServiceImpl) DescribeRules(ctx context.Context, input *elbv2.Descr if arnPtr == nil || *arnPtr == "" { continue } - r, err := s.store.GetRuleByArn(*arnPtr) + r, err := s.store.GetRuleByArn(ctx, *arnPtr) if err != nil { slog.ErrorContext(ctx, "DescribeRules: failed to get rule", "arn", *arnPtr, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) @@ -246,7 +246,7 @@ func (s *ELBv2ServiceImpl) DescribeRules(ctx context.Context, input *elbv2.Descr if r == nil { // Synthetic default rule: resolve by its parent listener. if lArn, isDefault := listenerArnFromDefaultRuleArn(*arnPtr); isDefault { - l, lErr := s.store.GetListenerByArn(lArn) + l, lErr := s.store.GetListenerByArn(ctx, lArn) if lErr != nil { slog.ErrorContext(ctx, "DescribeRules: failed to get listener", "arn", lArn, "err", lErr) return nil, errors.New(awserrors.ErrorServerInternal) @@ -292,7 +292,7 @@ func (s *ELBv2ServiceImpl) SetRulePriorities(ctx context.Context, input *elbv2.S if newPriority < RuleMinPriority || newPriority > RuleMaxPriority { return nil, errors.New(awserrors.ErrorELBv2InvalidRulePriority) } - r, err := s.store.GetRuleByArn(*rp.RuleArn) + r, err := s.store.GetRuleByArn(ctx, *rp.RuleArn) if err != nil { slog.ErrorContext(ctx, "SetRulePriorities: failed to get rule", "arn", *rp.RuleArn, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) @@ -318,7 +318,7 @@ func (s *ELBv2ServiceImpl) SetRulePriorities(ctx context.Context, input *elbv2.S } // Detect collisions with rules not being renumbered in this call. - all, err := s.store.ListRulesByListener(listenerArn) + all, err := s.store.ListRulesByListener(ctx, listenerArn) if err != nil { slog.ErrorContext(ctx, "SetRulePriorities: failed to list rules", "listenerArn", listenerArn, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) @@ -340,14 +340,14 @@ func (s *ELBv2ServiceImpl) SetRulePriorities(ctx context.Context, input *elbv2.S for _, p := range pending { next := *p.rule next.Priority = p.newPriority - if err := s.store.PutRule(&next); err != nil { + if err := s.store.PutRule(ctx, &next); err != nil { slog.ErrorContext(ctx, "SetRulePriorities: failed to persist", "ruleId", next.RuleID, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } updated = append(updated, ruleRecordToSDK(&next)) } - listener, err := s.store.GetListenerByArn(listenerArn) + listener, err := s.store.GetListenerByArn(ctx, listenerArn) if err == nil && listener != nil { if err := s.reloadListenerLB(ctx, listener); err != nil { return nil, err @@ -362,7 +362,7 @@ func (s *ELBv2ServiceImpl) SetRulePriorities(ctx context.Context, input *elbv2.S // reloadListenerLB resolves the load balancer that owns the listener and // regenerates its HAProxy config. func (s *ELBv2ServiceImpl) reloadListenerLB(ctx context.Context, listener *ListenerRecord) error { - lb, err := s.store.GetLoadBalancerByArn(listener.LoadBalancerArn) + lb, err := s.store.GetLoadBalancerByArn(ctx, listener.LoadBalancerArn) if err != nil { slog.ErrorContext(ctx, "rule reload: failed to get LB", "arn", listener.LoadBalancerArn, "err", err) return errors.New(awserrors.ErrorServerInternal) @@ -556,7 +556,7 @@ func stringPtrSlice(in []*string) []string { return out } -func (s *ELBv2ServiceImpl) validateAndConvertRuleActions(in []*elbv2.Action, listenerProto string) ([]ListenerAction, error) { +func (s *ELBv2ServiceImpl) validateAndConvertRuleActions(ctx context.Context, in []*elbv2.Action, listenerProto string) ([]ListenerAction, error) { if len(in) == 0 { return nil, errors.New(awserrors.ErrorMissingParameter) } @@ -575,7 +575,7 @@ func (s *ELBv2ServiceImpl) validateAndConvertRuleActions(in []*elbv2.Action, lis if action.TargetGroupArn == "" { return nil, errors.New(awserrors.ErrorMissingParameter) } - tg, err := s.store.GetTargetGroupByArn(action.TargetGroupArn) + tg, err := s.store.GetTargetGroupByArn(ctx, action.TargetGroupArn) if err != nil { slog.Error("rule action: failed to get target group", "arn", action.TargetGroupArn, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) diff --git a/spinifex/handlers/elbv2/service_impl_rules_extra_test.go b/spinifex/handlers/elbv2/service_impl_rules_extra_test.go index 2bf98d8a9..a51d83e49 100644 --- a/spinifex/handlers/elbv2/service_impl_rules_extra_test.go +++ b/spinifex/handlers/elbv2/service_impl_rules_extra_test.go @@ -411,11 +411,11 @@ func TestTargetGroupsForLB_IncludesRuleTGs(t *testing.T) { }, testAccountID) require.NoError(t, err) - lb, err := env.svc.store.GetLoadBalancerByArn(env.lbArn) + lb, err := env.svc.store.GetLoadBalancerByArn(t.Context(), env.lbArn) require.NoError(t, err) require.NotNil(t, lb) - tgs, err := env.svc.store.TargetGroupsForLB(lb.LoadBalancerID) + tgs, err := env.svc.store.TargetGroupsForLB(t.Context(), lb.LoadBalancerID) require.NoError(t, err) arns := make(map[string]bool, len(tgs)) diff --git a/spinifex/handlers/elbv2/service_impl_rules_test.go b/spinifex/handlers/elbv2/service_impl_rules_test.go index 2f9fb4561..5c49d9069 100644 --- a/spinifex/handlers/elbv2/service_impl_rules_test.go +++ b/spinifex/handlers/elbv2/service_impl_rules_test.go @@ -120,7 +120,7 @@ func TestDeleteLoadBalancer_CascadesRuleDeletion(t *testing.T) { require.NoError(t, err) // LB deletion must cascade rule deletion — no rule may survive. - rules, err := env.svc.store.ListRules() + rules, err := env.svc.store.ListRules(t.Context()) require.NoError(t, err) assert.Empty(t, rules, "LB deletion must cascade rule deletion") @@ -382,7 +382,7 @@ func TestDeleteListener_CascadesRules(t *testing.T) { _, err = env.svc.DeleteListener(context.Background(), &elbv2.DeleteListenerInput{ListenerArn: aws.String(env.listenerArn)}, testAccountID) require.NoError(t, err) - rules, err := env.svc.store.ListRules() + rules, err := env.svc.store.ListRules(t.Context()) require.NoError(t, err) assert.Empty(t, rules) } diff --git a/spinifex/handlers/elbv2/service_impl_tags.go b/spinifex/handlers/elbv2/service_impl_tags.go index e4ba744d7..c4069ebdf 100644 --- a/spinifex/handlers/elbv2/service_impl_tags.go +++ b/spinifex/handlers/elbv2/service_impl_tags.go @@ -21,7 +21,7 @@ type taggable struct { // resolveTaggable loads the ELBv2 resource by ARN and returns a tag read/write handle. // found is false when the resource doesn't exist; notFoundError is the per-type error. // Store failures are logged and returned as ErrorServerInternal. -func (s *ELBv2ServiceImpl) resolveTaggable(arn string) (h taggable, notFoundError string, found bool, err error) { +func (s *ELBv2ServiceImpl) resolveTaggable(ctx context.Context, arn string) (h taggable, notFoundError string, found bool, err error) { resourceType, terr := elbv2ResourceTypeFromArn(arn) if terr != nil { return taggable{}, "", false, terr @@ -30,7 +30,7 @@ func (s *ELBv2ServiceImpl) resolveTaggable(arn string) (h taggable, notFoundErro switch resourceType { case elbv2ResourceLoadBalancer: notFoundError = awserrors.ErrorELBv2LoadBalancerNotFound - lb, e := s.store.GetLoadBalancerByArn(arn) + lb, e := s.store.GetLoadBalancerByArn(ctx, arn) if e != nil { slog.Error("resolveTaggable: failed to get LB", "arn", arn, "err", e) return taggable{}, notFoundError, false, errors.New(awserrors.ErrorServerInternal) @@ -39,12 +39,12 @@ func (s *ELBv2ServiceImpl) resolveTaggable(arn string) (h taggable, notFoundErro found = true h = taggable{tags: lb.Tags, owner: lb.AccountID, save: func(t map[string]string) error { lb.Tags = t - return s.store.PutLoadBalancer(lb) + return s.store.PutLoadBalancer(ctx, lb) }} } case elbv2ResourceTargetGroup: notFoundError = awserrors.ErrorELBv2TargetGroupNotFound - tg, e := s.store.GetTargetGroupByArn(arn) + tg, e := s.store.GetTargetGroupByArn(ctx, arn) if e != nil { slog.Error("resolveTaggable: failed to get target group", "arn", arn, "err", e) return taggable{}, notFoundError, false, errors.New(awserrors.ErrorServerInternal) @@ -53,12 +53,12 @@ func (s *ELBv2ServiceImpl) resolveTaggable(arn string) (h taggable, notFoundErro found = true h = taggable{tags: tg.Tags, owner: tg.AccountID, save: func(t map[string]string) error { tg.Tags = t - return s.store.PutTargetGroup(tg) + return s.store.PutTargetGroup(ctx, tg) }} } case elbv2ResourceListener: notFoundError = awserrors.ErrorELBv2ListenerNotFound - l, e := s.store.GetListenerByArn(arn) + l, e := s.store.GetListenerByArn(ctx, arn) if e != nil { slog.Error("resolveTaggable: failed to get listener", "arn", arn, "err", e) return taggable{}, notFoundError, false, errors.New(awserrors.ErrorServerInternal) @@ -67,12 +67,12 @@ func (s *ELBv2ServiceImpl) resolveTaggable(arn string) (h taggable, notFoundErro found = true h = taggable{tags: l.Tags, owner: l.AccountID, save: func(t map[string]string) error { l.Tags = t - return s.store.PutListener(l) + return s.store.PutListener(ctx, l) }} } case elbv2ResourceListenerRule: notFoundError = awserrors.ErrorELBv2RuleNotFound - r, e := s.store.GetRuleByArn(arn) + r, e := s.store.GetRuleByArn(ctx, arn) if e != nil { slog.Error("resolveTaggable: failed to get rule", "arn", arn, "err", e) return taggable{}, notFoundError, false, errors.New(awserrors.ErrorServerInternal) @@ -81,7 +81,7 @@ func (s *ELBv2ServiceImpl) resolveTaggable(arn string) (h taggable, notFoundErro found = true h = taggable{tags: r.Tags, owner: r.AccountID, save: func(t map[string]string) error { r.Tags = t - return s.store.PutRule(r) + return s.store.PutRule(ctx, r) }} } } @@ -115,7 +115,7 @@ func (s *ELBv2ServiceImpl) AddTags(ctx context.Context, input *elbv2.AddTagsInpu } arn := *arnPtr - h, notFoundError, found, err := s.resolveTaggable(arn) + h, notFoundError, found, err := s.resolveTaggable(ctx, arn) if err != nil { return nil, err } @@ -159,7 +159,7 @@ func (s *ELBv2ServiceImpl) RemoveTags(ctx context.Context, input *elbv2.RemoveTa } arn := *arnPtr - h, notFoundError, found, err := s.resolveTaggable(arn) + h, notFoundError, found, err := s.resolveTaggable(ctx, arn) if err != nil { return nil, err } diff --git a/spinifex/handlers/elbv2/service_impl_test.go b/spinifex/handlers/elbv2/service_impl_test.go index d64cc820d..18d3d2b07 100644 --- a/spinifex/handlers/elbv2/service_impl_test.go +++ b/spinifex/handlers/elbv2/service_impl_test.go @@ -145,7 +145,7 @@ func TestCreateLoadBalancer_NetworkType_AcceptsSecurityGroups(t *testing.T) { }, testAccountID) require.NoError(t, err) - rec, err := svc.store.GetLoadBalancerByName("nlb-with-sg", testAccountID) + rec, err := svc.store.GetLoadBalancerByName(t.Context(), "nlb-with-sg", testAccountID) require.NoError(t, err) assert.Equal(t, []string{"sg-111", "sg-222"}, rec.SecurityGroups) assert.Empty(t, rec.NLBManagedSGID, "NLB created with customer SGs must not mint a managed SG") @@ -176,7 +176,7 @@ func TestCreateLoadBalancer_NLBCrossZoneAttribute(t *testing.T) { Type: aws.String("network"), }, testAccountID) require.NoError(t, err) - nlbRec, err := svc.store.GetLoadBalancerByName("nlb-cz", testAccountID) + nlbRec, err := svc.store.GetLoadBalancerByName(t.Context(), "nlb-cz", testAccountID) require.NoError(t, err) assert.Nil(t, nlbRec.Attributes, "NLB should store nil attributes — defaults come from the handler") @@ -199,7 +199,7 @@ func TestCreateLoadBalancer_ALBCrossZoneAttribute(t *testing.T) { Name: aws.String("alb-cz"), }, testAccountID) require.NoError(t, err) - albRec, err := svc.store.GetLoadBalancerByName("alb-cz", testAccountID) + albRec, err := svc.store.GetLoadBalancerByName(t.Context(), "alb-cz", testAccountID) require.NoError(t, err) assert.Nil(t, albRec.Attributes, "ALB should no longer seed attributes at create time") @@ -818,7 +818,7 @@ func TestRegisterTargets_IPType(t *testing.T) { // ip targets must carry the supplied IP as PrivateIP, not an empty // ENI-resolution result — otherwise HAProxy/health-check silently drop them. - tg, err := svc.store.GetTargetGroupByArn(*tgArn) + tg, err := svc.store.GetTargetGroupByArn(t.Context(), *tgArn) require.NoError(t, err) require.Len(t, tg.Targets, 2) ipByID := make(map[string]string) @@ -1939,7 +1939,7 @@ func TestLBAgentHeartbeat_TransitionsProvisioningToActive(t *testing.T) { VPCIP: "10.0.1.100", AccountID: testAccountID, } - require.NoError(t, svc.store.PutLoadBalancer(lb)) + require.NoError(t, svc.store.PutLoadBalancer(t.Context(), lb)) out, err := svc.LBAgentHeartbeat(context.Background(), &LBAgentHeartbeatInput{ LBID: aws.String("lb-hb1"), @@ -1947,7 +1947,7 @@ func TestLBAgentHeartbeat_TransitionsProvisioningToActive(t *testing.T) { require.NoError(t, err) assert.Equal(t, StateActive, *out.Status) - stored, err := svc.store.GetLoadBalancer("lb-hb1") + stored, err := svc.store.GetLoadBalancer(t.Context(), "lb-hb1") require.NoError(t, err) assert.Equal(t, StateActive, stored.State) assert.False(t, stored.LastHeartbeat.IsZero()) @@ -1965,7 +1965,7 @@ func TestLBAgentHeartbeat_ReturnsConfigHash(t *testing.T) { ConfigHash: "abc123def456", AccountID: testAccountID, } - require.NoError(t, svc.store.PutLoadBalancer(lb)) + require.NoError(t, svc.store.PutLoadBalancer(t.Context(), lb)) out, err := svc.LBAgentHeartbeat(context.Background(), &LBAgentHeartbeatInput{ LBID: aws.String("lb-hash1"), @@ -1985,7 +1985,7 @@ func TestLBAgentHeartbeat_ProcessesHealthReport(t *testing.T) { InstanceID: "i-sys-hr1", AccountID: testAccountID, } - require.NoError(t, svc.store.PutLoadBalancer(lb)) + require.NoError(t, svc.store.PutLoadBalancer(t.Context(), lb)) tg := &TargetGroupRecord{ TargetGroupArn: "arn:aws:elasticloadbalancing:us-east-1:123456789012:targetgroup/health-tg/tg-hr1", @@ -1997,10 +1997,10 @@ func TestLBAgentHeartbeat_ProcessesHealthReport(t *testing.T) { }, AccountID: testAccountID, } - require.NoError(t, svc.store.PutTargetGroup(tg)) + require.NoError(t, svc.store.PutTargetGroup(t.Context(), tg)) // Wire LB → listener → TG so the health checker can resolve the TG from the LBID. - require.NoError(t, svc.store.PutListener(&ListenerRecord{ + require.NoError(t, svc.store.PutListener(t.Context(), &ListenerRecord{ ListenerArn: lb.LoadBalancerArn + "/listener-1", ListenerID: "lst-hr1", LoadBalancerArn: lb.LoadBalancerArn, @@ -2019,7 +2019,7 @@ func TestLBAgentHeartbeat_ProcessesHealthReport(t *testing.T) { }, testAccountID) require.NoError(t, err) - stored, err := svc.store.GetTargetGroup("tg-hr1") + stored, err := svc.store.GetTargetGroup(t.Context(), "tg-hr1") require.NoError(t, err) assert.Equal(t, TargetHealthHealthy, stored.Targets[0].HealthState) } @@ -2041,7 +2041,7 @@ func TestLBAgentHeartbeat_BuildsConfigOnActivation(t *testing.T) { VPCIP: "10.0.1.100", AccountID: testAccountID, } - require.NoError(t, svc.store.PutLoadBalancer(lb)) + require.NoError(t, svc.store.PutLoadBalancer(t.Context(), lb)) tg := &TargetGroupRecord{ TargetGroupArn: "arn:aws:elasticloadbalancing:us-east-1:123456789012:targetgroup/act-tg/tg-act1", @@ -2054,9 +2054,9 @@ func TestLBAgentHeartbeat_BuildsConfigOnActivation(t *testing.T) { }, AccountID: testAccountID, } - require.NoError(t, svc.store.PutTargetGroup(tg)) + require.NoError(t, svc.store.PutTargetGroup(t.Context(), tg)) - require.NoError(t, svc.store.PutListener(&ListenerRecord{ + require.NoError(t, svc.store.PutListener(t.Context(), &ListenerRecord{ ListenerArn: lb.LoadBalancerArn + "/listener-1", ListenerID: "lst-act1", LoadBalancerArn: lb.LoadBalancerArn, @@ -2067,7 +2067,7 @@ func TestLBAgentHeartbeat_BuildsConfigOnActivation(t *testing.T) { })) // Pre-condition: config was never built during provisioning. - pre, err := svc.store.GetLoadBalancer("lb-act1") + pre, err := svc.store.GetLoadBalancer(t.Context(), "lb-act1") require.NoError(t, err) require.Empty(t, pre.ConfigHash, "config must be empty before activation") @@ -2078,7 +2078,7 @@ func TestLBAgentHeartbeat_BuildsConfigOnActivation(t *testing.T) { assert.Equal(t, StateActive, *out.Status) assert.NotEmpty(t, aws.StringValue(out.ConfigHash), "first heartbeat must return a built ConfigHash") - stored, err := svc.store.GetLoadBalancer("lb-act1") + stored, err := svc.store.GetLoadBalancer(t.Context(), "lb-act1") require.NoError(t, err) assert.NotEmpty(t, stored.ConfigText, "data-plane config must be built on activation") assert.NotEmpty(t, stored.ConfigHash) @@ -2117,7 +2117,7 @@ func TestGetLBConfig_ReturnsStoredConfig(t *testing.T) { ConfigHash: "deadbeef", AccountID: testAccountID, } - require.NoError(t, svc.store.PutLoadBalancer(lb)) + require.NoError(t, svc.store.PutLoadBalancer(t.Context(), lb)) out, err := svc.GetLBConfig(context.Background(), &GetLBConfigInput{ LBID: aws.String("lb-cfg1"), @@ -2143,7 +2143,7 @@ func TestGetLBConfig_DeliversHealthTargetsForNLB(t *testing.T) { {ServerName: "srv_i-1", Address: "10.0.0.1:80", Protocol: ProtocolTCP}, }, } - require.NoError(t, svc.store.PutLoadBalancer(lb)) + require.NoError(t, svc.store.PutLoadBalancer(t.Context(), lb)) out, err := svc.GetLBConfig(context.Background(), &GetLBConfigInput{LBID: aws.String("lb-nlbhc")}, testAccountID) require.NoError(t, err) @@ -2156,7 +2156,7 @@ func TestGetLBConfig_DeliversHealthTargetsForNLB(t *testing.T) { func TestGetLBConfig_HealthTargetsWireShape(t *testing.T) { svc := setupTestService(t) - require.NoError(t, svc.store.PutLoadBalancer(&LoadBalancerRecord{ + require.NoError(t, svc.store.PutLoadBalancer(t.Context(), &LoadBalancerRecord{ LoadBalancerID: "lb-wire", Type: LoadBalancerTypeNetwork, AccountID: testAccountID, @@ -2204,7 +2204,7 @@ func TestGetLBConfig_NoHealthTargetsForALB(t *testing.T) { ConfigHash: "hc2", AccountID: testAccountID, } - require.NoError(t, svc.store.PutLoadBalancer(lb)) + require.NoError(t, svc.store.PutLoadBalancer(t.Context(), lb)) out, err := svc.GetLBConfig(context.Background(), &GetLBConfigInput{LBID: aws.String("lb-albhc")}, testAccountID) require.NoError(t, err) @@ -2238,7 +2238,7 @@ func TestLBAgentHeartbeat_WrongAccount(t *testing.T) { State: StateActive, AccountID: testAccountID, } - require.NoError(t, svc.store.PutLoadBalancer(lb)) + require.NoError(t, svc.store.PutLoadBalancer(t.Context(), lb)) _, err := svc.LBAgentHeartbeat(context.Background(), &LBAgentHeartbeatInput{ LBID: aws.String("lb-auth1"), @@ -2257,7 +2257,7 @@ func TestLBAgentHeartbeat_SystemAccountAllowed(t *testing.T) { State: StateActive, AccountID: testAccountID, } - require.NoError(t, svc.store.PutLoadBalancer(lb)) + require.NoError(t, svc.store.PutLoadBalancer(t.Context(), lb)) out, err := svc.LBAgentHeartbeat(context.Background(), &LBAgentHeartbeatInput{ LBID: aws.String("lb-sys1"), @@ -2278,7 +2278,7 @@ func TestLBAgentHeartbeat_SkipsWriteWhenHeartbeatFresh(t *testing.T) { LastHeartbeat: recentHeartbeat, AccountID: testAccountID, } - require.NoError(t, svc.store.PutLoadBalancer(lb)) + require.NoError(t, svc.store.PutLoadBalancer(t.Context(), lb)) _, err := svc.LBAgentHeartbeat(context.Background(), &LBAgentHeartbeatInput{ LBID: aws.String("lb-fresh1"), @@ -2286,7 +2286,7 @@ func TestLBAgentHeartbeat_SkipsWriteWhenHeartbeatFresh(t *testing.T) { require.NoError(t, err) // LastHeartbeat should NOT have been updated because the stored value is fresh. - stored, err := svc.store.GetLoadBalancer("lb-fresh1") + stored, err := svc.store.GetLoadBalancer(t.Context(), "lb-fresh1") require.NoError(t, err) assert.Equal(t, recentHeartbeat, stored.LastHeartbeat) } @@ -2303,7 +2303,7 @@ func TestLBAgentHeartbeat_PersistsWhenHeartbeatStale(t *testing.T) { LastHeartbeat: staleHeartbeat, AccountID: testAccountID, } - require.NoError(t, svc.store.PutLoadBalancer(lb)) + require.NoError(t, svc.store.PutLoadBalancer(t.Context(), lb)) _, err := svc.LBAgentHeartbeat(context.Background(), &LBAgentHeartbeatInput{ LBID: aws.String("lb-stale1"), @@ -2311,7 +2311,7 @@ func TestLBAgentHeartbeat_PersistsWhenHeartbeatStale(t *testing.T) { require.NoError(t, err) // LastHeartbeat should have been refreshed. - stored, err := svc.store.GetLoadBalancer("lb-stale1") + stored, err := svc.store.GetLoadBalancer(t.Context(), "lb-stale1") require.NoError(t, err) assert.True(t, stored.LastHeartbeat.After(staleHeartbeat)) } @@ -2328,7 +2328,7 @@ func TestGetLBConfig_WrongAccount(t *testing.T) { ConfigHash: "aaa", AccountID: testAccountID, } - require.NoError(t, svc.store.PutLoadBalancer(lb)) + require.NoError(t, svc.store.PutLoadBalancer(t.Context(), lb)) _, err := svc.GetLBConfig(context.Background(), &GetLBConfigInput{ LBID: aws.String("lb-authcfg1"), @@ -2349,7 +2349,7 @@ func TestGetLBConfig_SystemAccountAllowed(t *testing.T) { ConfigHash: "bbb", AccountID: testAccountID, } - require.NoError(t, svc.store.PutLoadBalancer(lb)) + require.NoError(t, svc.store.PutLoadBalancer(t.Context(), lb)) out, err := svc.GetLBConfig(context.Background(), &GetLBConfigInput{ LBID: aws.String("lb-syscfg1"), @@ -2371,7 +2371,7 @@ func TestUpdateStoredConfig_StoresConfigAndHash(t *testing.T) { InstanceID: "i-sys-upd1", AccountID: testAccountID, } - require.NoError(t, svc.store.PutLoadBalancer(lb)) + require.NoError(t, svc.store.PutLoadBalancer(t.Context(), lb)) tg := &TargetGroupRecord{ TargetGroupArn: "arn:aws:elasticloadbalancing:us-east-1:123456789012:targetgroup/upd-tg/tg-upd1", @@ -2383,7 +2383,7 @@ func TestUpdateStoredConfig_StoresConfigAndHash(t *testing.T) { }, AccountID: testAccountID, } - require.NoError(t, svc.store.PutTargetGroup(tg)) + require.NoError(t, svc.store.PutTargetGroup(t.Context(), tg)) listener := &ListenerRecord{ ListenerArn: "arn:aws:elasticloadbalancing:us-east-1:123456789012:listener/app/upd-lb/lb-upd1/lst-upd1", @@ -2394,11 +2394,11 @@ func TestUpdateStoredConfig_StoresConfigAndHash(t *testing.T) { DefaultActions: []ListenerAction{{Type: ActionTypeForward, TargetGroupArn: tg.TargetGroupArn}}, AccountID: testAccountID, } - require.NoError(t, svc.store.PutListener(listener)) + require.NoError(t, svc.store.PutListener(t.Context(), listener)) require.NoError(t, svc.updateStoredConfig(context.Background(), lb)) - stored, err := svc.store.GetLoadBalancer("lb-upd1") + stored, err := svc.store.GetLoadBalancer(t.Context(), "lb-upd1") require.NoError(t, err) assert.NotEmpty(t, stored.ConfigText) assert.NotEmpty(t, stored.ConfigHash) @@ -2415,11 +2415,11 @@ func TestUpdateStoredConfig_SkipsWhenNoInstance(t *testing.T) { State: StateActive, AccountID: testAccountID, } - require.NoError(t, svc.store.PutLoadBalancer(lb)) + require.NoError(t, svc.store.PutLoadBalancer(t.Context(), lb)) require.NoError(t, svc.updateStoredConfig(context.Background(), lb)) - stored, err := svc.store.GetLoadBalancer("lb-noinst") + stored, err := svc.store.GetLoadBalancer(t.Context(), "lb-noinst") require.NoError(t, err) assert.Empty(t, stored.ConfigText) assert.Empty(t, stored.ConfigHash) @@ -2514,7 +2514,7 @@ func TestUpdateStoredConfig_MissingTargetGroup(t *testing.T) { InstanceID: "i-sys-misstg", AccountID: testAccountID, } - require.NoError(t, svc.store.PutLoadBalancer(lb)) + require.NoError(t, svc.store.PutLoadBalancer(t.Context(), lb)) listener := &ListenerRecord{ ListenerArn: "arn:aws:elasticloadbalancing:us-east-1:123456789012:listener/app/miss-tg/lb-misstg/lst-misstg", @@ -2525,12 +2525,12 @@ func TestUpdateStoredConfig_MissingTargetGroup(t *testing.T) { DefaultActions: []ListenerAction{{Type: ActionTypeForward, TargetGroupArn: "arn:aws:elasticloadbalancing:us-east-1:123456789012:targetgroup/gone/tg-gone"}}, AccountID: testAccountID, } - require.NoError(t, svc.store.PutListener(listener)) + require.NoError(t, svc.store.PutListener(t.Context(), listener)) // Should not error — skips missing TG and generates config with no backends require.NoError(t, svc.updateStoredConfig(context.Background(), lb)) - stored, err := svc.store.GetLoadBalancer("lb-misstg") + stored, err := svc.store.GetLoadBalancer(t.Context(), "lb-misstg") require.NoError(t, err) assert.NotEmpty(t, stored.ConfigHash) } @@ -2838,9 +2838,9 @@ func attrKinds() []attrKind { return pairs, nil }, revision: func(t *testing.T, svc *ELBv2ServiceImpl, arn string) uint64 { - tg, err := svc.store.GetTargetGroupByArn(arn) + tg, err := svc.store.GetTargetGroupByArn(t.Context(), arn) require.NoError(t, err) - entry, err := svc.store.kv.Get(KeyPrefixTG + tg.TargetGroupID) + entry, err := svc.store.kv.Get(t.Context(), KeyPrefixTG+tg.TargetGroupID) require.NoError(t, err) return entry.Revision() }, @@ -2892,9 +2892,9 @@ func attrKinds() []attrKind { return pairs, nil }, revision: func(t *testing.T, svc *ELBv2ServiceImpl, arn string) uint64 { - lb, err := svc.store.GetLoadBalancerByArn(arn) + lb, err := svc.store.GetLoadBalancerByArn(t.Context(), arn) require.NoError(t, err) - entry, err := svc.store.kv.Get(KeyPrefixLB + lb.LoadBalancerID) + entry, err := svc.store.kv.Get(t.Context(), KeyPrefixLB+lb.LoadBalancerID) require.NoError(t, err) return entry.Revision() }, diff --git a/spinifex/handlers/elbv2/service_impl_vpc_test.go b/spinifex/handlers/elbv2/service_impl_vpc_test.go index 6b3c978aa..451710c65 100644 --- a/spinifex/handlers/elbv2/service_impl_vpc_test.go +++ b/spinifex/handlers/elbv2/service_impl_vpc_test.go @@ -25,7 +25,7 @@ func setupTestServiceWithVPC(t *testing.T) (*ELBv2ServiceImpl, *handlers_ec2_vpc testutil.StubVpcdSGResponder(t, nc) // Create VPC service - vpcSvc, err := handlers_ec2_vpc.NewVPCServiceImplWithNATS(nil, nc) + vpcSvc, err := handlers_ec2_vpc.NewVPCServiceImplWithNATS(t.Context(), nil, nc) require.NoError(t, err) // Create ELBv2 service with VPC wired in. @@ -1006,7 +1006,7 @@ func TestCreateClusterNLBSync_CarriesCrossAccountENI(t *testing.T) { assert.Equal(t, "02:0a:01:23:45:67", launch.NICs[2].MAC) // And it is persisted for host-reboot recovery. - record, err := svc.store.GetLoadBalancerByName("eks-alpha", testAccountID) + record, err := svc.store.GetLoadBalancerByName(t.Context(), "eks-alpha", testAccountID) require.NoError(t, err) require.NotNil(t, record) require.Len(t, record.CrossAccountENIs, 1) diff --git a/spinifex/handlers/elbv2/store.go b/spinifex/handlers/elbv2/store.go index bf30d5230..f96550c95 100644 --- a/spinifex/handlers/elbv2/store.go +++ b/spinifex/handlers/elbv2/store.go @@ -1,6 +1,7 @@ package handlers_elbv2 import ( + "context" "encoding/json" "errors" "fmt" @@ -8,9 +9,11 @@ import ( "sort" "strings" + "github.com/mulgadc/spinifex/spinifex/kvutil" "github.com/mulgadc/spinifex/spinifex/migrate" "github.com/mulgadc/spinifex/spinifex/utils" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) const ( @@ -32,23 +35,28 @@ const ( ) // Store provides CRUD operations for ELBv2 resources backed by JetStream KV. +// Every method takes the caller's context as its leading parameter, so a read or +// write honors the caller's deadline and cancellation instead of the fixed +// internal wait the legacy KV API applied. type Store struct { - kv nats.KeyValue + kv jetstream.KeyValue } -// NewStore creates a new ELBv2 store using the provided NATS connection. -func NewStore(nc *nats.Conn) (*Store, error) { - js, err := nc.JetStream() +// NewStore creates a new ELBv2 store using the provided NATS connection. The +// context bounds bucket creation and the schema migration only; per-operation +// contexts come from the Store methods. +func NewStore(ctx context.Context, nc *nats.Conn) (*Store, error) { + js, err := jetstream.New(nc) if err != nil { return nil, fmt.Errorf("failed to get JetStream context: %w", err) } - kv, err := utils.GetOrCreateKVBucket(js, KVBucketELBv2, 1) + kv, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketELBv2, 1) if err != nil { return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketELBv2, err) } - if err := migrate.DefaultRegistry.RunKV(KVBucketELBv2, kv, KVBucketELBv2Version); err != nil { + if err := migrate.DefaultRegistry.RunKV(ctx, KVBucketELBv2, kv, KVBucketELBv2Version); err != nil { return nil, fmt.Errorf("migrate %s: %w", KVBucketELBv2, err) } @@ -66,24 +74,24 @@ func LBNameKey(name, accountID string) string { // ClaimLBName atomically claims the LB name; ok=true means this caller owns it, // dup=true means a live LB already holds it. An orphaned claim (owner resolves to // no record) is reclaimed via CAS. Idempotency barrier for CreateLoadBalancer. -func (s *Store) ClaimLBName(name, accountID, lbID string) (ok bool, dup bool, err error) { +func (s *Store) ClaimLBName(ctx context.Context, name, accountID, lbID string) (ok bool, dup bool, err error) { key := LBNameKey(name, accountID) for range maxLBNameClaimRetries { - if _, cerr := s.kv.Create(key, []byte(lbID)); cerr == nil { + if _, cerr := s.kv.Create(ctx, key, []byte(lbID)); cerr == nil { return true, false, nil - } else if !errors.Is(cerr, nats.ErrKeyExists) { + } else if !errors.Is(cerr, jetstream.ErrKeyExists) { return false, false, fmt.Errorf("kv create %s: %w", key, cerr) } - entry, gerr := s.kv.Get(key) + entry, gerr := s.kv.Get(ctx, key) if gerr != nil { - if errors.Is(gerr, nats.ErrKeyNotFound) { + if errors.Is(gerr, jetstream.ErrKeyNotFound) { continue // raced vanish; retry the create } return false, false, fmt.Errorf("kv get %s: %w", key, gerr) } ownerID := string(entry.Value()) if ownerID != "" && ownerID != lbID { - rec, rerr := s.GetLoadBalancer(ownerID) + rec, rerr := s.GetLoadBalancer(ctx, ownerID) if rerr != nil { return false, false, fmt.Errorf("resolve LB name owner %s: %w", ownerID, rerr) } @@ -92,8 +100,8 @@ func (s *Store) ClaimLBName(name, accountID, lbID string) (ok bool, dup bool, er } } // Orphaned (crashed prior create) or already ours: CAS-take the claim. - if _, uerr := s.kv.Update(key, []byte(lbID), entry.Revision()); uerr != nil { - if errors.Is(uerr, nats.ErrKeyExists) { + if _, uerr := s.kv.Update(ctx, key, []byte(lbID), entry.Revision()); uerr != nil { + if errors.Is(uerr, jetstream.ErrKeyExists) { continue // lost the CAS race; re-read } return false, false, fmt.Errorf("kv update %s: %w", key, uerr) @@ -105,29 +113,29 @@ func (s *Store) ClaimLBName(name, accountID, lbID string) (ok bool, dup bool, er // ReleaseLBName deletes the name claim. A missing key is success (idempotent), // so create-rollback paths and DeleteLoadBalancer can call it unconditionally. -func (s *Store) ReleaseLBName(name, accountID string) error { +func (s *Store) ReleaseLBName(ctx context.Context, name, accountID string) error { key := LBNameKey(name, accountID) - if err := s.kv.Delete(key); err != nil && !errors.Is(err, nats.ErrKeyNotFound) { + if err := s.kv.Delete(ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { return fmt.Errorf("kv delete %s: %w", key, err) } return nil } // PutLoadBalancer stores a load balancer record. -func (s *Store) PutLoadBalancer(lb *LoadBalancerRecord) error { +func (s *Store) PutLoadBalancer(ctx context.Context, lb *LoadBalancerRecord) error { data, err := json.Marshal(lb) if err != nil { return fmt.Errorf("marshal load balancer: %w", err) } - _, err = s.kv.Put(KeyPrefixLB+lb.LoadBalancerID, data) + _, err = s.kv.Put(ctx, KeyPrefixLB+lb.LoadBalancerID, data) return err } // GetLoadBalancer retrieves a load balancer by its short ID. -func (s *Store) GetLoadBalancer(lbID string) (*LoadBalancerRecord, error) { - entry, err := s.kv.Get(KeyPrefixLB + lbID) +func (s *Store) GetLoadBalancer(ctx context.Context, lbID string) (*LoadBalancerRecord, error) { + entry, err := s.kv.Get(ctx, KeyPrefixLB+lbID) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil, nil } return nil, err @@ -140,37 +148,37 @@ func (s *Store) GetLoadBalancer(lbID string) (*LoadBalancerRecord, error) { } // DeleteLoadBalancer removes a load balancer by its short ID. -func (s *Store) DeleteLoadBalancer(lbID string) error { - err := s.kv.Delete(KeyPrefixLB + lbID) - if err != nil && !errors.Is(err, nats.ErrKeyNotFound) { +func (s *Store) DeleteLoadBalancer(ctx context.Context, lbID string) error { + err := s.kv.Delete(ctx, KeyPrefixLB+lbID) + if err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { return err } return nil } // ListLoadBalancers returns all load balancer records. -func (s *Store) ListLoadBalancers() ([]*LoadBalancerRecord, error) { - return listByPrefix[LoadBalancerRecord](s.kv, KeyPrefixLB) +func (s *Store) ListLoadBalancers(ctx context.Context) ([]*LoadBalancerRecord, error) { + return listByPrefix[LoadBalancerRecord](ctx, s.kv, KeyPrefixLB) } // ListLoadBalancersStrict returns every load balancer record, failing if any // record cannot be unmarshalled so the caller never treats a partial list as a // complete inventory (the DNS reconcile prune authority requires the whole set). -func (s *Store) ListLoadBalancersStrict() ([]*LoadBalancerRecord, error) { - return listByPrefixStrict[LoadBalancerRecord](s.kv, KeyPrefixLB) +func (s *Store) ListLoadBalancersStrict(ctx context.Context) ([]*LoadBalancerRecord, error) { + return listByPrefixStrict[LoadBalancerRecord](ctx, s.kv, KeyPrefixLB) } // GetLoadBalancerByArn finds a load balancer by the short ID in the ARN's final // path segment. Returns (nil, nil) if unparseable or not found. O(1) — Terraform // hits Describe*Attributes on every plan/refresh. -func (s *Store) GetLoadBalancerByArn(arn string) (*LoadBalancerRecord, error) { +func (s *Store) GetLoadBalancerByArn(ctx context.Context, arn string) (*LoadBalancerRecord, error) { // ELBv2 LB ARN: arn:aws:elasticloadbalancing:{region}:{account}:loadbalancer/{app,net}/{name}/{lbID} idx := strings.LastIndex(arn, "/") if idx < 0 || idx == len(arn)-1 { return nil, nil } lbID := arn[idx+1:] - lb, err := s.GetLoadBalancer(lbID) + lb, err := s.GetLoadBalancer(ctx, lbID) if err != nil { return nil, err } @@ -187,8 +195,8 @@ func (s *Store) GetLoadBalancerByArn(arn string) (*LoadBalancerRecord, error) { } // GetLoadBalancerByName finds a load balancer by name within an account. -func (s *Store) GetLoadBalancerByName(name, accountID string) (*LoadBalancerRecord, error) { - lbs, err := s.ListLoadBalancers() +func (s *Store) GetLoadBalancerByName(ctx context.Context, name, accountID string) (*LoadBalancerRecord, error) { + lbs, err := s.ListLoadBalancers(ctx) if err != nil { return nil, err } @@ -203,20 +211,20 @@ func (s *Store) GetLoadBalancerByName(name, accountID string) (*LoadBalancerReco // --- Target Group CRUD --- // PutTargetGroup stores a target group record. -func (s *Store) PutTargetGroup(tg *TargetGroupRecord) error { +func (s *Store) PutTargetGroup(ctx context.Context, tg *TargetGroupRecord) error { data, err := json.Marshal(tg) if err != nil { return fmt.Errorf("marshal target group: %w", err) } - _, err = s.kv.Put(KeyPrefixTG+tg.TargetGroupID, data) + _, err = s.kv.Put(ctx, KeyPrefixTG+tg.TargetGroupID, data) return err } // GetTargetGroup retrieves a target group by its short ID. -func (s *Store) GetTargetGroup(tgID string) (*TargetGroupRecord, error) { - entry, err := s.kv.Get(KeyPrefixTG + tgID) +func (s *Store) GetTargetGroup(ctx context.Context, tgID string) (*TargetGroupRecord, error) { + entry, err := s.kv.Get(ctx, KeyPrefixTG+tgID) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil, nil } return nil, err @@ -229,29 +237,29 @@ func (s *Store) GetTargetGroup(tgID string) (*TargetGroupRecord, error) { } // DeleteTargetGroup removes a target group by its short ID. -func (s *Store) DeleteTargetGroup(tgID string) error { - err := s.kv.Delete(KeyPrefixTG + tgID) - if err != nil && !errors.Is(err, nats.ErrKeyNotFound) { +func (s *Store) DeleteTargetGroup(ctx context.Context, tgID string) error { + err := s.kv.Delete(ctx, KeyPrefixTG+tgID) + if err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { return err } return nil } // ListTargetGroups returns all target group records. -func (s *Store) ListTargetGroups() ([]*TargetGroupRecord, error) { - return listByPrefix[TargetGroupRecord](s.kv, KeyPrefixTG) +func (s *Store) ListTargetGroups(ctx context.Context) ([]*TargetGroupRecord, error) { + return listByPrefix[TargetGroupRecord](ctx, s.kv, KeyPrefixTG) } // GetTargetGroupByArn finds a target group by the short ID in the ARN's final // path segment. O(1) — see GetLoadBalancerByArn for motivation. -func (s *Store) GetTargetGroupByArn(arn string) (*TargetGroupRecord, error) { +func (s *Store) GetTargetGroupByArn(ctx context.Context, arn string) (*TargetGroupRecord, error) { // ELBv2 TG ARN: arn:aws:elasticloadbalancing:{region}:{account}:targetgroup/{name}/{tgID} idx := strings.LastIndex(arn, "/") if idx < 0 || idx == len(arn)-1 { return nil, nil } tgID := arn[idx+1:] - tg, err := s.GetTargetGroup(tgID) + tg, err := s.GetTargetGroup(ctx, tgID) if err != nil { return nil, err } @@ -268,8 +276,8 @@ func (s *Store) GetTargetGroupByArn(arn string) (*TargetGroupRecord, error) { // TargetGroupsForLB returns only the target groups attached to a load balancer // via its listeners. It follows LB ID → LB ARN → listeners → TG ARNs → TGs. -func (s *Store) TargetGroupsForLB(lbID string) ([]*TargetGroupRecord, error) { - lb, err := s.GetLoadBalancer(lbID) +func (s *Store) TargetGroupsForLB(ctx context.Context, lbID string) ([]*TargetGroupRecord, error) { + lb, err := s.GetLoadBalancer(ctx, lbID) if err != nil { return nil, fmt.Errorf("get load balancer %s: %w", lbID, err) } @@ -277,7 +285,7 @@ func (s *Store) TargetGroupsForLB(lbID string) ([]*TargetGroupRecord, error) { return nil, nil } - listeners, err := s.ListListenersByLB(lb.LoadBalancerArn) + listeners, err := s.ListListenersByLB(ctx, lb.LoadBalancerArn) if err != nil { return nil, fmt.Errorf("list listeners for %s: %w", lbID, err) } @@ -298,7 +306,7 @@ func (s *Store) TargetGroupsForLB(lbID string) ([]*TargetGroupRecord, error) { for _, a := range l.DefaultActions { collect(a.TargetGroupArn) } - rules, err := s.ListRulesByListener(l.ListenerArn) + rules, err := s.ListRulesByListener(ctx, l.ListenerArn) if err != nil { return nil, fmt.Errorf("list rules for listener %s: %w", l.ListenerArn, err) } @@ -311,7 +319,7 @@ func (s *Store) TargetGroupsForLB(lbID string) ([]*TargetGroupRecord, error) { tgs := make([]*TargetGroupRecord, 0, len(seen)) for tgID := range seen { - tg, err := s.GetTargetGroup(tgID) + tg, err := s.GetTargetGroup(ctx, tgID) if err != nil { return nil, fmt.Errorf("get target group %s: %w", tgID, err) } @@ -326,11 +334,11 @@ func (s *Store) TargetGroupsForLB(lbID string) ([]*TargetGroupRecord, error) { // forwards to tgArn. A target group not referenced by a load balancer serves no // traffic, so its targets are reported "unused" (AWS Target.NotInUse) instead of // staying stuck in "initial" with no health checker to advance them. -func (s *Store) TargetGroupInUse(tgArn string) (bool, error) { +func (s *Store) TargetGroupInUse(ctx context.Context, tgArn string) (bool, error) { if tgArn == "" { return false, nil } - listeners, err := s.ListListeners() + listeners, err := s.ListListeners(ctx) if err != nil { return false, err } @@ -341,7 +349,7 @@ func (s *Store) TargetGroupInUse(tgArn string) (bool, error) { } } } - rules, err := s.ListRules() + rules, err := s.ListRules(ctx) if err != nil { return false, err } @@ -356,8 +364,8 @@ func (s *Store) TargetGroupInUse(tgArn string) (bool, error) { } // GetTargetGroupByName finds a target group by name within a VPC. -func (s *Store) GetTargetGroupByName(name, vpcID string) (*TargetGroupRecord, error) { - tgs, err := s.ListTargetGroups() +func (s *Store) GetTargetGroupByName(ctx context.Context, name, vpcID string) (*TargetGroupRecord, error) { + tgs, err := s.ListTargetGroups(ctx) if err != nil { return nil, err } @@ -372,20 +380,20 @@ func (s *Store) GetTargetGroupByName(name, vpcID string) (*TargetGroupRecord, er // --- Listener CRUD --- // PutListener stores a listener record. -func (s *Store) PutListener(l *ListenerRecord) error { +func (s *Store) PutListener(ctx context.Context, l *ListenerRecord) error { data, err := json.Marshal(l) if err != nil { return fmt.Errorf("marshal listener: %w", err) } - _, err = s.kv.Put(KeyPrefixListener+l.ListenerID, data) + _, err = s.kv.Put(ctx, KeyPrefixListener+l.ListenerID, data) return err } // GetListener retrieves a listener by its short ID. -func (s *Store) GetListener(listenerID string) (*ListenerRecord, error) { - entry, err := s.kv.Get(KeyPrefixListener + listenerID) +func (s *Store) GetListener(ctx context.Context, listenerID string) (*ListenerRecord, error) { + entry, err := s.kv.Get(ctx, KeyPrefixListener+listenerID) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil, nil } return nil, err @@ -398,22 +406,22 @@ func (s *Store) GetListener(listenerID string) (*ListenerRecord, error) { } // DeleteListener removes a listener by its short ID. -func (s *Store) DeleteListener(listenerID string) error { - err := s.kv.Delete(KeyPrefixListener + listenerID) - if err != nil && !errors.Is(err, nats.ErrKeyNotFound) { +func (s *Store) DeleteListener(ctx context.Context, listenerID string) error { + err := s.kv.Delete(ctx, KeyPrefixListener+listenerID) + if err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { return err } return nil } // ListListeners returns all listener records. -func (s *Store) ListListeners() ([]*ListenerRecord, error) { - return listByPrefix[ListenerRecord](s.kv, KeyPrefixListener) +func (s *Store) ListListeners(ctx context.Context) ([]*ListenerRecord, error) { + return listByPrefix[ListenerRecord](ctx, s.kv, KeyPrefixListener) } // ListListenersByLB returns all listeners for a specific load balancer ARN. -func (s *Store) ListListenersByLB(lbArn string) ([]*ListenerRecord, error) { - all, err := s.ListListeners() +func (s *Store) ListListenersByLB(ctx context.Context, lbArn string) ([]*ListenerRecord, error) { + all, err := s.ListListeners(ctx) if err != nil { return nil, err } @@ -428,14 +436,14 @@ func (s *Store) ListListenersByLB(lbArn string) ([]*ListenerRecord, error) { // GetListenerByArn finds a listener by the short ID in the ARN's final path segment. // O(1) per ARN — Terraform calls DescribeTags for every listener on every plan. -func (s *Store) GetListenerByArn(arn string) (*ListenerRecord, error) { +func (s *Store) GetListenerByArn(ctx context.Context, arn string) (*ListenerRecord, error) { // ELBv2 listener ARN: arn:aws:elasticloadbalancing:{region}:{account}:listener/{app,net}/{name}/{lbID}/{listenerID} idx := strings.LastIndex(arn, "/") if idx < 0 || idx == len(arn)-1 { return nil, nil } listenerID := arn[idx+1:] - l, err := s.GetListener(listenerID) + l, err := s.GetListener(ctx, listenerID) if err != nil { return nil, err } @@ -448,20 +456,20 @@ func (s *Store) GetListenerByArn(arn string) (*ListenerRecord, error) { // --- Rule CRUD --- // PutRule stores a rule record. -func (s *Store) PutRule(r *RuleRecord) error { +func (s *Store) PutRule(ctx context.Context, r *RuleRecord) error { data, err := json.Marshal(r) if err != nil { return fmt.Errorf("marshal rule: %w", err) } - _, err = s.kv.Put(KeyPrefixRule+r.RuleID, data) + _, err = s.kv.Put(ctx, KeyPrefixRule+r.RuleID, data) return err } // GetRule retrieves a rule by its short ID. -func (s *Store) GetRule(ruleID string) (*RuleRecord, error) { - entry, err := s.kv.Get(KeyPrefixRule + ruleID) +func (s *Store) GetRule(ctx context.Context, ruleID string) (*RuleRecord, error) { + entry, err := s.kv.Get(ctx, KeyPrefixRule+ruleID) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil, nil } return nil, err @@ -474,24 +482,24 @@ func (s *Store) GetRule(ruleID string) (*RuleRecord, error) { } // DeleteRule removes a rule by its short ID. -func (s *Store) DeleteRule(ruleID string) error { - err := s.kv.Delete(KeyPrefixRule + ruleID) - if err != nil && !errors.Is(err, nats.ErrKeyNotFound) { +func (s *Store) DeleteRule(ctx context.Context, ruleID string) error { + err := s.kv.Delete(ctx, KeyPrefixRule+ruleID) + if err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { return err } return nil } // ListRules returns all rule records. -func (s *Store) ListRules() ([]*RuleRecord, error) { - return listByPrefix[RuleRecord](s.kv, KeyPrefixRule) +func (s *Store) ListRules(ctx context.Context) ([]*RuleRecord, error) { + return listByPrefix[RuleRecord](ctx, s.kv, KeyPrefixRule) } // ListRulesByListener returns all rules attached to a listener ARN, sorted by // ascending priority. Callers downstream of this method (HAProxy renderer, // SetRulePriorities) rely on the sort. -func (s *Store) ListRulesByListener(listenerArn string) ([]*RuleRecord, error) { - all, err := s.ListRules() +func (s *Store) ListRulesByListener(ctx context.Context, listenerArn string) ([]*RuleRecord, error) { + all, err := s.ListRules(ctx) if err != nil { return nil, err } @@ -507,8 +515,8 @@ func (s *Store) ListRulesByListener(listenerArn string) ([]*RuleRecord, error) { // GetRuleByArn finds a rule by its ARN via linear scan; the rule short ID is // embedded after several listener-specific segments, making direct parsing brittle. -func (s *Store) GetRuleByArn(arn string) (*RuleRecord, error) { - rules, err := s.ListRules() +func (s *Store) GetRuleByArn(ctx context.Context, arn string) (*RuleRecord, error) { + rules, err := s.ListRules(ctx) if err != nil { return nil, err } @@ -525,21 +533,21 @@ func (s *Store) GetRuleByArn(arn string) (*RuleRecord, error) { // listByPrefix returns all records with keys matching the given prefix. A record // that cannot be unmarshalled is logged and skipped so one corrupt entry does not // fail read/describe paths. -func listByPrefix[T any](kv nats.KeyValue, prefix string) ([]*T, error) { - return listByPrefixOpt[T](kv, prefix, false) +func listByPrefix[T any](ctx context.Context, kv jetstream.KeyValue, prefix string) ([]*T, error) { + return listByPrefixOpt[T](ctx, kv, prefix, false) } // listByPrefixStrict is listByPrefix but returns an error on the first record it // cannot unmarshal, so a caller that treats the result as a complete inventory // never mistakes a partial read for the whole set. -func listByPrefixStrict[T any](kv nats.KeyValue, prefix string) ([]*T, error) { - return listByPrefixOpt[T](kv, prefix, true) +func listByPrefixStrict[T any](ctx context.Context, kv jetstream.KeyValue, prefix string) ([]*T, error) { + return listByPrefixOpt[T](ctx, kv, prefix, true) } -func listByPrefixOpt[T any](kv nats.KeyValue, prefix string, strict bool) ([]*T, error) { - keys, err := kv.Keys() +func listByPrefixOpt[T any](ctx context.Context, kv jetstream.KeyValue, prefix string, strict bool) ([]*T, error) { + keys, err := kv.Keys(ctx) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return nil, nil } return nil, err @@ -554,9 +562,9 @@ func listByPrefixOpt[T any](kv nats.KeyValue, prefix string, strict bool) ([]*T, continue } - entry, err := kv.Get(key) + entry, err := kv.Get(ctx, key) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { continue } return nil, err diff --git a/spinifex/handlers/elbv2/store_test.go b/spinifex/handlers/elbv2/store_test.go index 6c1285c90..51eb304a6 100644 --- a/spinifex/handlers/elbv2/store_test.go +++ b/spinifex/handlers/elbv2/store_test.go @@ -15,7 +15,7 @@ func setupTestStore(t *testing.T) *Store { t.Helper() _, nc, _ := testutil.StartTestJetStream(t) - store, err := NewStore(nc) + store, err := NewStore(t.Context(), nc) require.NoError(t, err) return store } @@ -71,9 +71,9 @@ func newTestListener(id, lbArn string) *ListenerRecord { func TestLoadBalancerStoreLifecycle(t *testing.T) { t.Run("put and get", func(t *testing.T) { store := setupTestStore(t) - require.NoError(t, store.PutLoadBalancer(newTestLB("getput1", "lb-getput1"))) + require.NoError(t, store.PutLoadBalancer(t.Context(), newTestLB("getput1", "lb-getput1"))) - got, err := store.GetLoadBalancer("getput1") + got, err := store.GetLoadBalancer(t.Context(), "getput1") require.NoError(t, err) require.NotNil(t, got) assert.Equal(t, "getput1", got.LoadBalancerID) @@ -81,39 +81,39 @@ func TestLoadBalancerStoreLifecycle(t *testing.T) { t.Run("get not found", func(t *testing.T) { store := setupTestStore(t) - got, err := store.GetLoadBalancer("nonexistent") + got, err := store.GetLoadBalancer(t.Context(), "nonexistent") require.NoError(t, err) assert.Nil(t, got) }) t.Run("delete removes record", func(t *testing.T) { store := setupTestStore(t) - require.NoError(t, store.PutLoadBalancer(newTestLB("del1", "lb-del1"))) - require.NoError(t, store.DeleteLoadBalancer("del1")) + require.NoError(t, store.PutLoadBalancer(t.Context(), newTestLB("del1", "lb-del1"))) + require.NoError(t, store.DeleteLoadBalancer(t.Context(), "del1")) - got, err := store.GetLoadBalancer("del1") + got, err := store.GetLoadBalancer(t.Context(), "del1") require.NoError(t, err) assert.Nil(t, got) }) t.Run("delete idempotent", func(t *testing.T) { store := setupTestStore(t) - require.NoError(t, store.DeleteLoadBalancer("doesnt-exist")) + require.NoError(t, store.DeleteLoadBalancer(t.Context(), "doesnt-exist")) }) t.Run("list returns all", func(t *testing.T) { store := setupTestStore(t) - require.NoError(t, store.PutLoadBalancer(newTestLB("a", "lb-a"))) - require.NoError(t, store.PutLoadBalancer(newTestLB("b", "lb-b"))) + require.NoError(t, store.PutLoadBalancer(t.Context(), newTestLB("a", "lb-a"))) + require.NoError(t, store.PutLoadBalancer(t.Context(), newTestLB("b", "lb-b"))) - records, err := store.ListLoadBalancers() + records, err := store.ListLoadBalancers(t.Context()) require.NoError(t, err) assert.Len(t, records, 2) }) t.Run("list empty", func(t *testing.T) { store := setupTestStore(t) - records, err := store.ListLoadBalancers() + records, err := store.ListLoadBalancers(t.Context()) require.NoError(t, err) assert.Empty(t, records) }) @@ -122,9 +122,9 @@ func TestLoadBalancerStoreLifecycle(t *testing.T) { func TestTargetGroupStoreLifecycle(t *testing.T) { t.Run("put and get", func(t *testing.T) { store := setupTestStore(t) - require.NoError(t, store.PutTargetGroup(newTestTG("getput1", "tg-getput1"))) + require.NoError(t, store.PutTargetGroup(t.Context(), newTestTG("getput1", "tg-getput1"))) - got, err := store.GetTargetGroup("getput1") + got, err := store.GetTargetGroup(t.Context(), "getput1") require.NoError(t, err) require.NotNil(t, got) assert.Equal(t, "getput1", got.TargetGroupID) @@ -132,39 +132,39 @@ func TestTargetGroupStoreLifecycle(t *testing.T) { t.Run("get not found", func(t *testing.T) { store := setupTestStore(t) - got, err := store.GetTargetGroup("nonexistent") + got, err := store.GetTargetGroup(t.Context(), "nonexistent") require.NoError(t, err) assert.Nil(t, got) }) t.Run("delete removes record", func(t *testing.T) { store := setupTestStore(t) - require.NoError(t, store.PutTargetGroup(newTestTG("del1", "tg-del1"))) - require.NoError(t, store.DeleteTargetGroup("del1")) + require.NoError(t, store.PutTargetGroup(t.Context(), newTestTG("del1", "tg-del1"))) + require.NoError(t, store.DeleteTargetGroup(t.Context(), "del1")) - got, err := store.GetTargetGroup("del1") + got, err := store.GetTargetGroup(t.Context(), "del1") require.NoError(t, err) assert.Nil(t, got) }) t.Run("delete idempotent", func(t *testing.T) { store := setupTestStore(t) - require.NoError(t, store.DeleteTargetGroup("doesnt-exist")) + require.NoError(t, store.DeleteTargetGroup(t.Context(), "doesnt-exist")) }) t.Run("list returns all", func(t *testing.T) { store := setupTestStore(t) - require.NoError(t, store.PutTargetGroup(newTestTG("a", "tg-a"))) - require.NoError(t, store.PutTargetGroup(newTestTG("b", "tg-b"))) + require.NoError(t, store.PutTargetGroup(t.Context(), newTestTG("a", "tg-a"))) + require.NoError(t, store.PutTargetGroup(t.Context(), newTestTG("b", "tg-b"))) - records, err := store.ListTargetGroups() + records, err := store.ListTargetGroups(t.Context()) require.NoError(t, err) assert.Len(t, records, 2) }) t.Run("list empty", func(t *testing.T) { store := setupTestStore(t) - records, err := store.ListTargetGroups() + records, err := store.ListTargetGroups(t.Context()) require.NoError(t, err) assert.Empty(t, records) }) @@ -175,9 +175,9 @@ func TestListenerStoreLifecycle(t *testing.T) { t.Run("put and get", func(t *testing.T) { store := setupTestStore(t) - require.NoError(t, store.PutListener(newTestListener("getput1", lbArn))) + require.NoError(t, store.PutListener(t.Context(), newTestListener("getput1", lbArn))) - got, err := store.GetListener("getput1") + got, err := store.GetListener(t.Context(), "getput1") require.NoError(t, err) require.NotNil(t, got) assert.Equal(t, "getput1", got.ListenerID) @@ -185,39 +185,39 @@ func TestListenerStoreLifecycle(t *testing.T) { t.Run("get not found", func(t *testing.T) { store := setupTestStore(t) - got, err := store.GetListener("nonexistent") + got, err := store.GetListener(t.Context(), "nonexistent") require.NoError(t, err) assert.Nil(t, got) }) t.Run("delete removes record", func(t *testing.T) { store := setupTestStore(t) - require.NoError(t, store.PutListener(newTestListener("del1", lbArn))) - require.NoError(t, store.DeleteListener("del1")) + require.NoError(t, store.PutListener(t.Context(), newTestListener("del1", lbArn))) + require.NoError(t, store.DeleteListener(t.Context(), "del1")) - got, err := store.GetListener("del1") + got, err := store.GetListener(t.Context(), "del1") require.NoError(t, err) assert.Nil(t, got) }) t.Run("delete idempotent", func(t *testing.T) { store := setupTestStore(t) - require.NoError(t, store.DeleteListener("doesnt-exist")) + require.NoError(t, store.DeleteListener(t.Context(), "doesnt-exist")) }) t.Run("list returns all", func(t *testing.T) { store := setupTestStore(t) - require.NoError(t, store.PutListener(newTestListener("a", lbArn))) - require.NoError(t, store.PutListener(newTestListener("b", lbArn))) + require.NoError(t, store.PutListener(t.Context(), newTestListener("a", lbArn))) + require.NoError(t, store.PutListener(t.Context(), newTestListener("b", lbArn))) - records, err := store.ListListeners() + records, err := store.ListListeners(t.Context()) require.NoError(t, err) assert.Len(t, records, 2) }) t.Run("list empty", func(t *testing.T) { store := setupTestStore(t) - records, err := store.ListListeners() + records, err := store.ListListeners(t.Context()) require.NoError(t, err) assert.Empty(t, records) }) @@ -228,14 +228,14 @@ func TestListenerStoreLifecycle(t *testing.T) { func TestGetLoadBalancerByArn(t *testing.T) { store := setupTestStore(t) lb := newTestLB("arn123", "arn-test") - require.NoError(t, store.PutLoadBalancer(lb)) + require.NoError(t, store.PutLoadBalancer(t.Context(), lb)) - got, err := store.GetLoadBalancerByArn(lb.LoadBalancerArn) + got, err := store.GetLoadBalancerByArn(t.Context(), lb.LoadBalancerArn) require.NoError(t, err) require.NotNil(t, got) assert.Equal(t, lb.Name, got.Name) - got, err = store.GetLoadBalancerByArn("arn:nonexistent") + got, err = store.GetLoadBalancerByArn(t.Context(), "arn:nonexistent") require.NoError(t, err) assert.Nil(t, got) } @@ -243,9 +243,9 @@ func TestGetLoadBalancerByArn(t *testing.T) { func TestGetLoadBalancerByName(t *testing.T) { store := setupTestStore(t) lb := newTestLB("name123", "find-by-name") - require.NoError(t, store.PutLoadBalancer(lb)) + require.NoError(t, store.PutLoadBalancer(t.Context(), lb)) - got, err := store.GetLoadBalancerByName("find-by-name", testAccountID) + got, err := store.GetLoadBalancerByName(t.Context(), "find-by-name", testAccountID) require.NoError(t, err) require.NotNil(t, got) assert.Equal(t, lb.LoadBalancerID, got.LoadBalancerID) @@ -254,13 +254,13 @@ func TestGetLoadBalancerByName(t *testing.T) { func TestPutLoadBalancer_Update(t *testing.T) { store := setupTestStore(t) lb := newTestLB("upd123", "updatable") - require.NoError(t, store.PutLoadBalancer(lb)) + require.NoError(t, store.PutLoadBalancer(t.Context(), lb)) lb.State = StateFailed lb.ENIs = []string{"eni-111", "eni-222"} - require.NoError(t, store.PutLoadBalancer(lb)) + require.NoError(t, store.PutLoadBalancer(t.Context(), lb)) - got, err := store.GetLoadBalancer("upd123") + got, err := store.GetLoadBalancer(t.Context(), "upd123") require.NoError(t, err) require.NotNil(t, got) assert.Equal(t, StateFailed, got.State) @@ -272,9 +272,9 @@ func TestPutLoadBalancer_Update(t *testing.T) { func TestGetTargetGroupByArn(t *testing.T) { store := setupTestStore(t) tg := newTestTG("tgarn", "arn-tg") - require.NoError(t, store.PutTargetGroup(tg)) + require.NoError(t, store.PutTargetGroup(t.Context(), tg)) - got, err := store.GetTargetGroupByArn(tg.TargetGroupArn) + got, err := store.GetTargetGroupByArn(t.Context(), tg.TargetGroupArn) require.NoError(t, err) require.NotNil(t, got) assert.Equal(t, tg.Name, got.Name) @@ -283,14 +283,14 @@ func TestGetTargetGroupByArn(t *testing.T) { func TestGetTargetGroupByName(t *testing.T) { store := setupTestStore(t) tg := newTestTG("tgname", "named-tg") - require.NoError(t, store.PutTargetGroup(tg)) + require.NoError(t, store.PutTargetGroup(t.Context(), tg)) - got, err := store.GetTargetGroupByName("named-tg", "vpc-test123") + got, err := store.GetTargetGroupByName(t.Context(), "named-tg", "vpc-test123") require.NoError(t, err) require.NotNil(t, got) // Wrong VPC should not find it - got, err = store.GetTargetGroupByName("named-tg", "vpc-other") + got, err = store.GetTargetGroupByName(t.Context(), "named-tg", "vpc-other") require.NoError(t, err) assert.Nil(t, got) } @@ -302,9 +302,9 @@ func TestTargetGroupWithTargets(t *testing.T) { {Id: "i-aaa111", Port: 8080, HealthState: TargetHealthInitial, PrivateIP: "10.0.1.10"}, {Id: "i-bbb222", Port: 0, HealthState: TargetHealthHealthy, PrivateIP: "10.0.1.11"}, } - require.NoError(t, store.PutTargetGroup(tg)) + require.NoError(t, store.PutTargetGroup(t.Context(), tg)) - got, err := store.GetTargetGroup("tgtargets") + got, err := store.GetTargetGroup(t.Context(), "tgtargets") require.NoError(t, err) require.NotNil(t, got) assert.Len(t, got.Targets, 2) @@ -327,15 +327,15 @@ func TestListListenersByLB(t *testing.T) { l3 := newTestListener("lst3", lbArn2) l3.Port = 80 - require.NoError(t, store.PutListener(l1)) - require.NoError(t, store.PutListener(l2)) - require.NoError(t, store.PutListener(l3)) + require.NoError(t, store.PutListener(t.Context(), l1)) + require.NoError(t, store.PutListener(t.Context(), l2)) + require.NoError(t, store.PutListener(t.Context(), l3)) - listeners, err := store.ListListenersByLB(lbArn1) + listeners, err := store.ListListenersByLB(t.Context(), lbArn1) require.NoError(t, err) assert.Len(t, listeners, 2) - listeners, err = store.ListListenersByLB(lbArn2) + listeners, err = store.ListListenersByLB(t.Context(), lbArn2) require.NoError(t, err) assert.Len(t, listeners, 1) } @@ -344,20 +344,20 @@ func TestGetListenerByArn(t *testing.T) { store := setupTestStore(t) lbArn := "arn:aws:elasticloadbalancing:us-east-1:" + testAccountID + ":loadbalancer/app/test/lb1" l := newTestListener("lstarn", lbArn) - require.NoError(t, store.PutListener(l)) + require.NoError(t, store.PutListener(t.Context(), l)) - got, err := store.GetListenerByArn(l.ListenerArn) + got, err := store.GetListenerByArn(t.Context(), l.ListenerArn) require.NoError(t, err) require.NotNil(t, got) assert.Equal(t, l.ListenerID, got.ListenerID) - got, err = store.GetListenerByArn("arn:nonexistent") + got, err = store.GetListenerByArn(t.Context(), "arn:nonexistent") require.NoError(t, err) assert.Nil(t, got) // ARN that parses to a real listener ID but a different ARN must not match // (defence-in-depth against the parse trick serving a wrong record). - got, err = store.GetListenerByArn("arn:aws:elasticloadbalancing:us-east-1:" + testAccountID + ":listener/app/other/lbX/" + l.ListenerID) + got, err = store.GetListenerByArn(t.Context(), "arn:aws:elasticloadbalancing:us-east-1:"+testAccountID+":listener/app/other/lbX/"+l.ListenerID) require.NoError(t, err) assert.Nil(t, got) } @@ -371,30 +371,30 @@ func TestResourceIsolation(t *testing.T) { tg := newTestTG("shared1", "tg-shared") // Same ID as LB l := newTestListener("shared1", lb.LoadBalancerArn) - require.NoError(t, store.PutLoadBalancer(lb)) - require.NoError(t, store.PutTargetGroup(tg)) - require.NoError(t, store.PutListener(l)) + require.NoError(t, store.PutLoadBalancer(t.Context(), lb)) + require.NoError(t, store.PutTargetGroup(t.Context(), tg)) + require.NoError(t, store.PutListener(t.Context(), l)) - gotLB, err := store.GetLoadBalancer("shared1") + gotLB, err := store.GetLoadBalancer(t.Context(), "shared1") require.NoError(t, err) require.NotNil(t, gotLB) assert.Equal(t, "alb-shared", gotLB.Name) - gotTG, err := store.GetTargetGroup("shared1") + gotTG, err := store.GetTargetGroup(t.Context(), "shared1") require.NoError(t, err) require.NotNil(t, gotTG) assert.Equal(t, "tg-shared", gotTG.Name) - gotL, err := store.GetListener("shared1") + gotL, err := store.GetListener(t.Context(), "shared1") require.NoError(t, err) require.NotNil(t, gotL) assert.Equal(t, ProtocolHTTP, gotL.Protocol) // Deleting one type should not affect others - require.NoError(t, store.DeleteLoadBalancer("shared1")) - gotTG, _ = store.GetTargetGroup("shared1") + require.NoError(t, store.DeleteLoadBalancer(t.Context(), "shared1")) + gotTG, _ = store.GetTargetGroup(t.Context(), "shared1") assert.NotNil(t, gotTG) - gotL, _ = store.GetListener("shared1") + gotL, _ = store.GetListener(t.Context(), "shared1") assert.NotNil(t, gotL) } @@ -402,14 +402,14 @@ func TestTargetGroupsForLB(t *testing.T) { store := setupTestStore(t) // Non-existent LB returns nil, nil - tgs, err := store.TargetGroupsForLB("nonexistent") + tgs, err := store.TargetGroupsForLB(t.Context(), "nonexistent") require.NoError(t, err) assert.Nil(t, tgs) lb := newTestLB("tgflb1", "my-alb") tg := newTestTG("tg001", "my-tg") - require.NoError(t, store.PutLoadBalancer(lb)) - require.NoError(t, store.PutTargetGroup(tg)) + require.NoError(t, store.PutLoadBalancer(t.Context(), lb)) + require.NoError(t, store.PutTargetGroup(t.Context(), tg)) listener := &ListenerRecord{ ListenerArn: "arn:aws:elasticloadbalancing:us-east-1:" + testAccountID + ":listener/app/my-alb/tgflb1/lst1", @@ -423,9 +423,9 @@ func TestTargetGroupsForLB(t *testing.T) { }, AccountID: testAccountID, } - require.NoError(t, store.PutListener(listener)) + require.NoError(t, store.PutListener(t.Context(), listener)) - tgs, err = store.TargetGroupsForLB("tgflb1") + tgs, err = store.TargetGroupsForLB(t.Context(), "tgflb1") require.NoError(t, err) require.Len(t, tgs, 1) assert.Equal(t, "tg001", tgs[0].TargetGroupID) diff --git a/spinifex/handlers/iam/account_test.go b/spinifex/handlers/iam/account_test.go index 39d74e3b2..7241c045f 100644 --- a/spinifex/handlers/iam/account_test.go +++ b/spinifex/handlers/iam/account_test.go @@ -408,7 +408,7 @@ func TestCreateAccount_PublishesEvent(t *testing.T) { masterKey, err := GenerateMasterKey() require.NoError(t, err) - svc, err := NewIAMServiceImpl(nc, masterKey, 1) + svc, err := NewIAMServiceImpl(t.Context(), nc, masterKey, 1) require.NoError(t, err) eventCh := make(chan *nats.Msg, 1) diff --git a/spinifex/handlers/iam/groups.go b/spinifex/handlers/iam/groups.go index 04febc95f..0dc1e08fb 100644 --- a/spinifex/handlers/iam/groups.go +++ b/spinifex/handlers/iam/groups.go @@ -1,6 +1,7 @@ package handlers_iam import ( + "context" "encoding/json" "errors" "fmt" @@ -12,9 +13,11 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/iam" + "github.com/nats-io/nats.go/jetstream" + "github.com/mulgadc/spinifex/spinifex/awserrors" + "github.com/mulgadc/spinifex/spinifex/kvutil" "github.com/mulgadc/spinifex/spinifex/utils" - "github.com/nats-io/nats.go" ) // maxGroupsPerUser caps how many groups a single user may belong to. Mirrors the @@ -23,6 +26,7 @@ import ( const maxGroupsPerUser = 10 func (s *IAMServiceImpl) CreateGroup(accountID string, input *iam.CreateGroupInput) (*iam.CreateGroupOutput, error) { + ctx := context.Background() groupName := *input.GroupName if err := validateGroupName(groupName); err != nil { return nil, errors.New(awserrors.ErrorIAMInvalidInput) @@ -58,8 +62,8 @@ func (s *IAMServiceImpl) CreateGroup(accountID string, input *iam.CreateGroupInp } kvKey := accountID + "." + groupName - if _, err := s.groupsBucket.Create(kvKey, data); err != nil { - if errors.Is(err, nats.ErrKeyExists) { + if _, err := s.groupsBucket.Create(ctx, kvKey, data); err != nil { + if errors.Is(err, jetstream.ErrKeyExists) { return nil, errors.New(awserrors.ErrorIAMEntityAlreadyExists) } return nil, fmt.Errorf("store group: %w", err) @@ -71,12 +75,13 @@ func (s *IAMServiceImpl) CreateGroup(accountID string, input *iam.CreateGroupInp } func (s *IAMServiceImpl) GetGroup(accountID string, input *iam.GetGroupInput) (*iam.GetGroupOutput, error) { - group, err := s.getGroup(accountID, *input.GroupName) + ctx := context.Background() + group, err := s.getGroup(ctx, accountID, *input.GroupName) if err != nil { return nil, err } - members, err := s.findGroupMembers(accountID, group.GroupName) + members, err := s.findGroupMembers(ctx, accountID, group.GroupName) if err != nil { return nil, fmt.Errorf("find group members: %w", err) } @@ -101,9 +106,10 @@ func (s *IAMServiceImpl) GetGroup(accountID string, input *iam.GetGroupInput) (* } func (s *IAMServiceImpl) ListGroups(accountID string, input *iam.ListGroupsInput) (*iam.ListGroupsOutput, error) { - keys, err := s.groupsBucket.Keys() + ctx := context.Background() + keys, err := kvutil.Keys(ctx, s.groupsBucket) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return &iam.ListGroupsOutput{ Groups: []*iam.Group{}, IsTruncated: aws.Bool(false), @@ -127,9 +133,9 @@ func (s *IAMServiceImpl) ListGroups(accountID string, input *iam.ListGroupsInput continue } - entry, err := s.groupsBucket.Get(key) + entry, err := s.groupsBucket.Get(ctx, key) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { slog.Debug("ListGroups: group key disappeared (concurrent delete)", "key", key) } else { slog.Warn("ListGroups: failed to get group", "key", key, "err", err) @@ -157,9 +163,10 @@ func (s *IAMServiceImpl) ListGroups(accountID string, input *iam.ListGroupsInput } func (s *IAMServiceImpl) DeleteGroup(accountID string, input *iam.DeleteGroupInput) (*iam.DeleteGroupOutput, error) { + ctx := context.Background() groupName := *input.GroupName - group, err := s.getGroup(accountID, groupName) + group, err := s.getGroup(ctx, accountID, groupName) if err != nil { return nil, err } @@ -172,7 +179,7 @@ func (s *IAMServiceImpl) DeleteGroup(accountID string, input *iam.DeleteGroupInp return nil, errors.New(awserrors.ErrorIAMDeleteConflict) } - members, err := s.findGroupMembers(accountID, groupName) + members, err := s.findGroupMembers(ctx, accountID, groupName) if err != nil { return nil, fmt.Errorf("check group members: %w", err) } @@ -180,7 +187,7 @@ func (s *IAMServiceImpl) DeleteGroup(accountID string, input *iam.DeleteGroupInp return nil, errors.New(awserrors.ErrorIAMDeleteConflict) } - if err := s.groupsBucket.Delete(accountID + "." + groupName); err != nil { + if err := s.groupsBucket.Delete(ctx, accountID+"."+groupName); err != nil { return nil, fmt.Errorf("delete group: %w", err) } @@ -193,15 +200,16 @@ func (s *IAMServiceImpl) DeleteGroup(accountID string, input *iam.DeleteGroupInp // --------------------------------------------------------------------------- func (s *IAMServiceImpl) AddUserToGroup(accountID string, input *iam.AddUserToGroupInput) (*iam.AddUserToGroupOutput, error) { + ctx := context.Background() groupName := *input.GroupName userName := *input.UserName userKVKey := accountID + "." + userName - if _, err := s.getGroup(accountID, groupName); err != nil { + if _, err := s.getGroup(ctx, accountID, groupName); err != nil { return nil, err } - user, err := s.getUser(accountID, userName) + user, err := s.getUser(ctx, accountID, userName) if err != nil { return nil, err } @@ -219,7 +227,7 @@ func (s *IAMServiceImpl) AddUserToGroup(accountID string, input *iam.AddUserToGr return nil, fmt.Errorf("marshal user: %w", err) } - if _, err := s.usersBucket.Put(userKVKey, userData); err != nil { + if _, err := s.usersBucket.Put(ctx, userKVKey, userData); err != nil { return nil, fmt.Errorf("update user: %w", err) } @@ -228,11 +236,12 @@ func (s *IAMServiceImpl) AddUserToGroup(accountID string, input *iam.AddUserToGr } func (s *IAMServiceImpl) RemoveUserFromGroup(accountID string, input *iam.RemoveUserFromGroupInput) (*iam.RemoveUserFromGroupOutput, error) { + ctx := context.Background() groupName := *input.GroupName userName := *input.UserName userKVKey := accountID + "." + userName - user, err := s.getUser(accountID, userName) + user, err := s.getUser(ctx, accountID, userName) if err != nil { return nil, err } @@ -250,7 +259,7 @@ func (s *IAMServiceImpl) RemoveUserFromGroup(accountID string, input *iam.Remove return nil, fmt.Errorf("marshal user: %w", err) } - if _, err := s.usersBucket.Put(userKVKey, userData); err != nil { + if _, err := s.usersBucket.Put(ctx, userKVKey, userData); err != nil { return nil, fmt.Errorf("update user: %w", err) } @@ -259,14 +268,15 @@ func (s *IAMServiceImpl) RemoveUserFromGroup(accountID string, input *iam.Remove } func (s *IAMServiceImpl) ListGroupsForUser(accountID string, input *iam.ListGroupsForUserInput) (*iam.ListGroupsForUserOutput, error) { - user, err := s.getUser(accountID, *input.UserName) + ctx := context.Background() + user, err := s.getUser(ctx, accountID, *input.UserName) if err != nil { return nil, err } var groups []*iam.Group for _, name := range user.Groups { - group, err := s.getGroup(accountID, name) + group, err := s.getGroup(ctx, accountID, name) if err != nil { if err.Error() == awserrors.ErrorIAMNoSuchEntity { // A dangling pointer to a vanished group is inert; skip it. @@ -291,18 +301,19 @@ func (s *IAMServiceImpl) ListGroupsForUser(accountID string, input *iam.ListGrou // --------------------------------------------------------------------------- func (s *IAMServiceImpl) AttachGroupPolicy(accountID string, input *iam.AttachGroupPolicyInput) (*iam.AttachGroupPolicyOutput, error) { + ctx := context.Background() groupName := *input.GroupName policyARN := *input.PolicyArn kvKey := accountID + "." + groupName // AWS-managed ARNs are stored opaquely (like roles); customer-managed ARNs must exist. if !isAWSManagedPolicyARN(policyARN) { - if _, err := s.getPolicyByARN(accountID, policyARN); err != nil { + if _, err := s.getPolicyByARN(ctx, accountID, policyARN); err != nil { return nil, err } } - group, err := s.getGroup(accountID, groupName) + group, err := s.getGroup(ctx, accountID, groupName) if err != nil { return nil, err } @@ -317,7 +328,7 @@ func (s *IAMServiceImpl) AttachGroupPolicy(accountID string, input *iam.AttachGr return nil, fmt.Errorf("marshal group: %w", err) } - if _, err := s.groupsBucket.Put(kvKey, data); err != nil { + if _, err := s.groupsBucket.Put(ctx, kvKey, data); err != nil { return nil, fmt.Errorf("update group: %w", err) } @@ -326,11 +337,12 @@ func (s *IAMServiceImpl) AttachGroupPolicy(accountID string, input *iam.AttachGr } func (s *IAMServiceImpl) DetachGroupPolicy(accountID string, input *iam.DetachGroupPolicyInput) (*iam.DetachGroupPolicyOutput, error) { + ctx := context.Background() groupName := *input.GroupName policyARN := *input.PolicyArn kvKey := accountID + "." + groupName - group, err := s.getGroup(accountID, groupName) + group, err := s.getGroup(ctx, accountID, groupName) if err != nil { return nil, err } @@ -346,7 +358,7 @@ func (s *IAMServiceImpl) DetachGroupPolicy(accountID string, input *iam.DetachGr return nil, fmt.Errorf("marshal group: %w", err) } - if _, err := s.groupsBucket.Put(kvKey, data); err != nil { + if _, err := s.groupsBucket.Put(ctx, kvKey, data); err != nil { return nil, fmt.Errorf("update group: %w", err) } @@ -355,7 +367,8 @@ func (s *IAMServiceImpl) DetachGroupPolicy(accountID string, input *iam.DetachGr } func (s *IAMServiceImpl) ListAttachedGroupPolicies(accountID string, input *iam.ListAttachedGroupPoliciesInput) (*iam.ListAttachedGroupPoliciesOutput, error) { - group, err := s.getGroup(accountID, *input.GroupName) + ctx := context.Background() + group, err := s.getGroup(ctx, accountID, *input.GroupName) if err != nil { return nil, err } @@ -369,7 +382,7 @@ func (s *IAMServiceImpl) ListAttachedGroupPolicies(accountID string, input *iam. }) continue } - policy, err := s.getPolicyByARN(accountID, arn) + policy, err := s.getPolicyByARN(ctx, accountID, arn) if err != nil { slog.Warn("ListAttachedGroupPolicies: policy not found for ARN", "arn", arn, "err", err) continue @@ -394,6 +407,7 @@ func (s *IAMServiceImpl) ListAttachedGroupPolicies(accountID string, input *iam. // Idempotent upsert: a same-name policy is overwritten, mirroring AWS. Uses a // blind read-modify-write Put like the other group writers (no CAS). func (s *IAMServiceImpl) PutGroupPolicy(accountID string, input *iam.PutGroupPolicyInput) (*iam.PutGroupPolicyOutput, error) { + ctx := context.Background() groupName := *input.GroupName policyName := *input.PolicyName policyDoc := *input.PolicyDocument @@ -406,7 +420,7 @@ func (s *IAMServiceImpl) PutGroupPolicy(accountID string, input *iam.PutGroupPol return nil, errors.New(awserrors.ErrorIAMMalformedPolicyDocument) } - group, err := s.getGroup(accountID, groupName) + group, err := s.getGroup(ctx, accountID, groupName) if err != nil { return nil, err } @@ -421,7 +435,7 @@ func (s *IAMServiceImpl) PutGroupPolicy(accountID string, input *iam.PutGroupPol return nil, fmt.Errorf("marshal group: %w", err) } - if _, err := s.groupsBucket.Put(kvKey, data); err != nil { + if _, err := s.groupsBucket.Put(ctx, kvKey, data); err != nil { return nil, fmt.Errorf("update group: %w", err) } @@ -432,10 +446,11 @@ func (s *IAMServiceImpl) PutGroupPolicy(accountID string, input *iam.PutGroupPol // GetGroupPolicy returns a group's inline policy document by name as a raw JSON // string, matching the in-repo convention used by GetRolePolicy. func (s *IAMServiceImpl) GetGroupPolicy(accountID string, input *iam.GetGroupPolicyInput) (*iam.GetGroupPolicyOutput, error) { + ctx := context.Background() groupName := *input.GroupName policyName := *input.PolicyName - group, err := s.getGroup(accountID, groupName) + group, err := s.getGroup(ctx, accountID, groupName) if err != nil { return nil, err } @@ -455,11 +470,12 @@ func (s *IAMServiceImpl) GetGroupPolicy(accountID string, input *iam.GetGroupPol // DeleteGroupPolicy removes a group's inline policy by name. A missing name // yields NoSuchEntity, matching AWS. Blind Put like the other group writers. func (s *IAMServiceImpl) DeleteGroupPolicy(accountID string, input *iam.DeleteGroupPolicyInput) (*iam.DeleteGroupPolicyOutput, error) { + ctx := context.Background() groupName := *input.GroupName policyName := *input.PolicyName kvKey := accountID + "." + groupName - group, err := s.getGroup(accountID, groupName) + group, err := s.getGroup(ctx, accountID, groupName) if err != nil { return nil, err } @@ -474,7 +490,7 @@ func (s *IAMServiceImpl) DeleteGroupPolicy(accountID string, input *iam.DeleteGr return nil, fmt.Errorf("marshal group: %w", err) } - if _, err := s.groupsBucket.Put(kvKey, data); err != nil { + if _, err := s.groupsBucket.Put(ctx, kvKey, data); err != nil { return nil, fmt.Errorf("update group: %w", err) } @@ -485,7 +501,8 @@ func (s *IAMServiceImpl) DeleteGroupPolicy(accountID string, input *iam.DeleteGr // ListGroupPolicies returns the names of a group's inline policies, sorted for // deterministic output. Pagination is not implemented: IsTruncated is always false. func (s *IAMServiceImpl) ListGroupPolicies(accountID string, input *iam.ListGroupPoliciesInput) (*iam.ListGroupPoliciesOutput, error) { - group, err := s.getGroup(accountID, *input.GroupName) + ctx := context.Background() + group, err := s.getGroup(ctx, accountID, *input.GroupName) if err != nil { return nil, err } @@ -507,10 +524,10 @@ func (s *IAMServiceImpl) ListGroupPolicies(accountID string, input *iam.ListGrou // Helpers // --------------------------------------------------------------------------- -func (s *IAMServiceImpl) getGroup(accountID, groupName string) (*Group, error) { - entry, err := s.groupsBucket.Get(accountID + "." + groupName) +func (s *IAMServiceImpl) getGroup(ctx context.Context, accountID, groupName string) (*Group, error) { + entry, err := s.groupsBucket.Get(ctx, accountID+"."+groupName) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil, errors.New(awserrors.ErrorIAMNoSuchEntity) } return nil, fmt.Errorf("get group: %w", err) @@ -527,10 +544,10 @@ func (s *IAMServiceImpl) getGroup(accountID, groupName string) (*Group, error) { // User.Groups references the given group. Fails closed on per-key Get or // unmarshal errors so DeleteGroup cannot succeed while a real-but-unreadable // member exists. Mirrors findInstanceProfilesForRole. -func (s *IAMServiceImpl) findGroupMembers(accountID, groupName string) ([]*User, error) { - keys, err := s.usersBucket.Keys() +func (s *IAMServiceImpl) findGroupMembers(ctx context.Context, accountID, groupName string) ([]*User, error) { + keys, err := kvutil.Keys(ctx, s.usersBucket) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return nil, nil } return nil, fmt.Errorf("list user keys: %w", err) @@ -546,9 +563,9 @@ func (s *IAMServiceImpl) findGroupMembers(accountID, groupName string) ([]*User, continue } - entry, err := s.usersBucket.Get(key) + entry, err := s.usersBucket.Get(ctx, key) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { slog.Debug("findGroupMembers: user key disappeared", "key", key) continue } diff --git a/spinifex/handlers/iam/groups_test.go b/spinifex/handlers/iam/groups_test.go index 626da62b3..512d43f4b 100644 --- a/spinifex/handlers/iam/groups_test.go +++ b/spinifex/handlers/iam/groups_test.go @@ -394,7 +394,7 @@ func TestRemoveUserFromGroup_DanglingPointer(t *testing.T) { require.NoError(t, err) // Delete the group record directly, bypassing the non-empty-group guard. - require.NoError(t, svc.groupsBucket.Delete(testAccountID+".doomed")) + require.NoError(t, svc.groupsBucket.Delete(t.Context(), testAccountID+".doomed")) _, err = svc.RemoveUserFromGroup(testAccountID, &iam.RemoveUserFromGroupInput{ GroupName: aws.String("doomed"), @@ -471,7 +471,7 @@ func TestListGroupsForUser_SkipsMissingGroup(t *testing.T) { require.NoError(t, err) } - require.NoError(t, svc.groupsBucket.Delete(testAccountID+".vanishing")) + require.NoError(t, svc.groupsBucket.Delete(t.Context(), testAccountID+".vanishing")) out, err := svc.ListGroupsForUser(testAccountID, &iam.ListGroupsForUserInput{ UserName: aws.String("henry"), @@ -1030,7 +1030,7 @@ func TestGetUserPolicies_SkipsMissingGroup(t *testing.T) { require.NoError(t, err) // Delete the group record directly, leaving a dangling User.Groups pointer. - require.NoError(t, svc.groupsBucket.Delete(testAccountID+".soon-gone")) + require.NoError(t, svc.groupsBucket.Delete(t.Context(), testAccountID+".soon-gone")) docs, err := svc.GetUserPolicies(testAccountID, "kim") require.NoError(t, err, "a missing group must be skipped, not fail closed") @@ -1050,7 +1050,7 @@ func TestGetUserPolicies_SkipsMissingGroup(t *testing.T) { // corrupt stored document that the write path would otherwise reject. func putRawGroupInlinePolicy(t *testing.T, svc *IAMServiceImpl, groupName, policyName, raw string) { t.Helper() - group, err := svc.getGroup(testAccountID, groupName) + group, err := svc.getGroup(t.Context(), testAccountID, groupName) require.NoError(t, err) if group.InlinePolicies == nil { group.InlinePolicies = map[string]string{} @@ -1058,7 +1058,7 @@ func putRawGroupInlinePolicy(t *testing.T, svc *IAMServiceImpl, groupName, polic group.InlinePolicies[policyName] = raw data, err := json.Marshal(group) require.NoError(t, err) - _, err = svc.groupsBucket.Put(testAccountID+"."+groupName, data) + _, err = svc.groupsBucket.Put(t.Context(), testAccountID+"."+groupName, data) require.NoError(t, err) } diff --git a/spinifex/handlers/iam/instance_profiles.go b/spinifex/handlers/iam/instance_profiles.go index 58abddc2d..8047ff2af 100644 --- a/spinifex/handlers/iam/instance_profiles.go +++ b/spinifex/handlers/iam/instance_profiles.go @@ -1,6 +1,7 @@ package handlers_iam import ( + "context" "encoding/json" "errors" "fmt" @@ -10,12 +11,15 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/iam" + "github.com/nats-io/nats.go/jetstream" + "github.com/mulgadc/spinifex/spinifex/awserrors" + "github.com/mulgadc/spinifex/spinifex/kvutil" "github.com/mulgadc/spinifex/spinifex/utils" - "github.com/nats-io/nats.go" ) func (s *IAMServiceImpl) CreateInstanceProfile(accountID string, input *iam.CreateInstanceProfileInput) (*iam.CreateInstanceProfileOutput, error) { + ctx := context.Background() profileName := *input.InstanceProfileName if err := validatePolicyName(profileName); err != nil { @@ -50,8 +54,8 @@ func (s *IAMServiceImpl) CreateInstanceProfile(accountID string, input *iam.Crea return nil, fmt.Errorf("marshal instance profile: %w", err) } - if _, err := s.instanceProfilesBucket.Create(accountID+"."+profileName, data); err != nil { - if errors.Is(err, nats.ErrKeyExists) { + if _, err := s.instanceProfilesBucket.Create(ctx, accountID+"."+profileName, data); err != nil { + if errors.Is(err, jetstream.ErrKeyExists) { return nil, errors.New(awserrors.ErrorIAMEntityAlreadyExists) } return nil, fmt.Errorf("store instance profile: %w", err) @@ -60,7 +64,7 @@ func (s *IAMServiceImpl) CreateInstanceProfile(accountID string, input *iam.Crea slog.Info("IAM instance profile created", "accountID", accountID, "instanceProfileName", profileName, "instanceProfileID", profile.InstanceProfileID) - sdkProfile, err := s.profileToSDK(accountID, &profile) + sdkProfile, err := s.profileToSDK(ctx, accountID, &profile) if err != nil { return nil, err } @@ -68,12 +72,13 @@ func (s *IAMServiceImpl) CreateInstanceProfile(accountID string, input *iam.Crea } func (s *IAMServiceImpl) GetInstanceProfile(accountID string, input *iam.GetInstanceProfileInput) (*iam.GetInstanceProfileOutput, error) { - profile, err := s.getInstanceProfile(accountID, *input.InstanceProfileName) + ctx := context.Background() + profile, err := s.getInstanceProfile(ctx, accountID, *input.InstanceProfileName) if err != nil { return nil, err } - sdkProfile, err := s.profileToSDK(accountID, profile) + sdkProfile, err := s.profileToSDK(ctx, accountID, profile) if err != nil { return nil, err } @@ -81,9 +86,10 @@ func (s *IAMServiceImpl) GetInstanceProfile(accountID string, input *iam.GetInst } func (s *IAMServiceImpl) ListInstanceProfiles(accountID string, input *iam.ListInstanceProfilesInput) (*iam.ListInstanceProfilesOutput, error) { - keys, err := s.instanceProfilesBucket.Keys() + ctx := context.Background() + keys, err := kvutil.Keys(ctx, s.instanceProfilesBucket) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return &iam.ListInstanceProfilesOutput{ InstanceProfiles: []*iam.InstanceProfile{}, IsTruncated: aws.Bool(false), @@ -107,9 +113,9 @@ func (s *IAMServiceImpl) ListInstanceProfiles(accountID string, input *iam.ListI continue } - entry, err := s.instanceProfilesBucket.Get(key) + entry, err := s.instanceProfilesBucket.Get(ctx, key) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { slog.Debug("ListInstanceProfiles: profile key disappeared (concurrent delete)", "key", key) } else { slog.Warn("ListInstanceProfiles: failed to get profile", "key", key, "err", err) @@ -127,7 +133,7 @@ func (s *IAMServiceImpl) ListInstanceProfiles(accountID string, input *iam.ListI continue } - sdkProfile, err := s.profileToSDK(accountID, &profile) + sdkProfile, err := s.profileToSDK(ctx, accountID, &profile) if err != nil { return nil, err } @@ -141,9 +147,10 @@ func (s *IAMServiceImpl) ListInstanceProfiles(accountID string, input *iam.ListI } func (s *IAMServiceImpl) DeleteInstanceProfile(accountID string, input *iam.DeleteInstanceProfileInput) (*iam.DeleteInstanceProfileOutput, error) { + ctx := context.Background() profileName := *input.InstanceProfileName - profile, err := s.getInstanceProfile(accountID, profileName) + profile, err := s.getInstanceProfile(ctx, accountID, profileName) if err != nil { return nil, err } @@ -152,7 +159,7 @@ func (s *IAMServiceImpl) DeleteInstanceProfile(accountID string, input *iam.Dele return nil, errors.New(awserrors.ErrorIAMDeleteConflict) } - if err := s.instanceProfilesBucket.Delete(accountID + "." + profileName); err != nil { + if err := s.instanceProfilesBucket.Delete(ctx, accountID+"."+profileName); err != nil { return nil, fmt.Errorf("delete instance profile: %w", err) } @@ -161,14 +168,15 @@ func (s *IAMServiceImpl) DeleteInstanceProfile(accountID string, input *iam.Dele } func (s *IAMServiceImpl) AddRoleToInstanceProfile(accountID string, input *iam.AddRoleToInstanceProfileInput) (*iam.AddRoleToInstanceProfileOutput, error) { + ctx := context.Background() profileName := *input.InstanceProfileName roleName := *input.RoleName - if _, err := s.getRole(accountID, roleName); err != nil { + if _, err := s.getRole(ctx, accountID, roleName); err != nil { return nil, err } - profile, err := s.getInstanceProfile(accountID, profileName) + profile, err := s.getInstanceProfile(ctx, accountID, profileName) if err != nil { return nil, err } @@ -182,7 +190,7 @@ func (s *IAMServiceImpl) AddRoleToInstanceProfile(accountID string, input *iam.A if err != nil { return nil, fmt.Errorf("marshal instance profile: %w", err) } - if _, err := s.instanceProfilesBucket.Put(accountID+"."+profileName, data); err != nil { + if _, err := s.instanceProfilesBucket.Put(ctx, accountID+"."+profileName, data); err != nil { return nil, fmt.Errorf("update instance profile: %w", err) } @@ -192,10 +200,11 @@ func (s *IAMServiceImpl) AddRoleToInstanceProfile(accountID string, input *iam.A } func (s *IAMServiceImpl) RemoveRoleFromInstanceProfile(accountID string, input *iam.RemoveRoleFromInstanceProfileInput) (*iam.RemoveRoleFromInstanceProfileOutput, error) { + ctx := context.Background() profileName := *input.InstanceProfileName roleName := *input.RoleName - profile, err := s.getInstanceProfile(accountID, profileName) + profile, err := s.getInstanceProfile(ctx, accountID, profileName) if err != nil { return nil, err } @@ -209,7 +218,7 @@ func (s *IAMServiceImpl) RemoveRoleFromInstanceProfile(accountID string, input * if err != nil { return nil, fmt.Errorf("marshal instance profile: %w", err) } - if _, err := s.instanceProfilesBucket.Put(accountID+"."+profileName, data); err != nil { + if _, err := s.instanceProfilesBucket.Put(ctx, accountID+"."+profileName, data); err != nil { return nil, fmt.Errorf("update instance profile: %w", err) } @@ -219,20 +228,21 @@ func (s *IAMServiceImpl) RemoveRoleFromInstanceProfile(accountID string, input * } func (s *IAMServiceImpl) ListInstanceProfilesForRole(accountID string, input *iam.ListInstanceProfilesForRoleInput) (*iam.ListInstanceProfilesForRoleOutput, error) { + ctx := context.Background() roleName := *input.RoleName - if _, err := s.getRole(accountID, roleName); err != nil { + if _, err := s.getRole(ctx, accountID, roleName); err != nil { return nil, err } - profiles, err := s.findInstanceProfilesForRole(accountID, roleName) + profiles, err := s.findInstanceProfilesForRole(ctx, accountID, roleName) if err != nil { return nil, fmt.Errorf("find instance profiles for role: %w", err) } out := make([]*iam.InstanceProfile, 0, len(profiles)) for _, p := range profiles { - sdkProfile, err := s.profileToSDK(accountID, p) + sdkProfile, err := s.profileToSDK(ctx, accountID, p) if err != nil { return nil, err } @@ -248,12 +258,13 @@ func (s *IAMServiceImpl) ListInstanceProfilesForRole(accountID string, input *ia // ResolveInstanceProfile resolves an instance-profile name or ARN to its record. // When given an ARN, the embedded account ID must match accountID. func (s *IAMServiceImpl) ResolveInstanceProfile(accountID, nameOrARN string) (*InstanceProfile, error) { + ctx := context.Background() if nameOrARN == "" { return nil, errors.New(awserrors.ErrorIAMInvalidInput) } if !strings.HasPrefix(nameOrARN, "arn:") { - return s.getInstanceProfile(accountID, nameOrARN) + return s.getInstanceProfile(ctx, accountID, nameOrARN) } profileAccountID, profileName, err := parseInstanceProfileARN(nameOrARN) @@ -263,7 +274,7 @@ func (s *IAMServiceImpl) ResolveInstanceProfile(accountID, nameOrARN string) (*I if profileAccountID != accountID { return nil, errors.New(awserrors.ErrorAccessDenied) } - return s.getInstanceProfile(accountID, profileName) + return s.getInstanceProfile(ctx, accountID, profileName) } // parseInstanceProfileARN extracts accountID and profile name from an IAM instance-profile ARN. @@ -293,12 +304,13 @@ func parseInstanceProfileARN(arn string) (accountID, name string, err error) { // TagInstanceProfile upserts tags on an instance profile. Blind // read-modify-write Put like the other instance-profile writers (no CAS). func (s *IAMServiceImpl) TagInstanceProfile(accountID string, input *iam.TagInstanceProfileInput) (*iam.TagInstanceProfileOutput, error) { + ctx := context.Background() if err := validateTags(input.Tags); err != nil { return nil, err } profileName := *input.InstanceProfileName - profile, err := s.getInstanceProfile(accountID, profileName) + profile, err := s.getInstanceProfile(ctx, accountID, profileName) if err != nil { return nil, err } @@ -313,7 +325,7 @@ func (s *IAMServiceImpl) TagInstanceProfile(accountID string, input *iam.TagInst if err != nil { return nil, fmt.Errorf("marshal instance profile: %w", err) } - if _, err := s.instanceProfilesBucket.Put(accountID+"."+profileName, data); err != nil { + if _, err := s.instanceProfilesBucket.Put(ctx, accountID+"."+profileName, data); err != nil { return nil, fmt.Errorf("update instance profile: %w", err) } @@ -324,8 +336,9 @@ func (s *IAMServiceImpl) TagInstanceProfile(accountID string, input *iam.TagInst // UntagInstanceProfile removes the named tag keys from an instance profile; // unknown keys are a no-op. func (s *IAMServiceImpl) UntagInstanceProfile(accountID string, input *iam.UntagInstanceProfileInput) (*iam.UntagInstanceProfileOutput, error) { + ctx := context.Background() profileName := *input.InstanceProfileName - profile, err := s.getInstanceProfile(accountID, profileName) + profile, err := s.getInstanceProfile(ctx, accountID, profileName) if err != nil { return nil, err } @@ -336,7 +349,7 @@ func (s *IAMServiceImpl) UntagInstanceProfile(accountID string, input *iam.Untag if err != nil { return nil, fmt.Errorf("marshal instance profile: %w", err) } - if _, err := s.instanceProfilesBucket.Put(accountID+"."+profileName, data); err != nil { + if _, err := s.instanceProfilesBucket.Put(ctx, accountID+"."+profileName, data); err != nil { return nil, fmt.Errorf("update instance profile: %w", err) } @@ -347,7 +360,8 @@ func (s *IAMServiceImpl) UntagInstanceProfile(accountID string, input *iam.Untag // ListInstanceProfileTags returns an instance profile's tags. Pagination is // not implemented: IsTruncated is always false. func (s *IAMServiceImpl) ListInstanceProfileTags(accountID string, input *iam.ListInstanceProfileTagsInput) (*iam.ListInstanceProfileTagsOutput, error) { - profile, err := s.getInstanceProfile(accountID, *input.InstanceProfileName) + ctx := context.Background() + profile, err := s.getInstanceProfile(ctx, accountID, *input.InstanceProfileName) if err != nil { return nil, err } @@ -357,10 +371,10 @@ func (s *IAMServiceImpl) ListInstanceProfileTags(accountID string, input *iam.Li }, nil } -func (s *IAMServiceImpl) getInstanceProfile(accountID, profileName string) (*InstanceProfile, error) { - entry, err := s.instanceProfilesBucket.Get(accountID + "." + profileName) +func (s *IAMServiceImpl) getInstanceProfile(ctx context.Context, accountID, profileName string) (*InstanceProfile, error) { + entry, err := s.instanceProfilesBucket.Get(ctx, accountID+"."+profileName) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil, errors.New(awserrors.ErrorIAMNoSuchEntity) } return nil, fmt.Errorf("get instance profile: %w", err) @@ -375,7 +389,7 @@ func (s *IAMServiceImpl) getInstanceProfile(accountID, profileName string) (*Ins // profileToSDK converts the internal InstanceProfile to the AWS SDK shape. // Dereferences the attached role when present; propagates lookup errors. -func (s *IAMServiceImpl) profileToSDK(accountID string, p *InstanceProfile) (*iam.InstanceProfile, error) { +func (s *IAMServiceImpl) profileToSDK(ctx context.Context, accountID string, p *InstanceProfile) (*iam.InstanceProfile, error) { out := &iam.InstanceProfile{ InstanceProfileName: aws.String(p.InstanceProfileName), InstanceProfileId: aws.String(p.InstanceProfileID), @@ -391,7 +405,7 @@ func (s *IAMServiceImpl) profileToSDK(accountID string, p *InstanceProfile) (*ia }) } if p.RoleName != "" { - role, err := s.getRole(accountID, p.RoleName) + role, err := s.getRole(ctx, accountID, p.RoleName) if err != nil { return nil, fmt.Errorf("resolve attached role %q on profile %q: %w", p.RoleName, p.InstanceProfileName, err) } diff --git a/spinifex/handlers/iam/invariants_test.go b/spinifex/handlers/iam/invariants_test.go index 238a71a71..8a00e4e7d 100644 --- a/spinifex/handlers/iam/invariants_test.go +++ b/spinifex/handlers/iam/invariants_test.go @@ -36,11 +36,11 @@ func TestInvariant_AccessKeysBucket_RejectsNonAKIAPrefix(t *testing.T) { payload := []byte(`{}`) for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - err := svc.putAccessKey(tc.akid, payload) + err := svc.putAccessKey(t.Context(), tc.akid, payload) require.Error(t, err, "putAccessKey accepted invalid prefix %q", tc.akid) assert.Contains(t, err.Error(), LongLivedAccessKeyIDPrefix) - err = svc.createAccessKey(tc.akid, payload) + err = svc.createAccessKey(t.Context(), tc.akid, payload) require.Error(t, err, "createAccessKey accepted invalid prefix %q", tc.akid) assert.Contains(t, err.Error(), LongLivedAccessKeyIDPrefix) }) @@ -51,9 +51,9 @@ func TestInvariant_AccessKeysBucket_AcceptsAKIAPrefix(t *testing.T) { svc := setupTestIAMService(t) akid := LongLivedAccessKeyIDPrefix + "0123456789ABCDEF" - require.NoError(t, svc.putAccessKey(akid, []byte(`{}`))) + require.NoError(t, svc.putAccessKey(t.Context(), akid, []byte(`{}`))) - entry, err := svc.accessKeysBucket.Get(akid) + entry, err := svc.accessKeysBucket.Get(t.Context(), akid) require.NoError(t, err) assert.Equal(t, []byte(`{}`), entry.Value()) } diff --git a/spinifex/handlers/iam/managed_policies.go b/spinifex/handlers/iam/managed_policies.go index b3d6118b0..fea4706cb 100644 --- a/spinifex/handlers/iam/managed_policies.go +++ b/spinifex/handlers/iam/managed_policies.go @@ -1,6 +1,7 @@ package handlers_iam import ( + "context" "encoding/json" "fmt" ) @@ -219,14 +220,14 @@ func builtinManagedPolicyDoc(arn string) (PolicyDocument, bool) { // role carrying an unmodeled managed policy is denied that grant rather than // failing the whole request. Customer-managed ARNs are fetched from KV and // fail closed (error) when unresolvable. -func (s *IAMServiceImpl) resolveAttachedPolicy(accountID, arn string) (doc PolicyDocument, include bool, err error) { +func (s *IAMServiceImpl) resolveAttachedPolicy(ctx context.Context, accountID, arn string) (doc PolicyDocument, include bool, err error) { if isAWSManagedPolicyARN(arn) { if d, ok := builtinManagedPolicyDoc(arn); ok { return d, true, nil } return PolicyDocument{}, false, nil } - policy, err := s.getPolicyByARN(accountID, arn) + policy, err := s.getPolicyByARN(ctx, accountID, arn) if err != nil { return PolicyDocument{}, false, fmt.Errorf("resolve policy %s: %w", arn, err) } diff --git a/spinifex/handlers/iam/oidc_providers.go b/spinifex/handlers/iam/oidc_providers.go index 8a44249ec..6d5055df5 100644 --- a/spinifex/handlers/iam/oidc_providers.go +++ b/spinifex/handlers/iam/oidc_providers.go @@ -1,6 +1,7 @@ package handlers_iam import ( + "context" "crypto/sha256" "encoding/hex" "encoding/json" @@ -13,9 +14,11 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/iam" + "github.com/nats-io/nats.go/jetstream" + "github.com/mulgadc/spinifex/spinifex/awserrors" + "github.com/mulgadc/spinifex/spinifex/kvutil" "github.com/mulgadc/spinifex/spinifex/utils" - "github.com/nats-io/nats.go" ) // Per-account IAM bucket holds account-scoped resources (currently the OIDC provider registry). @@ -46,8 +49,8 @@ func OIDCProviderARN(accountID, issuerHostPath string) string { } // GetOrCreateIAMAccountBucket opens the per-account IAM bucket, creating it on first use. -func GetOrCreateIAMAccountBucket(js nats.JetStreamContext, accountID string, replicas int) (nats.KeyValue, error) { - return getOrCreateBucket(js, IAMAccountBucketName(accountID), 1, max(replicas, 1)) +func GetOrCreateIAMAccountBucket(ctx context.Context, js jetstream.JetStream, accountID string, replicas int) (jetstream.KeyValue, error) { + return kvutil.GetOrCreateBucketWithReplicas(ctx, js, IAMAccountBucketName(accountID), 1, max(replicas, 1)) } // OIDCProviderRecord is the stored shape of a registered OIDC identity provider. @@ -101,6 +104,7 @@ func awsStrings(in []*string) []string { } func (s *IAMServiceImpl) CreateOpenIDConnectProvider(accountID string, input *iam.CreateOpenIDConnectProviderInput) (*iam.CreateOpenIDConnectProviderOutput, error) { + ctx := context.Background() issuer := aws.StringValue(input.Url) if err := validateOIDCProviderURL(issuer); err != nil { slog.Debug("CreateOpenIDConnectProvider: invalid Url", "url", issuer, "err", err) @@ -119,13 +123,13 @@ func (s *IAMServiceImpl) CreateOpenIDConnectProvider(accountID string, input *ia return nil, fmt.Errorf("marshal OIDC provider: %w", err) } - kv, err := GetOrCreateIAMAccountBucket(s.js, accountID, s.replicas) + kv, err := GetOrCreateIAMAccountBucket(ctx, s.js, accountID, s.replicas) if err != nil { return nil, fmt.Errorf("open IAM account bucket: %w", err) } - if _, err := kv.Create(OIDCProviderKey(issuer), data); err != nil { - if errors.Is(err, nats.ErrKeyExists) { + if _, err := kv.Create(ctx, OIDCProviderKey(issuer), data); err != nil { + if errors.Is(err, jetstream.ErrKeyExists) { return nil, errors.New(awserrors.ErrorIAMEntityAlreadyExists) } return nil, fmt.Errorf("store OIDC provider: %w", err) @@ -140,7 +144,8 @@ func (s *IAMServiceImpl) CreateOpenIDConnectProvider(accountID string, input *ia } func (s *IAMServiceImpl) GetOpenIDConnectProvider(accountID string, input *iam.GetOpenIDConnectProviderInput) (*iam.GetOpenIDConnectProviderOutput, error) { - record, err := s.getOIDCProvider(accountID, aws.StringValue(input.OpenIDConnectProviderArn)) + ctx := context.Background() + record, err := s.getOIDCProvider(ctx, accountID, aws.StringValue(input.OpenIDConnectProviderArn)) if err != nil { return nil, err } @@ -154,21 +159,22 @@ func (s *IAMServiceImpl) GetOpenIDConnectProvider(accountID string, input *iam.G } func (s *IAMServiceImpl) ListOpenIDConnectProviders(accountID string, _ *iam.ListOpenIDConnectProvidersInput) (*iam.ListOpenIDConnectProvidersOutput, error) { + ctx := context.Background() out := &iam.ListOpenIDConnectProvidersOutput{ OpenIDConnectProviderList: []*iam.OpenIDConnectProviderListEntry{}, } - kv, err := s.js.KeyValue(IAMAccountBucketName(accountID)) + kv, err := s.js.KeyValue(ctx, IAMAccountBucketName(accountID)) if err != nil { - if errors.Is(err, nats.ErrBucketNotFound) { + if errors.Is(err, jetstream.ErrBucketNotFound) { return out, nil } return nil, fmt.Errorf("open IAM account bucket: %w", err) } - keys, err := kv.Keys() + keys, err := kvutil.Keys(ctx, kv) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return out, nil } return nil, fmt.Errorf("list OIDC provider keys: %w", err) @@ -178,9 +184,9 @@ func (s *IAMServiceImpl) ListOpenIDConnectProviders(accountID string, _ *iam.Lis if key == utils.VersionKey || !strings.HasPrefix(key, oidcProvidersKeyPrefix) { continue } - entry, err := kv.Get(key) + entry, err := kv.Get(ctx, key) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { continue } slog.Warn("ListOpenIDConnectProviders: get failed", "key", key, "err", err) @@ -200,28 +206,29 @@ func (s *IAMServiceImpl) ListOpenIDConnectProviders(accountID string, _ *iam.Lis } func (s *IAMServiceImpl) DeleteOpenIDConnectProvider(accountID string, input *iam.DeleteOpenIDConnectProviderInput) (*iam.DeleteOpenIDConnectProviderOutput, error) { + ctx := context.Background() arn := aws.StringValue(input.OpenIDConnectProviderArn) issuer, err := issuerFromOIDCProviderARN(arn) if err != nil { return nil, errors.New(awserrors.ErrorIAMNoSuchEntity) } - kv, err := s.js.KeyValue(IAMAccountBucketName(accountID)) + kv, err := s.js.KeyValue(ctx, IAMAccountBucketName(accountID)) if err != nil { - if errors.Is(err, nats.ErrBucketNotFound) { + if errors.Is(err, jetstream.ErrBucketNotFound) { return nil, errors.New(awserrors.ErrorIAMNoSuchEntity) } return nil, fmt.Errorf("open IAM account bucket: %w", err) } key := OIDCProviderKey(issuer) - if _, err := kv.Get(key); err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if _, err := kv.Get(ctx, key); err != nil { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil, errors.New(awserrors.ErrorIAMNoSuchEntity) } return nil, fmt.Errorf("get OIDC provider: %w", err) } - if err := kv.Delete(key); err != nil { + if err := kv.Delete(ctx, key); err != nil { return nil, fmt.Errorf("delete OIDC provider: %w", err) } @@ -232,12 +239,13 @@ func (s *IAMServiceImpl) DeleteOpenIDConnectProvider(accountID string, input *ia // TagOpenIDConnectProvider upserts tags on an OIDC provider. Blind // read-modify-write Put like the other writers here (no CAS). func (s *IAMServiceImpl) TagOpenIDConnectProvider(accountID string, input *iam.TagOpenIDConnectProviderInput) (*iam.TagOpenIDConnectProviderOutput, error) { + ctx := context.Background() if err := validateTags(input.Tags); err != nil { return nil, err } arn := aws.StringValue(input.OpenIDConnectProviderArn) - record, err := s.getOIDCProvider(accountID, arn) + record, err := s.getOIDCProvider(ctx, accountID, arn) if err != nil { return nil, err } @@ -248,7 +256,7 @@ func (s *IAMServiceImpl) TagOpenIDConnectProvider(accountID string, input *iam.T } record.Tags = merged - if err := s.putOIDCProvider(accountID, record); err != nil { + if err := s.putOIDCProvider(ctx, accountID, record); err != nil { return nil, err } @@ -259,15 +267,16 @@ func (s *IAMServiceImpl) TagOpenIDConnectProvider(accountID string, input *iam.T // UntagOpenIDConnectProvider removes the named tag keys from an OIDC provider; // unknown keys are a no-op. func (s *IAMServiceImpl) UntagOpenIDConnectProvider(accountID string, input *iam.UntagOpenIDConnectProviderInput) (*iam.UntagOpenIDConnectProviderOutput, error) { + ctx := context.Background() arn := aws.StringValue(input.OpenIDConnectProviderArn) - record, err := s.getOIDCProvider(accountID, arn) + record, err := s.getOIDCProvider(ctx, accountID, arn) if err != nil { return nil, err } record.Tags = removeTagKeys(record.Tags, input.TagKeys) - if err := s.putOIDCProvider(accountID, record); err != nil { + if err := s.putOIDCProvider(ctx, accountID, record); err != nil { return nil, err } @@ -278,7 +287,8 @@ func (s *IAMServiceImpl) UntagOpenIDConnectProvider(accountID string, input *iam // ListOpenIDConnectProviderTags returns an OIDC provider's tags. Pagination is // not implemented: IsTruncated is always false. func (s *IAMServiceImpl) ListOpenIDConnectProviderTags(accountID string, input *iam.ListOpenIDConnectProviderTagsInput) (*iam.ListOpenIDConnectProviderTagsOutput, error) { - record, err := s.getOIDCProvider(accountID, aws.StringValue(input.OpenIDConnectProviderArn)) + ctx := context.Background() + record, err := s.getOIDCProvider(ctx, accountID, aws.StringValue(input.OpenIDConnectProviderArn)) if err != nil { return nil, err } @@ -290,39 +300,39 @@ func (s *IAMServiceImpl) ListOpenIDConnectProviderTags(accountID string, input * // putOIDCProvider overwrites an existing provider record, keyed by its issuer // URL. The record must have been read from the bucket first. -func (s *IAMServiceImpl) putOIDCProvider(accountID string, record *OIDCProviderRecord) error { +func (s *IAMServiceImpl) putOIDCProvider(ctx context.Context, accountID string, record *OIDCProviderRecord) error { data, err := json.Marshal(record) if err != nil { return fmt.Errorf("marshal OIDC provider: %w", err) } - kv, err := s.js.KeyValue(IAMAccountBucketName(accountID)) + kv, err := s.js.KeyValue(ctx, IAMAccountBucketName(accountID)) if err != nil { - if errors.Is(err, nats.ErrBucketNotFound) { + if errors.Is(err, jetstream.ErrBucketNotFound) { return errors.New(awserrors.ErrorIAMNoSuchEntity) } return fmt.Errorf("open IAM account bucket: %w", err) } - if _, err := kv.Put(OIDCProviderKey(record.Url), data); err != nil { + if _, err := kv.Put(ctx, OIDCProviderKey(record.Url), data); err != nil { return fmt.Errorf("store OIDC provider: %w", err) } return nil } -func (s *IAMServiceImpl) getOIDCProvider(accountID, arn string) (*OIDCProviderRecord, error) { +func (s *IAMServiceImpl) getOIDCProvider(ctx context.Context, accountID, arn string) (*OIDCProviderRecord, error) { issuer, err := issuerFromOIDCProviderARN(arn) if err != nil { return nil, errors.New(awserrors.ErrorIAMNoSuchEntity) } - kv, err := s.js.KeyValue(IAMAccountBucketName(accountID)) + kv, err := s.js.KeyValue(ctx, IAMAccountBucketName(accountID)) if err != nil { - if errors.Is(err, nats.ErrBucketNotFound) { + if errors.Is(err, jetstream.ErrBucketNotFound) { return nil, errors.New(awserrors.ErrorIAMNoSuchEntity) } return nil, fmt.Errorf("open IAM account bucket: %w", err) } - entry, err := kv.Get(OIDCProviderKey(issuer)) + entry, err := kv.Get(ctx, OIDCProviderKey(issuer)) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil, errors.New(awserrors.ErrorIAMNoSuchEntity) } return nil, fmt.Errorf("get OIDC provider: %w", err) diff --git a/spinifex/handlers/iam/oidc_providers_test.go b/spinifex/handlers/iam/oidc_providers_test.go index d7c22e22f..e7d5e2fa2 100644 --- a/spinifex/handlers/iam/oidc_providers_test.go +++ b/spinifex/handlers/iam/oidc_providers_test.go @@ -29,11 +29,11 @@ func TestCreateOpenIDConnectProvider(t *testing.T) { } // The registry key STS reads must now exist for this exact issuer. - kv, err := svc.js.KeyValue(IAMAccountBucketName(acct)) + kv, err := svc.js.KeyValue(t.Context(), IAMAccountBucketName(acct)) if err != nil { t.Fatalf("open account bucket: %v", err) } - if _, err := kv.Get(OIDCProviderKey(testIssuer)); err != nil { + if _, err := kv.Get(t.Context(), OIDCProviderKey(testIssuer)); err != nil { t.Fatalf("provider key not written for issuer: %v", err) } } diff --git a/spinifex/handlers/iam/retry.go b/spinifex/handlers/iam/retry.go index 130806a8b..b7d3243d0 100644 --- a/spinifex/handlers/iam/retry.go +++ b/spinifex/handlers/iam/retry.go @@ -1,6 +1,7 @@ package handlers_iam import ( + "context" "fmt" "log/slog" "time" @@ -14,7 +15,7 @@ import ( // leave the service nil for the process lifetime. Blocks up to maxWait, then // returns the last error. Callers that legitimately run without a master key // must guard the call themselves. -func NewIAMServiceWithRetry(natsConn *nats.Conn, masterKey []byte, clusterSize int) (*IAMServiceImpl, error) { +func NewIAMServiceWithRetry(ctx context.Context, natsConn *nats.Conn, masterKey []byte, clusterSize int) (*IAMServiceImpl, error) { const maxWait = 5 * time.Minute retryDelay := 500 * time.Millisecond start := time.Now() @@ -22,7 +23,7 @@ func NewIAMServiceWithRetry(natsConn *nats.Conn, masterKey []byte, clusterSize i for { attempt++ - svc, err := NewIAMServiceImpl(natsConn, masterKey, clusterSize) + svc, err := NewIAMServiceImpl(ctx, natsConn, masterKey, clusterSize) if err == nil { if attempt > 1 { slog.Info("IAM service initialized after retry", "attempts", attempt, "elapsed", time.Since(start).Round(time.Second)) diff --git a/spinifex/handlers/iam/roles.go b/spinifex/handlers/iam/roles.go index efd3ec751..3f2092406 100644 --- a/spinifex/handlers/iam/roles.go +++ b/spinifex/handlers/iam/roles.go @@ -1,6 +1,7 @@ package handlers_iam import ( + "context" "encoding/json" "errors" "fmt" @@ -12,9 +13,11 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/iam" + "github.com/nats-io/nats.go/jetstream" + "github.com/mulgadc/spinifex/spinifex/awserrors" + "github.com/mulgadc/spinifex/spinifex/kvutil" "github.com/mulgadc/spinifex/spinifex/utils" - "github.com/nats-io/nats.go" ) const ( @@ -33,6 +36,7 @@ const ( ) func (s *IAMServiceImpl) CreateRole(accountID string, input *iam.CreateRoleInput) (*iam.CreateRoleOutput, error) { + ctx := context.Background() roleName := *input.RoleName if err := validateUserName(roleName); err != nil { return nil, errors.New(awserrors.ErrorIAMInvalidInput) @@ -84,8 +88,8 @@ func (s *IAMServiceImpl) CreateRole(accountID string, input *iam.CreateRoleInput } kvKey := accountID + "." + roleName - if _, err := s.rolesBucket.Create(kvKey, data); err != nil { - if errors.Is(err, nats.ErrKeyExists) { + if _, err := s.rolesBucket.Create(ctx, kvKey, data); err != nil { + if errors.Is(err, jetstream.ErrKeyExists) { return nil, errors.New(awserrors.ErrorIAMEntityAlreadyExists) } return nil, fmt.Errorf("store role: %w", err) @@ -97,7 +101,8 @@ func (s *IAMServiceImpl) CreateRole(accountID string, input *iam.CreateRoleInput } func (s *IAMServiceImpl) GetRole(accountID string, input *iam.GetRoleInput) (*iam.GetRoleOutput, error) { - role, err := s.getRole(accountID, *input.RoleName) + ctx := context.Background() + role, err := s.getRole(ctx, accountID, *input.RoleName) if err != nil { return nil, err } @@ -106,9 +111,10 @@ func (s *IAMServiceImpl) GetRole(accountID string, input *iam.GetRoleInput) (*ia } func (s *IAMServiceImpl) ListRoles(accountID string, input *iam.ListRolesInput) (*iam.ListRolesOutput, error) { - keys, err := s.rolesBucket.Keys() + ctx := context.Background() + keys, err := kvutil.Keys(ctx, s.rolesBucket) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return &iam.ListRolesOutput{ Roles: []*iam.Role{}, IsTruncated: aws.Bool(false), @@ -132,9 +138,9 @@ func (s *IAMServiceImpl) ListRoles(accountID string, input *iam.ListRolesInput) continue } - entry, err := s.rolesBucket.Get(key) + entry, err := s.rolesBucket.Get(ctx, key) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { slog.Debug("ListRoles: role key disappeared (concurrent delete)", "key", key) } else { slog.Warn("ListRoles: failed to get role", "key", key, "err", err) @@ -162,9 +168,10 @@ func (s *IAMServiceImpl) ListRoles(accountID string, input *iam.ListRolesInput) } func (s *IAMServiceImpl) DeleteRole(accountID string, input *iam.DeleteRoleInput) (*iam.DeleteRoleOutput, error) { + ctx := context.Background() roleName := *input.RoleName - role, err := s.getRole(accountID, roleName) + role, err := s.getRole(ctx, accountID, roleName) if err != nil { return nil, err } @@ -177,7 +184,7 @@ func (s *IAMServiceImpl) DeleteRole(accountID string, input *iam.DeleteRoleInput return nil, errors.New(awserrors.ErrorIAMDeleteConflict) } - profiles, err := s.findInstanceProfilesForRole(accountID, roleName) + profiles, err := s.findInstanceProfilesForRole(ctx, accountID, roleName) if err != nil { return nil, fmt.Errorf("check role instance profiles: %w", err) } @@ -185,7 +192,7 @@ func (s *IAMServiceImpl) DeleteRole(accountID string, input *iam.DeleteRoleInput return nil, errors.New(awserrors.ErrorIAMDeleteConflict) } - if err := s.rolesBucket.Delete(accountID + "." + roleName); err != nil { + if err := s.rolesBucket.Delete(ctx, accountID+"."+roleName); err != nil { return nil, fmt.Errorf("delete role: %w", err) } @@ -194,9 +201,10 @@ func (s *IAMServiceImpl) DeleteRole(accountID string, input *iam.DeleteRoleInput } func (s *IAMServiceImpl) UpdateRole(accountID string, input *iam.UpdateRoleInput) (*iam.UpdateRoleOutput, error) { + ctx := context.Background() roleName := *input.RoleName - role, err := s.getRole(accountID, roleName) + role, err := s.getRole(ctx, accountID, roleName) if err != nil { return nil, err } @@ -216,7 +224,7 @@ func (s *IAMServiceImpl) UpdateRole(accountID string, input *iam.UpdateRoleInput if err != nil { return nil, fmt.Errorf("marshal role: %w", err) } - if _, err := s.rolesBucket.Put(accountID+"."+roleName, data); err != nil { + if _, err := s.rolesBucket.Put(ctx, accountID+"."+roleName, data); err != nil { return nil, fmt.Errorf("update role: %w", err) } @@ -225,6 +233,7 @@ func (s *IAMServiceImpl) UpdateRole(accountID string, input *iam.UpdateRoleInput } func (s *IAMServiceImpl) UpdateAssumeRolePolicy(accountID string, input *iam.UpdateAssumeRolePolicyInput) (*iam.UpdateAssumeRolePolicyOutput, error) { + ctx := context.Background() roleName := *input.RoleName if _, err := ValidateTrustPolicyDocument(*input.PolicyDocument); err != nil { @@ -232,7 +241,7 @@ func (s *IAMServiceImpl) UpdateAssumeRolePolicy(accountID string, input *iam.Upd return nil, errors.New(awserrors.ErrorIAMMalformedPolicyDocument) } - role, err := s.getRole(accountID, roleName) + role, err := s.getRole(ctx, accountID, roleName) if err != nil { return nil, err } @@ -243,7 +252,7 @@ func (s *IAMServiceImpl) UpdateAssumeRolePolicy(accountID string, input *iam.Upd if err != nil { return nil, fmt.Errorf("marshal role: %w", err) } - if _, err := s.rolesBucket.Put(accountID+"."+roleName, data); err != nil { + if _, err := s.rolesBucket.Put(ctx, accountID+"."+roleName, data); err != nil { return nil, fmt.Errorf("update role trust policy: %w", err) } @@ -252,6 +261,7 @@ func (s *IAMServiceImpl) UpdateAssumeRolePolicy(accountID string, input *iam.Upd } func (s *IAMServiceImpl) AttachRolePolicy(accountID string, input *iam.AttachRolePolicyInput) (*iam.AttachRolePolicyOutput, error) { + ctx := context.Background() roleName := *input.RoleName policyARN := *input.PolicyArn @@ -261,12 +271,12 @@ func (s *IAMServiceImpl) AttachRolePolicy(accountID string, input *iam.AttachRol // so ListAttachedRolePolicies / DescribeNodegroup round-trip instead of // failing NoSuchEntity. Customer-managed ARNs must still exist. if !isAWSManagedPolicyARN(policyARN) { - if _, err := s.getPolicyByARN(accountID, policyARN); err != nil { + if _, err := s.getPolicyByARN(ctx, accountID, policyARN); err != nil { return nil, err } } - err := s.updateRoleCAS(accountID, roleName, func(role *Role) (bool, error) { + err := s.updateRoleCAS(ctx, accountID, roleName, func(role *Role) (bool, error) { if slices.Contains(role.AttachedPolicies, policyARN) { return false, nil } @@ -282,10 +292,11 @@ func (s *IAMServiceImpl) AttachRolePolicy(accountID string, input *iam.AttachRol } func (s *IAMServiceImpl) DetachRolePolicy(accountID string, input *iam.DetachRolePolicyInput) (*iam.DetachRolePolicyOutput, error) { + ctx := context.Background() roleName := *input.RoleName policyARN := *input.PolicyArn - err := s.updateRoleCAS(accountID, roleName, func(role *Role) (bool, error) { + err := s.updateRoleCAS(ctx, accountID, roleName, func(role *Role) (bool, error) { idx := slices.Index(role.AttachedPolicies, policyARN) if idx < 0 { return false, errors.New(awserrors.ErrorIAMNoSuchEntity) @@ -302,7 +313,8 @@ func (s *IAMServiceImpl) DetachRolePolicy(accountID string, input *iam.DetachRol } func (s *IAMServiceImpl) ListAttachedRolePolicies(accountID string, input *iam.ListAttachedRolePoliciesInput) (*iam.ListAttachedRolePoliciesOutput, error) { - role, err := s.getRole(accountID, *input.RoleName) + ctx := context.Background() + role, err := s.getRole(ctx, accountID, *input.RoleName) if err != nil { return nil, err } @@ -316,7 +328,7 @@ func (s *IAMServiceImpl) ListAttachedRolePolicies(accountID string, input *iam.L }) continue } - policy, err := s.getPolicyByARN(accountID, arn) + policy, err := s.getPolicyByARN(ctx, accountID, arn) if err != nil { slog.Warn("ListAttachedRolePolicies: policy not found for ARN", "arn", arn, "err", err) continue @@ -336,6 +348,7 @@ func (s *IAMServiceImpl) ListAttachedRolePolicies(accountID string, input *iam.L // PutRolePolicy embeds an inline policy document in a role, keyed by PolicyName. // Idempotent upsert: a same-name policy is overwritten, mirroring AWS. func (s *IAMServiceImpl) PutRolePolicy(accountID string, input *iam.PutRolePolicyInput) (*iam.PutRolePolicyOutput, error) { + ctx := context.Background() roleName := *input.RoleName policyName := *input.PolicyName policyDoc := *input.PolicyDocument @@ -347,7 +360,7 @@ func (s *IAMServiceImpl) PutRolePolicy(accountID string, input *iam.PutRolePolic return nil, errors.New(awserrors.ErrorIAMMalformedPolicyDocument) } - err := s.updateRoleCAS(accountID, roleName, func(role *Role) (bool, error) { + err := s.updateRoleCAS(ctx, accountID, roleName, func(role *Role) (bool, error) { if role.InlinePolicies == nil { role.InlinePolicies = map[string]string{} } @@ -366,10 +379,11 @@ func (s *IAMServiceImpl) PutRolePolicy(accountID string, input *iam.PutRolePolic // Returns the document as a raw JSON string, matching how GetRole returns // AssumeRolePolicyDocument; AWS URL-encodes it, we follow the in-repo convention. func (s *IAMServiceImpl) GetRolePolicy(accountID string, input *iam.GetRolePolicyInput) (*iam.GetRolePolicyOutput, error) { + ctx := context.Background() roleName := *input.RoleName policyName := *input.PolicyName - role, err := s.getRole(accountID, roleName) + role, err := s.getRole(ctx, accountID, roleName) if err != nil { return nil, err } @@ -389,10 +403,11 @@ func (s *IAMServiceImpl) GetRolePolicy(accountID string, input *iam.GetRolePolic // DeleteRolePolicy removes a role's inline policy by name. A missing name yields // NoSuchEntity, matching AWS. func (s *IAMServiceImpl) DeleteRolePolicy(accountID string, input *iam.DeleteRolePolicyInput) (*iam.DeleteRolePolicyOutput, error) { + ctx := context.Background() roleName := *input.RoleName policyName := *input.PolicyName - err := s.updateRoleCAS(accountID, roleName, func(role *Role) (bool, error) { + err := s.updateRoleCAS(ctx, accountID, roleName, func(role *Role) (bool, error) { if _, ok := role.InlinePolicies[policyName]; !ok { return false, errors.New(awserrors.ErrorIAMNoSuchEntity) } @@ -410,7 +425,8 @@ func (s *IAMServiceImpl) DeleteRolePolicy(accountID string, input *iam.DeleteRol // ListRolePolicies returns the names of a role's inline policies, sorted for // deterministic output. Pagination is not implemented: IsTruncated is always false. func (s *IAMServiceImpl) ListRolePolicies(accountID string, input *iam.ListRolePoliciesInput) (*iam.ListRolePoliciesOutput, error) { - role, err := s.getRole(accountID, *input.RoleName) + ctx := context.Background() + role, err := s.getRole(ctx, accountID, *input.RoleName) if err != nil { return nil, err } @@ -430,12 +446,13 @@ func (s *IAMServiceImpl) ListRolePolicies(accountID string, input *iam.ListRoleP // TagRole upserts tags on a role under CAS, like the other role writers. func (s *IAMServiceImpl) TagRole(accountID string, input *iam.TagRoleInput) (*iam.TagRoleOutput, error) { + ctx := context.Background() if err := validateTags(input.Tags); err != nil { return nil, err } roleName := *input.RoleName - err := s.updateRoleCAS(accountID, roleName, func(role *Role) (bool, error) { + err := s.updateRoleCAS(ctx, accountID, roleName, func(role *Role) (bool, error) { merged := mergeTags(role.Tags, input.Tags) if len(merged) > maxTagsPerResource { return false, errors.New(awserrors.ErrorIAMLimitExceeded) @@ -453,8 +470,9 @@ func (s *IAMServiceImpl) TagRole(accountID string, input *iam.TagRoleInput) (*ia // UntagRole removes the named tag keys from a role; unknown keys are a no-op. func (s *IAMServiceImpl) UntagRole(accountID string, input *iam.UntagRoleInput) (*iam.UntagRoleOutput, error) { + ctx := context.Background() roleName := *input.RoleName - err := s.updateRoleCAS(accountID, roleName, func(role *Role) (bool, error) { + err := s.updateRoleCAS(ctx, accountID, roleName, func(role *Role) (bool, error) { role.Tags = removeTagKeys(role.Tags, input.TagKeys) return true, nil }) @@ -469,7 +487,8 @@ func (s *IAMServiceImpl) UntagRole(accountID string, input *iam.UntagRoleInput) // ListRoleTags returns a role's tags. Pagination is not implemented: // IsTruncated is always false. func (s *IAMServiceImpl) ListRoleTags(accountID string, input *iam.ListRoleTagsInput) (*iam.ListRoleTagsOutput, error) { - role, err := s.getRole(accountID, *input.RoleName) + ctx := context.Background() + role, err := s.getRole(ctx, accountID, *input.RoleName) if err != nil { return nil, err } @@ -483,14 +502,15 @@ func (s *IAMServiceImpl) ListRoleTags(accountID string, input *iam.ListRoleTagsI // Used by the gateway for policy evaluation. Fails closed: any unresolvable // policy returns an error so the caller denies access rather than using a partial set. func (s *IAMServiceImpl) GetRolePolicies(accountID, roleName string) ([]PolicyDocument, error) { - role, err := s.getRole(accountID, roleName) + ctx := context.Background() + role, err := s.getRole(ctx, accountID, roleName) if err != nil { return nil, err } var docs []PolicyDocument for _, arn := range role.AttachedPolicies { - doc, include, err := s.resolveAttachedPolicy(accountID, arn) + doc, include, err := s.resolveAttachedPolicy(ctx, accountID, arn) if err != nil { return nil, err // fail closed } @@ -528,10 +548,10 @@ func managedPolicyNameFromARN(arn string) string { return name } -func (s *IAMServiceImpl) getRole(accountID, roleName string) (*Role, error) { - entry, err := s.rolesBucket.Get(accountID + "." + roleName) +func (s *IAMServiceImpl) getRole(ctx context.Context, accountID, roleName string) (*Role, error) { + entry, err := s.rolesBucket.Get(ctx, accountID+"."+roleName) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil, errors.New(awserrors.ErrorIAMNoSuchEntity) } return nil, fmt.Errorf("get role: %w", err) @@ -550,12 +570,12 @@ func (s *IAMServiceImpl) getRole(accountID, roleName string) (*Role, error) { // loses updates when callers (e.g. Terraform attaching several managed policies // to one role at once) write the same record concurrently. mutate reports // whether it changed the record; a false return commits nothing. -func (s *IAMServiceImpl) updateRoleCAS(accountID, roleName string, mutate func(*Role) (bool, error)) error { +func (s *IAMServiceImpl) updateRoleCAS(ctx context.Context, accountID, roleName string, mutate func(*Role) (bool, error)) error { key := accountID + "." + roleName for range roleCASMaxRetries { - entry, err := s.rolesBucket.Get(key) + entry, err := s.rolesBucket.Get(ctx, key) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return errors.New(awserrors.ErrorIAMNoSuchEntity) } return fmt.Errorf("get role: %w", err) @@ -578,8 +598,8 @@ func (s *IAMServiceImpl) updateRoleCAS(accountID, roleName string, mutate func(* if err != nil { return fmt.Errorf("marshal role: %w", err) } - if _, err := s.rolesBucket.Update(key, data, entry.Revision()); err != nil { - if errors.Is(err, nats.ErrKeyExists) { + if _, err := s.rolesBucket.Update(ctx, key, data, entry.Revision()); err != nil { + if errors.Is(err, jetstream.ErrKeyExists) { continue // CAS conflict — another writer won, re-read and retry. } return fmt.Errorf("update role: %w", err) @@ -593,10 +613,10 @@ func (s *IAMServiceImpl) updateRoleCAS(accountID, roleName string, mutate func(* // profile in the account that references the given role. Fails closed on // per-key Get or unmarshal errors so DeleteRole cannot succeed while a // real-but-unreadable reference exists. -func (s *IAMServiceImpl) findInstanceProfilesForRole(accountID, roleName string) ([]*InstanceProfile, error) { - keys, err := s.instanceProfilesBucket.Keys() +func (s *IAMServiceImpl) findInstanceProfilesForRole(ctx context.Context, accountID, roleName string) ([]*InstanceProfile, error) { + keys, err := kvutil.Keys(ctx, s.instanceProfilesBucket) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return nil, nil } return nil, fmt.Errorf("list instance profile keys: %w", err) @@ -612,9 +632,9 @@ func (s *IAMServiceImpl) findInstanceProfilesForRole(accountID, roleName string) continue } - entry, err := s.instanceProfilesBucket.Get(key) + entry, err := s.instanceProfilesBucket.Get(ctx, key) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { slog.Debug("findInstanceProfilesForRole: profile key disappeared", "key", key) continue } diff --git a/spinifex/handlers/iam/roles_test.go b/spinifex/handlers/iam/roles_test.go index a634e4004..a1ad16f1a 100644 --- a/spinifex/handlers/iam/roles_test.go +++ b/spinifex/handlers/iam/roles_test.go @@ -705,12 +705,12 @@ func TestGetRolePolicies_UnresolvablePolicy(t *testing.T) { // Overwrite the role record with an attachment to a policy that does not // exist, simulating a managed policy deleted out from under a live // attachment. - role, err := svc.getRole(testAccountID, "dangling-role") + role, err := svc.getRole(t.Context(), testAccountID, "dangling-role") require.NoError(t, err) role.AttachedPolicies = []string{"arn:aws:iam::" + testAccountID + ":policy/Ghost"} data, err := json.Marshal(role) require.NoError(t, err) - _, err = svc.rolesBucket.Put(testAccountID+".dangling-role", data) + _, err = svc.rolesBucket.Put(t.Context(), testAccountID+".dangling-role", data) require.NoError(t, err) _, err = svc.GetRolePolicies(testAccountID, "dangling-role") diff --git a/spinifex/handlers/iam/service_impl.go b/spinifex/handlers/iam/service_impl.go index 73bcf77fd..e6144dfc6 100644 --- a/spinifex/handlers/iam/service_impl.go +++ b/spinifex/handlers/iam/service_impl.go @@ -1,6 +1,7 @@ package handlers_iam import ( + "context" "crypto/rand" "encoding/hex" "encoding/json" @@ -15,12 +16,15 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/iam" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/mulgadc/predastore/pkg/masterkey" "github.com/mulgadc/spinifex/spinifex/admin" "github.com/mulgadc/spinifex/spinifex/awserrors" + "github.com/mulgadc/spinifex/spinifex/kvutil" "github.com/mulgadc/spinifex/spinifex/migrate" "github.com/mulgadc/spinifex/spinifex/utils" - "github.com/nats-io/nats.go" ) const ( @@ -54,12 +58,12 @@ const ( // putAccessKey writes an access-key record after enforcing the AKIA-prefix // invariant. All writers to accessKeysBucket MUST go through this helper. -func (s *IAMServiceImpl) putAccessKey(accessKeyID string, data []byte) error { +func (s *IAMServiceImpl) putAccessKey(ctx context.Context, accessKeyID string, data []byte) error { if !strings.HasPrefix(accessKeyID, LongLivedAccessKeyIDPrefix) { return fmt.Errorf("access key ID must start with %q, got %q", LongLivedAccessKeyIDPrefix, accessKeyID) } - if _, err := s.accessKeysBucket.Put(accessKeyID, data); err != nil { + if _, err := s.accessKeysBucket.Put(ctx, accessKeyID, data); err != nil { return err } return nil @@ -67,29 +71,36 @@ func (s *IAMServiceImpl) putAccessKey(accessKeyID string, data []byte) error { // createAccessKey writes an access-key record with CAS semantics (fails if the // key already exists). Enforces the AKIA-prefix invariant. -func (s *IAMServiceImpl) createAccessKey(accessKeyID string, data []byte) error { +func (s *IAMServiceImpl) createAccessKey(ctx context.Context, accessKeyID string, data []byte) error { if !strings.HasPrefix(accessKeyID, LongLivedAccessKeyIDPrefix) { return fmt.Errorf("access key ID must start with %q, got %q", LongLivedAccessKeyIDPrefix, accessKeyID) } - if _, err := s.accessKeysBucket.Create(accessKeyID, data); err != nil { + if _, err := s.accessKeysBucket.Create(ctx, accessKeyID, data); err != nil { return err } return nil } // IAMServiceImpl implements IAM operations using NATS JetStream KV. +// +// IAMService carries no context: the gateway calls it in-process and vpcd over +// NATS, and neither boundary passes one through. Each exported method therefore +// binds context.Background() for its KV work, which leaves the jetstream +// package's own 5s API timeout in force — the same wait the legacy KV API +// applied. Every helper below it takes the context as its leading parameter, so +// the day the contract gains one only those binding lines change. type IAMServiceImpl struct { - js nats.JetStreamContext + js jetstream.JetStream natsConn *nats.Conn - usersBucket nats.KeyValue - accessKeysBucket nats.KeyValue - policiesBucket nats.KeyValue - accountsBucket nats.KeyValue - accountCounterBucket nats.KeyValue - rolesBucket nats.KeyValue - instanceProfilesBucket nats.KeyValue - groupsBucket nats.KeyValue + usersBucket jetstream.KeyValue + accessKeysBucket jetstream.KeyValue + policiesBucket jetstream.KeyValue + accountsBucket jetstream.KeyValue + accountCounterBucket jetstream.KeyValue + rolesBucket jetstream.KeyValue + instanceProfilesBucket jetstream.KeyValue + groupsBucket jetstream.KeyValue key *masterkey.Key // replicas is the JetStream replication factor for lazily-created per-account buckets. replicas int @@ -99,79 +110,80 @@ var _ IAMService = (*IAMServiceImpl)(nil) // NewIAMServiceImpl creates a new IAM service backed by NATS JetStream KV. // clusterSize sets the replication factor; pass 1 for single-node or test setups. -func NewIAMServiceImpl(natsConn *nats.Conn, masterKey []byte, clusterSize int) (*IAMServiceImpl, error) { +// The context bounds bucket creation and the schema migrations only. +func NewIAMServiceImpl(ctx context.Context, natsConn *nats.Conn, masterKey []byte, clusterSize int) (*IAMServiceImpl, error) { if len(masterKey) != 32 { return nil, fmt.Errorf("master key must be 32 bytes, got %d", len(masterKey)) } replicas := max(clusterSize, 1) - js, err := natsConn.JetStream() + js, err := jetstream.New(natsConn) if err != nil { return nil, fmt.Errorf("get JetStream context: %w", err) } - usersBucket, err := getOrCreateBucket(js, KVBucketUsers, 10, replicas) + usersBucket, err := kvutil.GetOrCreateBucketWithReplicas(ctx, js, KVBucketUsers, 10, replicas) if err != nil { return nil, fmt.Errorf("init users bucket: %w", err) } - if err := migrate.DefaultRegistry.RunKV(KVBucketUsers, usersBucket, KVBucketUsersVersion); err != nil { + if err := migrate.DefaultRegistry.RunKV(ctx, KVBucketUsers, usersBucket, KVBucketUsersVersion); err != nil { return nil, fmt.Errorf("migrate %s: %w", KVBucketUsers, err) } - accessKeysBucket, err := getOrCreateBucket(js, KVBucketAccessKeys, 5, replicas) + accessKeysBucket, err := kvutil.GetOrCreateBucketWithReplicas(ctx, js, KVBucketAccessKeys, 5, replicas) if err != nil { return nil, fmt.Errorf("init access keys bucket: %w", err) } - if err := migrate.DefaultRegistry.RunKV(KVBucketAccessKeys, accessKeysBucket, KVBucketAccessKeysVersion); err != nil { + if err := migrate.DefaultRegistry.RunKV(ctx, KVBucketAccessKeys, accessKeysBucket, KVBucketAccessKeysVersion); err != nil { return nil, fmt.Errorf("migrate %s: %w", KVBucketAccessKeys, err) } - policiesBucket, err := getOrCreateBucket(js, KVBucketPolicies, 10, replicas) + policiesBucket, err := kvutil.GetOrCreateBucketWithReplicas(ctx, js, KVBucketPolicies, 10, replicas) if err != nil { return nil, fmt.Errorf("init policies bucket: %w", err) } - if err := migrate.DefaultRegistry.RunKV(KVBucketPolicies, policiesBucket, KVBucketPoliciesVersion); err != nil { + if err := migrate.DefaultRegistry.RunKV(ctx, KVBucketPolicies, policiesBucket, KVBucketPoliciesVersion); err != nil { return nil, fmt.Errorf("migrate %s: %w", KVBucketPolicies, err) } - accountsBucket, err := getOrCreateBucket(js, KVBucketAccounts, 5, replicas) + accountsBucket, err := kvutil.GetOrCreateBucketWithReplicas(ctx, js, KVBucketAccounts, 5, replicas) if err != nil { return nil, fmt.Errorf("init accounts bucket: %w", err) } - if err := migrate.DefaultRegistry.RunKV(KVBucketAccounts, accountsBucket, KVBucketAccountsVersion); err != nil { + if err := migrate.DefaultRegistry.RunKV(ctx, KVBucketAccounts, accountsBucket, KVBucketAccountsVersion); err != nil { return nil, fmt.Errorf("migrate %s: %w", KVBucketAccounts, err) } - accountCounterBucket, err := getOrCreateBucket(js, KVBucketAccountCounter, 5, replicas) + accountCounterBucket, err := kvutil.GetOrCreateBucketWithReplicas(ctx, js, KVBucketAccountCounter, 5, replicas) if err != nil { return nil, fmt.Errorf("init account counter bucket: %w", err) } - if err := migrate.DefaultRegistry.RunKV(KVBucketAccountCounter, accountCounterBucket, KVBucketAccountCounterVersion); err != nil { + if err := migrate.DefaultRegistry.RunKV(ctx, KVBucketAccountCounter, accountCounterBucket, KVBucketAccountCounterVersion); err != nil { return nil, fmt.Errorf("migrate %s: %w", KVBucketAccountCounter, err) } - rolesBucket, err := getOrCreateBucket(js, KVBucketRoles, 10, replicas) + rolesBucket, err := kvutil.GetOrCreateBucketWithReplicas(ctx, js, KVBucketRoles, 10, replicas) if err != nil { return nil, fmt.Errorf("init roles bucket: %w", err) } - if err := migrate.DefaultRegistry.RunKV(KVBucketRoles, rolesBucket, KVBucketRolesVersion); err != nil { + if err := migrate.DefaultRegistry.RunKV(ctx, KVBucketRoles, rolesBucket, KVBucketRolesVersion); err != nil { return nil, fmt.Errorf("migrate %s: %w", KVBucketRoles, err) } - instanceProfilesBucket, err := getOrCreateBucket(js, KVBucketInstanceProfiles, 10, replicas) + instanceProfilesBucket, err := kvutil.GetOrCreateBucketWithReplicas(ctx, js, KVBucketInstanceProfiles, 10, replicas) if err != nil { return nil, fmt.Errorf("init instance profiles bucket: %w", err) } - if err := migrate.DefaultRegistry.RunKV(KVBucketInstanceProfiles, instanceProfilesBucket, KVBucketInstanceProfilesVersion); err != nil { + if err := migrate.DefaultRegistry.RunKV(ctx, KVBucketInstanceProfiles, instanceProfilesBucket, KVBucketInstanceProfilesVersion); err != nil { return nil, fmt.Errorf("migrate %s: %w", KVBucketInstanceProfiles, err) } - groupsBucket, err := getOrCreateBucket(js, KVBucketGroups, 10, replicas) + groupsBucket, err := kvutil.GetOrCreateBucketWithReplicas(ctx, js, KVBucketGroups, 10, replicas) if err != nil { return nil, fmt.Errorf("init groups bucket: %w", err) } - if err := migrate.DefaultRegistry.RunKV(KVBucketGroups, groupsBucket, KVBucketGroupsVersion); err != nil { + if err := migrate.DefaultRegistry.RunKV(ctx, KVBucketGroups, groupsBucket, KVBucketGroupsVersion); err != nil { return nil, fmt.Errorf("migrate %s: %w", KVBucketGroups, err) } @@ -206,21 +218,6 @@ func NewIAMServiceImpl(natsConn *nats.Conn, masterKey []byte, clusterSize int) ( }, nil } -func getOrCreateBucket(js nats.JetStreamContext, name string, history uint8, replicas int) (nats.KeyValue, error) { - kv, err := js.CreateKeyValue(&nats.KeyValueConfig{ - Bucket: name, - History: history, - Replicas: replicas, - }) - if err != nil { - kv, err = js.KeyValue(name) - if err != nil { - return nil, err - } - } - return kv, nil -} - // copyTags converts SDK IAM tags into the stored Tag slice, skipping entries // with a nil key or value. Returns a non-nil (possibly empty) slice. func copyTags(tags []*iam.Tag) []Tag { @@ -313,6 +310,7 @@ func removeTagKeys(existing []Tag, keys []*string) []Tag { // --------------------------------------------------------------------------- func (s *IAMServiceImpl) CreateUser(accountID string, input *iam.CreateUserInput) (*iam.CreateUserOutput, error) { + ctx := context.Background() userName := *input.UserName if err := validateUserName(userName); err != nil { return nil, errors.New(awserrors.ErrorIAMInvalidInput) @@ -351,8 +349,8 @@ func (s *IAMServiceImpl) CreateUser(accountID string, input *iam.CreateUserInput } // Atomic create — fails if key already exists (race-safe) - if _, err := s.usersBucket.Create(kvKey, data); err != nil { - if errors.Is(err, nats.ErrKeyExists) { + if _, err := s.usersBucket.Create(ctx, kvKey, data); err != nil { + if errors.Is(err, jetstream.ErrKeyExists) { return nil, errors.New(awserrors.ErrorIAMEntityAlreadyExists) } return nil, fmt.Errorf("store user: %w", err) @@ -373,7 +371,8 @@ func (s *IAMServiceImpl) CreateUser(accountID string, input *iam.CreateUserInput } func (s *IAMServiceImpl) GetUser(accountID string, input *iam.GetUserInput) (*iam.GetUserOutput, error) { - user, err := s.getUser(accountID, *input.UserName) + ctx := context.Background() + user, err := s.getUser(ctx, accountID, *input.UserName) if err != nil { return nil, err } @@ -392,9 +391,10 @@ func (s *IAMServiceImpl) GetUser(accountID string, input *iam.GetUserInput) (*ia } func (s *IAMServiceImpl) ListUsers(accountID string, input *iam.ListUsersInput) (*iam.ListUsersOutput, error) { - keys, err := s.usersBucket.Keys() + ctx := context.Background() + keys, err := kvutil.Keys(ctx, s.usersBucket) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return &iam.ListUsersOutput{ Users: []*iam.User{}, IsTruncated: aws.Bool(false), @@ -418,9 +418,9 @@ func (s *IAMServiceImpl) ListUsers(accountID string, input *iam.ListUsersInput) continue } - entry, err := s.usersBucket.Get(key) + entry, err := s.usersBucket.Get(ctx, key) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { slog.Debug("ListUsers: user key disappeared (concurrent delete)", "key", key) } else { slog.Warn("ListUsers: failed to get user", "key", key, "err", err) @@ -455,10 +455,11 @@ func (s *IAMServiceImpl) ListUsers(accountID string, input *iam.ListUsersInput) } func (s *IAMServiceImpl) DeleteUser(accountID string, input *iam.DeleteUserInput) (*iam.DeleteUserOutput, error) { + ctx := context.Background() userName := *input.UserName kvKey := accountID + "." + userName - user, err := s.getUser(accountID, userName) + user, err := s.getUser(ctx, accountID, userName) if err != nil { return nil, err } @@ -476,7 +477,7 @@ func (s *IAMServiceImpl) DeleteUser(accountID string, input *iam.DeleteUserInput return nil, errors.New(awserrors.ErrorIAMDeleteConflict) } - if err := s.usersBucket.Delete(kvKey); err != nil { + if err := s.usersBucket.Delete(ctx, kvKey); err != nil { return nil, fmt.Errorf("delete user: %w", err) } @@ -489,10 +490,11 @@ func (s *IAMServiceImpl) DeleteUser(accountID string, input *iam.DeleteUserInput // --------------------------------------------------------------------------- func (s *IAMServiceImpl) CreateAccessKey(accountID string, input *iam.CreateAccessKeyInput) (*iam.CreateAccessKeyOutput, error) { + ctx := context.Background() userName := *input.UserName userKVKey := accountID + "." + userName - user, err := s.getUser(accountID, userName) + user, err := s.getUser(ctx, accountID, userName) if err != nil { return nil, err } @@ -529,21 +531,21 @@ func (s *IAMServiceImpl) CreateAccessKey(accountID string, input *iam.CreateAcce return nil, fmt.Errorf("marshal access key: %w", err) } - if err := s.putAccessKey(accessKeyID, akData); err != nil { + if err := s.putAccessKey(ctx, accessKeyID, akData); err != nil { return nil, fmt.Errorf("store access key: %w", err) } user.AccessKeys = append(user.AccessKeys, accessKeyID) userData, err := json.Marshal(user) if err != nil { - if rbErr := s.accessKeysBucket.Delete(accessKeyID); rbErr != nil { + if rbErr := s.accessKeysBucket.Delete(ctx, accessKeyID); rbErr != nil { slog.Error("Rollback failed: orphaned access key", "accessKeyID", accessKeyID, "err", rbErr) } return nil, fmt.Errorf("marshal user: %w", err) } - if _, err := s.usersBucket.Put(userKVKey, userData); err != nil { - if rbErr := s.accessKeysBucket.Delete(accessKeyID); rbErr != nil { + if _, err := s.usersBucket.Put(ctx, userKVKey, userData); err != nil { + if rbErr := s.accessKeysBucket.Delete(ctx, accessKeyID); rbErr != nil { slog.Error("Rollback failed: orphaned access key", "accessKeyID", accessKeyID, "err", rbErr) } return nil, fmt.Errorf("update user: %w", err) @@ -564,16 +566,17 @@ func (s *IAMServiceImpl) CreateAccessKey(accountID string, input *iam.CreateAcce } func (s *IAMServiceImpl) ListAccessKeys(accountID string, input *iam.ListAccessKeysInput) (*iam.ListAccessKeysOutput, error) { - user, err := s.getUser(accountID, *input.UserName) + ctx := context.Background() + user, err := s.getUser(ctx, accountID, *input.UserName) if err != nil { return nil, err } var metadata []*iam.AccessKeyMetadata for _, keyID := range user.AccessKeys { - entry, err := s.accessKeysBucket.Get(keyID) + entry, err := s.accessKeysBucket.Get(ctx, keyID) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { slog.Debug("ListAccessKeys: access key disappeared (concurrent delete)", "keyID", keyID) } else { slog.Warn("ListAccessKeys: failed to get access key", "keyID", keyID, "err", err) @@ -603,11 +606,12 @@ func (s *IAMServiceImpl) ListAccessKeys(accountID string, input *iam.ListAccessK } func (s *IAMServiceImpl) DeleteAccessKey(accountID string, input *iam.DeleteAccessKeyInput) (*iam.DeleteAccessKeyOutput, error) { + ctx := context.Background() userName := *input.UserName accessKeyID := *input.AccessKeyId userKVKey := accountID + "." + userName - user, err := s.getUser(accountID, userName) + user, err := s.getUser(ctx, accountID, userName) if err != nil { return nil, err } @@ -624,11 +628,11 @@ func (s *IAMServiceImpl) DeleteAccessKey(accountID string, input *iam.DeleteAcce return nil, fmt.Errorf("marshal user: %w", err) } - if _, err := s.usersBucket.Put(userKVKey, userData); err != nil { + if _, err := s.usersBucket.Put(ctx, userKVKey, userData); err != nil { return nil, fmt.Errorf("update user: %w", err) } - if err := s.accessKeysBucket.Delete(accessKeyID); err != nil { + if err := s.accessKeysBucket.Delete(ctx, accessKeyID); err != nil { return nil, fmt.Errorf("delete access key: %w", err) } @@ -637,6 +641,7 @@ func (s *IAMServiceImpl) DeleteAccessKey(accountID string, input *iam.DeleteAcce } func (s *IAMServiceImpl) UpdateAccessKey(accountID string, input *iam.UpdateAccessKeyInput) (*iam.UpdateAccessKeyOutput, error) { + ctx := context.Background() status := *input.Status if status != AccessKeyStatusActive && status != AccessKeyStatusInactive { return nil, errors.New(awserrors.ErrorIAMInvalidInput) @@ -644,9 +649,9 @@ func (s *IAMServiceImpl) UpdateAccessKey(accountID string, input *iam.UpdateAcce accessKeyID := *input.AccessKeyId - entry, err := s.accessKeysBucket.Get(accessKeyID) + entry, err := s.accessKeysBucket.Get(ctx, accessKeyID) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil, errors.New(awserrors.ErrorIAMNoSuchEntity) } return nil, fmt.Errorf("get access key: %w", err) @@ -667,7 +672,7 @@ func (s *IAMServiceImpl) UpdateAccessKey(accountID string, input *iam.UpdateAcce return nil, fmt.Errorf("marshal access key: %w", err) } - if err := s.putAccessKey(accessKeyID, data); err != nil { + if err := s.putAccessKey(ctx, accessKeyID, data); err != nil { return nil, fmt.Errorf("update access key: %w", err) } @@ -682,9 +687,10 @@ func (s *IAMServiceImpl) UpdateAccessKey(accountID string, input *iam.UpdateAcce // LookupAccessKey retrieves an access key by its ID. Returns the full record // including the encrypted secret, for use by the SigV4 middleware. func (s *IAMServiceImpl) LookupAccessKey(accessKeyID string) (*AccessKey, error) { - entry, err := s.accessKeysBucket.Get(accessKeyID) + ctx := context.Background() + entry, err := s.accessKeysBucket.Get(ctx, accessKeyID) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil, errors.New(awserrors.ErrorIAMNoSuchEntity) } return nil, fmt.Errorf("lookup access key: %w", err) @@ -706,6 +712,7 @@ func (s *IAMServiceImpl) DecryptSecret(ciphertext string) (string, error) { // SeedBootstrap seeds the system root user and optional admin account into NATS KV. // Uses conditional create for multi-node safety; first node wins, others skip silently. func (s *IAMServiceImpl) SeedBootstrap(data *BootstrapData) error { + ctx := context.Background() // --- Seed system account (000000000000) --- systemAccount := Account{ AccountID: utils.GlobalAccountID, @@ -717,7 +724,7 @@ func (s *IAMServiceImpl) SeedBootstrap(data *BootstrapData) error { if err != nil { return fmt.Errorf("marshal system account: %w", err) } - if _, err := s.accountsBucket.Create(utils.GlobalAccountID, accountData); err != nil && !errors.Is(err, nats.ErrKeyExists) { + if _, err := s.accountsBucket.Create(ctx, utils.GlobalAccountID, accountData); err != nil && !errors.Is(err, jetstream.ErrKeyExists) { return fmt.Errorf("seed system account: %w", err) } @@ -741,8 +748,8 @@ func (s *IAMServiceImpl) SeedBootstrap(data *BootstrapData) error { } // Conditional create — fails if key already exists (another node seeded first) - _, err = s.usersBucket.Create(kvKey, userData) - if errors.Is(err, nats.ErrKeyExists) { + _, err = s.usersBucket.Create(ctx, kvKey, userData) + if errors.Is(err, jetstream.ErrKeyExists) { slog.Info("Root user already seeded by another node, skipping") } else if err != nil { return fmt.Errorf("seed root user: %w", err) @@ -763,8 +770,8 @@ func (s *IAMServiceImpl) SeedBootstrap(data *BootstrapData) error { return fmt.Errorf("marshal root access key: %w", err) } - err = s.createAccessKey(data.AccessKeyID, akData) - if errors.Is(err, nats.ErrKeyExists) { + err = s.createAccessKey(ctx, data.AccessKeyID, akData) + if errors.Is(err, jetstream.ErrKeyExists) { slog.Info("Root access key already seeded by another node, skipping") } else if err != nil { return fmt.Errorf("seed root access key: %w", err) @@ -774,7 +781,7 @@ func (s *IAMServiceImpl) SeedBootstrap(data *BootstrapData) error { // --- Seed admin account (000000000001) if present --- if data.Admin != nil { - if err := s.seedAdminAccount(data.Admin); err != nil { + if err := s.seedAdminAccount(ctx, data.Admin); err != nil { return fmt.Errorf("seed admin account: %w", err) } } @@ -784,7 +791,7 @@ func (s *IAMServiceImpl) SeedBootstrap(data *BootstrapData) error { // seedAdminAccount creates the default admin account, user, access key, // AdministratorAccess policy, and sets the account counter to 2. -func (s *IAMServiceImpl) seedAdminAccount(admin *AdminBootstrapData) error { +func (s *IAMServiceImpl) seedAdminAccount(ctx context.Context, admin *AdminBootstrapData) error { now := time.Now().UTC().Format(time.RFC3339) // Create admin account record @@ -798,12 +805,12 @@ func (s *IAMServiceImpl) seedAdminAccount(admin *AdminBootstrapData) error { if err != nil { return fmt.Errorf("marshal admin account: %w", err) } - if _, err := s.accountsBucket.Create(admin.AccountID, accountData); err != nil && !errors.Is(err, nats.ErrKeyExists) { + if _, err := s.accountsBucket.Create(ctx, admin.AccountID, accountData); err != nil && !errors.Is(err, jetstream.ErrKeyExists) { return fmt.Errorf("store admin account: %w", err) } // Set account counter to 2 so next CreateAccount gets 000000000002 - if _, err := s.accountCounterBucket.Create("next_id", []byte("2")); err != nil && !errors.Is(err, nats.ErrKeyExists) { + if _, err := s.accountCounterBucket.Create(ctx, "next_id", []byte("2")); err != nil && !errors.Is(err, jetstream.ErrKeyExists) { return fmt.Errorf("seed account counter: %w", err) } @@ -830,7 +837,7 @@ func (s *IAMServiceImpl) seedAdminAccount(admin *AdminBootstrapData) error { return fmt.Errorf("marshal admin policy: %w", err) } policyKVKey := admin.AccountID + ".AdministratorAccess" - if _, err := s.policiesBucket.Create(policyKVKey, policyData); err != nil && !errors.Is(err, nats.ErrKeyExists) { + if _, err := s.policiesBucket.Create(ctx, policyKVKey, policyData); err != nil && !errors.Is(err, jetstream.ErrKeyExists) { return fmt.Errorf("store admin policy: %w", err) } @@ -857,7 +864,7 @@ func (s *IAMServiceImpl) seedAdminAccount(admin *AdminBootstrapData) error { if err != nil { return fmt.Errorf("marshal admin user: %w", err) } - if _, err := s.usersBucket.Create(kvKey, userData); err != nil && !errors.Is(err, nats.ErrKeyExists) { + if _, err := s.usersBucket.Create(ctx, kvKey, userData); err != nil && !errors.Is(err, jetstream.ErrKeyExists) { return fmt.Errorf("store admin user: %w", err) } @@ -874,7 +881,7 @@ func (s *IAMServiceImpl) seedAdminAccount(admin *AdminBootstrapData) error { if err != nil { return fmt.Errorf("marshal admin access key: %w", err) } - if err := s.createAccessKey(admin.AccessKeyID, akData); err != nil && !errors.Is(err, nats.ErrKeyExists) { + if err := s.createAccessKey(ctx, admin.AccessKeyID, akData); err != nil && !errors.Is(err, jetstream.ErrKeyExists) { return fmt.Errorf("store admin access key: %w", err) } @@ -902,9 +909,10 @@ func (s *IAMServiceImpl) seedAdminAccount(admin *AdminBootstrapData) error { // IsEmpty returns true if the users bucket has no entries. func (s *IAMServiceImpl) IsEmpty() (bool, error) { - keys, err := s.usersBucket.Keys() + ctx := context.Background() + keys, err := kvutil.Keys(ctx, s.usersBucket) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return true, nil } return false, fmt.Errorf("check users bucket: %w", err) @@ -923,18 +931,19 @@ func (s *IAMServiceImpl) IsEmpty() (bool, error) { // CreateAccount creates a new account with a sequentially assigned 12-digit ID using a CAS loop. func (s *IAMServiceImpl) CreateAccount(name string) (*Account, error) { + ctx := context.Background() if name == "" { return nil, errors.New(awserrors.ErrorIAMInvalidInput) } var accountID string for { - entry, err := s.accountCounterBucket.Get("next_id") - if errors.Is(err, nats.ErrKeyNotFound) { + entry, err := s.accountCounterBucket.Get(ctx, "next_id") + if errors.Is(err, jetstream.ErrKeyNotFound) { // First account ever; 000000000000 is the global account. accountID = fmt.Sprintf("%012d", 1) - if _, err := s.accountCounterBucket.Create("next_id", []byte("2")); err != nil { - if errors.Is(err, nats.ErrKeyExists) { + if _, err := s.accountCounterBucket.Create(ctx, "next_id", []byte("2")); err != nil { + if errors.Is(err, jetstream.ErrKeyExists) { continue // Another node raced us, retry } return nil, fmt.Errorf("init account counter: %w", err) @@ -952,8 +961,8 @@ func (s *IAMServiceImpl) CreateAccount(name string) (*Account, error) { accountID = fmt.Sprintf("%012d", nextID) newVal := []byte(strconv.FormatInt(nextID+1, 10)) - if _, err := s.accountCounterBucket.Update("next_id", newVal, entry.Revision()); err != nil { - if errors.Is(err, nats.ErrKeyExists) { + if _, err := s.accountCounterBucket.Update(ctx, "next_id", newVal, entry.Revision()); err != nil { + if errors.Is(err, jetstream.ErrKeyExists) { continue // CAS conflict, retry } return nil, fmt.Errorf("update account counter: %w", err) @@ -973,7 +982,7 @@ func (s *IAMServiceImpl) CreateAccount(name string) (*Account, error) { return nil, fmt.Errorf("marshal account: %w", err) } - if _, err := s.accountsBucket.Create(accountID, data); err != nil { + if _, err := s.accountsBucket.Create(ctx, accountID, data); err != nil { return nil, fmt.Errorf("store account: %w", err) } @@ -996,9 +1005,10 @@ func (s *IAMServiceImpl) CreateAccount(name string) (*Account, error) { // GetAccount retrieves an account by its 12-digit ID. func (s *IAMServiceImpl) GetAccount(accountID string) (*Account, error) { - entry, err := s.accountsBucket.Get(accountID) + ctx := context.Background() + entry, err := s.accountsBucket.Get(ctx, accountID) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil, errors.New(awserrors.ErrorIAMNoSuchEntity) } return nil, fmt.Errorf("get account: %w", err) @@ -1013,9 +1023,10 @@ func (s *IAMServiceImpl) GetAccount(accountID string) (*Account, error) { // ListAccounts returns all accounts. func (s *IAMServiceImpl) ListAccounts() ([]*Account, error) { - keys, err := s.accountsBucket.Keys() + ctx := context.Background() + keys, err := kvutil.Keys(ctx, s.accountsBucket) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return []*Account{}, nil } return nil, fmt.Errorf("list account keys: %w", err) @@ -1026,9 +1037,9 @@ func (s *IAMServiceImpl) ListAccounts() ([]*Account, error) { if key == utils.VersionKey { continue } - entry, err := s.accountsBucket.Get(key) + entry, err := s.accountsBucket.Get(ctx, key) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { continue } return nil, fmt.Errorf("get account %s: %w", key, err) @@ -1049,6 +1060,7 @@ func (s *IAMServiceImpl) ListAccounts() ([]*Account, error) { // --------------------------------------------------------------------------- func (s *IAMServiceImpl) CreatePolicy(accountID string, input *iam.CreatePolicyInput) (*iam.CreatePolicyOutput, error) { + ctx := context.Background() policyName := *input.PolicyName if err := validatePolicyName(policyName); err != nil { return nil, errors.New(awserrors.ErrorIAMInvalidInput) @@ -1089,8 +1101,8 @@ func (s *IAMServiceImpl) CreatePolicy(accountID string, input *iam.CreatePolicyI return nil, fmt.Errorf("marshal policy: %w", err) } - if _, err := s.policiesBucket.Create(kvKey, data); err != nil { - if errors.Is(err, nats.ErrKeyExists) { + if _, err := s.policiesBucket.Create(ctx, kvKey, data); err != nil { + if errors.Is(err, jetstream.ErrKeyExists) { return nil, errors.New(awserrors.ErrorIAMEntityAlreadyExists) } return nil, fmt.Errorf("store policy: %w", err) @@ -1115,12 +1127,13 @@ func (s *IAMServiceImpl) CreatePolicy(accountID string, input *iam.CreatePolicyI } func (s *IAMServiceImpl) GetPolicy(accountID string, input *iam.GetPolicyInput) (*iam.GetPolicyOutput, error) { - policy, err := s.getPolicyByARN(accountID, *input.PolicyArn) + ctx := context.Background() + policy, err := s.getPolicyByARN(ctx, accountID, *input.PolicyArn) if err != nil { return nil, err } - counts, err := s.buildAttachmentCounts(accountID) + counts, err := s.buildAttachmentCounts(ctx, accountID) if err != nil { return nil, fmt.Errorf("check policy attachments: %w", err) } @@ -1143,7 +1156,8 @@ func (s *IAMServiceImpl) GetPolicy(accountID string, input *iam.GetPolicyInput) } func (s *IAMServiceImpl) GetPolicyVersion(accountID string, input *iam.GetPolicyVersionInput) (*iam.GetPolicyVersionOutput, error) { - policy, err := s.getPolicyByARN(accountID, *input.PolicyArn) + ctx := context.Background() + policy, err := s.getPolicyByARN(ctx, accountID, *input.PolicyArn) if err != nil { return nil, err } @@ -1165,7 +1179,8 @@ func (s *IAMServiceImpl) GetPolicyVersion(accountID string, input *iam.GetPolicy } func (s *IAMServiceImpl) ListPolicyVersions(accountID string, input *iam.ListPolicyVersionsInput) (*iam.ListPolicyVersionsOutput, error) { - policy, err := s.getPolicyByARN(accountID, *input.PolicyArn) + ctx := context.Background() + policy, err := s.getPolicyByARN(ctx, accountID, *input.PolicyArn) if err != nil { return nil, err } @@ -1183,9 +1198,10 @@ func (s *IAMServiceImpl) ListPolicyVersions(accountID string, input *iam.ListPol } func (s *IAMServiceImpl) ListPolicies(accountID string, input *iam.ListPoliciesInput) (*iam.ListPoliciesOutput, error) { - keys, err := s.policiesBucket.Keys() + ctx := context.Background() + keys, err := kvutil.Keys(ctx, s.policiesBucket) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return &iam.ListPoliciesOutput{ Policies: []*iam.Policy{}, IsTruncated: aws.Bool(false), @@ -1194,7 +1210,7 @@ func (s *IAMServiceImpl) ListPolicies(accountID string, input *iam.ListPoliciesI return nil, fmt.Errorf("list policy keys: %w", err) } - attachCounts, err := s.buildAttachmentCounts(accountID) + attachCounts, err := s.buildAttachmentCounts(ctx, accountID) if err != nil { return nil, fmt.Errorf("build attachment counts: %w", err) } @@ -1209,9 +1225,9 @@ func (s *IAMServiceImpl) ListPolicies(accountID string, input *iam.ListPoliciesI continue } - entry, err := s.policiesBucket.Get(key) + entry, err := s.policiesBucket.Get(ctx, key) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { slog.Debug("ListPolicies: policy key disappeared (concurrent delete)", "key", key) } else { slog.Warn("ListPolicies: failed to get policy", "key", key, "err", err) @@ -1245,12 +1261,13 @@ func (s *IAMServiceImpl) ListPolicies(accountID string, input *iam.ListPoliciesI } func (s *IAMServiceImpl) DeletePolicy(accountID string, input *iam.DeletePolicyInput) (*iam.DeletePolicyOutput, error) { - policy, err := s.getPolicyByARN(accountID, *input.PolicyArn) + ctx := context.Background() + policy, err := s.getPolicyByARN(ctx, accountID, *input.PolicyArn) if err != nil { return nil, err } - counts, err := s.buildAttachmentCounts(accountID) + counts, err := s.buildAttachmentCounts(ctx, accountID) if err != nil { return nil, fmt.Errorf("check policy attachments: %w", err) } @@ -1259,7 +1276,7 @@ func (s *IAMServiceImpl) DeletePolicy(accountID string, input *iam.DeletePolicyI } kvKey := accountID + "." + policy.PolicyName - if err := s.policiesBucket.Delete(kvKey); err != nil { + if err := s.policiesBucket.Delete(ctx, kvKey); err != nil { return nil, fmt.Errorf("delete policy: %w", err) } @@ -1272,6 +1289,7 @@ func (s *IAMServiceImpl) DeletePolicy(accountID string, input *iam.DeletePolicyI // --------------------------------------------------------------------------- func (s *IAMServiceImpl) AttachUserPolicy(accountID string, input *iam.AttachUserPolicyInput) (*iam.AttachUserPolicyOutput, error) { + ctx := context.Background() userName := *input.UserName policyARN := *input.PolicyArn userKVKey := accountID + "." + userName @@ -1281,12 +1299,12 @@ func (s *IAMServiceImpl) AttachUserPolicy(accountID string, input *iam.AttachUse // grant document is modeled in builtinManagedPolicyDoc and resolved at // evaluation time. Customer-managed ARNs must still exist. if !isAWSManagedPolicyARN(policyARN) { - if _, err := s.getPolicyByARN(accountID, policyARN); err != nil { + if _, err := s.getPolicyByARN(ctx, accountID, policyARN); err != nil { return nil, err } } - user, err := s.getUser(accountID, userName) + user, err := s.getUser(ctx, accountID, userName) if err != nil { return nil, err } @@ -1301,7 +1319,7 @@ func (s *IAMServiceImpl) AttachUserPolicy(accountID string, input *iam.AttachUse return nil, fmt.Errorf("marshal user: %w", err) } - if _, err := s.usersBucket.Put(userKVKey, userData); err != nil { + if _, err := s.usersBucket.Put(ctx, userKVKey, userData); err != nil { return nil, fmt.Errorf("update user: %w", err) } @@ -1310,11 +1328,12 @@ func (s *IAMServiceImpl) AttachUserPolicy(accountID string, input *iam.AttachUse } func (s *IAMServiceImpl) DetachUserPolicy(accountID string, input *iam.DetachUserPolicyInput) (*iam.DetachUserPolicyOutput, error) { + ctx := context.Background() userName := *input.UserName policyARN := *input.PolicyArn userKVKey := accountID + "." + userName - user, err := s.getUser(accountID, userName) + user, err := s.getUser(ctx, accountID, userName) if err != nil { return nil, err } @@ -1330,7 +1349,7 @@ func (s *IAMServiceImpl) DetachUserPolicy(accountID string, input *iam.DetachUse return nil, fmt.Errorf("marshal user: %w", err) } - if _, err := s.usersBucket.Put(userKVKey, userData); err != nil { + if _, err := s.usersBucket.Put(ctx, userKVKey, userData); err != nil { return nil, fmt.Errorf("update user: %w", err) } @@ -1339,7 +1358,8 @@ func (s *IAMServiceImpl) DetachUserPolicy(accountID string, input *iam.DetachUse } func (s *IAMServiceImpl) ListAttachedUserPolicies(accountID string, input *iam.ListAttachedUserPoliciesInput) (*iam.ListAttachedUserPoliciesOutput, error) { - user, err := s.getUser(accountID, *input.UserName) + ctx := context.Background() + user, err := s.getUser(ctx, accountID, *input.UserName) if err != nil { return nil, err } @@ -1355,7 +1375,7 @@ func (s *IAMServiceImpl) ListAttachedUserPolicies(accountID string, input *iam.L }) continue } - policy, err := s.getPolicyByARN(accountID, arn) + policy, err := s.getPolicyByARN(ctx, accountID, arn) if err != nil { slog.Warn("ListAttachedUserPolicies: policy not found for ARN", "arn", arn, "err", err) continue @@ -1380,6 +1400,7 @@ func (s *IAMServiceImpl) ListAttachedUserPolicies(accountID string, input *iam.L // Idempotent upsert: a same-name policy is overwritten, mirroring AWS. Uses a // blind read-modify-write Put like the other user writers (no CAS). func (s *IAMServiceImpl) PutUserPolicy(accountID string, input *iam.PutUserPolicyInput) (*iam.PutUserPolicyOutput, error) { + ctx := context.Background() userName := *input.UserName policyName := *input.PolicyName policyDoc := *input.PolicyDocument @@ -1392,7 +1413,7 @@ func (s *IAMServiceImpl) PutUserPolicy(accountID string, input *iam.PutUserPolic return nil, errors.New(awserrors.ErrorIAMMalformedPolicyDocument) } - user, err := s.getUser(accountID, userName) + user, err := s.getUser(ctx, accountID, userName) if err != nil { return nil, err } @@ -1407,7 +1428,7 @@ func (s *IAMServiceImpl) PutUserPolicy(accountID string, input *iam.PutUserPolic return nil, fmt.Errorf("marshal user: %w", err) } - if _, err := s.usersBucket.Put(userKVKey, userData); err != nil { + if _, err := s.usersBucket.Put(ctx, userKVKey, userData); err != nil { return nil, fmt.Errorf("update user: %w", err) } @@ -1418,10 +1439,11 @@ func (s *IAMServiceImpl) PutUserPolicy(accountID string, input *iam.PutUserPolic // GetUserPolicy returns a user's inline policy document by name as a raw JSON // string, matching the in-repo convention used by GetGroupPolicy. func (s *IAMServiceImpl) GetUserPolicy(accountID string, input *iam.GetUserPolicyInput) (*iam.GetUserPolicyOutput, error) { + ctx := context.Background() userName := *input.UserName policyName := *input.PolicyName - user, err := s.getUser(accountID, userName) + user, err := s.getUser(ctx, accountID, userName) if err != nil { return nil, err } @@ -1441,11 +1463,12 @@ func (s *IAMServiceImpl) GetUserPolicy(accountID string, input *iam.GetUserPolic // DeleteUserPolicy removes a user's inline policy by name. A missing name // yields NoSuchEntity, matching AWS. Blind Put like the other user writers. func (s *IAMServiceImpl) DeleteUserPolicy(accountID string, input *iam.DeleteUserPolicyInput) (*iam.DeleteUserPolicyOutput, error) { + ctx := context.Background() userName := *input.UserName policyName := *input.PolicyName userKVKey := accountID + "." + userName - user, err := s.getUser(accountID, userName) + user, err := s.getUser(ctx, accountID, userName) if err != nil { return nil, err } @@ -1460,7 +1483,7 @@ func (s *IAMServiceImpl) DeleteUserPolicy(accountID string, input *iam.DeleteUse return nil, fmt.Errorf("marshal user: %w", err) } - if _, err := s.usersBucket.Put(userKVKey, userData); err != nil { + if _, err := s.usersBucket.Put(ctx, userKVKey, userData); err != nil { return nil, fmt.Errorf("update user: %w", err) } @@ -1471,7 +1494,8 @@ func (s *IAMServiceImpl) DeleteUserPolicy(accountID string, input *iam.DeleteUse // ListUserPolicies returns the names of a user's inline policies, sorted for // deterministic output. Pagination is not implemented: IsTruncated is always false. func (s *IAMServiceImpl) ListUserPolicies(accountID string, input *iam.ListUserPoliciesInput) (*iam.ListUserPoliciesOutput, error) { - user, err := s.getUser(accountID, *input.UserName) + ctx := context.Background() + user, err := s.getUser(ctx, accountID, *input.UserName) if err != nil { return nil, err } @@ -1496,12 +1520,13 @@ func (s *IAMServiceImpl) ListUserPolicies(accountID string, input *iam.ListUserP // TagUser upserts tags on a user. Blind read-modify-write Put like the other // user writers (no CAS). func (s *IAMServiceImpl) TagUser(accountID string, input *iam.TagUserInput) (*iam.TagUserOutput, error) { + ctx := context.Background() if err := validateTags(input.Tags); err != nil { return nil, err } userName := *input.UserName - user, err := s.getUser(accountID, userName) + user, err := s.getUser(ctx, accountID, userName) if err != nil { return nil, err } @@ -1516,7 +1541,7 @@ func (s *IAMServiceImpl) TagUser(accountID string, input *iam.TagUserInput) (*ia if err != nil { return nil, fmt.Errorf("marshal user: %w", err) } - if _, err := s.usersBucket.Put(accountID+"."+userName, data); err != nil { + if _, err := s.usersBucket.Put(ctx, accountID+"."+userName, data); err != nil { return nil, fmt.Errorf("update user: %w", err) } @@ -1526,8 +1551,9 @@ func (s *IAMServiceImpl) TagUser(accountID string, input *iam.TagUserInput) (*ia // UntagUser removes the named tag keys from a user; unknown keys are a no-op. func (s *IAMServiceImpl) UntagUser(accountID string, input *iam.UntagUserInput) (*iam.UntagUserOutput, error) { + ctx := context.Background() userName := *input.UserName - user, err := s.getUser(accountID, userName) + user, err := s.getUser(ctx, accountID, userName) if err != nil { return nil, err } @@ -1538,7 +1564,7 @@ func (s *IAMServiceImpl) UntagUser(accountID string, input *iam.UntagUserInput) if err != nil { return nil, fmt.Errorf("marshal user: %w", err) } - if _, err := s.usersBucket.Put(accountID+"."+userName, data); err != nil { + if _, err := s.usersBucket.Put(ctx, accountID+"."+userName, data); err != nil { return nil, fmt.Errorf("update user: %w", err) } @@ -1549,7 +1575,8 @@ func (s *IAMServiceImpl) UntagUser(accountID string, input *iam.UntagUserInput) // ListUserTags returns a user's tags. Pagination is not implemented: // IsTruncated is always false. func (s *IAMServiceImpl) ListUserTags(accountID string, input *iam.ListUserTagsInput) (*iam.ListUserTagsOutput, error) { - user, err := s.getUser(accountID, *input.UserName) + ctx := context.Background() + user, err := s.getUser(ctx, accountID, *input.UserName) if err != nil { return nil, err } @@ -1561,11 +1588,12 @@ func (s *IAMServiceImpl) ListUserTags(accountID string, input *iam.ListUserTagsI // TagPolicy upserts tags on a customer-managed policy, resolved by ARN. func (s *IAMServiceImpl) TagPolicy(accountID string, input *iam.TagPolicyInput) (*iam.TagPolicyOutput, error) { + ctx := context.Background() if err := validateTags(input.Tags); err != nil { return nil, err } - policy, err := s.getPolicyByARN(accountID, *input.PolicyArn) + policy, err := s.getPolicyByARN(ctx, accountID, *input.PolicyArn) if err != nil { return nil, err } @@ -1580,7 +1608,7 @@ func (s *IAMServiceImpl) TagPolicy(accountID string, input *iam.TagPolicyInput) if err != nil { return nil, fmt.Errorf("marshal policy: %w", err) } - if _, err := s.policiesBucket.Put(accountID+"."+policy.PolicyName, data); err != nil { + if _, err := s.policiesBucket.Put(ctx, accountID+"."+policy.PolicyName, data); err != nil { return nil, fmt.Errorf("update policy: %w", err) } @@ -1590,7 +1618,8 @@ func (s *IAMServiceImpl) TagPolicy(accountID string, input *iam.TagPolicyInput) // UntagPolicy removes the named tag keys from a policy; unknown keys are a no-op. func (s *IAMServiceImpl) UntagPolicy(accountID string, input *iam.UntagPolicyInput) (*iam.UntagPolicyOutput, error) { - policy, err := s.getPolicyByARN(accountID, *input.PolicyArn) + ctx := context.Background() + policy, err := s.getPolicyByARN(ctx, accountID, *input.PolicyArn) if err != nil { return nil, err } @@ -1601,7 +1630,7 @@ func (s *IAMServiceImpl) UntagPolicy(accountID string, input *iam.UntagPolicyInp if err != nil { return nil, fmt.Errorf("marshal policy: %w", err) } - if _, err := s.policiesBucket.Put(accountID+"."+policy.PolicyName, data); err != nil { + if _, err := s.policiesBucket.Put(ctx, accountID+"."+policy.PolicyName, data); err != nil { return nil, fmt.Errorf("update policy: %w", err) } @@ -1612,7 +1641,8 @@ func (s *IAMServiceImpl) UntagPolicy(accountID string, input *iam.UntagPolicyInp // ListPolicyTags returns a policy's tags. Pagination is not implemented: // IsTruncated is always false. func (s *IAMServiceImpl) ListPolicyTags(accountID string, input *iam.ListPolicyTagsInput) (*iam.ListPolicyTagsOutput, error) { - policy, err := s.getPolicyByARN(accountID, *input.PolicyArn) + ctx := context.Background() + policy, err := s.getPolicyByARN(ctx, accountID, *input.PolicyArn) if err != nil { return nil, err } @@ -1625,7 +1655,8 @@ func (s *IAMServiceImpl) ListPolicyTags(accountID string, input *iam.ListPolicyT // GetUserPolicies resolves all policy documents attached to a user. // Used internally by the gateway for policy evaluation. func (s *IAMServiceImpl) GetUserPolicies(accountID, userName string) ([]PolicyDocument, error) { - user, err := s.getUser(accountID, userName) + ctx := context.Background() + user, err := s.getUser(ctx, accountID, userName) if err != nil { return nil, err } @@ -1633,7 +1664,7 @@ func (s *IAMServiceImpl) GetUserPolicies(accountID, userName string) ([]PolicyDo var docs []PolicyDocument // Direct user-attached managed policies. for _, arn := range user.AttachedPolicies { - doc, include, err := s.resolveAttachedPolicy(accountID, arn) + doc, include, err := s.resolveAttachedPolicy(ctx, accountID, arn) if err != nil { // Fail closed: unresolvable policy means we cannot make a safe access decision. return nil, err @@ -1660,7 +1691,7 @@ func (s *IAMServiceImpl) GetUserPolicies(accountID, userName string) ([]PolicyDo // grants — never over-permitting. A missing/corrupt policy within a // resolvable group still fails closed, mirroring the direct-policy handling. for _, groupName := range user.Groups { - group, err := s.getGroup(accountID, groupName) + group, err := s.getGroup(ctx, accountID, groupName) if err != nil { if err.Error() == awserrors.ErrorIAMNoSuchEntity { slog.Warn("GetUserPolicies: member references missing group; skipping", @@ -1670,7 +1701,7 @@ func (s *IAMServiceImpl) GetUserPolicies(accountID, userName string) ([]PolicyDo return nil, err // transient/corrupt → fail closed } for _, arn := range group.AttachedPolicies { - doc, include, err := s.resolveAttachedPolicy(accountID, arn) + doc, include, err := s.resolveAttachedPolicy(ctx, accountID, arn) if err != nil { return nil, err // fail closed } @@ -1696,7 +1727,7 @@ func (s *IAMServiceImpl) GetUserPolicies(accountID, userName string) ([]PolicyDo // Helpers // --------------------------------------------------------------------------- -func (s *IAMServiceImpl) getPolicyByARN(accountID, policyARN string) (*Policy, error) { +func (s *IAMServiceImpl) getPolicyByARN(ctx context.Context, accountID, policyARN string) (*Policy, error) { parts := strings.SplitN(policyARN, ":policy", 2) if len(parts) != 2 || parts[1] == "" { return nil, errors.New(awserrors.ErrorIAMNoSuchEntity) @@ -1708,9 +1739,9 @@ func (s *IAMServiceImpl) getPolicyByARN(accountID, policyARN string) (*Policy, e } kvKey := accountID + "." + policyName - entry, err := s.policiesBucket.Get(kvKey) + entry, err := s.policiesBucket.Get(ctx, kvKey) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil, errors.New(awserrors.ErrorIAMNoSuchEntity) } return nil, fmt.Errorf("get policy: %w", err) @@ -1731,10 +1762,10 @@ func (s *IAMServiceImpl) getPolicyByARN(accountID, policyARN string) (*Policy, e // buildAttachmentCounts fetches all users for the account once and returns a // map of policyARN -> number of users that have it attached, avoiding a // per-policy full scan when listing or inspecting policies. -func (s *IAMServiceImpl) buildAttachmentCounts(accountID string) (map[string]int64, error) { - keys, err := s.usersBucket.Keys() +func (s *IAMServiceImpl) buildAttachmentCounts(ctx context.Context, accountID string) (map[string]int64, error) { + keys, err := kvutil.Keys(ctx, s.usersBucket) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return nil, nil } return nil, fmt.Errorf("build attachment counts: %w", err) @@ -1750,9 +1781,9 @@ func (s *IAMServiceImpl) buildAttachmentCounts(accountID string) (map[string]int continue } - entry, err := s.usersBucket.Get(key) + entry, err := s.usersBucket.Get(ctx, key) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { slog.Debug("buildAttachmentCounts: user key disappeared", "key", key) continue } @@ -1771,11 +1802,11 @@ func (s *IAMServiceImpl) buildAttachmentCounts(accountID string) (map[string]int return counts, nil } -func (s *IAMServiceImpl) getUser(accountID, userName string) (*User, error) { +func (s *IAMServiceImpl) getUser(ctx context.Context, accountID, userName string) (*User, error) { kvKey := accountID + "." + userName - entry, err := s.usersBucket.Get(kvKey) + entry, err := s.usersBucket.Get(ctx, kvKey) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil, errors.New(awserrors.ErrorIAMNoSuchEntity) } return nil, fmt.Errorf("get user: %w", err) @@ -1931,10 +1962,10 @@ var summaryQuotaDefaults = map[string]int64{ // are keyed "accountID.name"; counting is by key prefix only (no Get/unmarshal) // so it stays cheap with many resources. The version key is skipped and a // missing bucket counts as zero. -func countBucket(bucket nats.KeyValue, accountID string) (int64, error) { - keys, err := bucket.Keys() +func countBucket(ctx context.Context, bucket jetstream.KeyValue, accountID string) (int64, error) { + keys, err := kvutil.Keys(ctx, bucket) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return 0, nil } return 0, err @@ -1957,7 +1988,8 @@ func countBucket(bucket nats.KeyValue, accountID string) (int64, error) { // values as a SummaryMap. Counts are scoped to accountID by key prefix; quota // values are static informational constants (see summaryQuotaDefaults). func (s *IAMServiceImpl) GetAccountSummary(accountID string, _ *iam.GetAccountSummaryInput) (*iam.GetAccountSummaryOutput, error) { - counted := map[string]nats.KeyValue{ + ctx := context.Background() + counted := map[string]jetstream.KeyValue{ "Users": s.usersBucket, "Groups": s.groupsBucket, "Roles": s.rolesBucket, @@ -1970,7 +2002,7 @@ func (s *IAMServiceImpl) GetAccountSummary(accountID string, _ *iam.GetAccountSu summary[key] = aws.Int64(value) } for key, bucket := range counted { - n, err := countBucket(bucket, accountID) + n, err := countBucket(ctx, bucket, accountID) if err != nil { return nil, fmt.Errorf("count %s: %w", key, err) } diff --git a/spinifex/handlers/iam/service_impl_test.go b/spinifex/handlers/iam/service_impl_test.go index 8eb15b125..53940f24c 100644 --- a/spinifex/handlers/iam/service_impl_test.go +++ b/spinifex/handlers/iam/service_impl_test.go @@ -29,7 +29,7 @@ func setupTestIAMService(t *testing.T) *IAMServiceImpl { masterKey, err := GenerateMasterKey() require.NoError(t, err) - svc, err := NewIAMServiceImpl(nc, masterKey, 1) + svc, err := NewIAMServiceImpl(t.Context(), nc, masterKey, 1) require.NoError(t, err) return svc } @@ -1598,7 +1598,7 @@ func TestSensitiveDataNotLogged_MasterKey(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) masterKey, err := GenerateMasterKey() require.NoError(t, err) - _, err = NewIAMServiceImpl(nc, masterKey, 1) + _, err = NewIAMServiceImpl(t.Context(), nc, masterKey, 1) require.NoError(t, err) logOutput := buf.String() diff --git a/spinifex/handlers/iam/user_inline_test.go b/spinifex/handlers/iam/user_inline_test.go index 4ba5faa6a..dd838ba82 100644 --- a/spinifex/handlers/iam/user_inline_test.go +++ b/spinifex/handlers/iam/user_inline_test.go @@ -268,7 +268,7 @@ func TestDeleteUserPolicy_UserNotFound(t *testing.T) { // corrupt stored document that the write path would otherwise reject. func putRawUserInlinePolicy(t *testing.T, svc *IAMServiceImpl, userName, policyName, raw string) { t.Helper() - user, err := svc.getUser(testAccountID, userName) + user, err := svc.getUser(t.Context(), testAccountID, userName) require.NoError(t, err) if user.InlinePolicies == nil { user.InlinePolicies = map[string]string{} @@ -276,7 +276,7 @@ func putRawUserInlinePolicy(t *testing.T, svc *IAMServiceImpl, userName, policyN user.InlinePolicies[policyName] = raw data, err := json.Marshal(user) require.NoError(t, err) - _, err = svc.usersBucket.Put(testAccountID+"."+userName, data) + _, err = svc.usersBucket.Put(t.Context(), testAccountID+"."+userName, data) require.NoError(t, err) } diff --git a/spinifex/handlers/imds/resolver.go b/spinifex/handlers/imds/resolver.go index ddd9a1e22..7d03166bd 100644 --- a/spinifex/handlers/imds/resolver.go +++ b/spinifex/handlers/imds/resolver.go @@ -9,7 +9,7 @@ import ( "strings" "time" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // kvBucketENIs and kvBucketSecurityGroups are duplicated as literals to avoid an import cycle. @@ -85,10 +85,10 @@ type eniRecord struct { // metadataResolver resolves a tap's ENI ID to ENI + instance facts. // Resolution chain: eniID → ENIRecord (account recovered from the bucket key) → instanceFacts. type metadataResolver struct { - eniKV nats.KeyValue // spinifex-vpc-enis - sgKV nats.KeyValue // spinifex-vpc-security-groups (nil-safe: degrades to IDs) - subnetKV nats.KeyValue // spinifex-vpc-subnets (nil-safe: CIDR leaf 404s) - vpcKV nats.KeyValue // spinifex-vpc-vpcs (nil-safe: CIDR leaf 404s) + eniKV jetstream.KeyValue // spinifex-vpc-enis + sgKV jetstream.KeyValue // spinifex-vpc-security-groups (nil-safe: degrades to IDs) + subnetKV jetstream.KeyValue // spinifex-vpc-subnets (nil-safe: CIDR leaf 404s) + vpcKV jetstream.KeyValue // spinifex-vpc-vpcs (nil-safe: CIDR leaf 404s) lookup instanceLookup } @@ -102,7 +102,7 @@ func (r *metadataResolver) resolveENIByID(ctx context.Context, eniID string) (*e if eniID == "" { return nil, nil } - accountID, raw, err := r.findENIByID(eniID) + accountID, raw, err := r.findENIByID(ctx, eniID) if err != nil { return nil, err } @@ -120,10 +120,10 @@ func (r *metadataResolver) resolveENIByID(ctx context.Context, eniID string) (*e // findENIByID scans the ENI bucket for the record whose key ends in ".{eniID}", // returning the owning account and the raw record bytes, or ("", nil, nil) on // miss. ENI IDs are globally unique, so at most one key matches. -func (r *metadataResolver) findENIByID(eniID string) (string, []byte, error) { - keys, err := r.eniKV.Keys() +func (r *metadataResolver) findENIByID(ctx context.Context, eniID string) (string, []byte, error) { + keys, err := r.eniKV.Keys(ctx) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return "", nil, nil } return "", nil, fmt.Errorf("list eni bucket: %w", err) @@ -134,9 +134,9 @@ func (r *metadataResolver) findENIByID(eniID string) (string, []byte, error) { if !strings.HasSuffix(key, suffix) { continue } - entry, err := r.eniKV.Get(key) + entry, err := r.eniKV.Get(ctx, key) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { continue // raced with a concurrent delete } return "", nil, fmt.Errorf("get eni record %s: %w", key, err) @@ -180,9 +180,9 @@ func (r *metadataResolver) resolveSGNames(ctx context.Context, accountID string, if r.sgKV == nil { continue } - raw, err := r.sgKV.Get(accountID + "." + id) + raw, err := r.sgKV.Get(ctx, accountID+"."+id) if err != nil { - if !errors.Is(err, nats.ErrKeyNotFound) { + if !errors.Is(err, jetstream.ErrKeyNotFound) { slog.WarnContext(ctx, "IMDS: security-group name lookup failed", "account_id", accountID, "sg_id", id, "err", err) } continue @@ -204,24 +204,24 @@ type cidrRecord struct { CidrBlock string `json:"cidr_block"` } -func (r *metadataResolver) resolveSubnetCIDR(_ context.Context, accountID, subnetID string) (string, error) { - return cidrFromKV(r.subnetKV, accountID, subnetID) +func (r *metadataResolver) resolveSubnetCIDR(ctx context.Context, accountID, subnetID string) (string, error) { + return cidrFromKV(ctx, r.subnetKV, accountID, subnetID) } -func (r *metadataResolver) resolveVPCCIDR(_ context.Context, accountID, vpcID string) (string, error) { - return cidrFromKV(r.vpcKV, accountID, vpcID) +func (r *metadataResolver) resolveVPCCIDR(ctx context.Context, accountID, vpcID string) (string, error) { + return cidrFromKV(ctx, r.vpcKV, accountID, vpcID) } // cidrFromKV reads cidr_block from an account-scoped record. ("", nil) on a nil // bucket or key miss; the error on any other fault so the leaf 500s rather than // serving an empty CIDR a guest would mis-render into its network config. -func cidrFromKV(kv nats.KeyValue, accountID, id string) (string, error) { +func cidrFromKV(ctx context.Context, kv jetstream.KeyValue, accountID, id string) (string, error) { if kv == nil || id == "" { return "", nil } - entry, err := kv.Get(accountID + "." + id) + entry, err := kv.Get(ctx, accountID+"."+id) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return "", nil } return "", fmt.Errorf("get cidr record %s.%s: %w", accountID, id, err) diff --git a/spinifex/handlers/imds/resolver_test.go b/spinifex/handlers/imds/resolver_test.go index acb09e5d7..819b60f74 100644 --- a/spinifex/handlers/imds/resolver_test.go +++ b/spinifex/handlers/imds/resolver_test.go @@ -7,7 +7,7 @@ import ( "testing" "github.com/mulgadc/spinifex/spinifex/testutil" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -32,21 +32,22 @@ func (f *fakeLookup) describe(_ context.Context, accountID, _ string) (*instance // fake instance lookup the caller can program. func newTestResolver(t *testing.T) (*metadataResolver, *fakeLookup) { t.Helper() - _, _, js := testutil.StartTestJetStream(t) - eniKV, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: kvBucketENIs, History: 1}) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + eniKV, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: kvBucketENIs, History: 1}) require.NoError(t, err) - sgKV, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: kvBucketSecurityGroups, History: 1}) + sgKV, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: kvBucketSecurityGroups, History: 1}) require.NoError(t, err) lookup := &fakeLookup{} return &metadataResolver{eniKV: eniKV, sgKV: sgKV, lookup: lookup}, lookup } -func putJSON(t *testing.T, kv nats.KeyValue, key string, v any) { +func putJSON(t *testing.T, kv jetstream.KeyValue, key string, v any) { t.Helper() data, err := json.Marshal(v) require.NoError(t, err) - _, err = kv.Put(key, data) + _, err = kv.Put(t.Context(), key, data) require.NoError(t, err) } @@ -137,7 +138,7 @@ func TestResolveENIByID_EmptyIDIsNilNil(t *testing.T) { func TestResolveENIByID_BadJSONErrors(t *testing.T) { r, _ := newTestResolver(t) - _, err := r.eniKV.Put("111122223333.eni-aaa", []byte("not json")) + _, err := r.eniKV.Put(t.Context(), "111122223333.eni-aaa", []byte("not json")) require.NoError(t, err) _, err = r.resolveENIByID(context.Background(), "eni-aaa") @@ -215,7 +216,7 @@ func TestResolveSGNames_ResolvesAndFallsBack(t *testing.T) { func TestResolveSGNames_BadJSONFallsBackToID(t *testing.T) { r, _ := newTestResolver(t) - _, err := r.sgKV.Put("111122223333.sg-1", []byte("not json")) + _, err := r.sgKV.Put(t.Context(), "111122223333.sg-1", []byte("not json")) require.NoError(t, err) got := r.resolveSGNames(context.Background(), "111122223333", []string{"sg-1"}) diff --git a/spinifex/handlers/imds/service.go b/spinifex/handlers/imds/service.go index c1e7d737f..3dec08c9d 100644 --- a/spinifex/handlers/imds/service.go +++ b/spinifex/handlers/imds/service.go @@ -12,6 +12,7 @@ import ( "github.com/aws/aws-sdk-go/service/iam" handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // tokenSweepInterval is how often abandoned/expired IMDSv2 tokens are swept. @@ -77,7 +78,8 @@ type IMDSServiceImpl struct { // publishes. resolverIPs are the WAN IPs of nodes running northstar: when // non-empty, each per-tap responder also serves the VPC DNS shim on // 169.254.169.253:53, relaying to northstar's unprivileged wildcard listener. -func NewIMDSServiceImpl(natsConn *nats.Conn, sts stsAssumer, iamSvc profileLookup, pubKeys publicKeyLookup, expectedNodes int, listTaps listTapsFunc, baseDomain, internalDomain string, resolverIPs []string) (*IMDSServiceImpl, error) { +// ctx bounds the bucket opens only; each served request carries its own. +func NewIMDSServiceImpl(ctx context.Context, natsConn *nats.Conn, sts stsAssumer, iamSvc profileLookup, pubKeys publicKeyLookup, expectedNodes int, listTaps listTapsFunc, baseDomain, internalDomain string, resolverIPs []string) (*IMDSServiceImpl, error) { if natsConn == nil { return nil, errors.New("nil NATS connection") } @@ -94,18 +96,18 @@ func NewIMDSServiceImpl(natsConn *nats.Conn, sts stsAssumer, iamSvc profileLooku return nil, errors.New("nil tap enumerator") } - js, err := natsConn.JetStream() + js, err := jetstream.New(natsConn) if err != nil { return nil, fmt.Errorf("get JetStream context: %w", err) } - eniKV, err := js.KeyValue(kvBucketENIs) + eniKV, err := js.KeyValue(ctx, kvBucketENIs) if err != nil { return nil, fmt.Errorf("open %s bucket: %w", kvBucketENIs, err) } // Open SG bucket best-effort; IMDS starts fine without it, degrading to raw IDs. - sgKV, err := js.KeyValue(kvBucketSecurityGroups) + sgKV, err := js.KeyValue(ctx, kvBucketSecurityGroups) if err != nil { slog.Warn("IMDS: security-group bucket unavailable, /security-groups will serve IDs", "bucket", kvBucketSecurityGroups, "err", err) sgKV = nil @@ -113,12 +115,12 @@ func NewIMDSServiceImpl(natsConn *nats.Conn, sts stsAssumer, iamSvc profileLooku // Open subnet/VPC buckets best-effort; the network-interfaces CIDR leaves 404 // (and drop from the listing) when a bucket is unavailable, IMDS still starts. - subnetKV, err := js.KeyValue(kvBucketSubnets) + subnetKV, err := js.KeyValue(ctx, kvBucketSubnets) if err != nil { slog.Warn("IMDS: subnet bucket unavailable, network-interfaces subnet CIDR will 404", "bucket", kvBucketSubnets, "err", err) subnetKV = nil } - vpcKV, err := js.KeyValue(kvBucketVPCs) + vpcKV, err := js.KeyValue(ctx, kvBucketVPCs) if err != nil { slog.Warn("IMDS: vpc bucket unavailable, network-interfaces VPC CIDR will 404", "bucket", kvBucketVPCs, "err", err) vpcKV = nil diff --git a/spinifex/handlers/quota/quota.go b/spinifex/handlers/quota/quota.go index c281133ec..0fa16c123 100644 --- a/spinifex/handlers/quota/quota.go +++ b/spinifex/handlers/quota/quota.go @@ -8,7 +8,7 @@ import ( "time" "github.com/mulgadc/spinifex/spinifex/utils" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // ReconcileInterval is how often the gateway recomputes vCPU counters. It is @@ -44,13 +44,13 @@ type Service struct { // usage holds the per-account vCPU counters (key {accountID}). Nil when // quotas are disabled, in which case Exempt short-circuits every check // before the counter is touched. - usage nats.KeyValue + usage jetstream.KeyValue } // New constructs a quota Service from the configured limits and the gateway-owned // account-usage KV bucket. usage may be nil when quotas are disabled; Exempt then // short-circuits every check before the counter is read. -func New(limits Limits, usage nats.KeyValue) *Service { +func New(limits Limits, usage jetstream.KeyValue) *Service { return &Service{limits: limits, usage: usage} } diff --git a/spinifex/handlers/quota/reconcile.go b/spinifex/handlers/quota/reconcile.go index 8a7b90a87..19fa45364 100644 --- a/spinifex/handlers/quota/reconcile.go +++ b/spinifex/handlers/quota/reconcile.go @@ -72,7 +72,7 @@ func (s *Service) Reconcile(ctx context.Context, accounts AccountLister, list In } continue } - if err := s.reconcileVCPU(accountID, sumReservationVCPUs(reservations), complete); err != nil { + if err := s.reconcileVCPU(ctx, accountID, sumReservationVCPUs(reservations), complete); err != nil { slog.Warn("quota reconcile: counter overwrite failed", "account", accountID, "err", err) if firstErr == nil { firstErr = err diff --git a/spinifex/handlers/quota/reconcile_test.go b/spinifex/handlers/quota/reconcile_test.go index a1447d4a9..a69127574 100644 --- a/spinifex/handlers/quota/reconcile_test.go +++ b/spinifex/handlers/quota/reconcile_test.go @@ -47,7 +47,7 @@ func TestReconcileCorrectsOverCount(t *testing.T) { // Counter drifted high (e.g. a stale increment); the account truly holds one // m5.xlarge (4) plus one t3.micro (2) = 6. - if err := s.AddVCPU(testAccount, 16); err != nil { + if err := s.AddVCPU(t.Context(), testAccount, 16); err != nil { t.Fatalf("seed AddVCPU: %v", err) } lister := staticLister(map[string][]*ec2.Reservation{ @@ -68,7 +68,7 @@ func TestReconcileCorrectsOverCount(t *testing.T) { func TestReconcileCorrectsStaleTermination(t *testing.T) { s := newVCPUService(t, Limits{Enabled: true, VCPUs: 100}) - if err := s.AddVCPU(testAccount, 6); err != nil { + if err := s.AddVCPU(t.Context(), testAccount, 6); err != nil { t.Fatalf("seed AddVCPU: %v", err) } lister := staticLister(map[string][]*ec2.Reservation{ @@ -89,7 +89,7 @@ func TestReconcileCorrectsStaleTermination(t *testing.T) { func TestReconcileZeroesEmptyAccount(t *testing.T) { s := newVCPUService(t, Limits{Enabled: true, VCPUs: 100}) - if err := s.AddVCPU(testAccount, 8); err != nil { + if err := s.AddVCPU(t.Context(), testAccount, 8); err != nil { t.Fatalf("seed AddVCPU: %v", err) } lister := staticLister(map[string][]*ec2.Reservation{}, nil) // describes empty @@ -106,7 +106,7 @@ func TestReconcileSkipsSystemAccount(t *testing.T) { s := newVCPUService(t, Limits{Enabled: true, VCPUs: 100}) // Park a non-zero value directly under the system key; AddVCPU would no-op it. - if _, err := s.usage.PutString(utils.GlobalAccountID, "42"); err != nil { + if _, err := s.usage.PutString(t.Context(), utils.GlobalAccountID, "42"); err != nil { t.Fatalf("seed system counter: %v", err) } var seen []string @@ -133,7 +133,7 @@ func TestReconcileContinuesOnDescribeError(t *testing.T) { s := newVCPUService(t, Limits{Enabled: true, VCPUs: 100}) const bad, good = "111111111111", "222222222222" - if err := s.AddVCPU(bad, 8); err != nil { + if err := s.AddVCPU(t.Context(), bad, 8); err != nil { t.Fatalf("seed bad: %v", err) } sentinel := errors.New("describe boom") @@ -160,7 +160,7 @@ func TestReconcileIncompleteSweepDoesNotLower(t *testing.T) { // Counter holds 8 (two m5.xlarge across two nodes); a partial sweep sees only // one node's 4 vCPUs and reports complete=false. - if err := s.AddVCPU(testAccount, 8); err != nil { + if err := s.AddVCPU(t.Context(), testAccount, 8); err != nil { t.Fatalf("seed AddVCPU: %v", err) } partial := func(accountID string) ([]*ec2.Reservation, bool, error) { @@ -200,7 +200,7 @@ func TestReconcileDisabledNoop(t *testing.T) { func assertCounter(t *testing.T, s *Service, accountID string, want int) { t.Helper() - got, _, err := s.readVCPU(accountID) + got, _, err := s.readVCPU(t.Context(), accountID) if err != nil { t.Fatalf("readVCPU(%s): %v", accountID, err) } diff --git a/spinifex/handlers/quota/vcpu.go b/spinifex/handlers/quota/vcpu.go index d074a7f90..62b6a4d5d 100644 --- a/spinifex/handlers/quota/vcpu.go +++ b/spinifex/handlers/quota/vcpu.go @@ -12,6 +12,7 @@ import ( gateway_ec2_instance "github.com/mulgadc/spinifex/spinifex/gateway/ec2/instance" "github.com/mulgadc/spinifex/spinifex/instancetypes" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // vcpuCASRetries bounds AddVCPU's retry on a revision conflict. Each retry @@ -26,11 +27,11 @@ const vcpuCASRetries = 100 // caller increments via AddVCPU after the grow succeeds. Two concurrent checks // can both pass before either increments (the documented soft-cap window); the // per-node physical gate still backstops real overcommit. -func (s *Service) CheckVCPU(accountID string, want int) error { +func (s *Service) CheckVCPU(ctx context.Context, accountID string, want int) error { if s.Exempt(accountID) { return nil } - current, _, err := s.readVCPU(accountID) + current, _, err := s.readVCPU(ctx, accountID) if err != nil { return err } @@ -44,9 +45,9 @@ func (s *Service) CheckVCPU(accountID string, want int) error { // concurrent writers cannot lose updates. next maps the current value to the new // value and a skip flag; skip leaves the counter untouched and never creates a // key. A revision conflict is retried; exhausting the bound is a hard error. -func (s *Service) casVCPU(accountID string, next func(current int) (value int, skip bool)) error { +func (s *Service) casVCPU(ctx context.Context, accountID string, next func(current int) (value int, skip bool)) error { for range vcpuCASRetries { - current, revision, err := s.readVCPU(accountID) + current, revision, err := s.readVCPU(ctx, accountID) if err != nil { return err } @@ -59,9 +60,9 @@ func (s *Service) casVCPU(accountID string, next func(current int) (value int, s return err } if revision == 0 { - _, err = s.usage.Create(accountID, data) + _, err = s.usage.Create(ctx, accountID, data) } else { - _, err = s.usage.Update(accountID, data, revision) + _, err = s.usage.Update(ctx, accountID, data, revision) } if err == nil { return nil @@ -76,11 +77,11 @@ func (s *Service) casVCPU(accountID string, next func(current int) (value int, s // AddVCPU adds delta vCPUs to accountID's counter under CAS so concurrent grows // cannot lose updates. A non-positive delta is a no-op: shrinks are never charged // and are left to reconcile to lower the counter. -func (s *Service) AddVCPU(accountID string, delta int) error { +func (s *Service) AddVCPU(ctx context.Context, accountID string, delta int) error { if s.Exempt(accountID) || delta <= 0 { return nil } - return s.casVCPU(accountID, func(current int) (int, bool) { + return s.casVCPU(ctx, accountID, func(current int) (int, bool) { return current + delta, false }) } @@ -89,8 +90,8 @@ func (s *Service) AddVCPU(accountID string, delta int) error { // a counter. A counter already equal to value is left untouched, so a steady-state // pass writes nothing and an account with no usage and no key is never created. // Reconcile is the sole caller and runs under the leader lock. -func (s *Service) setVCPU(accountID string, value int) error { - return s.casVCPU(accountID, func(current int) (int, bool) { +func (s *Service) setVCPU(ctx context.Context, accountID string, value int) error { + return s.casVCPU(ctx, accountID, func(current int) (int, bool) { return value, current == value }) } @@ -101,18 +102,18 @@ func (s *Service) setVCPU(accountID string, value int) error { // counter: lowering from a partial view would under-count usage and lift the // cap, so a short count is dropped and left for the next clean pass. Raising is // always safe, since the instances actually observed do exist. -func (s *Service) reconcileVCPU(accountID string, value int, complete bool) error { +func (s *Service) reconcileVCPU(ctx context.Context, accountID string, value int, complete bool) error { if complete { - return s.setVCPU(accountID, value) + return s.setVCPU(ctx, accountID, value) } - current, _, err := s.readVCPU(accountID) + current, _, err := s.readVCPU(ctx, accountID) if err != nil { return err } if value <= current { return nil } - return s.setVCPU(accountID, value) + return s.setVCPU(ctx, accountID, value) } // isCASConflict reports whether err is a lost optimistic-concurrency race: a @@ -120,12 +121,12 @@ func (s *Service) reconcileVCPU(accountID string, value int, complete bool) erro // JetStream's wrong-last-sequence code and are retryable; any other error is a // genuine failure the caller must surface. func isCASConflict(err error) bool { - if errors.Is(err, nats.ErrKeyExists) { + if errors.Is(err, jetstream.ErrKeyExists) { return true } - var apiErr *nats.APIError + var apiErr *jetstream.APIError if errors.As(err, &apiErr) { - return apiErr.ErrorCode == nats.JSErrCodeStreamWrongLastSequence + return apiErr.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequence } return false } @@ -136,12 +137,12 @@ func isCASConflict(err error) bool { // ChargeLaunch. An unknown instance type contributes nothing and is left for the // daemon to reject as InvalidInstanceType; the Exempt short-circuit lives in // CheckVCPU. -func (s *Service) EnforceLaunch(accountID, instanceType string, maxCount int) error { +func (s *Service) EnforceLaunch(ctx context.Context, accountID, instanceType string, maxCount int) error { perType, ok := instancetypes.DefaultVCPUs(instanceType) if !ok { return nil } - return s.CheckVCPU(accountID, perType*maxCount) + return s.CheckVCPU(ctx, accountID, perType*maxCount) } // ChargeLaunch is the increment-after for a successful RunInstances: it adds the @@ -149,8 +150,8 @@ func (s *Service) EnforceLaunch(accountID, instanceType string, maxCount int) er // counter. The daemon may launch fewer than maxCount, so the charge is the real // reservation rather than the checked worst case. The caller treats a write // failure as drift for reconcile to correct, never failing the live launch. -func (s *Service) ChargeLaunch(accountID string, reservation *ec2.Reservation) error { - return s.AddVCPU(accountID, sumReservationVCPUs([]*ec2.Reservation{reservation})) +func (s *Service) ChargeLaunch(ctx context.Context, accountID string, reservation *ec2.Reservation) error { + return s.AddVCPU(ctx, accountID, sumReservationVCPUs([]*ec2.Reservation{reservation})) } // InstanceTypeResolver returns the current instance type of instanceID owned by @@ -200,7 +201,7 @@ func confirmInstanceType(reservations []*ec2.Reservation, instanceID string, com // to reconcile. Exempt accounts, an unknown newType (the daemon rejects it), and // an instance the resolver cannot find (the daemon rejects the modify) all return // 0 with no check. -func (s *Service) EnforceRetype(resolve InstanceTypeResolver, accountID, instanceID, newType string) (int, error) { +func (s *Service) EnforceRetype(ctx context.Context, resolve InstanceTypeResolver, accountID, instanceID, newType string) (int, error) { if s.Exempt(accountID) { return 0, nil } @@ -223,7 +224,7 @@ func (s *Service) EnforceRetype(resolve InstanceTypeResolver, accountID, instanc if delta <= 0 { return delta, nil } - if err := s.CheckVCPU(accountID, delta); err != nil { + if err := s.CheckVCPU(ctx, accountID, delta); err != nil { return 0, err } return delta, nil @@ -252,10 +253,10 @@ func instanceTypeFromReservations(reservations []*ec2.Reservation, instanceID st // readVCPU returns accountID's current reserved vCPU count and the KV revision // it was read at. A missing key is the zero counter at revision 0, which AddVCPU // treats as a create. -func (s *Service) readVCPU(accountID string) (count int, revision uint64, err error) { - entry, err := s.usage.Get(accountID) +func (s *Service) readVCPU(ctx context.Context, accountID string) (count int, revision uint64, err error) { + entry, err := s.usage.Get(ctx, accountID) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return 0, 0, nil } return 0, 0, err diff --git a/spinifex/handlers/quota/vcpu_launch_test.go b/spinifex/handlers/quota/vcpu_launch_test.go index d302ab8a7..e683bf820 100644 --- a/spinifex/handlers/quota/vcpu_launch_test.go +++ b/spinifex/handlers/quota/vcpu_launch_test.go @@ -44,11 +44,11 @@ func TestEnforceLaunch(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if tt.reserve > 0 { - if err := s.AddVCPU(tt.account, tt.reserve); err != nil { + if err := s.AddVCPU(t.Context(), tt.account, tt.reserve); err != nil { t.Fatalf("AddVCPU(%d): %v", tt.reserve, err) } } - err := s.EnforceLaunch(tt.account, tt.instanceType, tt.maxCount) + err := s.EnforceLaunch(t.Context(), tt.account, tt.instanceType, tt.maxCount) switch { case tt.wantErr == "" && err != nil: t.Fatalf("EnforceLaunch(%s, %d) = %v, want nil", tt.instanceType, tt.maxCount, err) @@ -65,14 +65,14 @@ func TestEnforceLaunch(t *testing.T) { func TestChargeLaunchActual(t *testing.T) { s := newVCPUService(t, Limits{Enabled: true, VCPUs: 8}) - if err := s.EnforceLaunch(testAccount, "t3.micro", 4); err != nil { + if err := s.EnforceLaunch(t.Context(), testAccount, "t3.micro", 4); err != nil { t.Fatalf("EnforceLaunch = %v, want nil", err) } // Daemon launched only 2; charge the actual 4 vCPUs, not the checked 8. - if err := s.ChargeLaunch(testAccount, reservationOf("t3.micro", 2)); err != nil { + if err := s.ChargeLaunch(t.Context(), testAccount, reservationOf("t3.micro", 2)); err != nil { t.Fatalf("ChargeLaunch = %v", err) } - got, _, err := s.readVCPU(testAccount) + got, _, err := s.readVCPU(t.Context(), testAccount) if err != nil { t.Fatalf("readVCPU: %v", err) } @@ -89,26 +89,26 @@ func TestLaunchSoftCapBounded(t *testing.T) { s := newVCPUService(t, Limits{Enabled: true, VCPUs: 8}) // Pre-charge to 6/8 so each 2-vCPU launch individually fits the headroom. - if err := s.AddVCPU(testAccount, 6); err != nil { + if err := s.AddVCPU(t.Context(), testAccount, 6); err != nil { t.Fatalf("AddVCPU(6): %v", err) } // Two launches enter the gate before either charges: both read 6/8 and pass. - if err := s.EnforceLaunch(testAccount, "t3.micro", 1); err != nil { + if err := s.EnforceLaunch(t.Context(), testAccount, "t3.micro", 1); err != nil { t.Fatalf("first EnforceLaunch = %v, want nil in soft-cap window", err) } - if err := s.EnforceLaunch(testAccount, "t3.micro", 1); err != nil { + if err := s.EnforceLaunch(t.Context(), testAccount, "t3.micro", 1); err != nil { t.Fatalf("second EnforceLaunch = %v, want nil in soft-cap window", err) } // Both charges then land under CAS without losing an update. for range 2 { - if err := s.ChargeLaunch(testAccount, reservationOf("t3.micro", 1)); err != nil { + if err := s.ChargeLaunch(t.Context(), testAccount, reservationOf("t3.micro", 1)); err != nil { t.Fatalf("ChargeLaunch = %v", err) } } - got, _, err := s.readVCPU(testAccount) + got, _, err := s.readVCPU(t.Context(), testAccount) if err != nil { t.Fatalf("readVCPU: %v", err) } @@ -128,10 +128,12 @@ func TestLaunchExemptShortCircuits(t *testing.T) { name string fn func() error }{ - {"enforce disabled", func() error { return disabled.EnforceLaunch(testAccount, "t3.micro", 1000) }}, - {"charge disabled", func() error { return disabled.ChargeLaunch(testAccount, reservationOf("t3.micro", 1000)) }}, - {"enforce system account", func() error { return enabled.EnforceLaunch(utils.GlobalAccountID, "t3.micro", 1000) }}, - {"charge system account", func() error { return enabled.ChargeLaunch(utils.GlobalAccountID, reservationOf("t3.micro", 1000)) }}, + {"enforce disabled", func() error { return disabled.EnforceLaunch(t.Context(), testAccount, "t3.micro", 1000) }}, + {"charge disabled", func() error { return disabled.ChargeLaunch(t.Context(), testAccount, reservationOf("t3.micro", 1000)) }}, + {"enforce system account", func() error { return enabled.EnforceLaunch(t.Context(), utils.GlobalAccountID, "t3.micro", 1000) }}, + {"charge system account", func() error { + return enabled.ChargeLaunch(t.Context(), utils.GlobalAccountID, reservationOf("t3.micro", 1000)) + }}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { diff --git a/spinifex/handlers/quota/vcpu_retype_test.go b/spinifex/handlers/quota/vcpu_retype_test.go index 518871f00..9e4f07921 100644 --- a/spinifex/handlers/quota/vcpu_retype_test.go +++ b/spinifex/handlers/quota/vcpu_retype_test.go @@ -50,11 +50,11 @@ func TestEnforceRetype(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if tt.reserve > 0 { - if err := s.AddVCPU(tt.account, tt.reserve); err != nil { + if err := s.AddVCPU(t.Context(), tt.account, tt.reserve); err != nil { t.Fatalf("AddVCPU(%d): %v", tt.reserve, err) } } - delta, err := s.EnforceRetype(staticResolver(tt.oldType, tt.resolveOK, nil), tt.account, "i-abc", tt.newType) + delta, err := s.EnforceRetype(t.Context(), staticResolver(tt.oldType, tt.resolveOK, nil), tt.account, "i-abc", tt.newType) switch { case tt.wantErr == "" && err != nil: t.Fatalf("EnforceRetype = %v, want nil", err) @@ -73,14 +73,14 @@ func TestEnforceRetype(t *testing.T) { // A retype from t3.micro (2) to m5.xlarge (4) on a 2/8 account lands at 4/8. func TestRetypeChargesDelta(t *testing.T) { s := newVCPUService(t, Limits{Enabled: true, VCPUs: 8}) - if err := s.AddVCPU(testAccount, 2); err != nil { // the t3.micro is already counted + if err := s.AddVCPU(t.Context(), testAccount, 2); err != nil { // the t3.micro is already counted t.Fatalf("seed AddVCPU: %v", err) } - delta, err := s.EnforceRetype(staticResolver("t3.micro", true, nil), testAccount, "i-a", "m5.xlarge") + delta, err := s.EnforceRetype(t.Context(), staticResolver("t3.micro", true, nil), testAccount, "i-a", "m5.xlarge") if err != nil { t.Fatalf("EnforceRetype: %v", err) } - if err := s.AddVCPU(testAccount, delta); err != nil { + if err := s.AddVCPU(t.Context(), testAccount, delta); err != nil { t.Fatalf("AddVCPU(%d): %v", delta, err) } assertCounter(t, s, testAccount, 4) @@ -91,17 +91,17 @@ func TestRetypeChargesDelta(t *testing.T) { // only path that lowers the counter. func TestRetypeDownDoesNotCharge(t *testing.T) { s := newVCPUService(t, Limits{Enabled: true, VCPUs: 8}) - if err := s.AddVCPU(testAccount, 4); err != nil { // the m5.xlarge is already counted + if err := s.AddVCPU(t.Context(), testAccount, 4); err != nil { // the m5.xlarge is already counted t.Fatalf("seed AddVCPU: %v", err) } - delta, err := s.EnforceRetype(staticResolver("m5.xlarge", true, nil), testAccount, "i-a", "t3.micro") + delta, err := s.EnforceRetype(t.Context(), staticResolver("m5.xlarge", true, nil), testAccount, "i-a", "t3.micro") if err != nil { t.Fatalf("EnforceRetype: %v", err) } if delta > 0 { t.Fatalf("delta = %d, want <= 0 (shrink left to reconcile)", delta) } - if err := s.AddVCPU(testAccount, delta); err != nil { + if err := s.AddVCPU(t.Context(), testAccount, delta); err != nil { t.Fatalf("AddVCPU(%d): %v", delta, err) } assertCounter(t, s, testAccount, 4) // unchanged: reconcile lowers, not the retype @@ -112,7 +112,7 @@ func TestRetypeDownDoesNotCharge(t *testing.T) { func TestEnforceRetypeResolverError(t *testing.T) { s := newVCPUService(t, Limits{Enabled: true, VCPUs: 8}) sentinel := errors.New("describe boom") - delta, err := s.EnforceRetype(staticResolver("t3.micro", false, sentinel), testAccount, "i-a", "m5.xlarge") + delta, err := s.EnforceRetype(t.Context(), staticResolver("t3.micro", false, sentinel), testAccount, "i-a", "m5.xlarge") if !errors.Is(err, sentinel) || delta != 0 { t.Fatalf("EnforceRetype = (%d, %v), want (0, %v)", delta, err, sentinel) } @@ -132,9 +132,11 @@ func TestEnforceRetypeExemptShortCircuits(t *testing.T) { name string fn func() (int, error) }{ - {"disabled", func() (int, error) { return disabled.EnforceRetype(failResolver, testAccount, "i-a", "m5.xlarge") }}, + {"disabled", func() (int, error) { + return disabled.EnforceRetype(t.Context(), failResolver, testAccount, "i-a", "m5.xlarge") + }}, {"system account", func() (int, error) { - return enabled.EnforceRetype(failResolver, utils.GlobalAccountID, "i-a", "m5.xlarge") + return enabled.EnforceRetype(t.Context(), failResolver, utils.GlobalAccountID, "i-a", "m5.xlarge") }}, } for _, tc := range cases { diff --git a/spinifex/handlers/quota/vcpu_test.go b/spinifex/handlers/quota/vcpu_test.go index a72912d5b..4c28edbc6 100644 --- a/spinifex/handlers/quota/vcpu_test.go +++ b/spinifex/handlers/quota/vcpu_test.go @@ -7,7 +7,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/awserrors" "github.com/mulgadc/spinifex/spinifex/testutil" "github.com/mulgadc/spinifex/spinifex/utils" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) const testAccount = "123456789012" @@ -16,8 +16,9 @@ const testAccount = "123456789012" // the way the gateway does, and returns a quota Service bound to it. func newVCPUService(t *testing.T, limits Limits) *Service { t.Helper() - _, _, js := testutil.StartTestJetStream(t) - kv, err := js.CreateKeyValue(&nats.KeyValueConfig{ + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + kv, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{ Bucket: KVBucketAccountUsage, History: 1, }) @@ -31,21 +32,21 @@ func TestCheckVCPUBoundaries(t *testing.T) { s := newVCPUService(t, Limits{Enabled: true, VCPUs: 8}) // Empty counter: at-limit passes, over-limit rejects. - if err := s.CheckVCPU(testAccount, 8); err != nil { + if err := s.CheckVCPU(t.Context(), testAccount, 8); err != nil { t.Fatalf("CheckVCPU(8) on empty = %v, want nil", err) } - if err := s.CheckVCPU(testAccount, 9); err == nil || err.Error() != awserrors.ErrorResourceLimitExceeded { + if err := s.CheckVCPU(t.Context(), testAccount, 9); err == nil || err.Error() != awserrors.ErrorResourceLimitExceeded { t.Fatalf("CheckVCPU(9) on empty = %v, want %q", err, awserrors.ErrorResourceLimitExceeded) } // Reserve 6, then the remaining headroom is 2. - if err := s.AddVCPU(testAccount, 6); err != nil { + if err := s.AddVCPU(t.Context(), testAccount, 6); err != nil { t.Fatalf("AddVCPU(6) = %v", err) } - if err := s.CheckVCPU(testAccount, 2); err != nil { + if err := s.CheckVCPU(t.Context(), testAccount, 2); err != nil { t.Fatalf("CheckVCPU(2) at 6/8 = %v, want nil", err) } - if err := s.CheckVCPU(testAccount, 3); err == nil || err.Error() != awserrors.ErrorResourceLimitExceeded { + if err := s.CheckVCPU(t.Context(), testAccount, 3); err == nil || err.Error() != awserrors.ErrorResourceLimitExceeded { t.Fatalf("CheckVCPU(3) at 6/8 = %v, want %q", err, awserrors.ErrorResourceLimitExceeded) } } @@ -59,10 +60,10 @@ func TestVCPUExemptShortCircuits(t *testing.T) { name string fn func() error }{ - {"check disabled", func() error { return disabled.CheckVCPU(testAccount, 1000) }}, - {"add disabled", func() error { return disabled.AddVCPU(testAccount, 1000) }}, - {"check system account", func() error { return enabled.CheckVCPU(utils.GlobalAccountID, 1000) }}, - {"add system account", func() error { return enabled.AddVCPU(utils.GlobalAccountID, 1000) }}, + {"check disabled", func() error { return disabled.CheckVCPU(t.Context(), testAccount, 1000) }}, + {"add disabled", func() error { return disabled.AddVCPU(t.Context(), testAccount, 1000) }}, + {"check system account", func() error { return enabled.CheckVCPU(t.Context(), utils.GlobalAccountID, 1000) }}, + {"add system account", func() error { return enabled.AddVCPU(t.Context(), utils.GlobalAccountID, 1000) }}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -77,12 +78,12 @@ func TestAddVCPUAccumulatesAndIgnoresShrinks(t *testing.T) { s := newVCPUService(t, Limits{Enabled: true, VCPUs: 100}) for _, delta := range []int{4, 2, 0, -3} { - if err := s.AddVCPU(testAccount, delta); err != nil { + if err := s.AddVCPU(t.Context(), testAccount, delta); err != nil { t.Fatalf("AddVCPU(%d) = %v", delta, err) } } // 4 + 2 land; the 0 and -3 are no-ops left to reconcile. - got, _, err := s.readVCPU(testAccount) + got, _, err := s.readVCPU(t.Context(), testAccount) if err != nil { t.Fatalf("readVCPU: %v", err) } @@ -101,7 +102,7 @@ func TestAddVCPUConcurrentNoLostUpdates(t *testing.T) { for range goroutines { wg.Go(func() { for range perGoroutine { - if err := s.AddVCPU(testAccount, 1); err != nil { + if err := s.AddVCPU(t.Context(), testAccount, 1); err != nil { t.Errorf("AddVCPU: %v", err) } } @@ -109,7 +110,7 @@ func TestAddVCPUConcurrentNoLostUpdates(t *testing.T) { } wg.Wait() - got, _, err := s.readVCPU(testAccount) + got, _, err := s.readVCPU(t.Context(), testAccount) if err != nil { t.Fatalf("readVCPU: %v", err) } diff --git a/spinifex/handlers/sts/assume_role.go b/spinifex/handlers/sts/assume_role.go index a9a1deac9..45b7e93c7 100644 --- a/spinifex/handlers/sts/assume_role.go +++ b/spinifex/handlers/sts/assume_role.go @@ -1,6 +1,7 @@ package handlers_sts import ( + "context" "crypto/hmac" "crypto/rand" "crypto/sha256" @@ -18,11 +19,12 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/iam" "github.com/aws/aws-sdk-go/service/sts" + "github.com/nats-io/nats.go/jetstream" + "github.com/mulgadc/predastore/auth" "github.com/mulgadc/spinifex/spinifex/awserrors" handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" "github.com/mulgadc/spinifex/spinifex/utils" - "github.com/nats-io/nats.go" ) const ( @@ -62,6 +64,7 @@ var roleSessionNameRegex = regexp.MustCompile(`^[A-Za-z0-9_+=,.@-]{2,64}$`) // AssumeRole mints temporary credentials after evaluating the target role's // trust policy against the caller. func (s *STSServiceImpl) AssumeRole(callerAccountID, callerARN, callerIdentity string, input *sts.AssumeRoleInput) (*sts.AssumeRoleOutput, error) { + ctx := context.Background() if input == nil { return nil, errors.New(awserrors.ErrorMissingParameter) } @@ -89,7 +92,7 @@ func (s *STSServiceImpl) AssumeRole(callerAccountID, callerARN, callerIdentity s duration = *input.DurationSeconds } - out, err := s.assumeRoleForCaller(callerARN, "", *input.RoleArn, sessionName, aws.StringValue(input.SourceIdentity), duration) + out, err := s.assumeRoleForCaller(ctx, callerARN, "", *input.RoleArn, sessionName, aws.StringValue(input.SourceIdentity), duration) if err != nil { return nil, err } @@ -111,6 +114,7 @@ func (s *STSServiceImpl) AssumeRole(callerAccountID, callerARN, callerIdentity s // AssumeRoleForInstance is the in-process IMDS entry point, not reachable over HTTPS. // Caller is synthesised as ec2.amazonaws.com; the session name is the instanceID. func (s *STSServiceImpl) AssumeRoleForInstance(accountID, roleARN, instanceID string, durationSeconds int64) (*sts.AssumeRoleOutput, error) { + ctx := context.Background() if accountID == "" || roleARN == "" || instanceID == "" { return nil, errors.New(awserrors.ErrorMissingParameter) } @@ -118,7 +122,7 @@ func (s *STSServiceImpl) AssumeRoleForInstance(accountID, roleARN, instanceID st return nil, errors.New(awserrors.ErrorValidationError) } - out, err := s.assumeRoleForCaller("", ec2ServicePrincipal, roleARN, instanceID, "", durationSeconds) + out, err := s.assumeRoleForCaller(ctx, "", ec2ServicePrincipal, roleARN, instanceID, "", durationSeconds) if err != nil { return nil, err } @@ -137,7 +141,7 @@ func (s *STSServiceImpl) AssumeRoleForInstance(accountID, roleARN, instanceID st // assumeRoleForCaller is the shared core of AssumeRole and AssumeRoleForInstance: // resolves the role, clamps the duration, evaluates the trust policy, and mints credentials. // principalSource is "" for HTTPS, ec2.amazonaws.com for IMDS. -func (s *STSServiceImpl) assumeRoleForCaller(callerARN, principalSource, roleARN, sessionName, sourceIdentity string, requestedDuration int64) (*sts.AssumeRoleOutput, error) { +func (s *STSServiceImpl) assumeRoleForCaller(ctx context.Context, callerARN, principalSource, roleARN, sessionName, sourceIdentity string, requestedDuration int64) (*sts.AssumeRoleOutput, error) { roleAccountID, roleName, err := auth.ParseRoleARN(roleARN) if err != nil { return nil, errors.New(awserrors.ErrorValidationError) @@ -171,7 +175,7 @@ func (s *STSServiceImpl) assumeRoleForCaller(callerARN, principalSource, roleARN } env := assumedRoleEnvelope(role, roleAccountID, sessionName, sourceIdentity) - cred, plainSecret, plainToken, err := s.mintSession(env, duration) + cred, plainSecret, plainToken, err := s.mintSession(ctx, env, duration) if err != nil { return nil, err } @@ -419,7 +423,7 @@ func assumedRoleEnvelope(role *iam.Role, roleAccountID, sessionName, sourceIdent // mintSession generates a fresh ASIA AKID, encrypts the secret, HMACs the token, // and persists the credential. Retries up to mintMaxAttempts on AKID collision. -func (s *STSServiceImpl) mintSession(env sessionEnvelope, duration int64) (*SessionCredential, string, string, error) { +func (s *STSServiceImpl) mintSession(ctx context.Context, env sessionEnvelope, duration int64) (*SessionCredential, string, string, error) { plainSecret, err := generateRandomBase64(sessionSecretBytes) if err != nil { return nil, "", "", err @@ -458,8 +462,8 @@ func (s *STSServiceImpl) mintSession(env sessionEnvelope, duration int64) (*Sess ExpiresAt: expiresAt, CreatedAt: now, } - if err := putSessionCredential(s.sessionsBucket, cred); err != nil { - if errors.Is(err, nats.ErrKeyExists) { + if err := putSessionCredential(ctx, s.sessionsBucket, cred); err != nil { + if errors.Is(err, jetstream.ErrKeyExists) { continue } return nil, "", "", fmt.Errorf("persist session credential: %w", err) diff --git a/spinifex/handlers/sts/assume_role_with_web_identity.go b/spinifex/handlers/sts/assume_role_with_web_identity.go index b4ccd5995..fa5749799 100644 --- a/spinifex/handlers/sts/assume_role_with_web_identity.go +++ b/spinifex/handlers/sts/assume_role_with_web_identity.go @@ -1,6 +1,7 @@ package handlers_sts import ( + "context" "crypto/ecdsa" "crypto/elliptic" "encoding/base64" @@ -14,10 +15,11 @@ import ( "github.com/aws/aws-sdk-go/service/iam" "github.com/aws/aws-sdk-go/service/sts" "github.com/golang-jwt/jwt/v5" + "github.com/nats-io/nats.go/jetstream" + "github.com/mulgadc/predastore/auth" "github.com/mulgadc/spinifex/spinifex/awserrors" handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" - "github.com/nats-io/nats.go" ) // webIdentityClaims is the RegisteredClaims subset the IRSA flow consumes (iss/sub/aud/exp). @@ -30,6 +32,7 @@ type webIdentityClaims struct { // verifies the JWT (ES256, registered OIDC provider, aud must contain sts.amazonaws.com), // evaluates the trust policy, and mints a session. Anonymous — identity is the JWT, not SigV4. func (s *STSServiceImpl) AssumeRoleWithWebIdentity(input *sts.AssumeRoleWithWebIdentityInput) (*sts.AssumeRoleWithWebIdentityOutput, error) { + ctx := context.Background() if input == nil { return nil, errors.New(awserrors.ErrorMissingParameter) } @@ -58,7 +61,7 @@ func (s *STSServiceImpl) AssumeRoleWithWebIdentity(input *sts.AssumeRoleWithWebI // Audience validated explicitly below; jwt/v5 default check is // string-equality, which misses set-membership semantics on multi-aud tokens. ) - _, err = parser.ParseWithClaims(rawToken, claims, s.webIdentityKeyFunc(roleAccountID)) + _, err = parser.ParseWithClaims(rawToken, claims, s.webIdentityKeyFunc(ctx, roleAccountID)) if err != nil { slog.Warn("AssumeRoleWithWebIdentity: token verify failed", "role_arn", roleARN, "err", err) @@ -99,18 +102,18 @@ func (s *STSServiceImpl) AssumeRoleWithWebIdentity(input *sts.AssumeRoleWithWebI issuerHostPath := strings.TrimPrefix(claims.Issuer, "https://") federatedARN := handlers_iam.OIDCProviderARN(roleAccountID, issuerHostPath) - ctx := webIdentityContext{ + identity := webIdentityContext{ federatedPrincipalARN: federatedARN, issuer: claims.Issuer, subject: claims.Subject, audience: []string(claims.Audience), } - if err := evalTrustPolicyForWebIdentity(aws.StringValue(role.AssumeRolePolicyDocument), ctx); err != nil { + if err := evalTrustPolicyForWebIdentity(aws.StringValue(role.AssumeRolePolicyDocument), identity); err != nil { return nil, err } env := assumedRoleEnvelope(role, roleAccountID, sessionName, claims.Subject) - cred, plainSecret, plainToken, err := s.mintSession(env, duration) + cred, plainSecret, plainToken, err := s.mintSession(ctx, env, duration) if err != nil { return nil, err } @@ -150,7 +153,7 @@ func (s *STSServiceImpl) AssumeRoleWithWebIdentity(input *sts.AssumeRoleWithWebI // webIdentityKeyFunc returns a jwt.Keyfunc that resolves iss → OIDC provider registry // (role account) → JWKS (issuer account) → JWK by kid → *ecdsa.PublicKey. // All failures map to ErrorInvalidIdentityToken at the call site. -func (s *STSServiceImpl) webIdentityKeyFunc(roleAccountID string) jwt.Keyfunc { +func (s *STSServiceImpl) webIdentityKeyFunc(ctx context.Context, roleAccountID string) jwt.Keyfunc { return func(t *jwt.Token) (any, error) { kid, _ := t.Header["kid"].(string) if kid == "" { @@ -168,10 +171,10 @@ func (s *STSServiceImpl) webIdentityKeyFunc(roleAccountID string) jwt.Keyfunc { if err != nil { return nil, fmt.Errorf("parse issuer: %w", err) } - if err := s.verifyOIDCProviderRegistered(roleAccountID, issuer); err != nil { + if err := s.verifyOIDCProviderRegistered(ctx, roleAccountID, issuer); err != nil { return nil, err } - jwks, err := FetchClusterJWKS(s.js, issuerAccountID, clusterName) + jwks, err := FetchClusterJWKS(ctx, s.js, issuerAccountID, clusterName) if err != nil { return nil, fmt.Errorf("fetch JWKS: %w", err) } @@ -189,18 +192,18 @@ func (s *STSServiceImpl) webIdentityKeyFunc(roleAccountID string) jwt.Keyfunc { // verifyOIDCProviderRegistered checks the issuer in the role-account's OIDC-provider // registry. Bucket-not-found and key-not-found return the same error to avoid leaking // account activation status to anonymous callers. -func (s *STSServiceImpl) verifyOIDCProviderRegistered(roleAccountID, issuer string) error { +func (s *STSServiceImpl) verifyOIDCProviderRegistered(ctx context.Context, roleAccountID, issuer string) error { bucketName := handlers_iam.IAMAccountBucketName(roleAccountID) - kv, err := s.js.KeyValue(bucketName) + kv, err := s.js.KeyValue(ctx, bucketName) if err != nil { - if errors.Is(err, nats.ErrBucketNotFound) { + if errors.Is(err, jetstream.ErrBucketNotFound) { return errors.New("OIDC provider not registered") } return fmt.Errorf("open IAM account bucket %s: %w", bucketName, err) } - _, err = kv.Get(handlers_iam.OIDCProviderKey(issuer)) + _, err = kv.Get(ctx, handlers_iam.OIDCProviderKey(issuer)) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return errors.New("OIDC provider not registered") } return fmt.Errorf("read OIDC provider: %w", err) diff --git a/spinifex/handlers/sts/assume_role_with_web_identity_test.go b/spinifex/handlers/sts/assume_role_with_web_identity_test.go index 6e8055ec8..d60f47f09 100644 --- a/spinifex/handlers/sts/assume_role_with_web_identity_test.go +++ b/spinifex/handlers/sts/assume_role_with_web_identity_test.go @@ -18,7 +18,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/awserrors" handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks" handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" - "github.com/mulgadc/spinifex/spinifex/utils" + "github.com/mulgadc/spinifex/spinifex/kvutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -74,20 +74,20 @@ func newWebIdentityFixture(t *testing.T, svc *STSServiceImpl, accountID string) federatedARN := handlers_iam.OIDCProviderARN(accountID, issuerHostPath) // Publish JWKS to the per-cluster EKS bucket. - kv, err := handlers_eks.GetOrCreateAccountBucket(svc.js, accountID, 1) + kv, err := handlers_eks.GetOrCreateAccountBucket(t.Context(), svc.js, accountID, 1) require.NoError(t, err) raw, err := json.Marshal(jwks) require.NoError(t, err) - _, err = kv.Put(handlers_eks.OIDCJWKSKey(testWebClusterName), raw) + _, err = kv.Put(t.Context(), handlers_eks.OIDCJWKSKey(testWebClusterName), raw) require.NoError(t, err) // Register the OIDC provider on the role-account's IAM bucket. Bucket is // created on-the-fly here — the IAM CreateOpenIDConnectProvider API lands // in Sprint 6e. - iamKV, err := utils.GetOrCreateKVBucket(svc.js, + iamKV, err := kvutil.GetOrCreateBucket(t.Context(), svc.js, handlers_iam.IAMAccountBucketName(accountID), handlers_iam.KVBucketIAMAccountVersion) require.NoError(t, err) - _, err = iamKV.Put(handlers_iam.OIDCProviderKey(issuer), []byte(`{"registered":true}`)) + _, err = iamKV.Put(t.Context(), handlers_iam.OIDCProviderKey(issuer), []byte(`{"registered":true}`)) require.NoError(t, err) return &webIdentityFixture{ @@ -364,14 +364,14 @@ func TestAssumeRoleWithWebIdentity_ProviderNotRegistered(t *testing.T) { federatedARN := handlers_iam.OIDCProviderARN(testCallerAccountID, issuerHostPath) // Publish JWKS — but skip the IAM provider registration. - kv, err := handlers_eks.GetOrCreateAccountBucket(svc.js, testCallerAccountID, 1) + kv, err := handlers_eks.GetOrCreateAccountBucket(t.Context(), svc.js, testCallerAccountID, 1) require.NoError(t, err) jwk := signingJWK(t, &priv.PublicKey) jwk.Alg, jwk.Use, jwk.Kid = "ES256", "sig", kid jwks := &JWKS{Keys: []JWK{*jwk}} raw, err := json.Marshal(jwks) require.NoError(t, err) - _, err = kv.Put(handlers_eks.OIDCJWKSKey("unregistered-cluster"), raw) + _, err = kv.Put(t.Context(), handlers_eks.OIDCJWKSKey("unregistered-cluster"), raw) require.NoError(t, err) role := createRoleForFixture(t, svc, testCallerAccountID, "irsa-unreg", diff --git a/spinifex/handlers/sts/get_session_token.go b/spinifex/handlers/sts/get_session_token.go index a1fbc2422..fb37d6cdb 100644 --- a/spinifex/handlers/sts/get_session_token.go +++ b/spinifex/handlers/sts/get_session_token.go @@ -1,6 +1,7 @@ package handlers_sts import ( + "context" "errors" "log/slog" "strings" @@ -21,6 +22,7 @@ const ( // (AKIA prefix, principalType "user") may call this; assumed-role and session callers // are rejected. Caller identity is resolved by the gateway and passed as plain strings. func (s *STSServiceImpl) GetSessionToken(callerAccountID, callerUserName, callerPrincipalType, callerAccessKeyID string, input *sts.GetSessionTokenInput) (*sts.GetSessionTokenOutput, error) { + ctx := context.Background() if input == nil { // An absent body is the common case from `aws sts get-session-token` with no args. input = &sts.GetSessionTokenInput{} @@ -56,7 +58,7 @@ func (s *STSServiceImpl) GetSessionToken(callerAccountID, callerUserName, caller duration = clampGetSessionTokenDuration(*input.DurationSeconds) } - cred, plainSecret, plainToken, err := s.mintSession(userEnvelope(callerAccountID, callerUserName), duration) + cred, plainSecret, plainToken, err := s.mintSession(ctx, userEnvelope(callerAccountID, callerUserName), duration) if err != nil { return nil, err } diff --git a/spinifex/handlers/sts/invariants_test.go b/spinifex/handlers/sts/invariants_test.go index 65016e657..dd1edfec3 100644 --- a/spinifex/handlers/sts/invariants_test.go +++ b/spinifex/handlers/sts/invariants_test.go @@ -34,7 +34,7 @@ func TestInvariant_SessionCredentialsBucket_RejectsNonASIAPrefix(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { cred := newTestSessionCredential(tc.akid) - err := putSessionCredential(bucket, cred) + err := putSessionCredential(t.Context(), bucket, cred) require.Error(t, err, "putSessionCredential accepted invalid prefix %q", tc.akid) assert.Contains(t, err.Error(), SessionAccessKeyIDPrefix) }) @@ -46,7 +46,7 @@ func TestInvariant_SessionCredentialsBucket_AcceptsASIAPrefix(t *testing.T) { akid := SessionAccessKeyIDPrefix + "0123456789ABCDEF" cred := newTestSessionCredential(akid) - require.NoError(t, putSessionCredential(bucket, cred)) + require.NoError(t, putSessionCredential(t.Context(), bucket, cred)) } // User sessions (minted by GetSessionToken) write to the same bucket through @@ -74,12 +74,12 @@ func TestInvariant_SessionCredentialsBucket_UserSessionPrefixEnforced(t *testing } // A non-ASIA user AKID is rejected at the writer, same as any other prefix. - err := putSessionCredential(bucket, userCred("AKIA0123456789ABCDEF")) + err := putSessionCredential(t.Context(), bucket, userCred("AKIA0123456789ABCDEF")) require.Error(t, err, "putSessionCredential accepted a non-ASIA user session") assert.Contains(t, err.Error(), SessionAccessKeyIDPrefix) // An ASIA user AKID is accepted, just like an assumed-role session. - require.NoError(t, putSessionCredential(bucket, userCred(SessionAccessKeyIDPrefix+"USERSESSION00001"))) + require.NoError(t, putSessionCredential(t.Context(), bucket, userCred(SessionAccessKeyIDPrefix+"USERSESSION00001"))) } // Cross-cluster anti-replay invariant. A presigned sts:GetCallerIdentity URL diff --git a/spinifex/handlers/sts/oidc_jwks.go b/spinifex/handlers/sts/oidc_jwks.go index 8eb837f78..427497d7c 100644 --- a/spinifex/handlers/sts/oidc_jwks.go +++ b/spinifex/handlers/sts/oidc_jwks.go @@ -1,6 +1,7 @@ package handlers_sts import ( + "context" "encoding/json" "errors" "fmt" @@ -8,8 +9,9 @@ import ( "slices" "strings" + "github.com/nats-io/nats.go/jetstream" + handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks" - "github.com/nats-io/nats.go" ) // JWK is one entry of an RFC 7517 JSON Web Key Set. EKS publishes EC P-256 keys; @@ -88,7 +90,7 @@ func ParseEKSIssuerURL(issuer string) (accountID, clusterName string, err error) // FetchClusterJWKS reads the JWKS for an EKS cluster from the per-account EKS KV bucket. // Returns (nil, nil) when the cluster has no JWKS published; callers map that to InvalidIdentityToken. -func FetchClusterJWKS(js nats.JetStreamContext, accountID, clusterName string) (*JWKS, error) { +func FetchClusterJWKS(ctx context.Context, js jetstream.JetStream, accountID, clusterName string) (*JWKS, error) { if js == nil { return nil, errors.New("nil JetStream context") } @@ -97,17 +99,17 @@ func FetchClusterJWKS(js nats.JetStreamContext, accountID, clusterName string) ( } bucketName := handlers_eks.AccountBucketName(accountID) - kv, err := js.KeyValue(bucketName) + kv, err := js.KeyValue(ctx, bucketName) if err != nil { - if errors.Is(err, nats.ErrBucketNotFound) { + if errors.Is(err, jetstream.ErrBucketNotFound) { return nil, nil } return nil, fmt.Errorf("open EKS account bucket %s: %w", bucketName, err) } - entry, err := kv.Get(handlers_eks.OIDCJWKSKey(clusterName)) + entry, err := kv.Get(ctx, handlers_eks.OIDCJWKSKey(clusterName)) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil, nil } return nil, fmt.Errorf("read JWKS for cluster %s: %w", clusterName, err) diff --git a/spinifex/handlers/sts/oidc_jwks_test.go b/spinifex/handlers/sts/oidc_jwks_test.go index 752ee4d0a..6c5b46da1 100644 --- a/spinifex/handlers/sts/oidc_jwks_test.go +++ b/spinifex/handlers/sts/oidc_jwks_test.go @@ -6,7 +6,6 @@ import ( handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks" "github.com/mulgadc/spinifex/spinifex/testutil" - "github.com/nats-io/nats.go" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -69,17 +68,19 @@ func TestJWKS_FindByKID(t *testing.T) { } func TestFetchClusterJWKS_NotFoundReturnsNil(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) - jwks, err := FetchClusterJWKS(js, "123456789012", "missing-cluster") + jwks, err := FetchClusterJWKS(t.Context(), js, "123456789012", "missing-cluster") require.NoError(t, err) assert.Nil(t, jwks) } func TestFetchClusterJWKS_RoundTrip(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) - kv, err := handlers_eks.GetOrCreateAccountBucket(js, "123456789012", 1) + kv, err := handlers_eks.GetOrCreateAccountBucket(t.Context(), js, "123456789012", 1) require.NoError(t, err) want := &JWKS{Keys: []JWK{ @@ -87,10 +88,10 @@ func TestFetchClusterJWKS_RoundTrip(t *testing.T) { }} raw, err := json.Marshal(want) require.NoError(t, err) - _, err = kv.Put(handlers_eks.OIDCJWKSKey("my-cluster"), raw) + _, err = kv.Put(t.Context(), handlers_eks.OIDCJWKSKey("my-cluster"), raw) require.NoError(t, err) - got, err := FetchClusterJWKS(js, "123456789012", "my-cluster") + got, err := FetchClusterJWKS(t.Context(), js, "123456789012", "my-cluster") require.NoError(t, err) require.NotNil(t, got) require.Len(t, got.Keys, 1) @@ -100,29 +101,31 @@ func TestFetchClusterJWKS_RoundTrip(t *testing.T) { } func TestFetchClusterJWKS_EmptyKeysIsError(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) - kv, err := handlers_eks.GetOrCreateAccountBucket(js, "123456789012", 1) + kv, err := handlers_eks.GetOrCreateAccountBucket(t.Context(), js, "123456789012", 1) require.NoError(t, err) - _, err = kv.Put(handlers_eks.OIDCJWKSKey("empty-cluster"), []byte(`{"keys":[]}`)) + _, err = kv.Put(t.Context(), handlers_eks.OIDCJWKSKey("empty-cluster"), []byte(`{"keys":[]}`)) require.NoError(t, err) - jwks, err := FetchClusterJWKS(js, "123456789012", "empty-cluster") + jwks, err := FetchClusterJWKS(t.Context(), js, "123456789012", "empty-cluster") require.Error(t, err) assert.Nil(t, jwks) } func TestFetchClusterJWKS_NilJetStream(t *testing.T) { - _, err := FetchClusterJWKS(nil, "123456789012", "c1") + _, err := FetchClusterJWKS(t.Context(), nil, "123456789012", "c1") require.Error(t, err) } func TestFetchClusterJWKS_EmptyArgs(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) - _, err := FetchClusterJWKS(js, "", "c1") + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + _, err := FetchClusterJWKS(t.Context(), js, "", "c1") require.Error(t, err) - _, err = FetchClusterJWKS(js, "123456789012", "") + _, err = FetchClusterJWKS(t.Context(), js, "123456789012", "") require.Error(t, err) } @@ -137,10 +140,3 @@ func TestJWK_RSAFieldsRoundTrip(t *testing.T) { require.NoError(t, json.Unmarshal(raw, &got)) assert.Equal(t, src, got) } - -// Ensure that a NATS error other than ErrBucketNotFound / ErrKeyNotFound is -// surfaced rather than masked. Without an injected JetStream stub there's no -// fault-injection path here, so the negative-cache cases above cover the two -// expected misses; this test documents the contract by exercising the nil -// pre-conditions only. -var _ = nats.ErrBucketNotFound diff --git a/spinifex/handlers/sts/presigned_url_verify.go b/spinifex/handlers/sts/presigned_url_verify.go index e5c211bb5..c6bfddf91 100644 --- a/spinifex/handlers/sts/presigned_url_verify.go +++ b/spinifex/handlers/sts/presigned_url_verify.go @@ -11,6 +11,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/iam" + "github.com/mulgadc/predastore/pkg/sigv4" "github.com/mulgadc/spinifex/spinifex/awserrors" handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" diff --git a/spinifex/handlers/sts/service_impl.go b/spinifex/handlers/sts/service_impl.go index 8386a3d4d..dcaede670 100644 --- a/spinifex/handlers/sts/service_impl.go +++ b/spinifex/handlers/sts/service_impl.go @@ -1,12 +1,14 @@ package handlers_sts import ( + "context" "errors" "fmt" "log/slog" handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) const masterKeySize = 32 // AES-256, must match handlers_iam. @@ -14,10 +16,19 @@ const masterKeySize = 32 // AES-256, must match handlers_iam. // STSServiceImpl implements STS operations backed by NATS JetStream KV. // Roles and IAM crypto are resolved through the in-process IAMService; the master key // is shared with IAMServiceImpl and not rotated by STS. +// +// STSService carries no context: the gateway calls it in-process off a SigV4 +// request and IMDS calls it directly, and neither boundary passes one through. +// Each exported method therefore binds context.Background() for its KV work, +// which leaves the jetstream package's own 5s API timeout in force — the same +// wait the legacy KV API applied. Every helper below it takes the context as its +// leading parameter, so the day the contract gains one only those binding lines +// change. RunJanitor is the exception: it already owns a real context and +// threads it into the sweep. type STSServiceImpl struct { natsConn *nats.Conn - js nats.JetStreamContext - sessionsBucket nats.KeyValue + js jetstream.JetStream + sessionsBucket jetstream.KeyValue iamSvc handlers_iam.IAMService masterKey []byte } @@ -26,8 +37,9 @@ var _ STSService = (*STSServiceImpl)(nil) // NewSTSServiceImpl constructs an STSServiceImpl. masterKey must be the 32-byte // key shared with IAMServiceImpl. clusterSize sets the JetStream replication factor; -// pass 1 for single-node or test setups. -func NewSTSServiceImpl(natsConn *nats.Conn, iamSvc handlers_iam.IAMService, masterKey []byte, clusterSize int) (*STSServiceImpl, error) { +// pass 1 for single-node or test setups. The context bounds bucket creation and +// the schema migration only. +func NewSTSServiceImpl(ctx context.Context, natsConn *nats.Conn, iamSvc handlers_iam.IAMService, masterKey []byte, clusterSize int) (*STSServiceImpl, error) { if natsConn == nil { return nil, errors.New("nil NATS connection") } @@ -40,12 +52,12 @@ func NewSTSServiceImpl(natsConn *nats.Conn, iamSvc handlers_iam.IAMService, mast replicas := max(clusterSize, 1) - js, err := natsConn.JetStream() + js, err := jetstream.New(natsConn) if err != nil { return nil, fmt.Errorf("get JetStream context: %w", err) } - sessionsBucket, err := initSessionCredentialsBucket(js, replicas) + sessionsBucket, err := initSessionCredentialsBucket(ctx, js, replicas) if err != nil { return nil, fmt.Errorf("init session credentials bucket: %w", err) } diff --git a/spinifex/handlers/sts/service_impl_test.go b/spinifex/handlers/sts/service_impl_test.go index 3b79159d5..a15e84b0b 100644 --- a/spinifex/handlers/sts/service_impl_test.go +++ b/spinifex/handlers/sts/service_impl_test.go @@ -24,17 +24,17 @@ func newTestSetup(t *testing.T) (*STSServiceImpl, *nats.Conn) { masterKey, err := handlers_iam.GenerateMasterKey() require.NoError(t, err) - iamSvc, err := handlers_iam.NewIAMServiceImpl(nc, masterKey, 1) + iamSvc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey, 1) require.NoError(t, err) - stsSvc, err := NewSTSServiceImpl(nc, iamSvc, masterKey, 1) + stsSvc, err := NewSTSServiceImpl(t.Context(), nc, iamSvc, masterKey, 1) require.NoError(t, err) return stsSvc, nc } func TestNewSTSServiceImpl_RejectsNilNATSConn(t *testing.T) { masterKey := bytes.Repeat([]byte{0x01}, masterKeySize) - _, err := NewSTSServiceImpl(nil, nopIAMService{}, masterKey, 1) + _, err := NewSTSServiceImpl(t.Context(), nil, nopIAMService{}, masterKey, 1) require.Error(t, err) assert.Contains(t, err.Error(), "NATS") } @@ -42,7 +42,7 @@ func TestNewSTSServiceImpl_RejectsNilNATSConn(t *testing.T) { func TestNewSTSServiceImpl_RejectsNilIAMService(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) masterKey := bytes.Repeat([]byte{0x01}, masterKeySize) - _, err := NewSTSServiceImpl(nc, nil, masterKey, 1) + _, err := NewSTSServiceImpl(t.Context(), nc, nil, masterKey, 1) require.Error(t, err) assert.Contains(t, err.Error(), "IAM") } @@ -51,7 +51,7 @@ func TestNewSTSServiceImpl_RejectsWrongMasterKeySize(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) for _, size := range []int{0, 1, 16, 31, 33, 64} { t.Run("size", func(t *testing.T) { - _, err := NewSTSServiceImpl(nc, nopIAMService{}, bytes.Repeat([]byte{0xaa}, size), 1) + _, err := NewSTSServiceImpl(t.Context(), nc, nopIAMService{}, bytes.Repeat([]byte{0xaa}, size), 1) require.Error(t, err) assert.Contains(t, err.Error(), "master key") }) @@ -68,10 +68,10 @@ func TestNewSTSServiceImpl_NormalisesNegativeClusterSize(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) masterKey, err := handlers_iam.GenerateMasterKey() require.NoError(t, err) - iamSvc, err := handlers_iam.NewIAMServiceImpl(nc, masterKey, 1) + iamSvc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey, 1) require.NoError(t, err) - svc, err := NewSTSServiceImpl(nc, iamSvc, masterKey, 0) + svc, err := NewSTSServiceImpl(t.Context(), nc, iamSvc, masterKey, 0) require.NoError(t, err) require.NotNil(t, svc) } @@ -116,7 +116,7 @@ func TestLookupSessionCredential_HitRoundTrips(t *testing.T) { ExpiresAt: now.Add(time.Hour), CreatedAt: now, } - require.NoError(t, putSessionCredential(svc.sessionsBucket, cred)) + require.NoError(t, putSessionCredential(t.Context(), svc.sessionsBucket, cred)) got, err := svc.LookupSessionCredential(cred.AccessKeyID) require.NoError(t, err) @@ -134,7 +134,7 @@ func TestLookupSessionCredential_UnmarshalFailureSurfacesError(t *testing.T) { // the prefix is valid (so the lookup reaches the bucket) but the JSON // body is garbage. This guards against the "lookup returns nil silently // on parse failure" silent-failure mode. - _, err := svc.sessionsBucket.Put(akid, []byte("not json")) + _, err := svc.sessionsBucket.Put(t.Context(), akid, []byte("not json")) require.NoError(t, err) got, err := svc.LookupSessionCredential(akid) diff --git a/spinifex/handlers/sts/session_credentials.go b/spinifex/handlers/sts/session_credentials.go index f75d36697..be0445ebb 100644 --- a/spinifex/handlers/sts/session_credentials.go +++ b/spinifex/handlers/sts/session_credentials.go @@ -11,9 +11,10 @@ import ( "strings" "time" + "github.com/mulgadc/spinifex/spinifex/kvutil" "github.com/mulgadc/spinifex/spinifex/migrate" "github.com/mulgadc/spinifex/spinifex/utils" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) const ( @@ -57,25 +58,15 @@ type SessionCredential struct { // initSessionCredentialsBucket opens (or creates) the session-credentials KV bucket. // History is fixed at 1: credentials are write-once at mint and delete-once at expiry. -func initSessionCredentialsBucket(js nats.JetStreamContext, replicas int) (nats.KeyValue, error) { - if replicas < 1 { - replicas = 1 - } - - kv, err := js.CreateKeyValue(&nats.KeyValueConfig{ - Bucket: KVBucketSessionCredentials, - History: 1, - Replicas: replicas, - }) +func initSessionCredentialsBucket(ctx context.Context, js jetstream.JetStream, replicas int) (jetstream.KeyValue, error) { + // kvutil clamps replicas to a minimum of 1, so a zero clusterSize still creates. + kv, err := kvutil.GetOrCreateBucketWithReplicas(ctx, js, KVBucketSessionCredentials, 1, replicas) if err != nil { - kv, err = js.KeyValue(KVBucketSessionCredentials) - if err != nil { - return nil, fmt.Errorf("open session credentials bucket: %w", err) - } + return nil, fmt.Errorf("open session credentials bucket: %w", err) } if err := migrate.DefaultRegistry.RunKV( - KVBucketSessionCredentials, kv, KVBucketSessionCredentialsVersion, + ctx, KVBucketSessionCredentials, kv, KVBucketSessionCredentialsVersion, ); err != nil { return nil, fmt.Errorf("migrate %s: %w", KVBucketSessionCredentials, err) } @@ -84,8 +75,8 @@ func initSessionCredentialsBucket(js nats.JetStreamContext, replicas int) (nats. } // putSessionCredential persists a SessionCredential via CAS create, enforcing the ASIA-prefix -// invariant. Returns nats.ErrKeyExists on collision so callers can retry. -func putSessionCredential(bucket nats.KeyValue, cred *SessionCredential) error { +// invariant. Returns jetstream.ErrKeyExists on collision so callers can retry. +func putSessionCredential(ctx context.Context, bucket jetstream.KeyValue, cred *SessionCredential) error { if cred == nil { return errors.New("nil session credential") } @@ -97,7 +88,7 @@ func putSessionCredential(bucket nats.KeyValue, cred *SessionCredential) error { if err != nil { return fmt.Errorf("marshal session credential: %w", err) } - if _, err := bucket.Create(cred.AccessKeyID, data); err != nil { + if _, err := bucket.Create(ctx, cred.AccessKeyID, data); err != nil { return fmt.Errorf("store session credential: %w", err) } return nil @@ -125,12 +116,13 @@ func (s *STSServiceImpl) VerifySessionToken(cred *SessionCredential, wireToken s // LookupSessionCredential resolves an AKID to its stored SessionCredential. // Returns (nil, nil) when the AKID lacks the ASIA prefix or has no record. func (s *STSServiceImpl) LookupSessionCredential(accessKeyID string) (*SessionCredential, error) { + ctx := context.Background() if !strings.HasPrefix(accessKeyID, SessionAccessKeyIDPrefix) { return nil, nil } - entry, err := s.sessionsBucket.Get(accessKeyID) + entry, err := s.sessionsBucket.Get(ctx, accessKeyID) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil, nil } return nil, fmt.Errorf("get session credential: %w", err) @@ -158,19 +150,19 @@ func (s *STSServiceImpl) RunJanitor(ctx context.Context) { slog.Info("STS session credential janitor stopped") return case <-ticker.C: - s.sweepExpired(time.Now().UTC()) + s.sweepExpired(ctx, time.Now().UTC()) } } } // sweepExpired deletes all records whose ExpiresAt is past the grace period. // Per-key errors are logged and skipped; returns the delete count. -func (s *STSServiceImpl) sweepExpired(now time.Time) int { +func (s *STSServiceImpl) sweepExpired(ctx context.Context, now time.Time) int { cutoff := now.Add(-janitorGracePeriod) - keys, err := s.sessionsBucket.Keys() + keys, err := kvutil.Keys(ctx, s.sessionsBucket) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return 0 } slog.Warn("STS janitor: list session credential keys failed", "err", err) @@ -182,9 +174,9 @@ func (s *STSServiceImpl) sweepExpired(now time.Time) int { if key == utils.VersionKey { continue } - entry, err := s.sessionsBucket.Get(key) + entry, err := s.sessionsBucket.Get(ctx, key) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { continue } slog.Warn("STS janitor: get session credential failed", @@ -203,7 +195,7 @@ func (s *STSServiceImpl) sweepExpired(now time.Time) int { continue } - if err := s.sessionsBucket.Delete(key); err != nil { + if err := s.sessionsBucket.Delete(ctx, key); err != nil { slog.Warn("STS janitor: delete expired session credential failed", "key", key, "err", err) continue diff --git a/spinifex/handlers/sts/session_credentials_test.go b/spinifex/handlers/sts/session_credentials_test.go index 5fe33f272..7b6e004a5 100644 --- a/spinifex/handlers/sts/session_credentials_test.go +++ b/spinifex/handlers/sts/session_credentials_test.go @@ -7,17 +7,18 @@ import ( "testing" "time" + "github.com/mulgadc/spinifex/spinifex/kvutil" "github.com/mulgadc/spinifex/spinifex/testutil" "github.com/mulgadc/spinifex/spinifex/utils" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func setupBucket(t *testing.T) nats.KeyValue { +func setupBucket(t *testing.T) jetstream.KeyValue { t.Helper() - _, _, js := testutil.StartTestJetStream(t) - kv, err := initSessionCredentialsBucket(js, 1) + _, nc, _ := testutil.StartTestJetStream(t) + kv, err := initSessionCredentialsBucket(t.Context(), testutil.NewJetStream(t, nc), 1) require.NoError(t, err) return kv } @@ -40,24 +41,26 @@ func newTestSessionCredential(akid string) *SessionCredential { } func TestInitSessionCredentialsBucket_StampsVersion(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) - kv, err := initSessionCredentialsBucket(js, 1) + kv, err := initSessionCredentialsBucket(t.Context(), js, 1) require.NoError(t, err) - version, err := utils.ReadVersion(kv) + version, err := kvutil.ReadVersion(t.Context(), kv) require.NoError(t, err) assert.Equal(t, KVBucketSessionCredentialsVersion, version) } func TestInitSessionCredentialsBucket_Idempotent(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) - kv1, err := initSessionCredentialsBucket(js, 1) + kv1, err := initSessionCredentialsBucket(t.Context(), js, 1) require.NoError(t, err) // Reopen — must return the same bucket without error. - kv2, err := initSessionCredentialsBucket(js, 1) + kv2, err := initSessionCredentialsBucket(t.Context(), js, 1) require.NoError(t, err) assert.Equal(t, kv1.Bucket(), kv2.Bucket()) } @@ -66,9 +69,9 @@ func TestPutSessionCredential_RoundTrip(t *testing.T) { bucket := setupBucket(t) cred := newTestSessionCredential("ASIAEXAMPLEAAAAAAAAA") - require.NoError(t, putSessionCredential(bucket, cred)) + require.NoError(t, putSessionCredential(t.Context(), bucket, cred)) - entry, err := bucket.Get(cred.AccessKeyID) + entry, err := bucket.Get(t.Context(), cred.AccessKeyID) require.NoError(t, err) var got SessionCredential @@ -97,7 +100,7 @@ func TestPutSessionCredential_RejectsNonASIAPrefix(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { cred := newTestSessionCredential(tc.akid) - err := putSessionCredential(bucket, cred) + err := putSessionCredential(t.Context(), bucket, cred) require.Error(t, err) assert.Contains(t, err.Error(), "ASIA") }) @@ -105,8 +108,8 @@ func TestPutSessionCredential_RejectsNonASIAPrefix(t *testing.T) { // Bucket must contain no credentials — every put failed before reaching // bucket.Create, so no AKID should be present. - keys, err := bucket.Keys() - if !errors.Is(err, nats.ErrNoKeysFound) { + keys, err := bucket.Keys(t.Context()) + if !errors.Is(err, jetstream.ErrNoKeysFound) { require.NoError(t, err) for _, k := range keys { if k == utils.VersionKey { @@ -119,7 +122,7 @@ func TestPutSessionCredential_RejectsNonASIAPrefix(t *testing.T) { func TestPutSessionCredential_NilCredential(t *testing.T) { bucket := setupBucket(t) - err := putSessionCredential(bucket, nil) + err := putSessionCredential(t.Context(), bucket, nil) require.Error(t, err) } @@ -127,13 +130,13 @@ func TestPutSessionCredential_CollisionReturnsKeyExists(t *testing.T) { bucket := setupBucket(t) cred := newTestSessionCredential("ASIACOLLISIONAAAAAAA") - require.NoError(t, putSessionCredential(bucket, cred)) + require.NoError(t, putSessionCredential(t.Context(), bucket, cred)) - // Second create with the same AKID must surface nats.ErrKeyExists so the + // Second create with the same AKID must surface jetstream.ErrKeyExists so the // mint helper can retry with a freshly generated AKID. - err := putSessionCredential(bucket, cred) + err := putSessionCredential(t.Context(), bucket, cred) require.Error(t, err) - assert.ErrorIs(t, err, nats.ErrKeyExists) + assert.ErrorIs(t, err, jetstream.ErrKeyExists) } func TestVerifySessionToken_MatchAndMismatch(t *testing.T) { @@ -173,7 +176,7 @@ func putCredWithExpiry(t *testing.T, svc *STSServiceImpl, akid string, expiresAt t.Helper() cred := newTestSessionCredential(akid) cred.ExpiresAt = expiresAt - require.NoError(t, putSessionCredential(svc.sessionsBucket, cred)) + require.NoError(t, putSessionCredential(t.Context(), svc.sessionsBucket, cred)) } func TestSweepExpired_DeletesPastGraceOnly(t *testing.T) { @@ -185,24 +188,24 @@ func TestSweepExpired_DeletesPastGraceOnly(t *testing.T) { putCredWithExpiry(t, svc, "ASIAPASTGRACE0000003", now.Add(-janitorGracePeriod-time.Minute)) // past grace putCredWithExpiry(t, svc, "ASIAANCIENT000000004", now.Add(-24*time.Hour)) // long past grace - deleted := svc.sweepExpired(now) + deleted := svc.sweepExpired(t.Context(), now) assert.Equal(t, 2, deleted) // Live and within-grace must still exist; past-grace must be gone. - _, err := svc.sessionsBucket.Get("ASIALIVE000000000001") + _, err := svc.sessionsBucket.Get(t.Context(), "ASIALIVE000000000001") require.NoError(t, err) - _, err = svc.sessionsBucket.Get("ASIAJUSTEXPIRED00002") + _, err = svc.sessionsBucket.Get(t.Context(), "ASIAJUSTEXPIRED00002") require.NoError(t, err) - _, err = svc.sessionsBucket.Get("ASIAPASTGRACE0000003") - require.ErrorIs(t, err, nats.ErrKeyNotFound) - _, err = svc.sessionsBucket.Get("ASIAANCIENT000000004") - require.ErrorIs(t, err, nats.ErrKeyNotFound) + _, err = svc.sessionsBucket.Get(t.Context(), "ASIAPASTGRACE0000003") + require.ErrorIs(t, err, jetstream.ErrKeyNotFound) + _, err = svc.sessionsBucket.Get(t.Context(), "ASIAANCIENT000000004") + require.ErrorIs(t, err, jetstream.ErrKeyNotFound) } func TestSweepExpired_EmptyBucketIsNoop(t *testing.T) { svc, _ := newTestSetup(t) - assert.Equal(t, 0, svc.sweepExpired(time.Now().UTC())) + assert.Equal(t, 0, svc.sweepExpired(t.Context(), time.Now().UTC())) } func TestSweepExpired_SkipsCorruptRecord(t *testing.T) { @@ -212,18 +215,18 @@ func TestSweepExpired_SkipsCorruptRecord(t *testing.T) { svc, _ := newTestSetup(t) now := time.Now().UTC() - _, err := svc.sessionsBucket.Put("ASIACORRUPT000000001", []byte("not json")) + _, err := svc.sessionsBucket.Put(t.Context(), "ASIACORRUPT000000001", []byte("not json")) require.NoError(t, err) putCredWithExpiry(t, svc, "ASIAEXPIRED000000002", now.Add(-24*time.Hour)) - deleted := svc.sweepExpired(now) + deleted := svc.sweepExpired(t.Context(), now) assert.Equal(t, 1, deleted) - _, err = svc.sessionsBucket.Get("ASIAEXPIRED000000002") - require.ErrorIs(t, err, nats.ErrKeyNotFound) + _, err = svc.sessionsBucket.Get(t.Context(), "ASIAEXPIRED000000002") + require.ErrorIs(t, err, jetstream.ErrKeyNotFound) // Corrupt record is left in place — janitor is not authorised to delete // data it cannot interpret; an operator must inspect it. - _, err = svc.sessionsBucket.Get("ASIACORRUPT000000001") + _, err = svc.sessionsBucket.Get(t.Context(), "ASIACORRUPT000000001") require.NoError(t, err) } @@ -232,9 +235,9 @@ func TestSweepExpired_IgnoresVersionKey(t *testing.T) { // iterator must skip it (it's not a SessionCredential and unmarshal // would fail every sweep). svc, _ := newTestSetup(t) - assert.Equal(t, 0, svc.sweepExpired(time.Now().UTC())) + assert.Equal(t, 0, svc.sweepExpired(t.Context(), time.Now().UTC())) - _, err := svc.sessionsBucket.Get(utils.VersionKey) + _, err := svc.sessionsBucket.Get(t.Context(), utils.VersionKey) require.NoError(t, err, "version key must survive the sweep") } diff --git a/spinifex/handlers/sts/web_identity_trust_policy.go b/spinifex/handlers/sts/web_identity_trust_policy.go index 60602650d..a161b0749 100644 --- a/spinifex/handlers/sts/web_identity_trust_policy.go +++ b/spinifex/handlers/sts/web_identity_trust_policy.go @@ -37,7 +37,7 @@ type webIdentityContext struct { // evalTrustPolicyForWebIdentity evaluates an AssumeRoleWithWebIdentity trust policy. // Explicit-deny wins; a statement matches when Action, Principal.Federated, and all Conditions hold. // Returns nil on grant or AccessDenied/error on deny/corruption. -func evalTrustPolicyForWebIdentity(docJSON string, ctx webIdentityContext) error { +func evalTrustPolicyForWebIdentity(docJSON string, identity webIdentityContext) error { doc, err := handlers_iam.ValidateTrustPolicyDocument(docJSON) if err != nil { return fmt.Errorf("stored trust policy invalid: %w", err) @@ -47,7 +47,7 @@ func evalTrustPolicyForWebIdentity(docJSON string, ctx webIdentityContext) error if stmt.Effect != handlers_iam.PolicyEffectDeny { continue } - matched, err := matchWebIdentityStatement(stmt.Action, stmt.Principal, stmt.Condition, ctx) + matched, err := matchWebIdentityStatement(stmt.Action, stmt.Principal, stmt.Condition, identity) if err != nil { return err } @@ -60,7 +60,7 @@ func evalTrustPolicyForWebIdentity(docJSON string, ctx webIdentityContext) error if stmt.Effect != handlers_iam.PolicyEffectAllow { continue } - matched, err := matchWebIdentityStatement(stmt.Action, stmt.Principal, stmt.Condition, ctx) + matched, err := matchWebIdentityStatement(stmt.Action, stmt.Principal, stmt.Condition, identity) if err != nil { return err } @@ -72,18 +72,18 @@ func evalTrustPolicyForWebIdentity(docJSON string, ctx webIdentityContext) error return errors.New(awserrors.ErrorAccessDenied) } -func matchWebIdentityStatement(actions []string, principalRaw, conditionRaw json.RawMessage, ctx webIdentityContext) (bool, error) { +func matchWebIdentityStatement(actions []string, principalRaw, conditionRaw json.RawMessage, identity webIdentityContext) (bool, error) { if !matchWebIdentityAction(actions) { return false, nil } - principalMatch, err := matchFederatedPrincipal(principalRaw, ctx.federatedPrincipalARN) + principalMatch, err := matchFederatedPrincipal(principalRaw, identity.federatedPrincipalARN) if err != nil { return false, err } if !principalMatch { return false, nil } - if !conditionsHold(conditionRaw, ctx) { + if !conditionsHold(conditionRaw, identity) { return false, nil } return true, nil @@ -136,7 +136,7 @@ func matchFederatedPrincipal(raw json.RawMessage, expectedARN string) (bool, err // conditionsHold evaluates a Condition block. Only StringEquals is supported; // keys are {iss}:sub (equality) and {iss}:aud (set-membership). No Condition = grant. -func conditionsHold(raw json.RawMessage, ctx webIdentityContext) bool { +func conditionsHold(raw json.RawMessage, identity webIdentityContext) bool { if !isCondRawNonEmpty(raw) { return true } @@ -149,7 +149,7 @@ func conditionsHold(raw json.RawMessage, ctx webIdentityContext) bool { // Validator rejects other operators at write time; a non-StringEquals here means tampering. return false } - issPrefix := strings.TrimSuffix(ctx.issuer, "/") + issPrefix := strings.TrimSuffix(identity.issuer, "/") for key, expectedRaw := range kv { expected, ok := unmarshalStringOrArray(expectedRaw) if !ok { @@ -157,11 +157,11 @@ func conditionsHold(raw json.RawMessage, ctx webIdentityContext) bool { } switch key { case issPrefix + ":sub": - if !anyEquals(expected, []string{ctx.subject}) { + if !anyEquals(expected, []string{identity.subject}) { return false } case issPrefix + ":aud": - if !anyEquals(expected, ctx.audience) { + if !anyEquals(expected, identity.audience) { return false } default: diff --git a/spinifex/kvutil/kvutil.go b/spinifex/kvutil/kvutil.go new file mode 100644 index 000000000..fd1c83db6 --- /dev/null +++ b/spinifex/kvutil/kvutil.go @@ -0,0 +1,138 @@ +// Package kvutil holds the shared NATS KeyValue helpers: bucket get-or-create +// at the cluster's replica count, bucket enumeration, and schema-version +// stamping. It is built on github.com/nats-io/nats.go/jetstream, so every +// operation takes a context and honors the caller's deadline and cancellation +// rather than the legacy API's fixed internal wait. +// +// The helpers take a jetstream.KeyValueManager, which a jetstream.JetStream +// satisfies, so call sites pass their JetStream handle directly. +package kvutil + +import ( + "context" + "errors" + "fmt" + "strconv" + "time" + + "github.com/mulgadc/spinifex/spinifex/utils" + "github.com/nats-io/nats.go/jetstream" +) + +// GetOrCreateBucket creates or opens a KV bucket at the cluster's default +// replica count (see utils.SetDefaultKVReplicas), so buckets created lazily +// after boot are quorate on multi-node rather than stuck at R1. +func GetOrCreateBucket(ctx context.Context, js jetstream.KeyValueManager, bucket string, history int) (jetstream.KeyValue, error) { + return GetOrCreateBucketWithReplicas(ctx, js, bucket, history, utils.DefaultKVReplicas()) +} + +// GetOrCreateBucketWithReplicas creates or opens a KV bucket at the given +// replica count (clamped to a minimum of 1). The replica count applies to +// creation only: an existing bucket is opened with the config it already has, +// so one created before the cluster grew is upgraded on rebalance, not here. +func GetOrCreateBucketWithReplicas(ctx context.Context, js jetstream.KeyValueManager, bucket string, history, replicas int) (jetstream.KeyValue, error) { + kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{ + Bucket: bucket, + History: utils.SafeIntToUint8(history), + Replicas: max(replicas, 1), + }) + if err == nil { + return kv, nil + } + + // A previous boot or a concurrent daemon already created it, so opening it + // is the expected outcome. Every other create failure is real and surfaces. + if !errors.Is(err, jetstream.ErrBucketExists) { + return nil, fmt.Errorf("create KV bucket %s: %w", bucket, err) + } + kv, err = js.KeyValue(ctx, bucket) + if err != nil { + return nil, fmt.Errorf("open KV bucket %s: %w", bucket, err) + } + return kv, nil +} + +// DeleteBucketIfExists deletes a KV bucket, treating an already-absent bucket +// as success so teardown paths are idempotent. +func DeleteBucketIfExists(ctx context.Context, js jetstream.KeyValueManager, bucket string) error { + err := js.DeleteKeyValue(ctx, bucket) + if err == nil || errors.Is(err, jetstream.ErrBucketNotFound) || errors.Is(err, jetstream.ErrStreamNotFound) { + return nil + } + return fmt.Errorf("delete KV bucket %s: %w", bucket, err) +} + +// BucketNames returns the name of every KV bucket, or an error if the listing +// could not be completed. The lister closes its channel both when the listing +// is complete and when the underlying stream-names request fails, so the +// terminal Error() check is the only thing separating a full listing from a +// truncated one. Callers that prune state for resources whose bucket is absent +// must therefore treat an error as "unknown", never as "no buckets" — a +// timeout would otherwise read as a fleet-wide deletion. +// +// The names are bucket names, already stripped of the KV_ stream prefix. +func BucketNames(ctx context.Context, js jetstream.KeyValueManager) ([]string, error) { + lister := js.KeyValueStoreNames(ctx) + + // The lister goroutine blocks on an unbuffered send, so the channel is + // drained fully — an early return here would leak it until ctx expires. + var names []string + for name := range lister.Name() { + names = append(names, name) + } + if err := lister.Error(); err != nil { + return nil, fmt.Errorf("enumerate KV buckets: %w", err) + } + return names, nil +} + +// Keys lists a bucket's keys with the five-second bound used by the legacy API. +// The explicit timeout is required because the new Keys watcher otherwise uses +// a deadline-free Background context indefinitely. +func Keys(ctx context.Context, kv jetstream.KeyValue) ([]string, error) { + listCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + keys, err := kv.Keys(listCtx) + if ctxErr := listCtx.Err(); ctxErr != nil { + return nil, ctxErr + } + return keys, err +} + +// WriteVersion writes the schema version to a bucket, only if missing or older. +// Returns an error if the stored value is corrupt (non-integer). +func WriteVersion(ctx context.Context, kv jetstream.KeyValue, version int) error { + entry, err := kv.Get(ctx, utils.VersionKey) + if err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { + return fmt.Errorf("read current version: %w", err) + } + if err == nil { + stored, parseErr := strconv.Atoi(string(entry.Value())) + if parseErr != nil { + return fmt.Errorf("corrupted %s key (raw=%q): %w", utils.VersionKey, string(entry.Value()), parseErr) + } + if stored >= version { + return nil + } + } + _, err = kv.PutString(ctx, utils.VersionKey, strconv.Itoa(version)) + return err +} + +// ReadVersion reads the schema version from a bucket; returns 0 if not set. +// Errors distinguish network failures and corrupt values from "not set". +func ReadVersion(ctx context.Context, kv jetstream.KeyValue) (int, error) { + entry, err := kv.Get(ctx, utils.VersionKey) + if err != nil { + if errors.Is(err, jetstream.ErrKeyNotFound) { + return 0, nil + } + return 0, fmt.Errorf("read version: %w", err) + } + v, parseErr := strconv.Atoi(string(entry.Value())) + if parseErr != nil { + return 0, fmt.Errorf("corrupted %s key (raw=%q): %w", utils.VersionKey, string(entry.Value()), parseErr) + } + return v, nil +} diff --git a/spinifex/kvutil/kvutil_test.go b/spinifex/kvutil/kvutil_test.go new file mode 100644 index 000000000..02a0e1fee --- /dev/null +++ b/spinifex/kvutil/kvutil_test.go @@ -0,0 +1,210 @@ +package kvutil + +import ( + "context" + "testing" + + "github.com/mulgadc/spinifex/spinifex/testutil" + "github.com/mulgadc/spinifex/spinifex/utils" + "github.com/nats-io/nats.go/jetstream" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// startJetStream starts an embedded JetStream-enabled NATS server and returns a +// handle on the jetstream package API. +func startJetStream(t *testing.T) jetstream.JetStream { + t.Helper() + _, nc, _ := testutil.StartTestJetStream(t) + return testutil.NewJetStream(t, nc) +} + +// streamReplicas returns the replica count of the JetStream stream backing a KV +// bucket, so tests can assert on the config actually sent to the server. +func streamReplicas(t *testing.T, js jetstream.JetStream, bucket string) int { + t.Helper() + stream, err := js.Stream(t.Context(), "KV_"+bucket) + require.NoError(t, err) + info, err := stream.Info(t.Context()) + require.NoError(t, err) + return info.Config.Replicas +} + +func TestGetOrCreateBucket_CreatesAtDefaultReplicas(t *testing.T) { + js := startJetStream(t) + + kv, err := GetOrCreateBucket(t.Context(), js, "regression-bucket", 5) + require.NoError(t, err) + require.NotNil(t, kv) + assert.Equal(t, 1, streamReplicas(t, js, "regression-bucket")) +} + +// TestGetOrCreateBucket_OpensExisting covers the second-boot path: a bucket that +// already exists is opened with its stored contents rather than reset. +func TestGetOrCreateBucket_OpensExisting(t *testing.T) { + js := startJetStream(t) + + kv, err := GetOrCreateBucket(t.Context(), js, "existing-bucket", 5) + require.NoError(t, err) + _, err = kv.PutString(t.Context(), "survivor", "value") + require.NoError(t, err) + + // A differing history must not stop the reopen — the existing config wins. + reopened, err := GetOrCreateBucket(t.Context(), js, "existing-bucket", 1) + require.NoError(t, err) + entry, err := reopened.Get(t.Context(), "survivor") + require.NoError(t, err) + assert.Equal(t, "value", string(entry.Value())) +} + +func TestGetOrCreateBucketWithReplicas_ClampsBelowOne(t *testing.T) { + js := startJetStream(t) + + kv, err := GetOrCreateBucketWithReplicas(t.Context(), js, "clamped-zero", 1, 0) + require.NoError(t, err) + require.NotNil(t, kv) + assert.Equal(t, 1, streamReplicas(t, js, "clamped-zero")) + + kv, err = GetOrCreateBucketWithReplicas(t.Context(), js, "clamped-negative", 1, -3) + require.NoError(t, err) + require.NotNil(t, kv) + assert.Equal(t, 1, streamReplicas(t, js, "clamped-negative")) +} + +// TestGetOrCreateBucketWithReplicas_SurfacesCreateFailure pins the reason the +// open is scoped to "bucket exists": a create that fails for any other reason +// must report that reason, not the "bucket not found" a blind reopen produces. +func TestGetOrCreateBucketWithReplicas_SurfacesCreateFailure(t *testing.T) { + js := startJetStream(t) + + // The embedded single-node server rejects Replicas > 1. + _, err := GetOrCreateBucketWithReplicas(t.Context(), js, "over-replicated", 1, 3) + require.Error(t, err) + assert.NotErrorIs(t, err, jetstream.ErrBucketNotFound) + assert.Contains(t, err.Error(), "create KV bucket over-replicated") +} + +func TestDeleteBucketIfExists(t *testing.T) { + js := startJetStream(t) + + // Missing bucket is a no-op. + require.NoError(t, DeleteBucketIfExists(t.Context(), js, "ghost-bucket")) + + // Existing bucket gets deleted. + _, err := GetOrCreateBucket(t.Context(), js, "doomed-bucket", 1) + require.NoError(t, err) + require.NoError(t, DeleteBucketIfExists(t.Context(), js, "doomed-bucket")) + + _, err = js.KeyValue(t.Context(), "doomed-bucket") + require.ErrorIs(t, err, jetstream.ErrBucketNotFound, "bucket should be gone") + + // Calling again on the now-missing bucket is still a no-op (idempotent). + require.NoError(t, DeleteBucketIfExists(t.Context(), js, "doomed-bucket")) +} + +func TestBucketNames_ListsEveryBucket(t *testing.T) { + js := startJetStream(t) + + names, err := BucketNames(t.Context(), js) + require.NoError(t, err) + assert.Empty(t, names, "no buckets yet") + + for _, bucket := range []string{"names-alpha", "names-beta"} { + _, err := GetOrCreateBucket(t.Context(), js, bucket, 1) + require.NoError(t, err) + } + + names, err = BucketNames(t.Context(), js) + require.NoError(t, err) + assert.ElementsMatch(t, []string{"names-alpha", "names-beta"}, names, "names are unprefixed bucket names") +} + +// TestBucketNames_SurfacesEnumerationFailure is the reason this helper exists: +// the underlying lister closes its channel on failure exactly as it does on +// success, so a caller that ignores Error() sees a failed listing as an empty +// one and prunes every resource it was meant to keep. +func TestBucketNames_SurfacesEnumerationFailure(t *testing.T) { + js := startJetStream(t) + + _, err := GetOrCreateBucket(t.Context(), js, "should-not-vanish", 1) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + names, err := BucketNames(ctx, js) + require.Error(t, err, "a failed listing must not read as a complete one") + assert.Nil(t, names) +} + +func TestKeys_ListsKeysAndSurfacesCancellation(t *testing.T) { + js := startJetStream(t) + kv, err := GetOrCreateBucket(t.Context(), js, "bounded-keys", 1) + require.NoError(t, err) + _, err = kv.PutString(t.Context(), "key", "value") + require.NoError(t, err) + + keys, err := Keys(t.Context(), kv) + require.NoError(t, err) + require.Equal(t, []string{"key"}, keys) + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + keys, err = Keys(ctx, kv) + require.ErrorIs(t, err, context.Canceled) + require.Nil(t, keys) +} + +// TestVersionStateMachine covers unset→0, first write, idempotent same write, upgrade, and no-downgrade. +// One bucket is reused so each step runs against the prior state — the only way to catch unconditional-overwrite regressions. +func TestVersionStateMachine(t *testing.T) { + js := startJetStream(t) + kv, err := GetOrCreateBucket(t.Context(), js, "test-version-fsm", 1) + require.NoError(t, err) + + // Unset → 0. + v, err := ReadVersion(t.Context(), kv) + require.NoError(t, err) + assert.Equal(t, 0, v, "ReadVersion on unset bucket") + + steps := []struct { + name string + write int + want int // expected ReadVersion after the write + }{ + {"first write persists", 1, 1}, + {"same version is no-op", 1, 1}, + {"higher version upgrades", 2, 2}, + {"lower version is no-op (no downgrade)", 1, 2}, + {"larger jump upgrades", 5, 5}, + } + for _, step := range steps { + t.Run(step.name, func(t *testing.T) { + require.NoError(t, WriteVersion(t.Context(), kv, step.write)) + v, err := ReadVersion(t.Context(), kv) + require.NoError(t, err) + assert.Equal(t, step.want, v) + }) + } + + // Round-trip the raw KV value to confirm the encoding is what readers + // outside this package would expect. + entry, err := kv.Get(t.Context(), utils.VersionKey) + require.NoError(t, err) + assert.Equal(t, "5", string(entry.Value())) +} + +// TestVersionCorruptValue asserts a mangled stamp is reported rather than +// silently overwritten, which would hide that the bucket needs inspection. +func TestVersionCorruptValue(t *testing.T) { + js := startJetStream(t) + kv, err := GetOrCreateBucket(t.Context(), js, "test-version-corrupt", 1) + require.NoError(t, err) + _, err = kv.PutString(t.Context(), utils.VersionKey, "not-a-number") + require.NoError(t, err) + + _, err = ReadVersion(t.Context(), kv) + require.ErrorContains(t, err, "corrupted") + + require.ErrorContains(t, WriteVersion(t.Context(), kv, 2), "corrupted") +} diff --git a/spinifex/migrate/001_security_group.go b/spinifex/migrate/001_security_group.go index 61ca3679d..1d72d8ac5 100644 --- a/spinifex/migrate/001_security_group.go +++ b/spinifex/migrate/001_security_group.go @@ -1,13 +1,14 @@ package migrate import ( + "context" "encoding/json" "errors" "fmt" "time" "github.com/mulgadc/spinifex/spinifex/utils" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // sgBucket mirrors handlers/ec2/vpc.KVBucketSecurityGroups. Duplicated here to @@ -46,10 +47,10 @@ func init() { FromVersion: 1, ToVersion: 2, Description: "assign sgr- IDs to security group rules", - Run: func(ctx KVContext) error { - keys, err := ctx.KV.Keys() + Run: func(ctx context.Context, kvc KVContext) error { + keys, err := kvc.KV.Keys(ctx) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return nil } return fmt.Errorf("list keys: %w", err) @@ -59,7 +60,7 @@ func init() { if key == utils.VersionKey { continue } - if err := backfillSGRuleIDs(ctx, key); err != nil { + if err := backfillSGRuleIDs(ctx, kvc, key); err != nil { return err } } @@ -70,12 +71,12 @@ func init() { // backfillSGRuleIDs assigns sgr- IDs to any rule missing one. Skips records // already fully populated. CAS conflicts retry up to sgMigrationMaxRetries. -func backfillSGRuleIDs(ctx KVContext, key string) error { +func backfillSGRuleIDs(ctx context.Context, kvc KVContext, key string) error { var lastErr error for attempt := range sgMigrationMaxRetries { - entry, err := ctx.KV.Get(key) + entry, err := kvc.KV.Get(ctx, key) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil } return fmt.Errorf("get %s: %w", key, err) @@ -108,10 +109,10 @@ func backfillSGRuleIDs(ctx KVContext, key string) error { if err != nil { return fmt.Errorf("marshal %s: %w", key, err) } - if _, err := ctx.KV.Update(key, data, entry.Revision()); err != nil { - // nats.ErrKeyExists = revision mismatch (JSStreamWrongLastSequence). - if errors.Is(err, nats.ErrKeyExists) { - ctx.Logger.Warn("SG migration CAS conflict, retrying", "key", key, "attempt", attempt+1) + if _, err := kvc.KV.Update(ctx, key, data, entry.Revision()); err != nil { + // jetstream.ErrKeyExists = revision mismatch (JSStreamWrongLastSequence). + if errors.Is(err, jetstream.ErrKeyExists) { + kvc.Logger.Warn("SG migration CAS conflict, retrying", "key", key, "attempt", attempt+1) lastErr = err continue } diff --git a/spinifex/migrate/001_security_group_test.go b/spinifex/migrate/001_security_group_test.go index 655ec0703..93bc8f71e 100644 --- a/spinifex/migrate/001_security_group_test.go +++ b/spinifex/migrate/001_security_group_test.go @@ -1,30 +1,31 @@ package migrate import ( + "context" "encoding/json" "log/slog" "testing" "time" "github.com/mulgadc/spinifex/spinifex/utils" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) const sgTestAccountID = "123456789012" -func sgMigrationKV(t *testing.T) nats.KeyValue { +func sgMigrationKV(t *testing.T) jetstream.KeyValue { t.Helper() _, nc := startTestNATS(t) return createTestBucket(t, nc, "sg-migration-test") } -func runSGMigration(t *testing.T, kv nats.KeyValue) { +func runSGMigration(t *testing.T, kv jetstream.KeyValue) { t.Helper() for _, m := range DefaultRegistry.kvMigrations[sgBucket] { if m.FromVersion == 1 && m.ToVersion == 2 { - require.NoError(t, m.Run(KVContext{KV: kv, Logger: slog.Default()})) + require.NoError(t, m.Run(t.Context(), KVContext{KV: kv, Logger: slog.Default()})) return } } @@ -49,12 +50,12 @@ func TestSGMigration_AssignsBlankRuleIDs(t *testing.T) { } data, err := json.Marshal(record) require.NoError(t, err) - _, err = kv.Put(utils.AccountKey(sgTestAccountID, record.GroupId), data) + _, err = kv.Put(t.Context(), utils.AccountKey(sgTestAccountID, record.GroupId), data) require.NoError(t, err) runSGMigration(t, kv) - entry, err := kv.Get(utils.AccountKey(sgTestAccountID, record.GroupId)) + entry, err := kv.Get(t.Context(), utils.AccountKey(sgTestAccountID, record.GroupId)) require.NoError(t, err) var got sgRecord require.NoError(t, json.Unmarshal(entry.Value(), &got)) @@ -93,16 +94,16 @@ func TestSGMigration_IdempotentOnAlreadyMigrated(t *testing.T) { data, err := json.Marshal(record) require.NoError(t, err) key := utils.AccountKey(sgTestAccountID, record.GroupId) - _, err = kv.Put(key, data) + _, err = kv.Put(t.Context(), key, data) require.NoError(t, err) - entryBefore, err := kv.Get(key) + entryBefore, err := kv.Get(t.Context(), key) require.NoError(t, err) revBefore := entryBefore.Revision() runSGMigration(t, kv) - entryAfter, err := kv.Get(key) + entryAfter, err := kv.Get(t.Context(), key) require.NoError(t, err) assert.Equal(t, revBefore, entryAfter.Revision(), "fully-populated records must not be rewritten") } @@ -122,12 +123,12 @@ func TestSGMigration_PartialBackfill(t *testing.T) { } data, err := json.Marshal(record) require.NoError(t, err) - _, err = kv.Put(utils.AccountKey(sgTestAccountID, record.GroupId), data) + _, err = kv.Put(t.Context(), utils.AccountKey(sgTestAccountID, record.GroupId), data) require.NoError(t, err) runSGMigration(t, kv) - entry, err := kv.Get(utils.AccountKey(sgTestAccountID, record.GroupId)) + entry, err := kv.Get(t.Context(), utils.AccountKey(sgTestAccountID, record.GroupId)) require.NoError(t, err) var got sgRecord require.NoError(t, json.Unmarshal(entry.Value(), &got)) @@ -140,24 +141,24 @@ func TestSGMigration_EmptyBucket(t *testing.T) { runSGMigration(t, kv) } -// casConflictKV wraps a nats.KeyValue and forces the next failuresLeft Update -// calls to return nats.ErrKeyExists, simulating the JetStream +// casConflictKV wraps a jetstream.KeyValue and forces the next failuresLeft +// Update calls to return jetstream.ErrKeyExists, simulating the JetStream // wrong-last-sequence response that backfillSGRuleIDs retries on. type casConflictKV struct { - nats.KeyValue + jetstream.KeyValue failuresLeft int } -func (k *casConflictKV) Update(key string, value []byte, last uint64) (uint64, error) { +func (k *casConflictKV) Update(ctx context.Context, key string, value []byte, last uint64) (uint64, error) { if k.failuresLeft > 0 { k.failuresLeft-- - return 0, nats.ErrKeyExists + return 0, jetstream.ErrKeyExists } - return k.KeyValue.Update(key, value, last) + return k.KeyValue.Update(ctx, key, value, last) } -func seedSGRecord(t *testing.T, kv nats.KeyValue) string { +func seedSGRecord(t *testing.T, kv jetstream.KeyValue) string { t.Helper() record := sgRecord{ GroupId: "sg-0123456789abcdef0", @@ -171,7 +172,7 @@ func seedSGRecord(t *testing.T, kv nats.KeyValue) string { data, err := json.Marshal(record) require.NoError(t, err) key := utils.AccountKey(sgTestAccountID, record.GroupId) - _, err = kv.Put(key, data) + _, err = kv.Put(t.Context(), key, data) require.NoError(t, err) return key } @@ -181,11 +182,11 @@ func TestSGMigration_RetriesOnCASConflict(t *testing.T) { key := seedSGRecord(t, kv) stub := &casConflictKV{KeyValue: kv, failuresLeft: sgMigrationMaxRetries - 1} - err := backfillSGRuleIDs(KVContext{KV: stub, Logger: slog.Default()}, key) + err := backfillSGRuleIDs(t.Context(), KVContext{KV: stub, Logger: slog.Default()}, key) require.NoError(t, err) assert.Equal(t, 0, stub.failuresLeft, "all injected conflicts must have been consumed") - entry, err := kv.Get(key) + entry, err := kv.Get(t.Context(), key) require.NoError(t, err) var got sgRecord require.NoError(t, json.Unmarshal(entry.Value(), &got)) @@ -198,12 +199,12 @@ func TestSGMigration_FailsAfterCASRetryExhaustion(t *testing.T) { key := seedSGRecord(t, kv) stub := &casConflictKV{KeyValue: kv, failuresLeft: sgMigrationMaxRetries + 1} - err := backfillSGRuleIDs(KVContext{KV: stub, Logger: slog.Default()}, key) + err := backfillSGRuleIDs(t.Context(), KVContext{KV: stub, Logger: slog.Default()}, key) require.Error(t, err) assert.Contains(t, err.Error(), "exceeded 3 CAS retries") - assert.ErrorIs(t, err, nats.ErrKeyExists, "exhaustion error must wrap the underlying CAS conflict") + assert.ErrorIs(t, err, jetstream.ErrKeyExists, "exhaustion error must wrap the underlying CAS conflict") - entry, err := kv.Get(key) + entry, err := kv.Get(t.Context(), key) require.NoError(t, err) var got sgRecord require.NoError(t, json.Unmarshal(entry.Value(), &got)) diff --git a/spinifex/migrate/002_external_ipam_purpose.go b/spinifex/migrate/002_external_ipam_purpose.go index 68baf0b5c..c2948fb4f 100644 --- a/spinifex/migrate/002_external_ipam_purpose.go +++ b/spinifex/migrate/002_external_ipam_purpose.go @@ -1,12 +1,13 @@ package migrate import ( + "context" "encoding/json" "errors" "fmt" "github.com/mulgadc/spinifex/spinifex/utils" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // extIPAMBucket mirrors handlers/ec2/vpc.KVBucketExternalIPAM. Duplicated to @@ -57,10 +58,10 @@ func init() { FromVersion: 1, ToVersion: 2, Description: "rename ExternalIPAllocation.Type → .Purpose with enum mapping", - Run: func(ctx KVContext) error { - keys, err := ctx.KV.Keys() + Run: func(ctx context.Context, kvc KVContext) error { + keys, err := kvc.KV.Keys(ctx) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return nil } return fmt.Errorf("list keys: %w", err) @@ -70,7 +71,7 @@ func init() { if key == utils.VersionKey { continue } - if err := renameExternalIPAMType(ctx, key); err != nil { + if err := renameExternalIPAMType(ctx, kvc, key); err != nil { return err } } @@ -79,12 +80,12 @@ func init() { }) } -func renameExternalIPAMType(ctx KVContext, key string) error { +func renameExternalIPAMType(ctx context.Context, kvc KVContext, key string) error { var lastErr error for attempt := range extIPAMMaxRetries { - entry, err := ctx.KV.Get(key) + entry, err := kvc.KV.Get(ctx, key) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil } return fmt.Errorf("get %s: %w", key, err) @@ -102,7 +103,7 @@ func renameExternalIPAMType(ctx KVContext, key string) error { } purpose := extLegacyTypeToPurpose[alloc.Type] if purpose == "" { - ctx.Logger.Warn("external IPAM v1→v2 migration: unknown legacy type, tagging unknown", + kvc.Logger.Warn("external IPAM v1→v2 migration: unknown legacy type, tagging unknown", "pool", key, "ip", ip, "legacy_type", alloc.Type) purpose = "unknown" } @@ -120,9 +121,9 @@ func renameExternalIPAMType(ctx KVContext, key string) error { if err != nil { return fmt.Errorf("marshal %s: %w", key, err) } - if _, err := ctx.KV.Update(key, data, entry.Revision()); err != nil { - if errors.Is(err, nats.ErrKeyExists) { - ctx.Logger.Warn("external IPAM migration CAS conflict, retrying", "key", key, "attempt", attempt+1) + if _, err := kvc.KV.Update(ctx, key, data, entry.Revision()); err != nil { + if errors.Is(err, jetstream.ErrKeyExists) { + kvc.Logger.Warn("external IPAM migration CAS conflict, retrying", "key", key, "attempt", attempt+1) lastErr = err continue } diff --git a/spinifex/migrate/002_external_ipam_purpose_test.go b/spinifex/migrate/002_external_ipam_purpose_test.go index 2319807f0..0c0f047da 100644 --- a/spinifex/migrate/002_external_ipam_purpose_test.go +++ b/spinifex/migrate/002_external_ipam_purpose_test.go @@ -5,22 +5,22 @@ import ( "log/slog" "testing" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func extIPAMMigrationKV(t *testing.T) nats.KeyValue { +func extIPAMMigrationKV(t *testing.T) jetstream.KeyValue { t.Helper() _, nc := startTestNATS(t) return createTestBucket(t, nc, "ext-ipam-migration-test") } -func runExtIPAMMigration(t *testing.T, kv nats.KeyValue) { +func runExtIPAMMigration(t *testing.T, kv jetstream.KeyValue) { t.Helper() for _, m := range DefaultRegistry.kvMigrations[extIPAMBucket] { if m.FromVersion == 1 && m.ToVersion == 2 { - require.NoError(t, m.Run(KVContext{KV: kv, Logger: slog.Default()})) + require.NoError(t, m.Run(t.Context(), KVContext{KV: kv, Logger: slog.Default()})) return } } @@ -45,12 +45,12 @@ func TestExtIPAMMigration_RenamesType(t *testing.T) { } data, err := json.Marshal(record) require.NoError(t, err) - _, err = kv.Put("wan", data) + _, err = kv.Put(t.Context(), "wan", data) require.NoError(t, err) runExtIPAMMigration(t, kv) - entry, err := kv.Get("wan") + entry, err := kv.Get(t.Context(), "wan") require.NoError(t, err) var got extIPAMRecordV1V2 require.NoError(t, json.Unmarshal(entry.Value(), &got)) @@ -80,7 +80,7 @@ func TestExtIPAMMigration_IdempotentOnAlreadyMigrated(t *testing.T) { } data, err := json.Marshal(record) require.NoError(t, err) - _, err = kv.Put("wan", data) + _, err = kv.Put(t.Context(), "wan", data) require.NoError(t, err) rev := getKVRevision(t, kv, "wan") @@ -95,9 +95,9 @@ func TestExtIPAMMigration_EmptyBucket(t *testing.T) { runExtIPAMMigration(t, kv) } -func getKVRevision(t *testing.T, kv nats.KeyValue, key string) uint64 { +func getKVRevision(t *testing.T, kv jetstream.KeyValue, key string) uint64 { t.Helper() - entry, err := kv.Get(key) + entry, err := kv.Get(t.Context(), key) require.NoError(t, err) return entry.Revision() } diff --git a/spinifex/migrate/003_ipam_purpose.go b/spinifex/migrate/003_ipam_purpose.go index b1b98f838..9ca24fa34 100644 --- a/spinifex/migrate/003_ipam_purpose.go +++ b/spinifex/migrate/003_ipam_purpose.go @@ -1,12 +1,13 @@ package migrate import ( + "context" "encoding/json" "errors" "fmt" "github.com/mulgadc/spinifex/spinifex/utils" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // ipamBucket / eniBucket mirror handlers/ec2/vpc.KVBucketIPAM and @@ -54,15 +55,15 @@ func init() { FromVersion: 1, ToVersion: 2, Description: "convert subnet IPAM Allocated []string → []IPEntry; backfill Purpose+OwnerID from ENIs", - Run: func(ctx KVContext) error { - ownerIndex, err := buildIPAMOwnerIndex(ctx.JetStream) + Run: func(ctx context.Context, kvc KVContext) error { + ownerIndex, err := buildIPAMOwnerIndex(ctx, kvc.JetStream) if err != nil { return fmt.Errorf("build owner index: %w", err) } - keys, err := ctx.KV.Keys() + keys, err := kvc.KV.Keys(ctx) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return nil } return fmt.Errorf("list keys: %w", err) @@ -72,7 +73,7 @@ func init() { if key == utils.VersionKey { continue } - if err := convertIPAMRecord(ctx, key, ownerIndex); err != nil { + if err := convertIPAMRecord(ctx, kvc, key, ownerIndex); err != nil { return err } } @@ -84,24 +85,24 @@ func init() { // buildIPAMOwnerIndex returns map[subnetID]map[ip]eniID by walking the ENI // bucket. Returns an empty index if the ENI bucket is missing // (fresh-install case) so the migration still runs cleanly. -func buildIPAMOwnerIndex(js nats.JetStreamContext) (map[string]map[string]string, error) { +func buildIPAMOwnerIndex(ctx context.Context, js jetstream.JetStream) (map[string]map[string]string, error) { if js == nil { // Called via the bare RunKV path (e.g. in tests with no JS). Skip // owner attribution — all entries become Purpose=unknown so the // schema-only conversion still succeeds. return map[string]map[string]string{}, nil } - eniKV, err := js.KeyValue(eniBucket) + eniKV, err := js.KeyValue(ctx, eniBucket) if err != nil { - if errors.Is(err, nats.ErrBucketNotFound) { + if errors.Is(err, jetstream.ErrBucketNotFound) { return map[string]map[string]string{}, nil } return nil, fmt.Errorf("open ENI bucket: %w", err) } - keys, err := eniKV.Keys() + keys, err := eniKV.Keys(ctx) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return map[string]map[string]string{}, nil } return nil, fmt.Errorf("list ENI keys: %w", err) @@ -112,9 +113,9 @@ func buildIPAMOwnerIndex(js nats.JetStreamContext) (map[string]map[string]string if key == utils.VersionKey { continue } - entry, err := eniKV.Get(key) + entry, err := eniKV.Get(ctx, key) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { continue } return nil, fmt.Errorf("get ENI %s: %w", key, err) @@ -134,12 +135,12 @@ func buildIPAMOwnerIndex(js nats.JetStreamContext) (map[string]map[string]string return index, nil } -func convertIPAMRecord(ctx KVContext, key string, ownerIndex map[string]map[string]string) error { +func convertIPAMRecord(ctx context.Context, kvc KVContext, key string, ownerIndex map[string]map[string]string) error { var lastErr error for attempt := range ipamMigrationMaxRetries { - entry, err := ctx.KV.Get(key) + entry, err := kvc.KV.Get(ctx, key) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil } return fmt.Errorf("get %s: %w", key, err) @@ -172,7 +173,7 @@ func convertIPAMRecord(ctx KVContext, key string, ownerIndex map[string]map[stri if !found { ownerID = "" purpose = "unknown" - ctx.Logger.Warn("internal IPAM v1→v2 migration: no owning ENI, tagging unknown", + kvc.Logger.Warn("internal IPAM v1→v2 migration: no owning ENI, tagging unknown", "subnet", raw.SubnetId, "ip", ip) } converted = append(converted, ipEntryV2{IP: ip, Purpose: purpose, OwnerID: ownerID}) @@ -187,9 +188,9 @@ func convertIPAMRecord(ctx KVContext, key string, ownerIndex map[string]map[stri if err != nil { return fmt.Errorf("marshal %s: %w", key, err) } - if _, err := ctx.KV.Update(key, data, entry.Revision()); err != nil { - if errors.Is(err, nats.ErrKeyExists) { - ctx.Logger.Warn("internal IPAM migration CAS conflict, retrying", "key", key, "attempt", attempt+1) + if _, err := kvc.KV.Update(ctx, key, data, entry.Revision()); err != nil { + if errors.Is(err, jetstream.ErrKeyExists) { + kvc.Logger.Warn("internal IPAM migration CAS conflict, retrying", "key", key, "attempt", attempt+1) lastErr = err continue } diff --git a/spinifex/migrate/003_ipam_purpose_test.go b/spinifex/migrate/003_ipam_purpose_test.go index b479b0856..e0ec437b8 100644 --- a/spinifex/migrate/003_ipam_purpose_test.go +++ b/spinifex/migrate/003_ipam_purpose_test.go @@ -5,39 +5,39 @@ import ( "log/slog" "testing" - "github.com/nats-io/nats.go" + "github.com/mulgadc/spinifex/spinifex/testutil" + "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func setupIPAMMigrationKV(t *testing.T) (nats.KeyValue, nats.JetStreamContext) { +func setupIPAMMigrationKV(t *testing.T) (jetstream.KeyValue, jetstream.JetStream) { t.Helper() _, nc := startTestNATS(t) - js, err := nc.JetStream() - require.NoError(t, err) + js := testutil.NewJetStream(t, nc) kv := createTestBucket(t, nc, ipamBucket) return kv, js } -func runIPAMMigration(t *testing.T, kv nats.KeyValue, js nats.JetStreamContext) { +func runIPAMMigration(t *testing.T, kv jetstream.KeyValue, js jetstream.JetStream) { t.Helper() for _, m := range DefaultRegistry.kvMigrations[ipamBucket] { if m.FromVersion == 1 && m.ToVersion == 2 { - require.NoError(t, m.Run(KVContext{KV: kv, JetStream: js, Logger: slog.Default()})) + require.NoError(t, m.Run(t.Context(), KVContext{KV: kv, JetStream: js, Logger: slog.Default()})) return } } t.Fatalf("v1→v2 migration not registered for %s", ipamBucket) } -func seedENIBucket(t *testing.T, js nats.JetStreamContext, enis []eniRecordSnapshot) { +func seedENIBucket(t *testing.T, js jetstream.JetStream, enis []eniRecordSnapshot) { t.Helper() - kv, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: eniBucket, History: 1}) + kv, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: eniBucket, History: 1}) require.NoError(t, err) for _, eni := range enis { data, err := json.Marshal(eni) require.NoError(t, err) - _, err = kv.Put(eni.NetworkInterfaceId, data) + _, err = kv.Put(t.Context(), eni.NetworkInterfaceId, data) require.NoError(t, err) } } @@ -62,7 +62,7 @@ func TestIPAMMigration_ConvertsV1ToV2WithOwnerBackfill(t *testing.T) { } data, err := json.Marshal(subnet1) require.NoError(t, err) - _, err = kv.Put("subnet-1", data) + _, err = kv.Put(t.Context(), "subnet-1", data) require.NoError(t, err) subnet2 := legacyRecord{ @@ -72,12 +72,12 @@ func TestIPAMMigration_ConvertsV1ToV2WithOwnerBackfill(t *testing.T) { } data, err = json.Marshal(subnet2) require.NoError(t, err) - _, err = kv.Put("subnet-2", data) + _, err = kv.Put(t.Context(), "subnet-2", data) require.NoError(t, err) runIPAMMigration(t, kv, js) - entry, err := kv.Get("subnet-1") + entry, err := kv.Get(t.Context(), "subnet-1") require.NoError(t, err) var got ipamRecordV2 require.NoError(t, json.Unmarshal(entry.Value(), &got)) @@ -92,7 +92,7 @@ func TestIPAMMigration_ConvertsV1ToV2WithOwnerBackfill(t *testing.T) { assert.Equal(t, "unknown", got.Allocated[2].Purpose, "orphan IP tagged unknown") assert.Empty(t, got.Allocated[2].OwnerID) - entry, err = kv.Get("subnet-2") + entry, err = kv.Get(t.Context(), "subnet-2") require.NoError(t, err) require.NoError(t, json.Unmarshal(entry.Value(), &got)) require.Len(t, got.Allocated, 1) @@ -111,7 +111,7 @@ func TestIPAMMigration_IdempotentOnAlreadyV2(t *testing.T) { } data, err := json.Marshal(v2) require.NoError(t, err) - _, err = kv.Put("subnet-1", data) + _, err = kv.Put(t.Context(), "subnet-1", data) require.NoError(t, err) rev := getKVRevision(t, kv, "subnet-1") @@ -137,14 +137,14 @@ func TestIPAMMigration_NoJetStream_TagsUnknown(t *testing.T) { } data, err := json.Marshal(rec) require.NoError(t, err) - _, err = kv.Put("subnet-1", data) + _, err = kv.Put(t.Context(), "subnet-1", data) require.NoError(t, err) // Run migration with nil JetStream — schema conversion still runs but // every entry becomes Purpose=unknown. runIPAMMigration(t, kv, nil) - entry, err := kv.Get("subnet-1") + entry, err := kv.Get(t.Context(), "subnet-1") require.NoError(t, err) var got ipamRecordV2 require.NoError(t, json.Unmarshal(entry.Value(), &got)) diff --git a/spinifex/migrate/migrate.go b/spinifex/migrate/migrate.go index cafc84821..6f78df1cd 100644 --- a/spinifex/migrate/migrate.go +++ b/spinifex/migrate/migrate.go @@ -1,6 +1,7 @@ package migrate import ( + "context" "fmt" "log/slog" "os" @@ -8,8 +9,8 @@ import ( "sort" "time" - "github.com/mulgadc/spinifex/spinifex/utils" - "github.com/nats-io/nats.go" + "github.com/mulgadc/spinifex/spinifex/kvutil" + "github.com/nats-io/nats.go/jetstream" ) // KVMigration represents a versioned transformation of KV bucket data. @@ -17,15 +18,15 @@ type KVMigration struct { FromVersion int ToVersion int Description string - Run func(ctx KVContext) error + Run func(ctx context.Context, kvc KVContext) error } // KVContext provides KV migration functions access to the bucket being migrated. // JetStream is non-nil only when the caller used RunKVWithJetStream — used by // migrations that need to read sibling buckets (e.g. owner-attribution lookups). type KVContext struct { - KV nats.KeyValue - JetStream nats.JetStreamContext + KV jetstream.KeyValue + JetStream jetstream.JetStream Logger *slog.Logger } @@ -103,18 +104,18 @@ func (r *Registry) RegisterConfig(target string, m ConfigMigration) { // RunKVWithJetStream is RunKV with a JetStream handle attached to each // migration's KVContext, enabling cross-bucket reads (e.g. owner-attribution // during a backfill). Prefer plain RunKV when the migration is self-contained. -func (r *Registry) RunKVWithJetStream(bucket string, kv nats.KeyValue, js nats.JetStreamContext, targetVersion int) error { - return r.runKV(bucket, kv, js, targetVersion) +func (r *Registry) RunKVWithJetStream(ctx context.Context, bucket string, kv jetstream.KeyValue, js jetstream.JetStream, targetVersion int) error { + return r.runKV(ctx, bucket, kv, js, targetVersion) } // RunKV applies pending KV migrations up to targetVersion. Stamps directly when // no migrations are registered (fresh bucket). Errors if the chain is incomplete. -func (r *Registry) RunKV(bucket string, kv nats.KeyValue, targetVersion int) error { - return r.runKV(bucket, kv, nil, targetVersion) +func (r *Registry) RunKV(ctx context.Context, bucket string, kv jetstream.KeyValue, targetVersion int) error { + return r.runKV(ctx, bucket, kv, nil, targetVersion) } -func (r *Registry) runKV(bucket string, kv nats.KeyValue, js nats.JetStreamContext, targetVersion int) error { - current, err := utils.ReadVersion(kv) +func (r *Registry) runKV(ctx context.Context, bucket string, kv jetstream.KeyValue, js jetstream.JetStream, targetVersion int) error { + current, err := kvutil.ReadVersion(ctx, kv) if err != nil { return fmt.Errorf("read version for %s: %w", bucket, err) } @@ -127,7 +128,7 @@ func (r *Registry) runKV(bucket string, kv nats.KeyValue, js nats.JetStreamConte // Fresh bucket, no migrations: stamp directly (common first-init path). if current == 0 && len(all) == 0 { - return utils.WriteVersion(kv, targetVersion) + return kvutil.WriteVersion(ctx, kv, targetVersion) } // Fresh bucket with migrations: no v0 schema by convention; start at chain bottom. @@ -162,11 +163,11 @@ func (r *Registry) runKV(bucket string, kv nats.KeyValue, js nats.JetStreamConte logger := slog.Default() for _, m := range pending { logger.Info("Running KV migration", "bucket", bucket, "from", m.FromVersion, "to", m.ToVersion, "description", m.Description) - ctx := KVContext{KV: kv, JetStream: js, Logger: logger} - if err := m.Run(ctx); err != nil { + kvc := KVContext{KV: kv, JetStream: js, Logger: logger} + if err := m.Run(ctx, kvc); err != nil { return fmt.Errorf("KV migration %s %d→%d failed: %w", bucket, m.FromVersion, m.ToVersion, err) } - if err := utils.WriteVersion(kv, m.ToVersion); err != nil { + if err := kvutil.WriteVersion(ctx, kv, m.ToVersion); err != nil { return fmt.Errorf("stamp version %d on %s: %w", m.ToVersion, bucket, err) } } diff --git a/spinifex/migrate/migrate_test.go b/spinifex/migrate/migrate_test.go index 3fd3430c9..e79744c1a 100644 --- a/spinifex/migrate/migrate_test.go +++ b/spinifex/migrate/migrate_test.go @@ -1,6 +1,7 @@ package migrate import ( + "context" "errors" "os" "path/filepath" @@ -10,6 +11,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/testutil" "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -21,11 +23,10 @@ func startTestNATS(t *testing.T) (*server.Server, *nats.Conn) { return ns, nc } -func createTestBucket(t *testing.T, nc *nats.Conn, name string) nats.KeyValue { +func createTestBucket(t *testing.T, nc *nats.Conn, name string) jetstream.KeyValue { t.Helper() - js, err := nc.JetStream() - require.NoError(t, err) - kv, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: name, History: 1}) + js := testutil.NewJetStream(t, nc) + kv, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: name, History: 1}) require.NoError(t, err) return kv } @@ -34,33 +35,33 @@ func createTestBucket(t *testing.T, nc *nats.Conn, name string) nats.KeyValue { func TestRegistry_ValidatesChainNoGaps(t *testing.T) { r := NewRegistry() - r.RegisterKV("test-bucket", KVMigration{FromVersion: 1, ToVersion: 2, Description: "first", Run: func(KVContext) error { return nil }}) - r.RegisterKV("test-bucket", KVMigration{FromVersion: 3, ToVersion: 4, Description: "gap", Run: func(KVContext) error { return nil }}) + r.RegisterKV("test-bucket", KVMigration{FromVersion: 1, ToVersion: 2, Description: "first", Run: func(context.Context, KVContext) error { return nil }}) + r.RegisterKV("test-bucket", KVMigration{FromVersion: 3, ToVersion: 4, Description: "gap", Run: func(context.Context, KVContext) error { return nil }}) _, nc := startTestNATS(t) kv := createTestBucket(t, nc, "test-bucket") // Stamp version 1 to simulate existing bucket. - _, err := kv.PutString("_version", "1") + _, err := kv.PutString(t.Context(), "_version", "1") require.NoError(t, err) - err = r.RunKV("test-bucket", kv, 4) + err = r.RunKV(t.Context(), "test-bucket", kv, 4) assert.Error(t, err) assert.Contains(t, err.Error(), "gap") } func TestRegistry_RejectsDuplicateVersions(t *testing.T) { r := NewRegistry() - r.RegisterKV("test-bucket", KVMigration{FromVersion: 1, ToVersion: 2, Description: "first", Run: func(KVContext) error { return nil }}) - r.RegisterKV("test-bucket", KVMigration{FromVersion: 1, ToVersion: 2, Description: "duplicate", Run: func(KVContext) error { return nil }}) + r.RegisterKV("test-bucket", KVMigration{FromVersion: 1, ToVersion: 2, Description: "first", Run: func(context.Context, KVContext) error { return nil }}) + r.RegisterKV("test-bucket", KVMigration{FromVersion: 1, ToVersion: 2, Description: "duplicate", Run: func(context.Context, KVContext) error { return nil }}) _, nc := startTestNATS(t) kv := createTestBucket(t, nc, "test-bucket") - _, err := kv.PutString("_version", "1") + _, err := kv.PutString(t.Context(), "_version", "1") require.NoError(t, err) // Two migrations with the same FromVersion — chain validation will catch this // because after running first 1→2, the second 1→2 won't match expected=2. - err = r.RunKV("test-bucket", kv, 2) + err = r.RunKV(t.Context(), "test-bucket", kv, 2) // Should succeed (first 1→2 runs, second is filtered out since FromVersion < current after first runs). // Actually with our filtering, both have FromVersion=1 >= current=1 and ToVersion=2 <= target=2, // so both are in pending. Chain validation: expected=1, first.From=1 ✓, expected=2, second.From=1 ✗. @@ -76,11 +77,11 @@ func TestRunKV_NoPendingMigrations_NoOp(t *testing.T) { kv := createTestBucket(t, nc, "test-bucket") // Stamp version 1 directly. - _, err := kv.PutString("_version", "1") + _, err := kv.PutString(t.Context(), "_version", "1") require.NoError(t, err) // RunKV with target=1, current=1 → no-op. - err = r.RunKV("test-bucket", kv, 1) + err = r.RunKV(t.Context(), "test-bucket", kv, 1) assert.NoError(t, err) } @@ -89,7 +90,7 @@ func TestRunKV_FreshBucket_WithMigrations(t *testing.T) { r := NewRegistry() r.RegisterKV("test-bucket", KVMigration{ FromVersion: 1, ToVersion: 2, Description: "first kv migration", - Run: func(KVContext) error { ran = true; return nil }, + Run: func(context.Context, KVContext) error { ran = true; return nil }, }) _, nc := startTestNATS(t) @@ -97,11 +98,11 @@ func TestRunKV_FreshBucket_WithMigrations(t *testing.T) { // Fresh bucket (no _version key), targetVersion=2. The chain bottoms at // 1, so RunKV should accept the chain and run it without a "gap" error. - err := r.RunKV("test-bucket", kv, 2) + err := r.RunKV(t.Context(), "test-bucket", kv, 2) require.NoError(t, err) assert.True(t, ran, "migration should run on fresh bucket with registered chain") - entry, err := kv.Get("_version") + entry, err := kv.Get(t.Context(), "_version") require.NoError(t, err) assert.Equal(t, "2", string(entry.Value())) } @@ -112,11 +113,11 @@ func TestRunKV_FreshBucket_StampsVersion(t *testing.T) { kv := createTestBucket(t, nc, "test-bucket") // No version set → version 0 (fresh bucket). - err := r.RunKV("test-bucket", kv, 1) + err := r.RunKV(t.Context(), "test-bucket", kv, 1) assert.NoError(t, err) // Verify version was stamped. - entry, err := kv.Get("_version") + entry, err := kv.Get(t.Context(), "_version") require.NoError(t, err) assert.Equal(t, "1", string(entry.Value())) } @@ -126,24 +127,24 @@ func TestRunKV_ExecutesMigrationsInOrder(t *testing.T) { r := NewRegistry() r.RegisterKV("test-bucket", KVMigration{ FromVersion: 1, ToVersion: 2, Description: "step 1", - Run: func(KVContext) error { order = append(order, 1); return nil }, + Run: func(context.Context, KVContext) error { order = append(order, 1); return nil }, }) r.RegisterKV("test-bucket", KVMigration{ FromVersion: 2, ToVersion: 3, Description: "step 2", - Run: func(KVContext) error { order = append(order, 2); return nil }, + Run: func(context.Context, KVContext) error { order = append(order, 2); return nil }, }) _, nc := startTestNATS(t) kv := createTestBucket(t, nc, "test-bucket") - _, err := kv.PutString("_version", "1") + _, err := kv.PutString(t.Context(), "_version", "1") require.NoError(t, err) - err = r.RunKV("test-bucket", kv, 3) + err = r.RunKV(t.Context(), "test-bucket", kv, 3) assert.NoError(t, err) assert.Equal(t, []int{1, 2}, order) // Verify final version. - entry, err := kv.Get("_version") + entry, err := kv.Get(t.Context(), "_version") require.NoError(t, err) assert.Equal(t, "3", string(entry.Value())) } @@ -152,27 +153,27 @@ func TestRunKV_StampsAfterEachStep(t *testing.T) { r := NewRegistry() r.RegisterKV("test-bucket", KVMigration{ FromVersion: 1, ToVersion: 2, Description: "step 1", - Run: func(ctx KVContext) error { + Run: func(context.Context, KVContext) error { // After this runs, version should be stamped to 2 by RunKV. return nil }, }) r.RegisterKV("test-bucket", KVMigration{ FromVersion: 2, ToVersion: 3, Description: "step 2 fails", - Run: func(KVContext) error { return errors.New("boom") }, + Run: func(context.Context, KVContext) error { return errors.New("boom") }, }) _, nc := startTestNATS(t) kv := createTestBucket(t, nc, "test-bucket") - _, err := kv.PutString("_version", "1") + _, err := kv.PutString(t.Context(), "_version", "1") require.NoError(t, err) - err = r.RunKV("test-bucket", kv, 3) + err = r.RunKV(t.Context(), "test-bucket", kv, 3) assert.Error(t, err) assert.Contains(t, err.Error(), "boom") // Version should be 2 (stamped after step 1, before step 2 failed). - entry, err := kv.Get("_version") + entry, err := kv.Get(t.Context(), "_version") require.NoError(t, err) assert.Equal(t, "2", string(entry.Value())) } @@ -181,19 +182,19 @@ func TestRunKV_StopsOnFailure_VersionNotBumped(t *testing.T) { r := NewRegistry() r.RegisterKV("test-bucket", KVMigration{ FromVersion: 1, ToVersion: 2, Description: "fails", - Run: func(KVContext) error { return errors.New("migration error") }, + Run: func(context.Context, KVContext) error { return errors.New("migration error") }, }) _, nc := startTestNATS(t) kv := createTestBucket(t, nc, "test-bucket") - _, err := kv.PutString("_version", "1") + _, err := kv.PutString(t.Context(), "_version", "1") require.NoError(t, err) - err = r.RunKV("test-bucket", kv, 2) + err = r.RunKV(t.Context(), "test-bucket", kv, 2) assert.Error(t, err) // Version remains at 1. - entry, err := kv.Get("_version") + entry, err := kv.Get(t.Context(), "_version") require.NoError(t, err) assert.Equal(t, "1", string(entry.Value())) } @@ -204,10 +205,10 @@ func TestRunKV_RejectsMissingMigration(t *testing.T) { _, nc := startTestNATS(t) kv := createTestBucket(t, nc, "test-bucket") - _, err := kv.PutString("_version", "1") + _, err := kv.PutString(t.Context(), "_version", "1") require.NoError(t, err) - err = r.RunKV("test-bucket", kv, 2) + err = r.RunKV(t.Context(), "test-bucket", kv, 2) assert.Error(t, err) assert.Contains(t, err.Error(), "no migrations registered") } @@ -217,26 +218,26 @@ func TestRunKV_Idempotent(t *testing.T) { r := NewRegistry() r.RegisterKV("test-bucket", KVMigration{ FromVersion: 1, ToVersion: 2, Description: "add field", - Run: func(ctx KVContext) error { + Run: func(ctx context.Context, kvc KVContext) error { runCount++ // Idempotent: write a key, re-running writes the same value. - _, err := ctx.KV.PutString("data.key1", `{"field":"value"}`) + _, err := kvc.KV.PutString(ctx, "data.key1", `{"field":"value"}`) return err }, }) _, nc := startTestNATS(t) kv := createTestBucket(t, nc, "test-bucket") - _, err := kv.PutString("_version", "1") + _, err := kv.PutString(t.Context(), "_version", "1") require.NoError(t, err) // First run. - err = r.RunKV("test-bucket", kv, 2) + err = r.RunKV(t.Context(), "test-bucket", kv, 2) assert.NoError(t, err) assert.Equal(t, 1, runCount) // Second run — already at version 2, should be no-op. - err = r.RunKV("test-bucket", kv, 2) + err = r.RunKV(t.Context(), "test-bucket", kv, 2) assert.NoError(t, err) assert.Equal(t, 1, runCount) // Not incremented. } diff --git a/spinifex/network/external/dhcp/iface_mac_test.go b/spinifex/network/external/dhcp/iface_mac_test.go index a215fb431..586912399 100644 --- a/spinifex/network/external/dhcp/iface_mac_test.go +++ b/spinifex/network/external/dhcp/iface_mac_test.go @@ -10,14 +10,16 @@ import ( "github.com/mulgadc/spinifex/spinifex/network/external/dhcp" "github.com/mulgadc/spinifex/spinifex/testutil" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func newIfaceMACManager(t *testing.T, fake *dhcp.Fake, ifaceIPs func(string) ([]net.IP, error)) (*dhcp.Manager, *dhcp.Store, *nats.Conn) { t.Helper() - _, nc, js := testutil.StartTestJetStream(t) - store, err := dhcp.NewStore(js, "az1") + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + store, err := dhcp.NewStore(t.Context(), js, "az1") require.NoError(t, err) mgr, err := dhcp.NewManager(dhcp.ManagerConfig{Client: fake, Store: store, IfaceIPs: ifaceIPs}) require.NoError(t, err) @@ -57,7 +59,7 @@ func TestAcquireUseIfaceMACThreadedAndPersisted(t *testing.T) { assert.True(t, gotReq.UseIfaceMAC, "flag must reach the DHCP client") assert.True(t, lease.UseIfaceMAC, "flag must reflect back on the lease") - got, err := store.Get("eipalloc-A") + got, err := store.Get(t.Context(), "eipalloc-A") require.NoError(t, err) assert.True(t, got.Lease.UseIfaceMAC, "flag must round-trip through KV") } @@ -73,7 +75,7 @@ func TestAcquireDerivedMACDefault(t *testing.T) { }) require.NoError(t, err) assert.False(t, lease.UseIfaceMAC) - got, err := store.Get("eipalloc-B") + got, err := store.Get(t.Context(), "eipalloc-B") require.NoError(t, err) assert.False(t, got.Lease.UseIfaceMAC) } @@ -96,8 +98,8 @@ func TestAcquireIfaceMACCollisionWithOwnIP(t *testing.T) { require.Error(t, err) assert.Contains(t, err.Error(), `use source="static"`) assert.Equal(t, 1, fake.ReleaseCount(), "colliding lease must be released") - _, err = store.Get("eipalloc-C") - assert.ErrorIs(t, err, nats.ErrKeyNotFound, "colliding lease must not persist") + _, err = store.Get(t.Context(), "eipalloc-C") + assert.ErrorIs(t, err, jetstream.ErrKeyNotFound, "colliding lease must not persist") } // Two client-ids ACKing the same IP means the router ignores option 61. diff --git a/spinifex/network/external/dhcp/kv.go b/spinifex/network/external/dhcp/kv.go index fd21d317b..b4fbbb4c3 100644 --- a/spinifex/network/external/dhcp/kv.go +++ b/spinifex/network/external/dhcp/kv.go @@ -1,6 +1,7 @@ package dhcp import ( + "context" "encoding/json" "errors" "fmt" @@ -9,8 +10,9 @@ import ( "strings" "time" + "github.com/mulgadc/spinifex/spinifex/kvutil" "github.com/mulgadc/spinifex/spinifex/utils" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // KVBucketPrefix is the lease bucket's name prefix. The full bucket name @@ -41,13 +43,14 @@ type Entry struct { // NATS JetStream KV in production; tests can substitute via the // NewStoreWithKV constructor. type Store struct { - kv nats.KeyValue + kv jetstream.KeyValue az string } -// NewStore creates/opens the per-AZ lease bucket and returns a Store. -func NewStore(js nats.JetStreamContext, az string) (*Store, error) { - kv, err := utils.GetOrCreateKVBucket(js, BucketName(az), 5) +// NewStore creates/opens the per-AZ lease bucket and returns a Store. ctx bounds +// the bucket get-or-create only; each operation carries its own. +func NewStore(ctx context.Context, js jetstream.KeyValueManager, az string) (*Store, error) { + kv, err := kvutil.GetOrCreateBucket(ctx, js, BucketName(az), 5) if err != nil { return nil, fmt.Errorf("create dhcp lease KV bucket: %w", err) } @@ -55,7 +58,7 @@ func NewStore(js nats.JetStreamContext, az string) (*Store, error) { } // NewStoreWithKV is the test constructor — caller owns the bucket. -func NewStoreWithKV(kv nats.KeyValue, az string) *Store { +func NewStoreWithKV(kv jetstream.KeyValue, az string) *Store { return &Store{kv: kv, az: az} } @@ -63,11 +66,11 @@ func NewStoreWithKV(kv nats.KeyValue, az string) *Store { func (s *Store) AZ() string { return s.az } // KV exposes the underlying bucket. Used by tests that share buckets. -func (s *Store) KV() nats.KeyValue { return s.kv } +func (s *Store) KV() jetstream.KeyValue { return s.kv } // Put persists an entry keyed by Lease.ClientID. Overwrites any existing // entry for the same client-id. -func (s *Store) Put(e Entry) error { +func (s *Store) Put(ctx context.Context, e Entry) error { if e.Lease == nil { return errors.New("dhcp store put: lease is nil") } @@ -78,16 +81,16 @@ func (s *Store) Put(e Entry) error { if err != nil { return fmt.Errorf("marshal lease entry: %w", err) } - if _, err := s.kv.Put(e.Lease.ClientID, data); err != nil { + if _, err := s.kv.Put(ctx, e.Lease.ClientID, data); err != nil { return fmt.Errorf("put lease entry: %w", err) } return nil } -// Get returns the entry for clientID. Returns (nil, nats.ErrKeyNotFound) +// Get returns the entry for clientID. Returns (nil, jetstream.ErrKeyNotFound) // when absent. -func (s *Store) Get(clientID string) (*Entry, error) { - raw, err := s.kv.Get(clientID) +func (s *Store) Get(ctx context.Context, clientID string) (*Entry, error) { + raw, err := s.kv.Get(ctx, clientID) if err != nil { return nil, err } @@ -103,9 +106,9 @@ func (s *Store) Get(clientID string) (*Entry, error) { } // Delete removes the entry. Idempotent: missing keys return nil. -func (s *Store) Delete(clientID string) error { - if err := s.kv.Delete(clientID); err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { +func (s *Store) Delete(ctx context.Context, clientID string) error { + if err := s.kv.Delete(ctx, clientID); err != nil { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil } return fmt.Errorf("delete lease entry %q: %w", clientID, err) @@ -115,11 +118,11 @@ func (s *Store) Delete(clientID string) error { // LookupByIP scans the bucket for an entry whose lease IP matches ip (and // pool name when non-empty). O(N) is fine — only called on release. -func (s *Store) LookupByIP(poolName, ip string) (*Entry, error) { +func (s *Store) LookupByIP(ctx context.Context, poolName, ip string) (*Entry, error) { if ip == "" { return nil, errors.New("dhcp store lookup: ip required") } - entries, err := s.List() + entries, err := s.List(ctx) if err != nil { return nil, err } @@ -135,15 +138,15 @@ func (s *Store) LookupByIP(poolName, ip string) (*Entry, error) { return e, nil } } - return nil, nats.ErrKeyNotFound + return nil, jetstream.ErrKeyNotFound } // List returns every entry currently in the bucket. Skips the internal -// version key written by utils.GetOrCreateKVBucket. -func (s *Store) List() ([]Entry, error) { - keys, err := s.kv.Keys() +// schema-version key the migration registry stamps on the bucket. +func (s *Store) List(ctx context.Context) ([]Entry, error) { + keys, err := s.kv.Keys(ctx) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return nil, nil } return nil, fmt.Errorf("list lease keys: %w", err) @@ -153,9 +156,9 @@ func (s *Store) List() ([]Entry, error) { if k == utils.VersionKey { continue } - e, err := s.Get(k) + e, err := s.Get(ctx, k) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { continue } slog.Warn("dhcp store list: skipping unreadable entry", "key", k, "err", err) diff --git a/spinifex/network/external/dhcp/kv_test.go b/spinifex/network/external/dhcp/kv_test.go index c22d37952..af2111b4f 100644 --- a/spinifex/network/external/dhcp/kv_test.go +++ b/spinifex/network/external/dhcp/kv_test.go @@ -7,15 +7,16 @@ import ( "github.com/mulgadc/spinifex/spinifex/network/external/dhcp" "github.com/mulgadc/spinifex/spinifex/testutil" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func newTestStore(t *testing.T, az string) *dhcp.Store { t.Helper() - _, _, js := testutil.StartTestJetStream(t) - s, err := dhcp.NewStore(js, az) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + s, err := dhcp.NewStore(t.Context(), js, az) require.NoError(t, err) return s } @@ -55,9 +56,9 @@ func TestStorePutGetRoundtrip(t *testing.T) { PoolName: "default", Lease: sampleLease("eipalloc-0a1b2c3d"), } - require.NoError(t, s.Put(e)) + require.NoError(t, s.Put(t.Context(), e)) - got, err := s.Get("eipalloc-0a1b2c3d") + got, err := s.Get(t.Context(), "eipalloc-0a1b2c3d") require.NoError(t, err) require.NotNil(t, got) @@ -77,28 +78,28 @@ func TestStorePutGetRoundtrip(t *testing.T) { func TestStoreGetMissing(t *testing.T) { s := newTestStore(t, "az1") - _, err := s.Get("absent") - assert.ErrorIs(t, err, nats.ErrKeyNotFound, "got %v", err) + _, err := s.Get(t.Context(), "absent") + assert.ErrorIs(t, err, jetstream.ErrKeyNotFound, "got %v", err) } func TestStoreDeleteIdempotent(t *testing.T) { s := newTestStore(t, "az1") - require.NoError(t, s.Delete("never-existed")) + require.NoError(t, s.Delete(t.Context(), "never-existed")) - require.NoError(t, s.Put(dhcp.Entry{Purpose: "eip", PoolName: "default", Lease: sampleLease("x")})) - require.NoError(t, s.Delete("x")) - _, err := s.Get("x") - assert.ErrorIs(t, err, nats.ErrKeyNotFound) + require.NoError(t, s.Put(t.Context(), dhcp.Entry{Purpose: "eip", PoolName: "default", Lease: sampleLease("x")})) + require.NoError(t, s.Delete(t.Context(), "x")) + _, err := s.Get(t.Context(), "x") + assert.ErrorIs(t, err, jetstream.ErrKeyNotFound) } func TestStoreListSkipsVersionKey(t *testing.T) { s := newTestStore(t, "az1") for _, id := range []string{"eipalloc-a", "eipalloc-b", "eipalloc-c"} { - require.NoError(t, s.Put(dhcp.Entry{Purpose: "eip", PoolName: "default", Lease: sampleLease(id)})) + require.NoError(t, s.Put(t.Context(), dhcp.Entry{Purpose: "eip", PoolName: "default", Lease: sampleLease(id)})) } - entries, err := s.List() + entries, err := s.List(t.Context()) require.NoError(t, err) got := map[string]bool{} for _, e := range entries { @@ -112,13 +113,13 @@ func TestStoreListSkipsVersionKey(t *testing.T) { func TestStorePutRejectsNilLease(t *testing.T) { s := newTestStore(t, "az1") - err := s.Put(dhcp.Entry{Purpose: "eip", PoolName: "default", Lease: nil}) + err := s.Put(t.Context(), dhcp.Entry{Purpose: "eip", PoolName: "default", Lease: nil}) assert.Error(t, err) } func TestStorePutRejectsEmptyClientID(t *testing.T) { s := newTestStore(t, "az1") lease := sampleLease("") - err := s.Put(dhcp.Entry{Purpose: "eip", PoolName: "default", Lease: lease}) + err := s.Put(t.Context(), dhcp.Entry{Purpose: "eip", PoolName: "default", Lease: lease}) assert.Error(t, err) } diff --git a/spinifex/network/external/dhcp/lrp_allocator_test.go b/spinifex/network/external/dhcp/lrp_allocator_test.go index 307a335b2..058450fe2 100644 --- a/spinifex/network/external/dhcp/lrp_allocator_test.go +++ b/spinifex/network/external/dhcp/lrp_allocator_test.go @@ -35,7 +35,7 @@ func TestDHCPGatewayLRPAllocatorAllocate(t *testing.T) { assert.Equal(t, 24, prefix) assert.Equal(t, "192.0.2.1", nexthop, "nexthop must come from lease.Routers[0]") - entry, err := store.Get(dhcp.GatewayLRPClientID("vpc-1")) + entry, err := store.Get(t.Context(), dhcp.GatewayLRPClientID("vpc-1")) require.NoError(t, err) assert.Equal(t, dhcp.PurposeGatewayLRP, entry.Purpose) assert.Equal(t, "vpc-1", entry.VPCID) @@ -72,7 +72,7 @@ func TestDHCPGatewayLRPAllocatorRelease(t *testing.T) { require.True(t, ok) require.NoError(t, allocator.Release(context.Background(), "vpc-1")) - _, err = store.Get(dhcp.GatewayLRPClientID("vpc-1")) + _, err = store.Get(t.Context(), dhcp.GatewayLRPClientID("vpc-1")) assert.Error(t, err) assert.Equal(t, 1, fake.ReleaseCount()) } diff --git a/spinifex/network/external/dhcp/manager.go b/spinifex/network/external/dhcp/manager.go index 4641b8008..27241a101 100644 --- a/spinifex/network/external/dhcp/manager.go +++ b/spinifex/network/external/dhcp/manager.go @@ -12,6 +12,7 @@ import ( "time" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" "golang.org/x/sync/singleflight" ) @@ -136,7 +137,7 @@ func (m *Manager) Start(ctx context.Context) error { m.parentCancel = cancel m.mu.Unlock() - entries, err := m.store.List() + entries, err := m.store.List(ctx) if err != nil { return fmt.Errorf("list dhcp leases: %w", err) } @@ -146,7 +147,7 @@ func (m *Manager) Start(ctx context.Context) error { continue } if !e.Lease.ExpiresAt().After(now) { - if delErr := m.store.Delete(e.Lease.ClientID); delErr != nil { + if delErr := m.store.Delete(ctx, e.Lease.ClientID); delErr != nil { slog.Warn("dhcp manager: drop expired lease failed", "client_id", e.Lease.ClientID, "err", delErr) } continue @@ -318,7 +319,7 @@ func (m *Manager) doAcquire(ctx context.Context, e *Entry) error { return err } e.Lease = lease - if err := m.store.Put(*e); err != nil { + if err := m.store.Put(ctx, *e); err != nil { return fmt.Errorf("persist re-acquired lease: %w", err) } return nil @@ -376,7 +377,7 @@ func (m *Manager) doRenew(ctx context.Context, e *Entry) error { return err } e.Lease = renewed - if err := m.store.Put(*e); err != nil { + if err := m.store.Put(ctx, *e); err != nil { return fmt.Errorf("persist renewed lease: %w", err) } return nil @@ -415,13 +416,17 @@ func (m *Manager) handleReleaseMsg(msg *nats.Msg) { respondReleaseErr(msg, fmt.Sprintf("decode release: %v", err)) return } + // A release must run to completion once started — the upstream DHCPRELEASE + // and the KV delete have to stay paired — so it is not tied to a caller + // deadline or to manager shutdown. + ctx := context.Background() clientID := req.ClientID if clientID == "" && req.IP != "" { - entry, err := m.store.LookupByIP(req.PoolName, req.IP) + entry, err := m.store.LookupByIP(ctx, req.PoolName, req.IP) switch { case err == nil: clientID = entry.Lease.ClientID - case errors.Is(err, nats.ErrKeyNotFound): + case errors.Is(err, jetstream.ErrKeyNotFound): slog.Warn("dhcp manager: release for unknown IP", "pool", req.PoolName, "ip", req.IP) _ = msg.Respond(emptyReleaseReply) return @@ -430,7 +435,7 @@ func (m *Manager) handleReleaseMsg(msg *nats.Msg) { return } } - if err := m.handleRelease(context.Background(), clientID); err != nil { + if err := m.handleRelease(ctx, clientID); err != nil { respondReleaseErr(msg, err.Error()) return } @@ -458,13 +463,13 @@ func (m *Manager) handleAcquire(ctx context.Context, req acquireWireRequest) (*E } func (m *Manager) acquireLocked(ctx context.Context, req acquireWireRequest) (*Entry, error) { - existing, err := m.store.Get(req.ClientID) + existing, err := m.store.Get(ctx, req.ClientID) switch { case err == nil: if existing != nil && existing.Lease != nil && existing.Lease.ExpiresAt().After(m.now()) { return existing, nil } - case errors.Is(err, nats.ErrKeyNotFound): + case errors.Is(err, jetstream.ErrKeyNotFound): // fall through to fresh acquire default: return nil, fmt.Errorf("look up lease: %w", err) @@ -486,7 +491,7 @@ func (m *Manager) acquireLocked(ctx context.Context, req acquireWireRequest) (*E return nil, err } if req.UseIfaceMAC { - if cerr := m.checkIfaceMACCollision(req.Bridge, req.ClientID, lease.IP); cerr != nil { + if cerr := m.checkIfaceMACCollision(ctx, req.Bridge, req.ClientID, lease.IP); cerr != nil { if relErr := m.client.Release(ctx, lease); relErr != nil { slog.Warn("dhcp manager: release of colliding lease failed", "client_id", req.ClientID, "err", relErr) } @@ -494,7 +499,7 @@ func (m *Manager) acquireLocked(ctx context.Context, req acquireWireRequest) (*E } } entry := Entry{Purpose: req.Purpose, PoolName: req.PoolName, VPCID: req.VPCID, Lease: lease} - if err := m.store.Put(entry); err != nil { + if err := m.store.Put(ctx, entry); err != nil { return nil, fmt.Errorf("persist new lease: %w", err) } m.spawnLoop(entry, false) @@ -505,7 +510,7 @@ func (m *Manager) acquireLocked(ctx context.Context, req acquireWireRequest) (*E // ignore option 61: every interface-MAC client then ACKs the same IP. The // ACK IP matching the interface's own address or another client-id's live // lease is a hard error — the operator must switch the pool to a static range. -func (m *Manager) checkIfaceMACCollision(iface, clientID string, ip net.IP) error { +func (m *Manager) checkIfaceMACCollision(ctx context.Context, iface, clientID string, ip net.IP) error { if ip == nil { return nil } @@ -517,7 +522,7 @@ func (m *Manager) checkIfaceMACCollision(iface, clientID string, ip net.IP) erro } } } - entries, err := m.store.List() + entries, err := m.store.List(ctx) if err != nil { slog.Warn("dhcp manager: lease list for MAC-collision check failed", "err", err) return nil @@ -556,7 +561,7 @@ func defaultIfaceIPs(name string) ([]net.IP, error) { // the KV records. Best-effort (per-lease failures logged, not fatal). Used on // destructive teardown; Stop() preserves leases for adopt-on-reboot. func (m *Manager) DrainAll(ctx context.Context) (int, error) { - entries, err := m.store.List() + entries, err := m.store.List(ctx) if err != nil { return 0, fmt.Errorf("list leases for drain: %w", err) } @@ -597,9 +602,9 @@ func (m *Manager) handleRelease(ctx context.Context, clientID string) error { if clientID == "" { return errors.New("client_id required") } - entry, err := m.store.Get(clientID) + entry, err := m.store.Get(ctx, clientID) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { slog.Warn("dhcp manager: release for unknown client", "client_id", clientID) return nil } @@ -624,7 +629,7 @@ func (m *Manager) handleRelease(ctx context.Context, clientID string) error { if err := m.client.Release(ctx, entry.Lease); err != nil { slog.Warn("dhcp manager: client release failed; deleting KV entry anyway", "client_id", clientID, "err", err) } - return m.store.Delete(clientID) + return m.store.Delete(ctx, clientID) } // LoopCount returns the number of active renewal goroutines. Test diff --git a/spinifex/network/external/dhcp/manager_test.go b/spinifex/network/external/dhcp/manager_test.go index 17efcd473..4c202a561 100644 --- a/spinifex/network/external/dhcp/manager_test.go +++ b/spinifex/network/external/dhcp/manager_test.go @@ -14,14 +14,16 @@ import ( "github.com/mulgadc/spinifex/spinifex/network/external/dhcp" "github.com/mulgadc/spinifex/spinifex/testutil" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func newTestManager(t *testing.T, az string, fake *dhcp.Fake, now func() time.Time) (*dhcp.Manager, *dhcp.Store, *nats.Conn) { t.Helper() - _, nc, js := testutil.StartTestJetStream(t) - store, err := dhcp.NewStore(js, az) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + store, err := dhcp.NewStore(t.Context(), js, az) require.NoError(t, err) mgr, err := dhcp.NewManager(dhcp.ManagerConfig{Client: fake, Store: store, Now: now}) require.NoError(t, err) @@ -50,7 +52,7 @@ func TestManagerStartScansAndReaffirms(t *testing.T) { }) require.NoError(t, err) held, _ := fake.HeldLease(id) - require.NoError(t, store.Put(dhcp.Entry{Purpose: "eip", PoolName: "default", Lease: held})) + require.NoError(t, store.Put(t.Context(), dhcp.Entry{Purpose: "eip", PoolName: "default", Lease: held})) } require.Equal(t, 0, fake.RenewCount()) @@ -73,11 +75,11 @@ func TestManagerStartDropsExpiredLeases(t *testing.T) { AcquiredAt: fixed.Add(-2 * time.Hour), LeaseDuration: time.Hour, } - require.NoError(t, store.Put(dhcp.Entry{Purpose: "eip", PoolName: "default", Lease: expired})) + require.NoError(t, store.Put(t.Context(), dhcp.Entry{Purpose: "eip", PoolName: "default", Lease: expired})) require.NoError(t, mgr.Start(context.Background())) - _, err := store.Get("eipalloc-old") - assert.ErrorIs(t, err, nats.ErrKeyNotFound) + _, err := store.Get(t.Context(), "eipalloc-old") + assert.ErrorIs(t, err, jetstream.ErrKeyNotFound) assert.Equal(t, 0, fake.RenewCount()) assert.Equal(t, 0, mgr.LoopCount()) } @@ -113,15 +115,15 @@ func TestNATSClientAcquireReleaseRoundtrip(t *testing.T) { assert.Equal(t, "eipalloc-A", lease.ClientID) assert.NotNil(t, lease.IP) - got, err := store.Get("eipalloc-A") + got, err := store.Get(t.Context(), "eipalloc-A") require.NoError(t, err) require.NotNil(t, got) assert.Equal(t, "eip", got.Purpose) assert.Equal(t, "default", got.PoolName) require.NoError(t, client.RequestRelease(context.Background(), "eipalloc-A")) - _, err = store.Get("eipalloc-A") - assert.ErrorIs(t, err, nats.ErrKeyNotFound) + _, err = store.Get(t.Context(), "eipalloc-A") + assert.ErrorIs(t, err, jetstream.ErrKeyNotFound) assert.Equal(t, 1, fake.ReleaseCount()) } @@ -137,7 +139,7 @@ func seedLeases(t *testing.T, store *dhcp.Store, ids []string) { AcquiredAt: time.Now(), LeaseDuration: time.Hour, } - require.NoError(t, store.Put(dhcp.Entry{Purpose: "eip", PoolName: "default", Lease: lease})) + require.NoError(t, store.Put(t.Context(), dhcp.Entry{Purpose: "eip", PoolName: "default", Lease: lease})) } } @@ -157,7 +159,7 @@ func TestManagerDrainAll(t *testing.T) { assert.Equal(t, len(ids), n) assert.Equal(t, len(ids), fake.ReleaseCount()) - entries, err := store.List() + entries, err := store.List(t.Context()) require.NoError(t, err) assert.Empty(t, entries) assert.Equal(t, 0, mgr.LoopCount()) @@ -182,7 +184,7 @@ func TestManagerDrainAllBestEffortOnReleaseError(t *testing.T) { assert.Equal(t, len(ids), n) assert.Equal(t, len(ids), fake.ReleaseCount()) - entries, err := store.List() + entries, err := store.List(t.Context()) require.NoError(t, err) assert.Empty(t, entries) } @@ -207,7 +209,7 @@ func TestManagerDrainMsgWithoutReplyIsDropped(t *testing.T) { require.NoError(t, nc.Flush()) time.Sleep(100 * time.Millisecond) - entries, err := store.List() + entries, err := store.List(t.Context()) require.NoError(t, err) assert.Len(t, entries, 1) assert.Equal(t, 0, fake.ReleaseCount()) @@ -237,7 +239,7 @@ func TestManagerDrainRPCRoundtrip(t *testing.T) { assert.Empty(t, reply.Error) assert.Equal(t, 2, reply.Released) - entries, err := store.List() + entries, err := store.List(t.Context()) require.NoError(t, err) assert.Empty(t, entries) } @@ -395,8 +397,9 @@ func TestManagerAcquireRetriesUnderBackoff(t *testing.T) { } // Compress the schedule to keep the test sub-second. - _, nc, js := testutil.StartTestJetStream(t) - store, err := dhcp.NewStore(js, "az1") + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + store, err := dhcp.NewStore(t.Context(), js, "az1") require.NoError(t, err) mgr, err := dhcp.NewManager(dhcp.ManagerConfig{ Client: fake, @@ -430,8 +433,9 @@ func TestManagerAcquireRetriesUnderBackoff(t *testing.T) { // Managers against a shared NATS conn. Each acquire must fire exactly one // handler (queue-group semantics) — Bug 3 regression. func TestManagerSubscribeIsQueueGroup_ExactlyOneHandlerFires(t *testing.T) { - _, nc, js := testutil.StartTestJetStream(t) - store, err := dhcp.NewStore(js, "az1") + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + store, err := dhcp.NewStore(t.Context(), js, "az1") require.NoError(t, err) fakeA := dhcp.NewFake() diff --git a/spinifex/network/external/dhcp/pool_allocator_test.go b/spinifex/network/external/dhcp/pool_allocator_test.go index 0175d33e2..03f26b26a 100644 --- a/spinifex/network/external/dhcp/pool_allocator_test.go +++ b/spinifex/network/external/dhcp/pool_allocator_test.go @@ -43,13 +43,13 @@ func TestDHCPPoolAllocatorAllocateAndRelease(t *testing.T) { require.NoError(t, err) assert.Equal(t, "192.0.2.100", addr.String()) - entry, err := store.Get("eipalloc-1") + entry, err := store.Get(t.Context(), "eipalloc-1") require.NoError(t, err) assert.Equal(t, dhcp.PurposeEIP, entry.Purpose) assert.Equal(t, "wan", entry.PoolName) require.NoError(t, allocator.Release(context.Background(), "wan", addr, "")) - _, err = store.Get("eipalloc-1") + _, err = store.Get(t.Context(), "eipalloc-1") assert.Error(t, err) assert.Equal(t, 1, fake.ReleaseCount()) } @@ -71,7 +71,7 @@ func TestDHCPPoolAllocatorClientIDPrecedence(t *testing.T) { for _, tc := range cases { _, err := allocator.Allocate(context.Background(), tc.req) require.NoError(t, err, "case %+v", tc.req) - entry, err := store.Get(tc.clientID) + entry, err := store.Get(t.Context(), tc.clientID) require.NoError(t, err, "client id %q not persisted", tc.clientID) assert.Equal(t, tc.purpose, entry.Purpose) } diff --git a/spinifex/network/external/static_pool.go b/spinifex/network/external/static_pool.go index dd2785dab..921459882 100644 --- a/spinifex/network/external/static_pool.go +++ b/spinifex/network/external/static_pool.go @@ -10,9 +10,9 @@ import ( "net/netip" "github.com/mulgadc/spinifex/spinifex/awserrors" + "github.com/mulgadc/spinifex/spinifex/kvutil" "github.com/mulgadc/spinifex/spinifex/migrate" - "github.com/mulgadc/spinifex/spinifex/utils" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) const ( @@ -62,7 +62,7 @@ type PoolRecord struct { // bucket. Bucket schema and CAS semantics are unchanged from the pre-Q1 // ExternalIPAM. type StaticPoolAllocator struct { - kv nats.KeyValue + kv jetstream.KeyValue pools []ExternalPoolConfig } @@ -70,16 +70,16 @@ var _ Allocator = (*StaticPoolAllocator)(nil) // NewStaticPoolAllocator creates the KV bucket (if missing), runs pending // migrations, and seeds each pool's record. -func NewStaticPoolAllocator(js nats.JetStreamContext, pools []ExternalPoolConfig) (*StaticPoolAllocator, error) { - kv, err := utils.GetOrCreateKVBucket(js, KVBucketStaticPool, 5) +func NewStaticPoolAllocator(ctx context.Context, js jetstream.JetStream, pools []ExternalPoolConfig) (*StaticPoolAllocator, error) { + kv, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketStaticPool, 5) if err != nil { return nil, fmt.Errorf("create external IPAM KV bucket: %w", err) } - if err := migrate.DefaultRegistry.RunKV(KVBucketStaticPool, kv, KVBucketStaticPoolVersion); err != nil { + if err := migrate.DefaultRegistry.RunKV(ctx, KVBucketStaticPool, kv, KVBucketStaticPoolVersion); err != nil { return nil, fmt.Errorf("migrate %s: %w", KVBucketStaticPool, err) } a := &StaticPoolAllocator{kv: kv, pools: pools} - if err := a.initPools(); err != nil { + if err := a.initPools(ctx); err != nil { return nil, fmt.Errorf("init external IPAM pools: %w", err) } return a, nil @@ -87,22 +87,22 @@ func NewStaticPoolAllocator(js nats.JetStreamContext, pools []ExternalPoolConfig // NewStaticPoolAllocatorWithKV is the test constructor — skips bucket // creation and migrations. -func NewStaticPoolAllocatorWithKV(kv nats.KeyValue, pools []ExternalPoolConfig) *StaticPoolAllocator { +func NewStaticPoolAllocatorWithKV(kv jetstream.KeyValue, pools []ExternalPoolConfig) *StaticPoolAllocator { return &StaticPoolAllocator{kv: kv, pools: pools} } // KV exposes the underlying bucket so callers that share the bucket // (notably ExternalIPAM facade tests) can construct sibling allocators. -func (a *StaticPoolAllocator) KV() nats.KeyValue { return a.kv } +func (a *StaticPoolAllocator) KV() jetstream.KeyValue { return a.kv } // Pools returns the allocator's pool list. Used by the ExternalIPAM // facade to satisfy pool-by-region lookups. func (a *StaticPoolAllocator) Pools() []ExternalPoolConfig { return a.pools } // Allocate implements Allocator. -func (a *StaticPoolAllocator) Allocate(_ context.Context, req AllocateRequest) (netip.Addr, error) { +func (a *StaticPoolAllocator) Allocate(ctx context.Context, req AllocateRequest) (netip.Addr, error) { for attempt := range staticPoolCASRetries { - record, revision, err := a.getRecord(req.PoolName) + record, revision, err := a.getRecord(ctx, req.PoolName) if err != nil { return netip.Addr{}, fmt.Errorf("get external IPAM record: %w", err) } @@ -125,7 +125,7 @@ func (a *StaticPoolAllocator) Allocate(_ context.Context, req AllocateRequest) ( return netip.Addr{}, fmt.Errorf("marshal external IPAM record: %w", err) } - if _, err := a.kv.Update(req.PoolName, data, revision); err != nil { + if _, err := a.kv.Update(ctx, req.PoolName, data, revision); err != nil { slog.Debug("external IPAM CAS conflict, retrying", "pool", req.PoolName, "attempt", attempt) continue } @@ -141,10 +141,10 @@ func (a *StaticPoolAllocator) Allocate(_ context.Context, req AllocateRequest) ( } // Release implements Allocator. -func (a *StaticPoolAllocator) Release(_ context.Context, poolName string, ip netip.Addr, ownerENIID string) error { +func (a *StaticPoolAllocator) Release(ctx context.Context, poolName string, ip netip.Addr, ownerENIID string) error { target := ip.String() for attempt := range staticPoolCASRetries { - record, revision, err := a.getRecord(poolName) + record, revision, err := a.getRecord(ctx, poolName) if err != nil { return fmt.Errorf("get external IPAM record for release: %w", err) } @@ -178,7 +178,7 @@ func (a *StaticPoolAllocator) Release(_ context.Context, poolName string, ip net return fmt.Errorf("marshal external IPAM record: %w", err) } - if _, err := a.kv.Update(poolName, data, revision); err != nil { + if _, err := a.kv.Update(ctx, poolName, data, revision); err != nil { slog.Debug("external IPAM release CAS conflict, retrying", "pool", poolName, "attempt", attempt) continue } @@ -190,24 +190,24 @@ func (a *StaticPoolAllocator) Release(_ context.Context, poolName string, ip net } // GetPoolRecord returns the current pool record. -func (a *StaticPoolAllocator) GetPoolRecord(poolName string) (*PoolRecord, error) { - rec, _, err := a.getRecord(poolName) +func (a *StaticPoolAllocator) GetPoolRecord(ctx context.Context, poolName string) (*PoolRecord, error) { + rec, _, err := a.getRecord(ctx, poolName) return rec, err } -func (a *StaticPoolAllocator) initPools() error { +func (a *StaticPoolAllocator) initPools(ctx context.Context) error { for _, pool := range a.pools { - if err := a.initPool(pool); err != nil { + if err := a.initPool(ctx, pool); err != nil { return fmt.Errorf("init pool %q: %w", pool.Name, err) } } return nil } -func (a *StaticPoolAllocator) initPool(pool ExternalPoolConfig) error { - chk, revision, err := a.getRecord(pool.Name) +func (a *StaticPoolAllocator) initPool(ctx context.Context, pool ExternalPoolConfig) error { + chk, revision, err := a.getRecord(ctx, pool.Name) - if err != nil && !errors.Is(err, nats.ErrKeyNotFound) { + if err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { return err } @@ -230,7 +230,7 @@ func (a *StaticPoolAllocator) initPool(pool ExternalPoolConfig) error { return fmt.Errorf("marshal external IPAM record: %w", err) } - if _, err := a.kv.Update(pool.Name, data, revision); err != nil { + if _, err := a.kv.Update(ctx, pool.Name, data, revision); err != nil { slog.Warn("external IPAM update failed", "pool", pool.Name, "err", err) return err } @@ -268,8 +268,8 @@ func (a *StaticPoolAllocator) initPool(pool ExternalPoolConfig) error { return fmt.Errorf("marshal pool record: %w", err) } - if _, err := a.kv.Create(pool.Name, data); err != nil { - if errors.Is(err, nats.ErrKeyExists) { + if _, err := a.kv.Create(ctx, pool.Name, data); err != nil { + if errors.Is(err, jetstream.ErrKeyExists) { return nil } return fmt.Errorf("create pool KV entry: %w", err) @@ -279,8 +279,8 @@ func (a *StaticPoolAllocator) initPool(pool ExternalPoolConfig) error { return nil } -func (a *StaticPoolAllocator) getRecord(poolName string) (*PoolRecord, uint64, error) { - entry, err := a.kv.Get(poolName) +func (a *StaticPoolAllocator) getRecord(ctx context.Context, poolName string) (*PoolRecord, uint64, error) { + entry, err := a.kv.Get(ctx, poolName) if err != nil { return nil, 0, err } diff --git a/spinifex/network/external/static_pool_test.go b/spinifex/network/external/static_pool_test.go index 5ec329013..e93f676af 100644 --- a/spinifex/network/external/static_pool_test.go +++ b/spinifex/network/external/static_pool_test.go @@ -14,8 +14,9 @@ import ( func newStaticAllocator(t *testing.T, pools []ExternalPoolConfig) *StaticPoolAllocator { t.Helper() - _, _, js := testutil.StartTestJetStream(t) - a, err := NewStaticPoolAllocator(js, pools) + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + a, err := NewStaticPoolAllocator(t.Context(), js, pools) require.NoError(t, err) return a } @@ -53,7 +54,7 @@ func TestStaticPool_AllocateSequential(t *testing.T) { func TestStaticPool_GatewayReserved(t *testing.T) { a := newStaticAllocator(t, []ExternalPoolConfig{wanPool()}) - rec, err := a.GetPoolRecord("wan") + rec, err := a.GetPoolRecord(t.Context(), "wan") require.NoError(t, err) alloc, ok := rec.Allocated["192.168.1.150"] require.True(t, ok) @@ -161,7 +162,7 @@ func TestStaticPool_Release_OwnershipScoped(t *testing.T) { // Stale teardown of a prior owner must not free the live lease. require.NoError(t, a.Release(ctx, "wan", ip, "eni-stale")) - rec, err := a.GetPoolRecord("wan") + rec, err := a.GetPoolRecord(ctx, "wan") require.NoError(t, err) alloc, ok := rec.Allocated[ip.String()] require.True(t, ok, "foreign-ENI release must not free the live lease") @@ -169,7 +170,7 @@ func TestStaticPool_Release_OwnershipScoped(t *testing.T) { // The current owner releases it. require.NoError(t, a.Release(ctx, "wan", ip, "eni-live")) - rec, err = a.GetPoolRecord("wan") + rec, err = a.GetPoolRecord(ctx, "wan") require.NoError(t, err) _, ok = rec.Allocated[ip.String()] assert.False(t, ok, "owner-scoped release must free the lease") diff --git a/spinifex/network/reconcile/drift.go b/spinifex/network/reconcile/drift.go index 57b28010e..de4041633 100644 --- a/spinifex/network/reconcile/drift.go +++ b/spinifex/network/reconcile/drift.go @@ -6,6 +6,7 @@ import ( "time" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // DriftInterval is the gap between drift passes. Var (not const) so @@ -15,7 +16,7 @@ var DriftInterval = 5 * time.Minute // DriftLoop runs Reconcile every DriftInterval, gated on AcquireLeader so // only one vpcd scans at a time. Returns when ctx is cancelled. func DriftLoop(ctx context.Context, rec Reconciler, nc *nats.Conn, localAZ, holder string) { - js, err := nc.JetStream() + js, err := jetstream.New(nc) if err != nil { slog.Error("reconcile/drift: JetStream context unavailable, drift loop disabled", "err", err) return @@ -35,8 +36,8 @@ func DriftLoop(ctx context.Context, rec Reconciler, nc *nats.Conn, localAZ, hold } // runDriftCycle is one tick body, split out so tests can drive it directly. -func runDriftCycle(ctx context.Context, rec Reconciler, nc *nats.Conn, js nats.JetStreamContext, localAZ, holder string) { - release, elected := AcquireLeader(nc, KVBucketVPCDReconcile, holder) +func runDriftCycle(ctx context.Context, rec Reconciler, nc *nats.Conn, js jetstream.JetStream, localAZ, holder string) { + release, elected := AcquireLeader(ctx, nc, KVBucketVPCDReconcile, holder) if !elected { return } diff --git a/spinifex/network/reconcile/intent.go b/spinifex/network/reconcile/intent.go index caa509798..f4cbb5c23 100644 --- a/spinifex/network/reconcile/intent.go +++ b/spinifex/network/reconcile/intent.go @@ -19,7 +19,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/network/policy" "github.com/mulgadc/spinifex/spinifex/network/topology" "github.com/mulgadc/spinifex/spinifex/utils" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // IntentState is the desired OVN state derived from the JetStream KV @@ -52,7 +52,7 @@ func subnetEgressKey(subnetID string, prefix netip.Prefix) string { // LoadIntentFromKV assembles IntentState for localAZ. Missing buckets are empty. // AZ filter: `vpc.AZ == "" || vpc.AZ == localAZ`; children inherit it transitively. -func LoadIntentFromKV(ctx context.Context, js nats.JetStreamContext, localAZ string) (IntentState, error) { +func LoadIntentFromKV(ctx context.Context, js jetstream.JetStream, localAZ string) (IntentState, error) { intent := IntentState{ VPCs: make(map[string]topology.VPCSpec), Subnets: make(map[string]topology.SubnetSpec), @@ -66,36 +66,35 @@ func LoadIntentFromKV(ctx context.Context, js nats.JetStreamContext, localAZ str DropGates: make(map[string]SubnetEgressIntent), } - localVPCs, err := loadVPCs(js, localAZ, intent.VPCs) + localVPCs, err := loadVPCs(ctx, js, localAZ, intent.VPCs) if err != nil { return IntentState{}, err } - if err := loadSubnets(js, localVPCs, intent.Subnets); err != nil { + if err := loadSubnets(ctx, js, localVPCs, intent.Subnets); err != nil { return IntentState{}, err } - if err := loadSGs(js, localVPCs, intent.SGs); err != nil { + if err := loadSGs(ctx, js, localVPCs, intent.SGs); err != nil { return IntentState{}, err } - if err := loadPorts(js, localVPCs, intent.Ports); err != nil { + if err := loadPorts(ctx, js, localVPCs, intent.Ports); err != nil { return IntentState{}, err } - if err := loadIGWs(js, localVPCs, intent.IGWs); err != nil { + if err := loadIGWs(ctx, js, localVPCs, intent.IGWs); err != nil { return IntentState{}, err } - if err := loadEIPs(js, localVPCs, intent.EIPs); err != nil { + if err := loadEIPs(ctx, js, localVPCs, intent.EIPs); err != nil { return IntentState{}, err } - routeTables, err := loadRouteTables(js, localVPCs) + routeTables, err := loadRouteTables(ctx, js, localVPCs) if err != nil { return IntentState{}, err } - if err := loadNATGWs(js, localVPCs, intent.Subnets, routeTables, intent.NATGWs); err != nil { + if err := loadNATGWs(ctx, js, localVPCs, intent.Subnets, routeTables, intent.NATGWs); err != nil { return IntentState{}, err } loadSubnetEgressRoutes(localVPCs, intent.Subnets, routeTables, intent.IGWRoutes, intent.NATGWRoutes) loadSubnetDropGates(localVPCs, intent.Subnets, intent.IGWs, intent.IGWRoutes, intent.NATGWRoutes, intent.DropGates) - _ = ctx return intent, nil } @@ -218,17 +217,17 @@ func matchesLocalAZ(vpcAZ, localAZ string) bool { func keyIsVersion(key string) bool { return key == utils.VersionKey } -func loadVPCs(js nats.JetStreamContext, localAZ string, out map[string]topology.VPCSpec) (map[string]struct{}, error) { +func loadVPCs(ctx context.Context, js jetstream.JetStream, localAZ string, out map[string]topology.VPCSpec) (map[string]struct{}, error) { localVPCs := make(map[string]struct{}) - kv, err := js.KeyValue(handlers_ec2_vpc.KVBucketVPCs) + kv, err := js.KeyValue(ctx, handlers_ec2_vpc.KVBucketVPCs) if err != nil { slog.Debug("reconcile/intent: VPC bucket not available, skipping", "err", err) return localVPCs, nil } - keys, err := kv.Keys() + keys, err := kv.Keys(ctx) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return localVPCs, nil } return nil, fmt.Errorf("list VPC keys: %w", err) @@ -237,7 +236,7 @@ func loadVPCs(js nats.JetStreamContext, localAZ string, out map[string]topology. if keyIsVersion(key) { continue } - entry, err := kv.Get(key) + entry, err := kv.Get(ctx, key) if err != nil { slog.Warn("reconcile/intent: VPC read failed", "key", key, "err", err) continue @@ -265,15 +264,15 @@ func loadVPCs(js nats.JetStreamContext, localAZ string, out map[string]topology. return localVPCs, nil } -func loadSubnets(js nats.JetStreamContext, localVPCs map[string]struct{}, out map[string]topology.SubnetSpec) error { - kv, err := js.KeyValue(handlers_ec2_vpc.KVBucketSubnets) +func loadSubnets(ctx context.Context, js jetstream.JetStream, localVPCs map[string]struct{}, out map[string]topology.SubnetSpec) error { + kv, err := js.KeyValue(ctx, handlers_ec2_vpc.KVBucketSubnets) if err != nil { slog.Debug("reconcile/intent: subnet bucket not available, skipping", "err", err) return nil } - keys, err := kv.Keys() + keys, err := kv.Keys(ctx) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return nil } return fmt.Errorf("list subnet keys: %w", err) @@ -282,7 +281,7 @@ func loadSubnets(js nats.JetStreamContext, localVPCs map[string]struct{}, out ma if keyIsVersion(key) { continue } - entry, err := kv.Get(key) + entry, err := kv.Get(ctx, key) if err != nil { slog.Warn("reconcile/intent: subnet read failed", "key", key, "err", err) continue @@ -309,15 +308,15 @@ func loadSubnets(js nats.JetStreamContext, localVPCs map[string]struct{}, out ma return nil } -func loadSGs(js nats.JetStreamContext, localVPCs map[string]struct{}, out map[string]policy.SGSpec) error { - kv, err := js.KeyValue(handlers_ec2_vpc.KVBucketSecurityGroups) +func loadSGs(ctx context.Context, js jetstream.JetStream, localVPCs map[string]struct{}, out map[string]policy.SGSpec) error { + kv, err := js.KeyValue(ctx, handlers_ec2_vpc.KVBucketSecurityGroups) if err != nil { slog.Debug("reconcile/intent: SG bucket not available, skipping", "err", err) return nil } - keys, err := kv.Keys() + keys, err := kv.Keys(ctx) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return nil } return fmt.Errorf("list SG keys: %w", err) @@ -326,7 +325,7 @@ func loadSGs(js nats.JetStreamContext, localVPCs map[string]struct{}, out map[st if keyIsVersion(key) { continue } - entry, err := kv.Get(key) + entry, err := kv.Get(ctx, key) if err != nil { slog.Warn("reconcile/intent: SG read failed", "key", key, "err", err) continue @@ -349,15 +348,15 @@ func loadSGs(js nats.JetStreamContext, localVPCs map[string]struct{}, out map[st return nil } -func loadPorts(js nats.JetStreamContext, localVPCs map[string]struct{}, out map[string]topology.PortSpec) error { - kv, err := js.KeyValue(handlers_ec2_vpc.KVBucketENIs) +func loadPorts(ctx context.Context, js jetstream.JetStream, localVPCs map[string]struct{}, out map[string]topology.PortSpec) error { + kv, err := js.KeyValue(ctx, handlers_ec2_vpc.KVBucketENIs) if err != nil { slog.Debug("reconcile/intent: ENI bucket not available, skipping", "err", err) return nil } - keys, err := kv.Keys() + keys, err := kv.Keys(ctx) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return nil } return fmt.Errorf("list ENI keys: %w", err) @@ -366,7 +365,7 @@ func loadPorts(js nats.JetStreamContext, localVPCs map[string]struct{}, out map[ if keyIsVersion(key) { continue } - entry, err := kv.Get(key) + entry, err := kv.Get(ctx, key) if err != nil { slog.Warn("reconcile/intent: ENI read failed", "key", key, "err", err) continue @@ -405,15 +404,15 @@ func loadPorts(js nats.JetStreamContext, localVPCs map[string]struct{}, out map[ return nil } -func loadIGWs(js nats.JetStreamContext, localVPCs map[string]struct{}, out map[string]external.IGWSpec) error { - kv, err := js.KeyValue(handlers_ec2_igw.KVBucketIGW) +func loadIGWs(ctx context.Context, js jetstream.JetStream, localVPCs map[string]struct{}, out map[string]external.IGWSpec) error { + kv, err := js.KeyValue(ctx, handlers_ec2_igw.KVBucketIGW) if err != nil { slog.Debug("reconcile/intent: IGW bucket not available, skipping", "err", err) return nil } - keys, err := kv.Keys() + keys, err := kv.Keys(ctx) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return nil } return fmt.Errorf("list IGW keys: %w", err) @@ -422,7 +421,7 @@ func loadIGWs(js nats.JetStreamContext, localVPCs map[string]struct{}, out map[s if keyIsVersion(key) { continue } - entry, err := kv.Get(key) + entry, err := kv.Get(ctx, key) if err != nil { slog.Warn("reconcile/intent: IGW read failed", "key", key, "err", err) continue @@ -446,15 +445,15 @@ func loadIGWs(js nats.JetStreamContext, localVPCs map[string]struct{}, out map[s return nil } -func loadEIPs(js nats.JetStreamContext, localVPCs map[string]struct{}, out map[string]policy.EIPSpec) error { - kv, err := js.KeyValue(handlers_ec2_eip.KVBucketEIPs) +func loadEIPs(ctx context.Context, js jetstream.JetStream, localVPCs map[string]struct{}, out map[string]policy.EIPSpec) error { + kv, err := js.KeyValue(ctx, handlers_ec2_eip.KVBucketEIPs) if err != nil { slog.Debug("reconcile/intent: EIP bucket not available, skipping", "err", err) return nil } - keys, err := kv.Keys() + keys, err := kv.Keys(ctx) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return nil } return fmt.Errorf("list EIP keys: %w", err) @@ -463,7 +462,7 @@ func loadEIPs(js nats.JetStreamContext, localVPCs map[string]struct{}, out map[s if keyIsVersion(key) { continue } - entry, err := kv.Get(key) + entry, err := kv.Get(ctx, key) if err != nil { slog.Warn("reconcile/intent: EIP read failed", "key", key, "err", err) continue @@ -495,15 +494,15 @@ func loadEIPs(js nats.JetStreamContext, localVPCs map[string]struct{}, out map[s } // loadRouteTables snapshots every local-VPC route table. -func loadRouteTables(js nats.JetStreamContext, localVPCs map[string]struct{}) ([]handlers_ec2_routetable.RouteTableRecord, error) { - kv, err := js.KeyValue(handlers_ec2_routetable.KVBucketRouteTables) +func loadRouteTables(ctx context.Context, js jetstream.JetStream, localVPCs map[string]struct{}) ([]handlers_ec2_routetable.RouteTableRecord, error) { + kv, err := js.KeyValue(ctx, handlers_ec2_routetable.KVBucketRouteTables) if err != nil { slog.Debug("reconcile/intent: route table bucket not available, skipping", "err", err) return nil, nil } - keys, err := kv.Keys() + keys, err := kv.Keys(ctx) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return nil, nil } return nil, fmt.Errorf("list route table keys: %w", err) @@ -513,7 +512,7 @@ func loadRouteTables(js nats.JetStreamContext, localVPCs map[string]struct{}) ([ if keyIsVersion(key) { continue } - entry, err := kv.Get(key) + entry, err := kv.Get(ctx, key) if err != nil { slog.Warn("reconcile/intent: route table read failed", "key", key, "err", err) continue @@ -539,20 +538,21 @@ func natgwSpecKey(natgwID, subnetCIDR string) string { // loadNATGWs emits one NATGWSpec per (NATGW, associated private subnet) pair. // SNAT rewrites traffic from the private subnets routed through the NATGW, not its home subnet. func loadNATGWs( - js nats.JetStreamContext, + ctx context.Context, + js jetstream.JetStream, localVPCs map[string]struct{}, subnets map[string]topology.SubnetSpec, routeTables []handlers_ec2_routetable.RouteTableRecord, out map[string]policy.NATGWSpec, ) error { - kv, err := js.KeyValue(handlers_ec2_natgw.KVBucketNatGateways) + kv, err := js.KeyValue(ctx, handlers_ec2_natgw.KVBucketNatGateways) if err != nil { slog.Debug("reconcile/intent: NAT GW bucket not available, skipping", "err", err) return nil } - keys, err := kv.Keys() + keys, err := kv.Keys(ctx) if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { + if errors.Is(err, jetstream.ErrNoKeysFound) { return nil } return fmt.Errorf("list NAT GW keys: %w", err) @@ -561,7 +561,7 @@ func loadNATGWs( if keyIsVersion(key) { continue } - entry, err := kv.Get(key) + entry, err := kv.Get(ctx, key) if err != nil { slog.Warn("reconcile/intent: NAT GW read failed", "key", key, "err", err) continue diff --git a/spinifex/network/reconcile/intent_test.go b/spinifex/network/reconcile/intent_test.go index be3b72b62..1f84757a0 100644 --- a/spinifex/network/reconcile/intent_test.go +++ b/spinifex/network/reconcile/intent_test.go @@ -13,8 +13,17 @@ import ( handlers_ec2_routetable "github.com/mulgadc/spinifex/spinifex/handlers/ec2/routetable" handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc" "github.com/mulgadc/spinifex/spinifex/testutil" + "github.com/nats-io/nats.go/jetstream" ) +// startKV returns a jetstream handle onto an embedded server, used both to seed +// the control-plane buckets and to load intent through them. +func startKV(t *testing.T) jetstream.JetStream { + t.Helper() + _, _, js := testutil.StartTestJetStream(t) + return js +} + func TestMatchesLocalAZ(t *testing.T) { cases := []struct { vpcAZ, localAZ string @@ -33,7 +42,7 @@ func TestMatchesLocalAZ(t *testing.T) { } func TestLoadIntentFromKV_AZFilter(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) + js := startKV(t) localVPC := handlers_ec2_vpc.VPCRecord{ VpcId: "vpc-local", CidrBlock: "10.0.0.0/16", State: "available", VNI: 100, AZ: "us-east-1a", CreatedAt: time.Now(), @@ -67,7 +76,7 @@ func TestLoadIntentFromKV_AZFilter(t *testing.T) { } func TestLoadIntentFromKV_TransitiveSubnetFilter(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) + js := startKV(t) testutil.SeedKV(t, js, handlers_ec2_vpc.KVBucketVPCs, map[string][]byte{ "acct/vpc-local": mustJSON(t, handlers_ec2_vpc.VPCRecord{ @@ -101,7 +110,7 @@ func TestLoadIntentFromKV_TransitiveSubnetFilter(t *testing.T) { } func TestLoadIntentFromKV_EIPStateFilter(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) + js := startKV(t) testutil.SeedKV(t, js, handlers_ec2_vpc.KVBucketVPCs, map[string][]byte{ "acct/vpc-local": mustJSON(t, handlers_ec2_vpc.VPCRecord{ @@ -133,7 +142,7 @@ func TestLoadIntentFromKV_EIPStateFilter(t *testing.T) { } func TestLoadIntentFromKV_IGWAttachedFilter(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) + js := startKV(t) testutil.SeedKV(t, js, handlers_ec2_vpc.KVBucketVPCs, map[string][]byte{ "acct/vpc-local": mustJSON(t, handlers_ec2_vpc.VPCRecord{ @@ -166,7 +175,7 @@ func TestLoadIntentFromKV_IGWAttachedFilter(t *testing.T) { // specs for associated *private* subnets, not the NATGW's own public home subnet. // Wrong CIDR corrupts conntrack reverse-NAT and causes 100% packet loss. func TestLoadIntentFromKV_NATGWUsesAssociatedSubnet(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) + js := startKV(t) testutil.SeedKV(t, js, handlers_ec2_vpc.KVBucketVPCs, map[string][]byte{ "acct/vpc-local": mustJSON(t, handlers_ec2_vpc.VPCRecord{ @@ -217,7 +226,7 @@ func TestLoadIntentFromKV_NATGWUsesAssociatedSubnet(t *testing.T) { // TestLoadIntentFromKV_NATGWNoAssociationSkips guards against SNAT install for // a NATGW with no route-table association; emitting the home-subnet CIDR would corrupt the SNAT table. func TestLoadIntentFromKV_NATGWNoAssociationSkips(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) + js := startKV(t) testutil.SeedKV(t, js, handlers_ec2_vpc.KVBucketVPCs, map[string][]byte{ "acct/vpc-local": mustJSON(t, handlers_ec2_vpc.VPCRecord{ @@ -244,7 +253,7 @@ func TestLoadIntentFromKV_NATGWNoAssociationSkips(t *testing.T) { } func TestLoadIntentFromKV_NoBucketsTolerated(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) + js := startKV(t) intent, err := LoadIntentFromKV(context.Background(), js, "us-east-1a") if err != nil { @@ -259,7 +268,7 @@ func TestLoadIntentFromKV_NoBucketsTolerated(t *testing.T) { // dropped before subscribers attach; reconcile must re-derive per-subnet egress // intent from the main RT, including implicit-main subnets. func TestLoadIntentFromKV_IGWRoutesFanOutMainRT(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) + js := startKV(t) testutil.SeedKV(t, js, handlers_ec2_vpc.KVBucketVPCs, map[string][]byte{ "acct/vpc-local": mustJSON(t, handlers_ec2_vpc.VPCRecord{ @@ -311,7 +320,7 @@ func TestLoadIntentFromKV_IGWRoutesFanOutMainRT(t *testing.T) { // TestLoadIntentFromKV_DropGatesForUnroutedSubnetWithIGW: a subnet with no // 0.0.0.0/0 route in an IGW-attached VPC must produce a DropGates intent. func TestLoadIntentFromKV_DropGatesForUnroutedSubnetWithIGW(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) + js := startKV(t) testutil.SeedKV(t, js, handlers_ec2_vpc.KVBucketVPCs, map[string][]byte{ "acct/vpc-local": mustJSON(t, handlers_ec2_vpc.VPCRecord{ @@ -367,7 +376,7 @@ func TestLoadIntentFromKV_DropGatesForUnroutedSubnetWithIGW(t *testing.T) { // TestLoadIntentFromKV_NoDropGateWithoutIGW: VPC with no IGW has no router-wide // default static route; lr_in_ip_routing already drops, so no drop policy needed. func TestLoadIntentFromKV_NoDropGateWithoutIGW(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) + js := startKV(t) testutil.SeedKV(t, js, handlers_ec2_vpc.KVBucketVPCs, map[string][]byte{ "acct/vpc-air-gapped": mustJSON(t, handlers_ec2_vpc.VPCRecord{ diff --git a/spinifex/network/reconcile/lock.go b/spinifex/network/reconcile/lock.go index 5dd33c82d..7e7e33f8d 100644 --- a/spinifex/network/reconcile/lock.go +++ b/spinifex/network/reconcile/lock.go @@ -1,11 +1,13 @@ package reconcile import ( + "context" "errors" "log/slog" "time" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // Single CAS-elected leader key; TTL bounds crash-recovery. @@ -13,6 +15,10 @@ const ( KVBucketVPCDReconcile = "spinifex-vpcd-reconcile" reconcileLeaderKey = "leader" reconcileLeaderTTL = 60 * time.Second + + // reconcileReleaseTimeout bounds the lock delete on the way out, which runs + // on a detached context and so cannot inherit the caller's deadline. + reconcileReleaseTimeout = 5 * time.Second ) // Bounded wait for JetStream quorum on cold multi-node start. Vars (not @@ -26,22 +32,22 @@ var ( // reconcile loops pass distinct buckets so they never share a single mutex: the // gateway quota reconcile must not block vpcd's network reconcile, and vice // versa. -func AcquireLeader(nc *nats.Conn, bucket, holder string) (func(), bool) { - js, err := nc.JetStream() +func AcquireLeader(ctx context.Context, nc *nats.Conn, bucket, holder string) (func(), bool) { + js, err := jetstream.New(nc) if err != nil { slog.Error("reconcile/lock: JetStream unavailable, skipping reconcile", "holder", holder, "bucket", bucket, "err", err) return nil, false } - var kv nats.KeyValue + var kv jetstream.KeyValue deadline := time.Now().Add(leaderRetryFor) for { // Get-or-create: CreateKeyValue returns "stream name already in use" if // the bucket exists; attach first, create only when genuinely absent. - kv, err = js.KeyValue(bucket) - if errors.Is(err, nats.ErrBucketNotFound) { - kv, err = js.CreateKeyValue(&nats.KeyValueConfig{ + kv, err = js.KeyValue(ctx, bucket) + if errors.Is(err, jetstream.ErrBucketNotFound) { + kv, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{ Bucket: bucket, History: 1, TTL: reconcileLeaderTTL, @@ -56,17 +62,28 @@ func AcquireLeader(nc *nats.Conn, bucket, holder string) (func(), bool) { return nil, false } slog.Debug("reconcile/lock: JetStream KV not ready, retrying", "holder", holder, "bucket", bucket, "err", err) - time.Sleep(leaderRetryStep) + // A shutdown mid-wait must not sit out the remaining retry window. + select { + case <-ctx.Done(): + slog.Info("reconcile/lock: cancelled while waiting for JetStream KV", "holder", holder, "bucket", bucket) + return nil, false + case <-time.After(leaderRetryStep): + } } - if _, err := kv.Create(reconcileLeaderKey, []byte(holder)); err != nil { + if _, err := kv.Create(ctx, reconcileLeaderKey, []byte(holder)); err != nil { slog.Info("reconcile/lock: another holder is leader, skipping reconcile", "holder", holder, "bucket", bucket, "err", err) return nil, false } slog.Info("reconcile/lock: elected", "holder", holder, "bucket", bucket) return func() { - if err := kv.Delete(reconcileLeaderKey); err != nil { + // Release outlives ctx: shutdown is the common reason to release, and + // skipping the delete would park the lock for the full TTL and stall + // every other node's reconcile. + releaseCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), reconcileReleaseTimeout) + defer cancel() + if err := kv.Delete(releaseCtx, reconcileLeaderKey); err != nil { slog.Warn("reconcile/lock: failed to release lock (TTL will reap)", "holder", holder, "bucket", bucket, "err", err) } }, true diff --git a/spinifex/network/reconcile/lock_test.go b/spinifex/network/reconcile/lock_test.go index 699f28094..121385b53 100644 --- a/spinifex/network/reconcile/lock_test.go +++ b/spinifex/network/reconcile/lock_test.go @@ -1,6 +1,7 @@ package reconcile import ( + "context" "testing" "github.com/mulgadc/spinifex/spinifex/testutil" @@ -14,20 +15,39 @@ import ( func TestAcquireLeader_GetOrCreateAcrossCalls(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) - release, ok := AcquireLeader(nc, KVBucketVPCDReconcile, "node-1") + release, ok := AcquireLeader(t.Context(), nc, KVBucketVPCDReconcile, "node-1") require.True(t, ok, "first acquire must win and create the bucket") require.NotNil(t, release) // Bucket now exists; the second acquire must reach the Create-key contention // path (and lose) rather than dead-ending on CreateKeyValue. - loserRelease, ok := AcquireLeader(nc, KVBucketVPCDReconcile, "node-2") + loserRelease, ok := AcquireLeader(t.Context(), nc, KVBucketVPCDReconcile, "node-2") assert.False(t, ok, "second acquire must lose while the lock is held") assert.Nil(t, loserRelease) // Releasing frees the key so a subsequent acquire can take over. release() - release2, ok := AcquireLeader(nc, KVBucketVPCDReconcile, "node-3") + release2, ok := AcquireLeader(t.Context(), nc, KVBucketVPCDReconcile, "node-3") require.True(t, ok, "acquire after release must win") require.NotNil(t, release2) release2() } + +// TestAcquireLeader_ReleaseSurvivesCancelledContext pins that the release runs +// on a context of its own. Shutdown cancels the acquiring context first and then +// releases, so a release bound to that context would silently no-op and park the +// lock for the full TTL, blocking every other node's reconcile. +func TestAcquireLeader_ReleaseSurvivesCancelledContext(t *testing.T) { + _, nc, _ := testutil.StartTestJetStream(t) + + ctx, cancel := context.WithCancel(t.Context()) + release, ok := AcquireLeader(ctx, nc, KVBucketVPCDReconcile, "node-1") + require.True(t, ok) + + cancel() + release() + + release2, ok := AcquireLeader(t.Context(), nc, KVBucketVPCDReconcile, "node-2") + require.True(t, ok, "lock must be free immediately after release, not held until the TTL reaps it") + release2() +} diff --git a/spinifex/network/reconcile/scenario_kv_live_test.go b/spinifex/network/reconcile/scenario_kv_live_test.go index 6c90bdedc..912d4eb3b 100644 --- a/spinifex/network/reconcile/scenario_kv_live_test.go +++ b/spinifex/network/reconcile/scenario_kv_live_test.go @@ -11,25 +11,24 @@ import ( "testing" vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc" - "github.com/mulgadc/spinifex/spinifex/testutil" - "github.com/mulgadc/spinifex/spinifex/utils" + "github.com/mulgadc/spinifex/spinifex/kvutil" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // seedKV marshals rec and stores it under key in the named JetStream KV bucket, // creating the bucket if absent — mirroring how the VPC handlers persist state. -func seedKV(t *testing.T, js nats.JetStreamContext, bucket, key string, rec any) { +func seedKV(t *testing.T, js jetstream.JetStream, bucket, key string, rec any) { t.Helper() - kv, err := utils.GetOrCreateKVBucket(js, bucket, 10) + kv, err := kvutil.GetOrCreateBucket(t.Context(), js, bucket, 10) if err != nil { - t.Fatalf("GetOrCreateKVBucket %s: %v", bucket, err) + t.Fatalf("GetOrCreateBucket %s: %v", bucket, err) } b, err := json.Marshal(rec) if err != nil { t.Fatalf("marshal %s/%s: %v", bucket, key, err) } - if _, err := kv.Put(key, b); err != nil { + if _, err := kv.Put(t.Context(), key, b); err != nil { t.Fatalf("kv.Put %s/%s: %v", bucket, key, err) } } @@ -38,7 +37,7 @@ func seedKV(t *testing.T, js nats.JetStreamContext, bucket, key string, rec any) // control-plane KV, loads intent from it, reconciles into a real NB DB, and // asserts the expected rows plus idempotency. func TestScenario_LoadIntentFromKV_Live(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) + js := startKV(t) seedKV(t, js, vpc.KVBucketVPCs, "vpc-a", vpc.VPCRecord{ VpcId: "vpc-a", CidrBlock: "10.0.0.0/16", VNI: 100, diff --git a/spinifex/services/awsgw/awsgw.go b/spinifex/services/awsgw/awsgw.go index d25e70e53..e765e933f 100644 --- a/spinifex/services/awsgw/awsgw.go +++ b/spinifex/services/awsgw/awsgw.go @@ -29,6 +29,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/objectstore" "github.com/mulgadc/spinifex/spinifex/utils" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" toml "github.com/pelletier/go-toml/v2" ) @@ -116,22 +117,26 @@ func loadAWSGWConfig(path string) (awsgwTOML, error) { // openAccountUsageBucket opens (or idempotently creates) the gateway-owned // per-account vCPU usage bucket. History is 1: each account key holds a single -// CAS-updated integer counter. It attaches first and creates only when the bucket -// is genuinely absent, so a transient create error is not masked by the fallback. -func openAccountUsageBucket(js nats.JetStreamContext, replicas int) (nats.KeyValue, error) { - if replicas < 1 { - replicas = 1 +// CAS-updated integer counter, so no revision beyond the latest is worth keeping. +func openAccountUsageBucket(ctx context.Context, js jetstream.KeyValueManager, replicas int) (jetstream.KeyValue, error) { + kv, err := js.KeyValue(ctx, handlers_quota.KVBucketAccountUsage) + if err == nil { + return kv, nil } - kv, err := js.KeyValue(handlers_quota.KVBucketAccountUsage) - if errors.Is(err, nats.ErrBucketNotFound) { - kv, err = js.CreateKeyValue(&nats.KeyValueConfig{ - Bucket: handlers_quota.KVBucketAccountUsage, - History: 1, - Replicas: replicas, - }) + if !errors.Is(err, jetstream.ErrBucketNotFound) { + return nil, fmt.Errorf("open account usage bucket: %w", err) + } + + kv, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{ + Bucket: handlers_quota.KVBucketAccountUsage, + History: 1, + Replicas: max(replicas, 1), + }) + if errors.Is(err, jetstream.ErrBucketExists) { + return js.KeyValue(ctx, handlers_quota.KVBucketAccountUsage) } if err != nil { - return nil, fmt.Errorf("open account usage bucket: %w", err) + return nil, fmt.Errorf("create account usage bucket: %w", err) } return kv, nil } @@ -157,26 +162,28 @@ func launchService(config *config.ClusterConfig) error { return fmt.Errorf("load IAM master key from %s: %w", masterKeyPath, err) } + // Bound to the process lifetime — the server below blocks until exit, so + // cancelling on return is sufficient to let the background goroutines drain. + // Every bucket the gateway opens at startup hangs off it too. + janitorCtx, cancelJanitor := context.WithCancel(context.Background()) + defer cancelJanitor() + // Initialize IAM service with NATS KV backend (required for auth). // On multi-node clusters, JetStream KV requires cluster quorum which may // not be available yet if nodes start concurrently. Retry with backoff. - iamService, err := handlers_iam.NewIAMServiceWithRetry(natsConn, masterKey, len(config.Nodes)) + iamService, err := handlers_iam.NewIAMServiceWithRetry(janitorCtx, natsConn, masterKey, len(config.Nodes)) if err != nil { return fmt.Errorf("initialize IAM service: %w", err) } // STS service shares the IAM master key (single envelope for at-rest // secrets + session-token HMACs) and resolves roles via IAMService. - stsService, err := handlers_sts.NewSTSServiceImpl(natsConn, iamService, masterKey, len(config.Nodes)) + stsService, err := handlers_sts.NewSTSServiceImpl(janitorCtx, natsConn, iamService, masterKey, len(config.Nodes)) if err != nil { return fmt.Errorf("initialize STS service: %w", err) } - // Janitor sweeps expired session credentials. Bound to the process - // lifetime — the server below blocks until exit, so cancelling on return - // is sufficient to let the goroutine drain. - janitorCtx, cancelJanitor := context.WithCancel(context.Background()) - defer cancelJanitor() + // Janitor sweeps expired session credentials. go stsService.RunJanitor(janitorCtx) // IMDS serves 169.254.169.254 to guest VMs from vpcd, which holds the network @@ -256,12 +263,13 @@ func launchService(config *config.ClusterConfig) error { // ECR auth bridge: load (or first-run create) the ES256 signing key from the // cluster-replicated awsgw-keys KV bucket, then build the token issuer - // (GetAuthorizationToken) and verifier (/v2 Authorization). - js, err := natsConn.JetStream() + // (GetAuthorizationToken) and verifier (/v2 Authorization). The bridge reads + // and rotates its keys under the janitor lifetime context. + js, err := jetstream.New(natsConn) if err != nil { - return fmt.Errorf("ECR auth bridge: JetStream context: %w", err) + return fmt.Errorf("jetstream client: %w", err) } - signingKey, verifyKeys, err := gateway_ecrauth.LoadOrCreateSigningKey(js, masterKey, len(config.Nodes)) + signingKey, verifyKeys, err := gateway_ecrauth.LoadOrCreateSigningKey(janitorCtx, js, masterKey, len(config.Nodes)) if err != nil { return fmt.Errorf("ECR auth bridge: load signing key: %w", err) } @@ -323,7 +331,7 @@ func launchService(config *config.ClusterConfig) error { // Rotate the ECR signing key on a 30-day cadence, retaining the previous keys // until their tokens expire. The rotator keeps the issuer/verifier current as // keys roll. Bound to the same lifetime context as the STS janitor. - keyRotator, err := gateway_ecrauth.NewRotator(js, masterKey, len(config.Nodes), gw.ECRTokenIssuer, gw.ECRTokenVerifier) + keyRotator, err := gateway_ecrauth.NewRotator(janitorCtx, js, masterKey, len(config.Nodes), gw.ECRTokenIssuer, gw.ECRTokenVerifier) if err != nil { return fmt.Errorf("ECR auth bridge: signing-key rotator: %w", err) } @@ -337,9 +345,9 @@ func launchService(config *config.ClusterConfig) error { // Per-account service quotas. Only the enabled path opens the gateway-owned // usage KV bucket, leaving existing default-off gateways untouched; a disabled // config builds a no-op Service whose Exempt short-circuits every check. - var usageBucket nats.KeyValue + var usageBucket jetstream.KeyValue if quotaCfg.Enabled { - usageBucket, err = openAccountUsageBucket(js, max(len(config.Nodes), 1)) + usageBucket, err = openAccountUsageBucket(janitorCtx, js, len(config.Nodes)) if err != nil { return fmt.Errorf("init account usage bucket: %w", err) } @@ -397,7 +405,7 @@ func runQuotaReconcile(ctx context.Context, quota *handlers_quota.Service, natsC list := handlers_quota.NATSInstanceLister(natsConn, expectedNodes) runPass := func() { - release, elected := reconcile.AcquireLeader(natsConn, handlers_quota.KVBucketQuotaReconcile, holder) + release, elected := reconcile.AcquireLeader(ctx, natsConn, handlers_quota.KVBucketQuotaReconcile, holder) if !elected { return } diff --git a/spinifex/services/awsgw/quota_reconcile_test.go b/spinifex/services/awsgw/quota_reconcile_test.go index 9935ccda0..7b058f7a6 100644 --- a/spinifex/services/awsgw/quota_reconcile_test.go +++ b/spinifex/services/awsgw/quota_reconcile_test.go @@ -12,6 +12,7 @@ import ( handlers_quota "github.com/mulgadc/spinifex/spinifex/handlers/quota" "github.com/mulgadc/spinifex/spinifex/testutil" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/require" ) @@ -28,12 +29,28 @@ func respondJSON(t *testing.T, nc *nats.Conn, subject string, out *ec2.DescribeI t.Cleanup(func() { _ = sub.Unsubscribe() }) } +func TestOpenAccountUsageBucketPreservesExistingConfiguration(t *testing.T) { + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + _, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{ + Bucket: handlers_quota.KVBucketAccountUsage, + History: 5, + }) + require.NoError(t, err) + + bucket, err := openAccountUsageBucket(t.Context(), js, 1) + require.NoError(t, err) + status, err := bucket.Status(t.Context()) + require.NoError(t, err) + require.EqualValues(t, 5, status.History()) +} + // TestRunQuotaReconcile drives the leader-locked reconcile loop end to end: the // startup pass elects this gateway, describes the account's instances over NATS // via the production NATSInstanceLister, and writes the recomputed vCPU counter. func TestRunQuotaReconcile(t *testing.T) { - _, nc, js := testutil.StartTestJetStream(t) - bucket, err := openAccountUsageBucket(js, 1) + _, nc, _ := testutil.StartTestJetStream(t) + bucket, err := openAccountUsageBucket(t.Context(), testutil.NewJetStream(t, nc), 1) require.NoError(t, err) quota := handlers_quota.New(handlers_quota.Limits{Enabled: true, VCPUs: 100}, bucket) @@ -70,9 +87,9 @@ func TestRunQuotaReconcile(t *testing.T) { // readUsageCounter reads an account's integer vCPU counter from the usage bucket, // returning -1 while the key is still absent so Eventually keeps polling. -func readUsageCounter(t *testing.T, bucket nats.KeyValue, account string) int { +func readUsageCounter(t *testing.T, bucket jetstream.KeyValue, account string) int { t.Helper() - entry, err := bucket.Get(account) + entry, err := bucket.Get(t.Context(), account) if err != nil { return -1 } diff --git a/spinifex/testutil/nats.go b/spinifex/testutil/nats.go index 8196542f5..36e4961e0 100644 --- a/spinifex/testutil/nats.go +++ b/spinifex/testutil/nats.go @@ -7,6 +7,7 @@ import ( "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/require" ) @@ -33,9 +34,9 @@ func StartTestNATS(t *testing.T) (*server.Server, *nats.Conn) { } // StartTestJetStream starts an embedded NATS server with JetStream enabled -// and returns the server, a connected client, and a JetStream context. +// and returns the server, a connected client, and a jetstream package handle. // All resources are cleaned up via t.Cleanup. -func StartTestJetStream(t *testing.T) (*server.Server, *nats.Conn, nats.JetStreamContext) { +func StartTestJetStream(t *testing.T) (*server.Server, *nats.Conn, jetstream.JetStream) { t.Helper() opts := &server.Options{ Host: "127.0.0.1", @@ -55,11 +56,21 @@ func StartTestJetStream(t *testing.T) (*server.Server, *nats.Conn, nats.JetStrea require.NoError(t, err) t.Cleanup(func() { nc.Close() }) - js, err := nc.JetStream() + js, err := jetstream.New(nc) require.NoError(t, err) return ns, nc, js } +// NewJetStream returns a jetstream package handle for an existing test +// connection, for tests that hold a *nats.Conn without the tuple +// StartTestJetStream returns. +func NewJetStream(t *testing.T, nc *nats.Conn) jetstream.JetStream { + t.Helper() + js, err := jetstream.New(nc) + require.NoError(t, err) + return js +} + // vpcdStubTopics lists every synchronous vpcd topic the stub auto-replies to. // All block on a 5 s round-trip, so missing any one causes test timeouts. var vpcdStubTopics = []string{ @@ -133,12 +144,12 @@ func registerVpcdStub(t *testing.T, nc *nats.Conn, defaultPayload []byte) { // SeedKV creates a KV bucket and populates it with the given entries. // Returns the KV handle for further use. -func SeedKV(t *testing.T, js nats.JetStreamContext, bucket string, entries map[string][]byte) nats.KeyValue { +func SeedKV(t *testing.T, js jetstream.JetStream, bucket string, entries map[string][]byte) jetstream.KeyValue { t.Helper() - kv, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: bucket, History: 1}) + kv, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: bucket, History: 1}) require.NoError(t, err) for k, v := range entries { - _, err := kv.Put(k, v) + _, err := kv.Put(t.Context(), k, v) require.NoError(t, err) } return kv diff --git a/spinifex/utils/account.go b/spinifex/utils/account.go index 85a4cf32b..823bfe4b2 100644 --- a/spinifex/utils/account.go +++ b/spinifex/utils/account.go @@ -1,12 +1,7 @@ package utils import ( - "errors" - "fmt" - "strconv" "sync/atomic" - - "github.com/nats-io/nats.go" ) // defaultKVReplicas is the replica count applied to buckets created via @@ -28,45 +23,10 @@ func DefaultKVReplicas() int { } // VersionKey is the well-known KV key used to store a bucket's schema version. +// The schema-version helpers that read and write it live in package kvutil, +// alongside the other context-aware KV helpers. const VersionKey = "_version" -// WriteVersion writes the schema version to a bucket, only if missing or older. -// Returns an error if the stored value is corrupt (non-integer). -func WriteVersion(kv nats.KeyValue, version int) error { - entry, err := kv.Get(VersionKey) - if err != nil && !errors.Is(err, nats.ErrKeyNotFound) { - return fmt.Errorf("read current version: %w", err) - } - if err == nil { - stored, parseErr := strconv.Atoi(string(entry.Value())) - if parseErr != nil { - return fmt.Errorf("corrupted _version key (raw=%q): %w", string(entry.Value()), parseErr) - } - if stored >= version { - return nil - } - } - _, err = kv.PutString(VersionKey, strconv.Itoa(version)) - return err -} - -// ReadVersion reads the schema version from a bucket; returns 0 if not set. -// Errors distinguish network failures and corrupt values from "not set". -func ReadVersion(kv nats.KeyValue) (int, error) { - entry, err := kv.Get(VersionKey) - if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { - return 0, nil - } - return 0, fmt.Errorf("read version: %w", err) - } - v, parseErr := strconv.Atoi(string(entry.Value())) - if parseErr != nil { - return 0, fmt.Errorf("corrupted _version key (raw=%q): %w", string(entry.Value()), parseErr) - } - return v, nil -} - // GlobalAccountID is the root/system account ID. const GlobalAccountID = "000000000000" @@ -87,41 +47,3 @@ func IsAccountID(s string) bool { } return true } - -// GetOrCreateKVBucket creates or retrieves a NATS KV bucket at the cluster's -// default replica count (see SetDefaultKVReplicas), so buckets created lazily -// after boot are quorate on multi-node rather than stuck at R1. -func GetOrCreateKVBucket(js nats.JetStreamContext, bucketName string, history int) (nats.KeyValue, error) { - return GetOrCreateKVBucketWithReplicas(js, bucketName, history, DefaultKVReplicas()) -} - -// GetOrCreateKVBucketWithReplicas creates or retrieves a NATS KV bucket at the -// given replica count (clamped to a minimum of 1). The vendored nats.go -// (v1.52.0) has no UpdateKeyValue to live-upgrade an existing under-replicated -// bucket's replica count; existing buckets are upgraded on rebalance elsewhere. -func GetOrCreateKVBucketWithReplicas(js nats.JetStreamContext, bucketName string, history int, replicas int) (nats.KeyValue, error) { - kv, err := js.CreateKeyValue(&nats.KeyValueConfig{ - Bucket: bucketName, - History: SafeIntToUint8(history), - Replicas: max(replicas, 1), - }) - if err != nil { - kv, err = js.KeyValue(bucketName) - if err != nil { - return nil, err - } - } - return kv, nil -} - -// DeleteKVBucketIfExists deletes a NATS KV bucket by name, returning nil if it does not exist. -func DeleteKVBucketIfExists(js nats.JetStreamContext, bucketName string) error { - err := js.DeleteKeyValue(bucketName) - if err == nil { - return nil - } - if errors.Is(err, nats.ErrBucketNotFound) || errors.Is(err, nats.ErrStreamNotFound) { - return nil - } - return err -} diff --git a/spinifex/utils/account_replicas_test.go b/spinifex/utils/account_replicas_test.go index ef6d04c02..7ba5ae27e 100644 --- a/spinifex/utils/account_replicas_test.go +++ b/spinifex/utils/account_replicas_test.go @@ -2,8 +2,6 @@ package utils import ( "testing" - - "github.com/mulgadc/spinifex/spinifex/testutil" ) func TestDefaultKVReplicas(t *testing.T) { @@ -21,23 +19,3 @@ func TestDefaultKVReplicas(t *testing.T) { t.Fatalf("SetDefaultKVReplicas(3) = %d, want 3", got) } } - -func TestGetOrCreateKVBucketUsesDefaultReplicas(t *testing.T) { - t.Cleanup(func() { defaultKVReplicas.Store(0) }) - _, _, js := testutil.StartTestJetStream(t) - - // A single-node test server can only host R1, so assert the default is - // threaded into the created stream config rather than the hardcoded 1. - // The R>1 case is exercised on a live multi-node cluster. - SetDefaultKVReplicas(1) - if _, err := GetOrCreateKVBucket(js, "test-default-replicas", 1); err != nil { - t.Fatalf("GetOrCreateKVBucket: %v", err) - } - info, err := js.StreamInfo("KV_test-default-replicas") - if err != nil { - t.Fatalf("StreamInfo: %v", err) - } - if info.Config.Replicas != 1 { - t.Fatalf("bucket replicas = %d, want 1", info.Config.Replicas) - } -} diff --git a/spinifex/utils/account_test.go b/spinifex/utils/account_test.go index 4a089730a..a59fbdecd 100644 --- a/spinifex/utils/account_test.go +++ b/spinifex/utils/account_test.go @@ -2,11 +2,6 @@ package utils import ( "testing" - - "github.com/mulgadc/spinifex/spinifex/testutil" - "github.com/nats-io/nats.go" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func TestAccountKey(t *testing.T) { @@ -59,112 +54,3 @@ func TestGlobalAccountID(t *testing.T) { t.Error("GlobalAccountID should be a valid account ID") } } - -// startJSNATSServer starts an embedded JetStream-enabled NATS server for testing. -func startJSNATSServer(t *testing.T) (*nats.Conn, nats.JetStreamContext) { - t.Helper() - _, nc, js := testutil.StartTestJetStream(t) - return nc, js -} - -// TestVersionStateMachine covers unset→0, first write, idempotent same write, upgrade, and no-downgrade. -// One bucket is reused so each step runs against the prior state — the only way to catch unconditional-overwrite regressions. -func TestVersionStateMachine(t *testing.T) { - _, js := startJSNATSServer(t) - kv, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: "test-version-fsm"}) - require.NoError(t, err) - - // Unset → 0. - v, err := ReadVersion(kv) - require.NoError(t, err) - assert.Equal(t, 0, v, "ReadVersion on unset bucket") - - steps := []struct { - name string - write int - want int // expected ReadVersion after the write - }{ - {"first write persists", 1, 1}, - {"same version is no-op", 1, 1}, - {"higher version upgrades", 2, 2}, - {"lower version is no-op (no downgrade)", 1, 2}, - {"larger jump upgrades", 5, 5}, - } - for _, step := range steps { - t.Run(step.name, func(t *testing.T) { - require.NoError(t, WriteVersion(kv, step.write)) - v, err := ReadVersion(kv) - require.NoError(t, err) - assert.Equal(t, step.want, v) - }) - } - - // Round-trip the raw KV value to confirm the encoding is what readers - // outside this package would expect. - entry, err := kv.Get(VersionKey) - require.NoError(t, err) - assert.Equal(t, "5", string(entry.Value())) -} - -// streamReplicas returns the replica count of the JetStream stream backing a -// KV bucket, so tests can assert on the config actually sent to the server. -func streamReplicas(t *testing.T, js nats.JetStreamContext, bucket string) int { - t.Helper() - si, err := js.StreamInfo("KV_" + bucket) - require.NoError(t, err) - return si.Config.Replicas -} - -func TestGetOrCreateKVBucket_CreatesAtReplicaOne(t *testing.T) { - _, js := startJSNATSServer(t) - - kv, err := GetOrCreateKVBucket(js, "regression-bucket", 5) - require.NoError(t, err) - require.NotNil(t, kv) - assert.Equal(t, 1, streamReplicas(t, js, "regression-bucket")) -} - -func TestGetOrCreateKVBucketWithReplicas_CreatesAtRequestedReplicas(t *testing.T) { - _, js := startJSNATSServer(t) - - // The embedded single-node test server rejects Replicas > 1 - // ("replicas > 1 not supported in non-clustered mode"), so the only - // replica count exercisable end-to-end here is 1; the clamping logic - // (max(replicas, 1)) is covered separately below. - kv, err := GetOrCreateKVBucketWithReplicas(js, "replicated-bucket", 5, 1) - require.NoError(t, err) - require.NotNil(t, kv) - assert.Equal(t, 1, streamReplicas(t, js, "replicated-bucket")) -} - -func TestGetOrCreateKVBucketWithReplicas_ClampsBelowOne(t *testing.T) { - _, js := startJSNATSServer(t) - - kv, err := GetOrCreateKVBucketWithReplicas(js, "clamped-zero", 1, 0) - require.NoError(t, err) - require.NotNil(t, kv) - assert.Equal(t, 1, streamReplicas(t, js, "clamped-zero")) - - kv, err = GetOrCreateKVBucketWithReplicas(js, "clamped-negative", 1, -3) - require.NoError(t, err) - require.NotNil(t, kv) - assert.Equal(t, 1, streamReplicas(t, js, "clamped-negative")) -} - -func TestDeleteKVBucketIfExists(t *testing.T) { - _, js := startJSNATSServer(t) - - // Missing bucket is a no-op. - require.NoError(t, DeleteKVBucketIfExists(js, "ghost-bucket")) - - // Existing bucket gets deleted. - _, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: "doomed-bucket"}) - require.NoError(t, err) - require.NoError(t, DeleteKVBucketIfExists(js, "doomed-bucket")) - - _, err = js.KeyValue("doomed-bucket") - require.Error(t, err, "bucket should be gone") - - // Calling again on the now-missing bucket is still a no-op (idempotent). - require.NoError(t, DeleteKVBucketIfExists(js, "doomed-bucket")) -} diff --git a/spinifex/utils/tagmirror.go b/spinifex/utils/tagmirror.go index b11bd0b9a..b742828e4 100644 --- a/spinifex/utils/tagmirror.go +++ b/spinifex/utils/tagmirror.go @@ -1,6 +1,7 @@ package utils import ( + "context" "encoding/json" "errors" "log/slog" @@ -8,7 +9,7 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" "github.com/mulgadc/spinifex/spinifex/awserrors" - "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // MergeTagsMut returns a mutator that merges the CreateTagsInput tags into a @@ -76,11 +77,11 @@ func RemoveTagsMut(input *ec2.DeleteTagsInput) func(map[string]string) { // UpdateKVRecordTags read-modify-writes a typed account-scoped KV record, // applying mut. A resource absent from this store is skipped (its tags live // elsewhere). -func UpdateKVRecordTags[R any](kv nats.KeyValue, accountID, resourceID string, mut func(*R)) error { +func UpdateKVRecordTags[R any](ctx context.Context, kv jetstream.KeyValue, accountID, resourceID string, mut func(*R)) error { key := AccountKey(accountID, resourceID) - entry, err := kv.Get(key) + entry, err := kv.Get(ctx, key) if err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { + if errors.Is(err, jetstream.ErrKeyNotFound) { return nil } slog.Error("UpdateKVRecordTags: KV read failed", "key", key, "err", err) @@ -97,7 +98,7 @@ func UpdateKVRecordTags[R any](kv nats.KeyValue, accountID, resourceID string, m slog.Error("UpdateKVRecordTags: record marshal failed", "key", key, "err", err) return errors.New(awserrors.ErrorServerInternal) } - if _, err := kv.Put(key, data); err != nil { + if _, err := kv.Put(ctx, key, data); err != nil { slog.Error("UpdateKVRecordTags: KV write failed", "key", key, "err", err) return errors.New(awserrors.ErrorServerInternal) } @@ -108,12 +109,12 @@ func UpdateKVRecordTags[R any](kv nats.KeyValue, accountID, resourceID string, m // resources carrying prefix, via UpdateKVRecordTags on the service's own KV // handle. tagsOf returns the address of the record's tag map so a nil map can // be initialized in place. Resource ids without prefix are skipped. -func MirrorKVRecordTags[R any](kv nats.KeyValue, accountID, prefix string, resources []*string, tagsOf func(*R) *map[string]string, mut func(map[string]string)) error { +func MirrorKVRecordTags[R any](ctx context.Context, kv jetstream.KeyValue, accountID, prefix string, resources []*string, tagsOf func(*R) *map[string]string, mut func(map[string]string)) error { for _, res := range resources { if res == nil || !strings.HasPrefix(*res, prefix) { continue } - if err := UpdateKVRecordTags(kv, accountID, *res, func(r *R) { + if err := UpdateKVRecordTags(ctx, kv, accountID, *res, func(r *R) { tp := tagsOf(r) if *tp == nil { *tp = map[string]string{} diff --git a/spinifex/utils/tagmirror_test.go b/spinifex/utils/tagmirror_test.go index 672127568..5c7db6dc2 100644 --- a/spinifex/utils/tagmirror_test.go +++ b/spinifex/utils/tagmirror_test.go @@ -8,6 +8,7 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" "github.com/mulgadc/spinifex/spinifex/awserrors" "github.com/mulgadc/spinifex/spinifex/testutil" + "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -17,6 +18,22 @@ type tagMirrorRecord struct { Tags map[string]string `json:"tags"` } +// seedKV creates a bucket on the jetstream API and populates it. Package utils +// cannot use kvutil's helpers — kvutil imports utils — so the bucket is created +// here directly. +func seedKV(t *testing.T, bucket string, entries map[string][]byte) jetstream.KeyValue { + t.Helper() + _, nc, _ := testutil.StartTestJetStream(t) + js := testutil.NewJetStream(t, nc) + kv, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: bucket, History: 1}) + require.NoError(t, err) + for k, v := range entries { + _, err := kv.Put(t.Context(), k, v) + require.NoError(t, err) + } + return kv +} + func TestMergeTagsMut(t *testing.T) { mut := MergeTagsMut(&ec2.CreateTagsInput{ Tags: []*ec2.Tag{ @@ -62,19 +79,18 @@ func TestRemoveTagsMut(t *testing.T) { } func TestUpdateKVRecordTags(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) rec, err := json.Marshal(&tagMirrorRecord{Name: "r1", Tags: map[string]string{"a": "1"}}) require.NoError(t, err) - kv := testutil.SeedKV(t, js, "tagmirror-test", map[string][]byte{ + kv := seedKV(t, "tagmirror-test", map[string][]byte{ AccountKey("acct", "res-1"): rec, AccountKey("acct", "res-bad"): []byte("not-json"), }) t.Run("mutates and persists", func(t *testing.T) { - require.NoError(t, UpdateKVRecordTags(kv, "acct", "res-1", func(r *tagMirrorRecord) { + require.NoError(t, UpdateKVRecordTags(t.Context(), kv, "acct", "res-1", func(r *tagMirrorRecord) { r.Tags["b"] = "2" })) - entry, err := kv.Get(AccountKey("acct", "res-1")) + entry, err := kv.Get(t.Context(), AccountKey("acct", "res-1")) require.NoError(t, err) var got tagMirrorRecord require.NoError(t, json.Unmarshal(entry.Value(), &got)) @@ -83,25 +99,24 @@ func TestUpdateKVRecordTags(t *testing.T) { }) t.Run("absent key is a no-op", func(t *testing.T) { - require.NoError(t, UpdateKVRecordTags(kv, "acct", "res-missing", func(r *tagMirrorRecord) { + require.NoError(t, UpdateKVRecordTags(t.Context(), kv, "acct", "res-missing", func(r *tagMirrorRecord) { t.Fatal("mutator must not run for absent record") })) }) t.Run("corrupt record returns internal error", func(t *testing.T) { - err := UpdateKVRecordTags(kv, "acct", "res-bad", func(r *tagMirrorRecord) {}) + err := UpdateKVRecordTags(t.Context(), kv, "acct", "res-bad", func(r *tagMirrorRecord) {}) require.Error(t, err) assert.Equal(t, awserrors.ErrorServerInternal, err.Error()) }) } func TestMirrorKVRecordTags(t *testing.T) { - _, _, js := testutil.StartTestJetStream(t) withTags, err := json.Marshal(&tagMirrorRecord{Tags: map[string]string{"drop": "v", "keep": "yes"}}) require.NoError(t, err) nilTags, err := json.Marshal(&tagMirrorRecord{}) require.NoError(t, err) - kv := testutil.SeedKV(t, js, "tagmirror-mirror-test", map[string][]byte{ + kv := seedKV(t, "tagmirror-mirror-test", map[string][]byte{ AccountKey("acct", "vol-1"): withTags, AccountKey("acct", "vol-2"): nilTags, }) @@ -109,7 +124,7 @@ func TestMirrorKVRecordTags(t *testing.T) { tagsOf := func(r *tagMirrorRecord) *map[string]string { return &r.Tags } readTags := func(t *testing.T, id string) map[string]string { - entry, err := kv.Get(AccountKey("acct", id)) + entry, err := kv.Get(t.Context(), AccountKey("acct", id)) require.NoError(t, err) var got tagMirrorRecord require.NoError(t, json.Unmarshal(entry.Value(), &got)) @@ -124,14 +139,14 @@ func TestMirrorKVRecordTags(t *testing.T) { nil, } - require.NoError(t, MirrorKVRecordTags(kv, "acct", "vol-", resources, tagsOf, + require.NoError(t, MirrorKVRecordTags(t.Context(), kv, "acct", "vol-", resources, tagsOf, MergeTagsMut(&ec2.CreateTagsInput{Tags: []*ec2.Tag{ {Key: aws.String("added"), Value: aws.String("1")}, }}))) assert.Equal(t, map[string]string{"drop": "v", "keep": "yes", "added": "1"}, readTags(t, "vol-1")) assert.Equal(t, map[string]string{"added": "1"}, readTags(t, "vol-2"), "nil tag map initialized in place") - require.NoError(t, MirrorKVRecordTags(kv, "acct", "vol-", resources, tagsOf, + require.NoError(t, MirrorKVRecordTags(t.Context(), kv, "acct", "vol-", resources, tagsOf, RemoveTagsMut(&ec2.DeleteTagsInput{Tags: []*ec2.Tag{ {Key: aws.String("drop"), Value: aws.String("v")}, {Key: aws.String("keep"), Value: aws.String("wrong")}, @@ -142,9 +157,9 @@ func TestMirrorKVRecordTags(t *testing.T) { // must never be touched even though it exists in this bucket. otherRec, err := json.Marshal(&tagMirrorRecord{Tags: map[string]string{"x": "y"}}) require.NoError(t, err) - _, err = kv.Put(AccountKey("acct", "snap-other"), otherRec) + _, err = kv.Put(t.Context(), AccountKey("acct", "snap-other"), otherRec) require.NoError(t, err) - require.NoError(t, MirrorKVRecordTags(kv, "acct", "vol-", resources, tagsOf, + require.NoError(t, MirrorKVRecordTags(t.Context(), kv, "acct", "vol-", resources, tagsOf, MergeTagsMut(&ec2.CreateTagsInput{Tags: []*ec2.Tag{ {Key: aws.String("added"), Value: aws.String("2")}, }}))) diff --git a/spinifex/vpcd/vpcd.go b/spinifex/vpcd/vpcd.go index 41a0f4139..f8e83e874 100644 --- a/spinifex/vpcd/vpcd.go +++ b/spinifex/vpcd/vpcd.go @@ -27,6 +27,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/network/topology" "github.com/mulgadc/spinifex/spinifex/utils" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // Bridge mode selects how the WAN NIC reaches the OVS bridge. @@ -489,7 +490,7 @@ func launchService(cfg *Config) error { } routeMgr := policy.NewRouteManager(liveClient) - js, err := nc.JetStream() + js, err := jetstream.New(nc) if err != nil { return fmt.Errorf("get JetStream context: %w", err) } @@ -511,6 +512,7 @@ func launchService(cfg *Config) error { return live, nil } imdsSvc, err := handlers_imds.NewIMDSServiceImpl( + ctx, nc, handlers_imds.NewNATSSTSAssumer(nc), handlers_imds.NewNATSProfileLookup(nc), @@ -612,7 +614,7 @@ func launchService(cfg *Config) error { // Elect one vpcd for startup reconcile; without this, concurrent Get-then-Create on Logical_Router // produces duplicate rows that ovn-nbctl rejects. Runtime events still fan out via queue groups. holder, _ := os.Hostname() - releaseLeader, isLeader := reconcile.AcquireLeader(nc, reconcile.KVBucketVPCDReconcile, holder) + releaseLeader, isLeader := reconcile.AcquireLeader(ctx, nc, reconcile.KVBucketVPCDReconcile, holder) subscriber, err := subscribers.New(subscribers.Config{ Topology: topoMgr, @@ -723,7 +725,7 @@ func pickDNSServer(pools []external.ExternalPoolConfig) string { // startDHCPManagerIfNeeded starts the per-AZ DHCP Manager when any pool has Source="dhcp". // Returns (nil, nil, nil) when not needed. -func startDHCPManagerIfNeeded(ctx context.Context, nc *nats.Conn, js nats.JetStreamContext, cfg *Config) (*dhcp.Manager, []*nats.Subscription, error) { +func startDHCPManagerIfNeeded(ctx context.Context, nc *nats.Conn, js jetstream.JetStream, cfg *Config) (*dhcp.Manager, []*nats.Subscription, error) { if cfg == nil || cfg.ExternalMode == "" { return nil, nil, nil } @@ -738,7 +740,7 @@ func startDHCPManagerIfNeeded(ctx context.Context, nc *nats.Conn, js nats.JetStr return nil, nil, nil } - store, err := dhcp.NewStore(js, cfg.AZ) + store, err := dhcp.NewStore(ctx, js, cfg.AZ) if err != nil { return nil, nil, fmt.Errorf("create dhcp lease store: %w", err) } diff --git a/tests/e2e/harness/eks_mgmt.go b/tests/e2e/harness/eks_mgmt.go index 8f6998359..64306d639 100644 --- a/tests/e2e/harness/eks_mgmt.go +++ b/tests/e2e/harness/eks_mgmt.go @@ -12,6 +12,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/config" handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks" "github.com/mulgadc/spinifex/spinifex/utils" + "github.com/nats-io/nats.go/jetstream" "github.com/spf13/viper" ) @@ -27,15 +28,15 @@ func ControlPlaneMgmtIP(t *testing.T, env *Env, accountID, clusterName string) s } defer nc.Close() - js, err := nc.JetStream() + js, err := jetstream.New(nc) if err != nil { t.Fatalf("jetstream: %v", err) } - kv, err := js.KeyValue(handlers_eks.AccountBucketName(accountID)) + kv, err := js.KeyValue(t.Context(), handlers_eks.AccountBucketName(accountID)) if err != nil { t.Fatalf("open EKS account KV for %s: %v", accountID, err) } - meta, err := handlers_eks.GetClusterMeta(kv, clusterName) + meta, err := handlers_eks.GetClusterMeta(t.Context(), kv, clusterName) if err != nil { t.Fatalf("get cluster meta %s: %v", clusterName, err) } diff --git a/tests/e2e/harness/jetstream.go b/tests/e2e/harness/jetstream.go index af0807ebb..f04f3587e 100644 --- a/tests/e2e/harness/jetstream.go +++ b/tests/e2e/harness/jetstream.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/mulgadc/spinifex/spinifex/utils" + "github.com/nats-io/nats.go/jetstream" ) // KVReplicaFactors connects to the cluster NATS and returns the configured @@ -22,14 +23,18 @@ func KVReplicaFactors(t *testing.T, env *Env, buckets ...string) map[string]int } defer nc.Close() - js, err := nc.JetStream() + js, err := jetstream.New(nc) if err != nil { t.Fatalf("jetstream context: %v", err) } out := make(map[string]int, len(buckets)) for _, b := range buckets { - info, err := js.StreamInfo("KV_" + b) + stream, err := js.Stream(t.Context(), "KV_"+b) + if err != nil { + t.Fatalf("open stream for KV bucket %q: %v", b, err) + } + info, err := stream.Info(t.Context()) if err != nil { t.Fatalf("stream info for KV bucket %q: %v", b, err) } diff --git a/tests/integration/daemon_lite.go b/tests/integration/daemon_lite.go index a5e709693..e29f766b2 100644 --- a/tests/integration/daemon_lite.go +++ b/tests/integration/daemon_lite.go @@ -110,19 +110,19 @@ func StartDaemonLite(t *testing.T, gw *Gateway, opts ...DaemonLiteOption) *Daemo keySvc := handlers_ec2_key.NewKeyServiceImplWithStore(memStore, cfg.Predastore.Bucket) tagsSvc := handlers_ec2_tags.NewTagsServiceImplWithStore(cfg, memStore) - vpcSvc, err := handlers_ec2_vpc.NewVPCServiceImplWithNATS(cfg, nc) + vpcSvc, err := handlers_ec2_vpc.NewVPCServiceImplWithNATS(t.Context(), cfg, nc) require.NoError(t, err, "construct VPC service") - rtbSvc, err := handlers_ec2_routetable.NewRouteTableServiceImplWithNATS(cfg, nc) + rtbSvc, err := handlers_ec2_routetable.NewRouteTableServiceImplWithNATS(t.Context(), cfg, nc) require.NoError(t, err, "construct route table service") - igwSvc, err := handlers_ec2_igw.NewIGWServiceImplWithNATS(cfg, nc) + igwSvc, err := handlers_ec2_igw.NewIGWServiceImplWithNATS(t.Context(), cfg, nc) require.NoError(t, err, "construct IGW service") - eigwSvc, err := handlers_ec2_eigw.NewEgressOnlyIGWServiceImplWithNATS(cfg, nc) + eigwSvc, err := handlers_ec2_eigw.NewEgressOnlyIGWServiceImplWithNATS(t.Context(), cfg, nc) require.NoError(t, err, "construct EIGW service") - acctSettingsSvc, err := handlers_ec2_account.NewAccountSettingsServiceImplWithNATS(cfg, nc) + acctSettingsSvc, err := handlers_ec2_account.NewAccountSettingsServiceImplWithNATS(t.Context(), cfg, nc) require.NoError(t, err, "construct account settings service") dl := &DaemonLite{ diff --git a/tests/integration/harness.go b/tests/integration/harness.go index c624689cf..240c94fbd 100644 --- a/tests/integration/harness.go +++ b/tests/integration/harness.go @@ -132,10 +132,10 @@ func StartGateway(t *testing.T, opts ...Option) *Gateway { masterKey, err := handlers_iam.GenerateMasterKey() require.NoError(t, err) - iamSvc, err := handlers_iam.NewIAMServiceWithRetry(nc, masterKey, 1) + iamSvc, err := handlers_iam.NewIAMServiceWithRetry(t.Context(), nc, masterKey, 1) require.NoError(t, err) - stsSvc, err := handlers_sts.NewSTSServiceImpl(nc, iamSvc, masterKey, 1) + stsSvc, err := handlers_sts.NewSTSServiceImpl(t.Context(), nc, iamSvc, masterKey, 1) require.NoError(t, err) encryptedSecret, err := handlers_iam.EncryptSecret(testSecretAccessKey, masterKey) @@ -150,7 +150,7 @@ func StartGateway(t *testing.T, opts ...Option) *Gateway { // ECR auth bridge signing key: reuses the IAM master key to encrypt the // signing key at rest in the same embedded JetStream KV, matching // production's awsgw-keys wiring (services/awsgw/awsgw.go). - signingKey, verifyKeys, err := gateway_ecrauth.LoadOrCreateSigningKey(js, masterKey, 1) + signingKey, verifyKeys, err := gateway_ecrauth.LoadOrCreateSigningKey(t.Context(), js, masterKey, 1) require.NoError(t, err) cfg := &gateway.GatewayConfig{ diff --git a/tests/integration/launchtemplate_daemon_lite.go b/tests/integration/launchtemplate_daemon_lite.go index 8a7cfac5b..af3c1f545 100644 --- a/tests/integration/launchtemplate_daemon_lite.go +++ b/tests/integration/launchtemplate_daemon_lite.go @@ -28,7 +28,7 @@ func StartLaunchTemplateDaemonLite(t *testing.T, gw *Gateway) *handlers_ec2_laun t.Helper() cfg := &config.Config{AZ: testAZ} - svc, err := handlers_ec2_launchtemplate.NewLaunchTemplateServiceImplWithNATS(cfg, gw.NATSConn) + svc, err := handlers_ec2_launchtemplate.NewLaunchTemplateServiceImplWithNATS(t.Context(), cfg, gw.NATSConn) require.NoError(t, err, "construct launch template service") nc := gw.NATSConn diff --git a/tests/integration/placementgroup_daemon_lite.go b/tests/integration/placementgroup_daemon_lite.go index 0bf711528..df011214c 100644 --- a/tests/integration/placementgroup_daemon_lite.go +++ b/tests/integration/placementgroup_daemon_lite.go @@ -29,7 +29,7 @@ func StartPlacementGroupDaemonLite(t *testing.T, gw *Gateway) *handlers_ec2_plac t.Helper() cfg := &config.Config{AZ: testAZ} - svc, err := handlers_ec2_placementgroup.NewPlacementGroupServiceImplWithNATS(cfg, gw.NATSConn) + svc, err := handlers_ec2_placementgroup.NewPlacementGroupServiceImplWithNATS(t.Context(), cfg, gw.NATSConn) require.NoError(t, err, "construct placement group service") nc := gw.NATSConn diff --git a/tests/integration/shared_nats.go b/tests/integration/shared_nats.go index 8ffcc110b..34ca5d6b1 100644 --- a/tests/integration/shared_nats.go +++ b/tests/integration/shared_nats.go @@ -13,6 +13,7 @@ import ( "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) // sharedNATSHarness is the package-wide handle TestMain installs before @@ -70,12 +71,12 @@ func startSharedNATS() (*sharedNATS, error) { // namespace no other test's account can see or collide with. The returned // connection is tracked and closed by TestMain once every test has // finished, not by the caller — see the TestMain doc comment for why. -func (h *sharedNATS) connectIsolated(name string) (*nats.Conn, nats.JetStreamContext, error) { +func (h *sharedNATS) connectIsolated(name string) (*nats.Conn, jetstream.JetStream, error) { nc, err := nats.Connect(h.srv.ClientURL(), nats.UserInfo(name, "x")) if err != nil { return nil, nil, fmt.Errorf("connect: %w", err) } - js, err := nc.JetStream() + js, err := jetstream.New(nc) if err != nil { nc.Close() return nil, nil, fmt.Errorf("jetstream: %w", err) diff --git a/tests/integration/spot_daemon_lite.go b/tests/integration/spot_daemon_lite.go index dad065678..034733563 100644 --- a/tests/integration/spot_daemon_lite.go +++ b/tests/integration/spot_daemon_lite.go @@ -26,7 +26,7 @@ import ( func StartSpotDaemonLite(t *testing.T, gw *Gateway) *handlers_ec2_spotinstance.SpotInstanceServiceImpl { t.Helper() - svc, err := handlers_ec2_spotinstance.NewSpotInstanceServiceImplWithNATS(nil, gw.NATSConn) + svc, err := handlers_ec2_spotinstance.NewSpotInstanceServiceImplWithNATS(t.Context(), nil, gw.NATSConn) require.NoError(t, err, "construct spot instance service") nc := gw.NATSConn