diff --git a/cmd/spinifex/cmd/service.go b/cmd/spinifex/cmd/service.go index 12d8b5644..50de4edb0 100644 --- a/cmd/spinifex/cmd/service.go +++ b/cmd/spinifex/cmd/service.go @@ -1014,6 +1014,7 @@ var vpcdStartCmd = &cobra.Command{ OVNNBAddr: nodeConfig.VPCD.OVNNBAddr, OVNSBAddr: nodeConfig.VPCD.OVNSBAddr, BaseDir: nodeConfig.BaseDir, + DataDir: nodeConfig.DataDir, Debug: false, ExternalMode: clusterConfig.Network.ExternalMode, ExternalPools: extPools, diff --git a/scripts/check-test-package.sh b/scripts/check-test-package.sh index dce3d8674..09d2854bf 100755 --- a/scripts/check-test-package.sh +++ b/scripts/check-test-package.sh @@ -50,9 +50,10 @@ if ! git rev-parse --verify "$BASE_REF" &>/dev/null; then fi # e2e and integration suites share a package-level fixture built in -# main_test.go, so their files cannot move individually. +# main_test.go, so their files cannot move individually. export_test.go is the +# remedy this check recommends, and it is in-package by definition. ADDED=$(git diff "$BASE_REF" HEAD --name-only --diff-filter=A -- '*_test.go' \ - ':!tests/e2e/*' ':!tests/integration/*' || true) + ':!tests/e2e/*' ':!tests/integration/*' ':!*export_test.go' || true) if [[ -z "$ADDED" ]]; then [[ -z "$QUIET" ]] && echo "No new test files — skipping test package check." diff --git a/spinifex/accountteardown/reapers_ec2.go b/spinifex/accountteardown/reapers_ec2.go index 1aecaa374..6d2df5290 100644 --- a/spinifex/accountteardown/reapers_ec2.go +++ b/spinifex/accountteardown/reapers_ec2.go @@ -76,7 +76,7 @@ func (r *instanceReaper) Stage() Stage { return StageCompute } func (r *instanceReaper) List(ctx context.Context, accountID string) ([]Resource, error) { // Checked: a partial answer here would read as "no instances left" and let // storage deletion start while a guest still holds a volume. - out, err := gateway_ec2_instance.DescribeInstancesChecked(ctx, &ec2.DescribeInstancesInput{}, r.nc, r.expectedNodes, accountID) + out, err := gateway_ec2_instance.DescribeInstancesChecked(ctx, &ec2.DescribeInstancesInput{}, r.nc, r.expectedNodes, nil, accountID) if err != nil { return nil, err } diff --git a/spinifex/daemon/daemon.go b/spinifex/daemon/daemon.go index e71de43a7..e9535c209 100644 --- a/spinifex/daemon/daemon.go +++ b/spinifex/daemon/daemon.go @@ -875,130 +875,130 @@ func (d *Daemon) subscribeAll() error { subs := []natsSub{ // ec2.RunInstances is handled by dynamic per-instance-type subscriptions // managed by ResourceManager.initSubscriptions() - {"ec2.CreateKeyPair", handleNATSRequest(d.keyService.CreateKeyPair), "spinifex-workers"}, - {"ec2.DeleteKeyPair", handleNATSRequest(d.keyService.DeleteKeyPair), "spinifex-workers"}, - {"ec2.DescribeKeyPairs", handleNATSRequest(d.keyService.DescribeKeyPairs), "spinifex-workers"}, - {"ec2.ImportKeyPair", handleNATSRequest(d.keyService.ImportKeyPair), "spinifex-workers"}, + {"ec2.CreateKeyPair", handleNATSRequest(d.node, d.keyService.CreateKeyPair), "spinifex-workers"}, + {"ec2.DeleteKeyPair", handleNATSRequest(d.node, d.keyService.DeleteKeyPair), "spinifex-workers"}, + {"ec2.DescribeKeyPairs", handleNATSRequest(d.node, d.keyService.DescribeKeyPairs), "spinifex-workers"}, + {"ec2.ImportKeyPair", handleNATSRequest(d.node, d.keyService.ImportKeyPair), "spinifex-workers"}, {"imds.ec2.get_public_key", d.handleIMDSGetPublicKey, "spinifex-workers"}, - {"ec2.DescribeImages", handleNATSRequest(d.imageService.DescribeImages), "spinifex-workers"}, + {"ec2.DescribeImages", handleNATSRequest(d.node, d.imageService.DescribeImages), "spinifex-workers"}, {"ec2.CreateImage", d.handleEC2CreateImage, ""}, - {"ec2.DeregisterImage", handleNATSRequest(d.imageService.DeregisterImage), "spinifex-workers"}, - {"ec2.RegisterImage", handleNATSRequest(d.imageService.RegisterImage), "spinifex-workers"}, - {"ec2.CopyImage", handleNATSRequest(d.imageService.CopyImage), "spinifex-workers"}, - {"ec2.DescribeImageAttribute", handleNATSRequest(d.imageService.DescribeImageAttribute), "spinifex-workers"}, - {"ec2.ModifyImageAttribute", handleNATSRequest(d.imageService.ModifyImageAttribute), "spinifex-workers"}, - {"ec2.ResetImageAttribute", handleNATSRequest(d.imageService.ResetImageAttribute), "spinifex-workers"}, - {"ec2.CreateVolume", handleNATSRequest(d.volumeService.CreateVolume), "spinifex-workers"}, - {"ec2.DescribeVolumes", handleNATSRequest(d.volumeService.DescribeVolumes), "spinifex-workers"}, + {"ec2.DeregisterImage", handleNATSRequest(d.node, d.imageService.DeregisterImage), "spinifex-workers"}, + {"ec2.RegisterImage", handleNATSRequest(d.node, d.imageService.RegisterImage), "spinifex-workers"}, + {"ec2.CopyImage", handleNATSRequest(d.node, d.imageService.CopyImage), "spinifex-workers"}, + {"ec2.DescribeImageAttribute", handleNATSRequest(d.node, d.imageService.DescribeImageAttribute), "spinifex-workers"}, + {"ec2.ModifyImageAttribute", handleNATSRequest(d.node, d.imageService.ModifyImageAttribute), "spinifex-workers"}, + {"ec2.ResetImageAttribute", handleNATSRequest(d.node, d.imageService.ResetImageAttribute), "spinifex-workers"}, + {"ec2.CreateVolume", handleNATSRequest(d.node, d.volumeService.CreateVolume), "spinifex-workers"}, + {"ec2.DescribeVolumes", handleNATSRequest(d.node, d.volumeService.DescribeVolumes), "spinifex-workers"}, {"ec2.ModifyVolume", d.handleEC2ModifyVolume, "spinifex-workers"}, - {"ec2.DeleteVolume", handleNATSRequest(d.volumeService.DeleteVolume), "spinifex-workers"}, + {"ec2.DeleteVolume", handleNATSRequest(d.node, d.volumeService.DeleteVolume), "spinifex-workers"}, // Cluster-wide on purpose: the QMP detach is instance-scoped, so a // volume held by an unreachable host has no other way out. - {"ec2.ForceDetachVolume", handleNATSRequest(d.volumeService.ForceDetachVolume), "spinifex-workers"}, - {"ec2.DescribeVolumeStatus", handleNATSRequest(d.volumeService.DescribeVolumeStatus), "spinifex-workers"}, - {"ec2.DescribeVolumesModifications", handleNATSRequest(d.volumeService.DescribeVolumesModifications), "spinifex-workers"}, - {"ec2.CreateSnapshot", handleNATSRequest(d.snapshotService.CreateSnapshot), "spinifex-workers"}, - {"ec2.DescribeSnapshots", handleNATSRequest(d.snapshotService.DescribeSnapshots), "spinifex-workers"}, - {"ec2.DeleteSnapshot", handleNATSRequest(d.snapshotService.DeleteSnapshot), "spinifex-workers"}, - {"ec2.CopySnapshot", handleNATSRequest(d.snapshotService.CopySnapshot), "spinifex-workers"}, - {"ec2.CreateTags", handleNATSRequest(d.createTags), "spinifex-workers"}, - {"ec2.DeleteTags", handleNATSRequest(d.deleteTags), "spinifex-workers"}, - {"ec2.DescribeTags", handleNATSRequest(d.tagsService.DescribeTags), "spinifex-workers"}, - {"ec2.CreateEgressOnlyInternetGateway", handleNATSRequest(d.eigwService.CreateEgressOnlyInternetGateway), "spinifex-workers"}, - {"ec2.DeleteEgressOnlyInternetGateway", handleNATSRequest(d.eigwService.DeleteEgressOnlyInternetGateway), "spinifex-workers"}, - {"ec2.DescribeEgressOnlyInternetGateways", handleNATSRequest(d.eigwService.DescribeEgressOnlyInternetGateways), "spinifex-workers"}, - {"ec2.CreateInternetGateway", handleNATSRequest(d.igwService.CreateInternetGateway), "spinifex-workers"}, - {"ec2.DeleteInternetGateway", handleNATSRequest(d.igwService.DeleteInternetGateway), "spinifex-workers"}, - {"ec2.DescribeInternetGateways", handleNATSRequest(d.igwService.DescribeInternetGateways), "spinifex-workers"}, - {"ec2.AttachInternetGateway", handleNATSRequest(d.igwService.AttachInternetGateway), "spinifex-workers"}, - {"ec2.DetachInternetGateway", handleNATSRequest(d.igwService.DetachInternetGateway), "spinifex-workers"}, - {"ec2.CreatePlacementGroup", handleNATSRequest(d.placementGroupService.CreatePlacementGroup), "spinifex-workers"}, - {"ec2.DeletePlacementGroup", handleNATSRequest(d.placementGroupService.DeletePlacementGroup), "spinifex-workers"}, - {"ec2.DescribePlacementGroups", handleNATSRequest(d.placementGroupService.DescribePlacementGroups), "spinifex-workers"}, - {"ec2.ReserveSpreadNodes", handleNATSRequest(d.placementGroupService.ReserveSpreadNodes), "spinifex-workers"}, - {"ec2.FinalizeSpreadInstances", handleNATSRequest(d.placementGroupService.FinalizeSpreadInstances), "spinifex-workers"}, - {"ec2.ReleaseSpreadNodes", handleNATSRequest(d.placementGroupService.ReleaseSpreadNodes), "spinifex-workers"}, - {"ec2.RemoveInstanceFromPlacementGroup", handleNATSRequest(d.placementGroupService.RemoveInstance), "spinifex-workers"}, - {"ec2.ReserveClusterNode", handleNATSRequest(d.placementGroupService.ReserveClusterNode), "spinifex-workers"}, - {"ec2.FinalizeClusterInstances", handleNATSRequest(d.placementGroupService.FinalizeClusterInstances), "spinifex-workers"}, - {"ec2.CreateLaunchTemplate", handleNATSRequest(d.launchTemplateService.CreateLaunchTemplate), "spinifex-workers"}, - {"ec2.CreateLaunchTemplateVersion", handleNATSRequest(d.launchTemplateService.CreateLaunchTemplateVersion), "spinifex-workers"}, - {"ec2.DeleteLaunchTemplate", handleNATSRequest(d.launchTemplateService.DeleteLaunchTemplate), "spinifex-workers"}, - {"ec2.DeleteLaunchTemplateVersions", handleNATSRequest(d.launchTemplateService.DeleteLaunchTemplateVersions), "spinifex-workers"}, - {"ec2.ModifyLaunchTemplate", handleNATSRequest(d.launchTemplateService.ModifyLaunchTemplate), "spinifex-workers"}, - {"ec2.DescribeLaunchTemplates", handleNATSRequest(d.launchTemplateService.DescribeLaunchTemplates), "spinifex-workers"}, - {"ec2.DescribeLaunchTemplateVersions", handleNATSRequest(d.launchTemplateService.DescribeLaunchTemplateVersions), "spinifex-workers"}, - {"ec2.PutSpotInstanceRequests", handleNATSRequest(d.spotInstanceService.PutSpotInstanceRequests), "spinifex-workers"}, - {"ec2.DescribeSpotInstanceRequests", handleNATSRequest(d.spotInstanceService.DescribeSpotInstanceRequests), "spinifex-workers"}, - {"ec2.CancelSpotInstanceRequests", handleNATSRequest(d.spotInstanceService.CancelSpotInstanceRequests), "spinifex-workers"}, + {"ec2.ForceDetachVolume", handleNATSRequest(d.node, d.volumeService.ForceDetachVolume), "spinifex-workers"}, + {"ec2.DescribeVolumeStatus", handleNATSRequest(d.node, d.volumeService.DescribeVolumeStatus), "spinifex-workers"}, + {"ec2.DescribeVolumesModifications", handleNATSRequest(d.node, d.volumeService.DescribeVolumesModifications), "spinifex-workers"}, + {"ec2.CreateSnapshot", handleNATSRequest(d.node, d.snapshotService.CreateSnapshot), "spinifex-workers"}, + {"ec2.DescribeSnapshots", handleNATSRequest(d.node, d.snapshotService.DescribeSnapshots), "spinifex-workers"}, + {"ec2.DeleteSnapshot", handleNATSRequest(d.node, d.snapshotService.DeleteSnapshot), "spinifex-workers"}, + {"ec2.CopySnapshot", handleNATSRequest(d.node, d.snapshotService.CopySnapshot), "spinifex-workers"}, + {"ec2.CreateTags", handleNATSRequest(d.node, d.createTags), "spinifex-workers"}, + {"ec2.DeleteTags", handleNATSRequest(d.node, d.deleteTags), "spinifex-workers"}, + {"ec2.DescribeTags", handleNATSRequest(d.node, d.tagsService.DescribeTags), "spinifex-workers"}, + {"ec2.CreateEgressOnlyInternetGateway", handleNATSRequest(d.node, d.eigwService.CreateEgressOnlyInternetGateway), "spinifex-workers"}, + {"ec2.DeleteEgressOnlyInternetGateway", handleNATSRequest(d.node, d.eigwService.DeleteEgressOnlyInternetGateway), "spinifex-workers"}, + {"ec2.DescribeEgressOnlyInternetGateways", handleNATSRequest(d.node, d.eigwService.DescribeEgressOnlyInternetGateways), "spinifex-workers"}, + {"ec2.CreateInternetGateway", handleNATSRequest(d.node, d.igwService.CreateInternetGateway), "spinifex-workers"}, + {"ec2.DeleteInternetGateway", handleNATSRequest(d.node, d.igwService.DeleteInternetGateway), "spinifex-workers"}, + {"ec2.DescribeInternetGateways", handleNATSRequest(d.node, d.igwService.DescribeInternetGateways), "spinifex-workers"}, + {"ec2.AttachInternetGateway", handleNATSRequest(d.node, d.igwService.AttachInternetGateway), "spinifex-workers"}, + {"ec2.DetachInternetGateway", handleNATSRequest(d.node, d.igwService.DetachInternetGateway), "spinifex-workers"}, + {"ec2.CreatePlacementGroup", handleNATSRequest(d.node, d.placementGroupService.CreatePlacementGroup), "spinifex-workers"}, + {"ec2.DeletePlacementGroup", handleNATSRequest(d.node, d.placementGroupService.DeletePlacementGroup), "spinifex-workers"}, + {"ec2.DescribePlacementGroups", handleNATSRequest(d.node, d.placementGroupService.DescribePlacementGroups), "spinifex-workers"}, + {"ec2.ReserveSpreadNodes", handleNATSRequest(d.node, d.placementGroupService.ReserveSpreadNodes), "spinifex-workers"}, + {"ec2.FinalizeSpreadInstances", handleNATSRequest(d.node, d.placementGroupService.FinalizeSpreadInstances), "spinifex-workers"}, + {"ec2.ReleaseSpreadNodes", handleNATSRequest(d.node, d.placementGroupService.ReleaseSpreadNodes), "spinifex-workers"}, + {"ec2.RemoveInstanceFromPlacementGroup", handleNATSRequest(d.node, d.placementGroupService.RemoveInstance), "spinifex-workers"}, + {"ec2.ReserveClusterNode", handleNATSRequest(d.node, d.placementGroupService.ReserveClusterNode), "spinifex-workers"}, + {"ec2.FinalizeClusterInstances", handleNATSRequest(d.node, d.placementGroupService.FinalizeClusterInstances), "spinifex-workers"}, + {"ec2.CreateLaunchTemplate", handleNATSRequest(d.node, d.launchTemplateService.CreateLaunchTemplate), "spinifex-workers"}, + {"ec2.CreateLaunchTemplateVersion", handleNATSRequest(d.node, d.launchTemplateService.CreateLaunchTemplateVersion), "spinifex-workers"}, + {"ec2.DeleteLaunchTemplate", handleNATSRequest(d.node, d.launchTemplateService.DeleteLaunchTemplate), "spinifex-workers"}, + {"ec2.DeleteLaunchTemplateVersions", handleNATSRequest(d.node, d.launchTemplateService.DeleteLaunchTemplateVersions), "spinifex-workers"}, + {"ec2.ModifyLaunchTemplate", handleNATSRequest(d.node, d.launchTemplateService.ModifyLaunchTemplate), "spinifex-workers"}, + {"ec2.DescribeLaunchTemplates", handleNATSRequest(d.node, d.launchTemplateService.DescribeLaunchTemplates), "spinifex-workers"}, + {"ec2.DescribeLaunchTemplateVersions", handleNATSRequest(d.node, d.launchTemplateService.DescribeLaunchTemplateVersions), "spinifex-workers"}, + {"ec2.PutSpotInstanceRequests", handleNATSRequest(d.node, d.spotInstanceService.PutSpotInstanceRequests), "spinifex-workers"}, + {"ec2.DescribeSpotInstanceRequests", handleNATSRequest(d.node, d.spotInstanceService.DescribeSpotInstanceRequests), "spinifex-workers"}, + {"ec2.CancelSpotInstanceRequests", handleNATSRequest(d.node, d.spotInstanceService.CancelSpotInstanceRequests), "spinifex-workers"}, // Capacity reservations: Create is node-targeted (gateway pins one node); // Describe fans out and Cancel broadcasts, so both use plain Subscribe. {fmt.Sprintf("ec2.CreateCapacityReservation.%s", d.node), d.handleEC2CreateCapacityReservation, ""}, {"ec2.DescribeCapacityReservations", d.handleEC2DescribeCapacityReservations, ""}, {"ec2.CancelCapacityReservation", d.handleEC2CancelCapacityReservation, ""}, - {"ec2.CreateNatGateway", handleNATSRequest(d.natGatewayService.CreateNatGateway), "spinifex-workers"}, - {"ec2.DeleteNatGateway", handleNATSRequest(d.natGatewayService.DeleteNatGateway), "spinifex-workers"}, - {"ec2.DescribeNatGateways", handleNATSRequest(d.natGatewayService.DescribeNatGateways), "spinifex-workers"}, - {"ec2.CreateRouteTable", handleNATSRequest(d.routeTableService.CreateRouteTable), "spinifex-workers"}, - {"ec2.DeleteRouteTable", handleNATSRequest(d.routeTableService.DeleteRouteTable), "spinifex-workers"}, - {"ec2.DescribeRouteTables", handleNATSRequest(d.routeTableService.DescribeRouteTables), "spinifex-workers"}, - {"ec2.CreateRoute", handleNATSRequest(d.routeTableService.CreateRoute), "spinifex-workers"}, - {"ec2.DeleteRoute", handleNATSRequest(d.routeTableService.DeleteRoute), "spinifex-workers"}, - {"ec2.ReplaceRoute", handleNATSRequest(d.routeTableService.ReplaceRoute), "spinifex-workers"}, - {"ec2.AssociateRouteTable", handleNATSRequest(d.routeTableService.AssociateRouteTable), "spinifex-workers"}, - {"ec2.DisassociateRouteTable", handleNATSRequest(d.routeTableService.DisassociateRouteTable), "spinifex-workers"}, - {"ec2.ReplaceRouteTableAssociation", handleNATSRequest(d.routeTableService.ReplaceRouteTableAssociation), "spinifex-workers"}, - {"ec2.CreateVpc", handleNATSRequest(d.vpcService.CreateVpc), "spinifex-workers"}, - {"ec2.DeleteVpc", handleNATSRequest(d.vpcService.DeleteVpc), "spinifex-workers"}, - {"ec2.DescribeVpcs", handleNATSRequest(d.vpcService.DescribeVpcs), "spinifex-workers"}, - {"ec2.CreateSubnet", handleNATSRequest(d.vpcService.CreateSubnet), "spinifex-workers"}, - {"ec2.DeleteSubnet", handleNATSRequest(d.vpcService.DeleteSubnet), "spinifex-workers"}, - {"ec2.DescribeSubnets", handleNATSRequest(d.vpcService.DescribeSubnets), "spinifex-workers"}, - {"ec2.ModifySubnetAttribute", handleNATSRequest(d.vpcService.ModifySubnetAttribute), "spinifex-workers"}, - {"ec2.ModifyVpcAttribute", handleNATSRequest(d.vpcService.ModifyVpcAttribute), "spinifex-workers"}, - {"ec2.DescribeVpcAttribute", handleNATSRequest(d.vpcService.DescribeVpcAttribute), "spinifex-workers"}, - {"ec2.CreateNetworkInterface", handleNATSRequest(d.vpcService.CreateNetworkInterface), "spinifex-workers"}, - {"ec2.DeleteNetworkInterface", handleNATSRequest(d.vpcService.DeleteNetworkInterface), "spinifex-workers"}, - {"ec2.DescribeNetworkInterfaces", handleNATSRequest(d.vpcService.DescribeNetworkInterfaces), "spinifex-workers"}, - {"ec2.ModifyNetworkInterfaceAttribute", handleNATSRequest(d.vpcService.ModifyNetworkInterfaceAttribute), "spinifex-workers"}, - {"ec2.CreateSecurityGroup", handleNATSRequest(d.vpcService.CreateSecurityGroup), "spinifex-workers"}, - {"ec2.DeleteSecurityGroup", handleNATSRequest(d.vpcService.DeleteSecurityGroup), "spinifex-workers"}, - {"ec2.DescribeSecurityGroups", handleNATSRequest(d.vpcService.DescribeSecurityGroups), "spinifex-workers"}, - {"ec2.DescribeSecurityGroupRules", handleNATSRequest(d.vpcService.DescribeSecurityGroupRules), "spinifex-workers"}, - {"ec2.AuthorizeSecurityGroupIngress", handleNATSRequest(d.vpcService.AuthorizeSecurityGroupIngress), "spinifex-workers"}, - {"ec2.AuthorizeSecurityGroupEgress", handleNATSRequest(d.vpcService.AuthorizeSecurityGroupEgress), "spinifex-workers"}, - {"ec2.RevokeSecurityGroupIngress", handleNATSRequest(d.vpcService.RevokeSecurityGroupIngress), "spinifex-workers"}, - {"ec2.RevokeSecurityGroupEgress", handleNATSRequest(d.vpcService.RevokeSecurityGroupEgress), "spinifex-workers"}, - {"ec2.ModifyInstanceAttribute", handleNATSRequest(d.instanceService.ModifyInstanceAttribute), "spinifex-workers"}, - {"ec2.ModifyInstanceMetadataOptions", handleNATSRequest(d.instanceService.ModifyInstanceMetadataOptions), "spinifex-workers"}, - {"ec2.MonitorInstances", handleNATSRequest(d.monitorInstances), "spinifex-workers"}, - {"ec2.UnmonitorInstances", handleNATSRequest(d.unmonitorInstances), "spinifex-workers"}, + {"ec2.CreateNatGateway", handleNATSRequest(d.node, d.natGatewayService.CreateNatGateway), "spinifex-workers"}, + {"ec2.DeleteNatGateway", handleNATSRequest(d.node, d.natGatewayService.DeleteNatGateway), "spinifex-workers"}, + {"ec2.DescribeNatGateways", handleNATSRequest(d.node, d.natGatewayService.DescribeNatGateways), "spinifex-workers"}, + {"ec2.CreateRouteTable", handleNATSRequest(d.node, d.routeTableService.CreateRouteTable), "spinifex-workers"}, + {"ec2.DeleteRouteTable", handleNATSRequest(d.node, d.routeTableService.DeleteRouteTable), "spinifex-workers"}, + {"ec2.DescribeRouteTables", handleNATSRequest(d.node, d.routeTableService.DescribeRouteTables), "spinifex-workers"}, + {"ec2.CreateRoute", handleNATSRequest(d.node, d.routeTableService.CreateRoute), "spinifex-workers"}, + {"ec2.DeleteRoute", handleNATSRequest(d.node, d.routeTableService.DeleteRoute), "spinifex-workers"}, + {"ec2.ReplaceRoute", handleNATSRequest(d.node, d.routeTableService.ReplaceRoute), "spinifex-workers"}, + {"ec2.AssociateRouteTable", handleNATSRequest(d.node, d.routeTableService.AssociateRouteTable), "spinifex-workers"}, + {"ec2.DisassociateRouteTable", handleNATSRequest(d.node, d.routeTableService.DisassociateRouteTable), "spinifex-workers"}, + {"ec2.ReplaceRouteTableAssociation", handleNATSRequest(d.node, d.routeTableService.ReplaceRouteTableAssociation), "spinifex-workers"}, + {"ec2.CreateVpc", handleNATSRequest(d.node, d.vpcService.CreateVpc), "spinifex-workers"}, + {"ec2.DeleteVpc", handleNATSRequest(d.node, d.vpcService.DeleteVpc), "spinifex-workers"}, + {"ec2.DescribeVpcs", handleNATSRequest(d.node, d.vpcService.DescribeVpcs), "spinifex-workers"}, + {"ec2.CreateSubnet", handleNATSRequest(d.node, d.vpcService.CreateSubnet), "spinifex-workers"}, + {"ec2.DeleteSubnet", handleNATSRequest(d.node, d.vpcService.DeleteSubnet), "spinifex-workers"}, + {"ec2.DescribeSubnets", handleNATSRequest(d.node, d.vpcService.DescribeSubnets), "spinifex-workers"}, + {"ec2.ModifySubnetAttribute", handleNATSRequest(d.node, d.vpcService.ModifySubnetAttribute), "spinifex-workers"}, + {"ec2.ModifyVpcAttribute", handleNATSRequest(d.node, d.vpcService.ModifyVpcAttribute), "spinifex-workers"}, + {"ec2.DescribeVpcAttribute", handleNATSRequest(d.node, d.vpcService.DescribeVpcAttribute), "spinifex-workers"}, + {"ec2.CreateNetworkInterface", handleNATSRequest(d.node, d.vpcService.CreateNetworkInterface), "spinifex-workers"}, + {"ec2.DeleteNetworkInterface", handleNATSRequest(d.node, d.vpcService.DeleteNetworkInterface), "spinifex-workers"}, + {"ec2.DescribeNetworkInterfaces", handleNATSRequest(d.node, d.vpcService.DescribeNetworkInterfaces), "spinifex-workers"}, + {"ec2.ModifyNetworkInterfaceAttribute", handleNATSRequest(d.node, d.vpcService.ModifyNetworkInterfaceAttribute), "spinifex-workers"}, + {"ec2.CreateSecurityGroup", handleNATSRequest(d.node, d.vpcService.CreateSecurityGroup), "spinifex-workers"}, + {"ec2.DeleteSecurityGroup", handleNATSRequest(d.node, d.vpcService.DeleteSecurityGroup), "spinifex-workers"}, + {"ec2.DescribeSecurityGroups", handleNATSRequest(d.node, d.vpcService.DescribeSecurityGroups), "spinifex-workers"}, + {"ec2.DescribeSecurityGroupRules", handleNATSRequest(d.node, d.vpcService.DescribeSecurityGroupRules), "spinifex-workers"}, + {"ec2.AuthorizeSecurityGroupIngress", handleNATSRequest(d.node, d.vpcService.AuthorizeSecurityGroupIngress), "spinifex-workers"}, + {"ec2.AuthorizeSecurityGroupEgress", handleNATSRequest(d.node, d.vpcService.AuthorizeSecurityGroupEgress), "spinifex-workers"}, + {"ec2.RevokeSecurityGroupIngress", handleNATSRequest(d.node, d.vpcService.RevokeSecurityGroupIngress), "spinifex-workers"}, + {"ec2.RevokeSecurityGroupEgress", handleNATSRequest(d.node, d.vpcService.RevokeSecurityGroupEgress), "spinifex-workers"}, + {"ec2.ModifyInstanceAttribute", handleNATSRequest(d.node, d.instanceService.ModifyInstanceAttribute), "spinifex-workers"}, + {"ec2.ModifyInstanceMetadataOptions", handleNATSRequest(d.node, d.instanceService.ModifyInstanceMetadataOptions), "spinifex-workers"}, + {"ec2.MonitorInstances", handleNATSRequest(d.node, d.monitorInstances), "spinifex-workers"}, + {"ec2.UnmonitorInstances", handleNATSRequest(d.node, d.unmonitorInstances), "spinifex-workers"}, {"ec2.start", d.handleEC2StartStoppedInstance, "spinifex-workers"}, // ec2.start.{node} is the node-targeted variant: it always starts locally // and never re-forwards, so it goes straight to the service (no routing loop). - {fmt.Sprintf("ec2.start.%s", d.node), handleNATSRequest(d.instanceService.StartStoppedInstance), ""}, - {"ec2.terminate", handleNATSRequest(d.instanceService.TerminateStoppedInstance), "spinifex-workers"}, - {"ec2.DescribeStoppedInstances", handleNATSRequest(d.instanceService.DescribeStoppedInstances), "spinifex-workers"}, - {"ec2.DescribeTerminatedInstances", handleNATSRequest(d.instanceService.DescribeTerminatedInstances), "spinifex-workers"}, + {fmt.Sprintf("ec2.start.%s", d.node), handleNATSRequest(d.node, d.instanceService.StartStoppedInstance), ""}, + {"ec2.terminate", handleNATSRequest(d.node, d.instanceService.TerminateStoppedInstance), "spinifex-workers"}, + {"ec2.DescribeStoppedInstances", handleNATSRequest(d.node, d.instanceService.DescribeStoppedInstances), "spinifex-workers"}, + {"ec2.DescribeTerminatedInstances", handleNATSRequest(d.node, d.instanceService.DescribeTerminatedInstances), "spinifex-workers"}, // these fan out to all nodes and gateway aggregates the results. The // handler only sees per-daemon local state (vmMgr/stoppedStore), so // any queue-grouped routing produces 1/N false NotFound responses. - {"ec2.DescribeInstances", handleNATSRequest(d.instanceService.DescribeInstances), ""}, - {"ec2.DescribeInstanceStatus", handleNATSRequest(d.instanceService.DescribeInstanceStatus), ""}, - {"ec2.DescribeInstanceTypes", handleNATSRequest(d.instanceService.DescribeInstanceTypes), ""}, - {"ec2.DescribeInstanceAttribute", handleNATSRequest(d.instanceService.DescribeInstanceAttribute), ""}, + {"ec2.DescribeInstances", handleNATSRequest(d.node, d.instanceService.DescribeInstances), ""}, + {"ec2.DescribeInstanceStatus", handleNATSRequest(d.node, d.instanceService.DescribeInstanceStatus), ""}, + {"ec2.DescribeInstanceTypes", handleNATSRequest(d.node, d.instanceService.DescribeInstanceTypes), ""}, + {"ec2.DescribeInstanceAttribute", handleNATSRequest(d.node, d.instanceService.DescribeInstanceAttribute), ""}, // IAM instance profile associations: Disassociate/Replace mutate the // owning daemon's vm.VM (non-owners NoOp with Found=false); Describe // returns per-daemon matches that the gateway concatenates. - {"ec2.IamProfileAssociation.disassociate", handleNATSRequest(d.instanceService.DisassociateIamProfileAssociation), ""}, - {"ec2.IamProfileAssociation.replace", handleNATSRequest(d.instanceService.ReplaceIamProfileAssociation), ""}, - {"ec2.IamProfileAssociation.describe", handleNATSRequest(d.instanceService.DescribeIamProfileAssociations), ""}, - {"ec2.EnableEbsEncryptionByDefault", handleNATSRequest(d.accountService.EnableEbsEncryptionByDefault), "spinifex-workers"}, - {"ec2.DisableEbsEncryptionByDefault", handleNATSRequest(d.accountService.DisableEbsEncryptionByDefault), "spinifex-workers"}, - {"ec2.GetEbsEncryptionByDefault", handleNATSRequest(d.accountService.GetEbsEncryptionByDefault), "spinifex-workers"}, - {"ec2.GetSerialConsoleAccessStatus", handleNATSRequest(d.accountService.GetSerialConsoleAccessStatus), "spinifex-workers"}, - {"ec2.EnableSerialConsoleAccess", handleNATSRequest(d.accountService.EnableSerialConsoleAccess), "spinifex-workers"}, - {"ec2.DisableSerialConsoleAccess", handleNATSRequest(d.accountService.DisableSerialConsoleAccess), "spinifex-workers"}, + {"ec2.IamProfileAssociation.disassociate", handleNATSRequest(d.node, d.instanceService.DisassociateIamProfileAssociation), ""}, + {"ec2.IamProfileAssociation.replace", handleNATSRequest(d.node, d.instanceService.ReplaceIamProfileAssociation), ""}, + {"ec2.IamProfileAssociation.describe", handleNATSRequest(d.node, d.instanceService.DescribeIamProfileAssociations), ""}, + {"ec2.EnableEbsEncryptionByDefault", handleNATSRequest(d.node, d.accountService.EnableEbsEncryptionByDefault), "spinifex-workers"}, + {"ec2.DisableEbsEncryptionByDefault", handleNATSRequest(d.node, d.accountService.DisableEbsEncryptionByDefault), "spinifex-workers"}, + {"ec2.GetEbsEncryptionByDefault", handleNATSRequest(d.node, d.accountService.GetEbsEncryptionByDefault), "spinifex-workers"}, + {"ec2.GetSerialConsoleAccessStatus", handleNATSRequest(d.node, d.accountService.GetSerialConsoleAccessStatus), "spinifex-workers"}, + {"ec2.EnableSerialConsoleAccess", handleNATSRequest(d.node, d.accountService.EnableSerialConsoleAccess), "spinifex-workers"}, + {"ec2.DisableSerialConsoleAccess", handleNATSRequest(d.node, d.accountService.DisableSerialConsoleAccess), "spinifex-workers"}, {fmt.Sprintf("spinifex.admin.%s.health", d.node), d.handleHealthCheck, ""}, {"spinifex.nodes.discover", d.handleNodeDiscover, ""}, {"spinifex.node.status", d.handleNodeStatus, ""}, @@ -1020,41 +1020,41 @@ func (d *Daemon) subscribeAll() error { // Without a subscriber the gateway returns nats.ErrNoResponders → ServiceUnavailable. if d.elbv2Service.GatewayURL != "" { subs = append(subs, - natsSub{"elbv2.CreateLoadBalancer", handleNATSRequest(d.elbv2Service.CreateLoadBalancer), "spinifex-workers"}, - natsSub{"elbv2.DeleteLoadBalancer", handleNATSRequest(d.elbv2Service.DeleteLoadBalancer), "spinifex-workers"}, - natsSub{"elbv2.DescribeLoadBalancers", handleNATSRequest(d.elbv2Service.DescribeLoadBalancers), "spinifex-workers"}, - natsSub{"elbv2.CreateTargetGroup", handleNATSRequest(d.elbv2Service.CreateTargetGroup), "spinifex-workers"}, - natsSub{"elbv2.ModifyTargetGroup", handleNATSRequest(d.elbv2Service.ModifyTargetGroup), "spinifex-workers"}, - natsSub{"elbv2.DeleteTargetGroup", handleNATSRequest(d.elbv2Service.DeleteTargetGroup), "spinifex-workers"}, - natsSub{"elbv2.DescribeTargetGroups", handleNATSRequest(d.elbv2Service.DescribeTargetGroups), "spinifex-workers"}, - natsSub{"elbv2.RegisterTargets", handleNATSRequest(d.elbv2Service.RegisterTargets), "spinifex-workers"}, - natsSub{"elbv2.DeregisterTargets", handleNATSRequest(d.elbv2Service.DeregisterTargets), "spinifex-workers"}, - natsSub{"elbv2.DescribeTargetHealth", handleNATSRequest(d.elbv2Service.DescribeTargetHealth), "spinifex-workers"}, - natsSub{"elbv2.CreateListener", handleNATSRequest(d.elbv2Service.CreateListener), "spinifex-workers"}, - natsSub{"elbv2.DeleteListener", handleNATSRequest(d.elbv2Service.DeleteListener), "spinifex-workers"}, - natsSub{"elbv2.ModifyListener", handleNATSRequest(d.elbv2Service.ModifyListener), "spinifex-workers"}, - natsSub{"elbv2.DescribeListeners", handleNATSRequest(d.elbv2Service.DescribeListeners), "spinifex-workers"}, - natsSub{"elbv2.CreateRule", handleNATSRequest(d.elbv2Service.CreateRule), "spinifex-workers"}, - natsSub{"elbv2.ModifyRule", handleNATSRequest(d.elbv2Service.ModifyRule), "spinifex-workers"}, - natsSub{"elbv2.DeleteRule", handleNATSRequest(d.elbv2Service.DeleteRule), "spinifex-workers"}, - natsSub{"elbv2.DescribeRules", handleNATSRequest(d.elbv2Service.DescribeRules), "spinifex-workers"}, - natsSub{"elbv2.SetRulePriorities", handleNATSRequest(d.elbv2Service.SetRulePriorities), "spinifex-workers"}, - natsSub{"elbv2.DescribeTags", handleNATSRequest(d.elbv2Service.DescribeTags), "spinifex-workers"}, - natsSub{"elbv2.AddTags", handleNATSRequest(d.elbv2Service.AddTags), "spinifex-workers"}, - natsSub{"elbv2.RemoveTags", handleNATSRequest(d.elbv2Service.RemoveTags), "spinifex-workers"}, - natsSub{"elbv2.LBAgentHeartbeat", handleNATSRequest(d.elbv2Service.LBAgentHeartbeat), "spinifex-workers"}, - natsSub{"elbv2.GetLBConfig", handleNATSRequest(d.elbv2Service.GetLBConfig), "spinifex-workers"}, - natsSub{"elbv2.ModifyTargetGroupAttributes", handleNATSRequest(d.elbv2Service.ModifyTargetGroupAttributes), "spinifex-workers"}, - natsSub{"elbv2.DescribeTargetGroupAttributes", handleNATSRequest(d.elbv2Service.DescribeTargetGroupAttributes), "spinifex-workers"}, - natsSub{"elbv2.ModifyLoadBalancerAttributes", handleNATSRequest(d.elbv2Service.ModifyLoadBalancerAttributes), "spinifex-workers"}, - natsSub{"elbv2.DescribeLoadBalancerAttributes", handleNATSRequest(d.elbv2Service.DescribeLoadBalancerAttributes), "spinifex-workers"}, - natsSub{"elbv2.SetSecurityGroups", handleNATSRequest(d.elbv2Service.SetSecurityGroups), "spinifex-workers"}, - natsSub{"elbv2.SetIpAddressType", handleNATSRequest(d.elbv2Service.SetIpAddressType), "spinifex-workers"}, - natsSub{"elbv2.SetSubnets", handleNATSRequest(d.elbv2Service.SetSubnets), "spinifex-workers"}, - natsSub{"elbv2.AddListenerCertificates", handleNATSRequest(d.elbv2Service.AddListenerCertificates), "spinifex-workers"}, - natsSub{"elbv2.RemoveListenerCertificates", handleNATSRequest(d.elbv2Service.RemoveListenerCertificates), "spinifex-workers"}, - natsSub{"elbv2.DescribeListenerCertificates", handleNATSRequest(d.elbv2Service.DescribeListenerCertificates), "spinifex-workers"}, - natsSub{"elbv2.DescribeSSLPolicies", handleNATSRequest(d.elbv2Service.DescribeSSLPolicies), "spinifex-workers"}, + natsSub{"elbv2.CreateLoadBalancer", handleNATSRequest(d.node, d.elbv2Service.CreateLoadBalancer), "spinifex-workers"}, + natsSub{"elbv2.DeleteLoadBalancer", handleNATSRequest(d.node, d.elbv2Service.DeleteLoadBalancer), "spinifex-workers"}, + natsSub{"elbv2.DescribeLoadBalancers", handleNATSRequest(d.node, d.elbv2Service.DescribeLoadBalancers), "spinifex-workers"}, + natsSub{"elbv2.CreateTargetGroup", handleNATSRequest(d.node, d.elbv2Service.CreateTargetGroup), "spinifex-workers"}, + natsSub{"elbv2.ModifyTargetGroup", handleNATSRequest(d.node, d.elbv2Service.ModifyTargetGroup), "spinifex-workers"}, + natsSub{"elbv2.DeleteTargetGroup", handleNATSRequest(d.node, d.elbv2Service.DeleteTargetGroup), "spinifex-workers"}, + natsSub{"elbv2.DescribeTargetGroups", handleNATSRequest(d.node, d.elbv2Service.DescribeTargetGroups), "spinifex-workers"}, + natsSub{"elbv2.RegisterTargets", handleNATSRequest(d.node, d.elbv2Service.RegisterTargets), "spinifex-workers"}, + natsSub{"elbv2.DeregisterTargets", handleNATSRequest(d.node, d.elbv2Service.DeregisterTargets), "spinifex-workers"}, + natsSub{"elbv2.DescribeTargetHealth", handleNATSRequest(d.node, d.elbv2Service.DescribeTargetHealth), "spinifex-workers"}, + natsSub{"elbv2.CreateListener", handleNATSRequest(d.node, d.elbv2Service.CreateListener), "spinifex-workers"}, + natsSub{"elbv2.DeleteListener", handleNATSRequest(d.node, d.elbv2Service.DeleteListener), "spinifex-workers"}, + natsSub{"elbv2.ModifyListener", handleNATSRequest(d.node, d.elbv2Service.ModifyListener), "spinifex-workers"}, + natsSub{"elbv2.DescribeListeners", handleNATSRequest(d.node, d.elbv2Service.DescribeListeners), "spinifex-workers"}, + natsSub{"elbv2.CreateRule", handleNATSRequest(d.node, d.elbv2Service.CreateRule), "spinifex-workers"}, + natsSub{"elbv2.ModifyRule", handleNATSRequest(d.node, d.elbv2Service.ModifyRule), "spinifex-workers"}, + natsSub{"elbv2.DeleteRule", handleNATSRequest(d.node, d.elbv2Service.DeleteRule), "spinifex-workers"}, + natsSub{"elbv2.DescribeRules", handleNATSRequest(d.node, d.elbv2Service.DescribeRules), "spinifex-workers"}, + natsSub{"elbv2.SetRulePriorities", handleNATSRequest(d.node, d.elbv2Service.SetRulePriorities), "spinifex-workers"}, + natsSub{"elbv2.DescribeTags", handleNATSRequest(d.node, d.elbv2Service.DescribeTags), "spinifex-workers"}, + natsSub{"elbv2.AddTags", handleNATSRequest(d.node, d.elbv2Service.AddTags), "spinifex-workers"}, + natsSub{"elbv2.RemoveTags", handleNATSRequest(d.node, d.elbv2Service.RemoveTags), "spinifex-workers"}, + natsSub{"elbv2.LBAgentHeartbeat", handleNATSRequest(d.node, d.elbv2Service.LBAgentHeartbeat), "spinifex-workers"}, + natsSub{"elbv2.GetLBConfig", handleNATSRequest(d.node, d.elbv2Service.GetLBConfig), "spinifex-workers"}, + natsSub{"elbv2.ModifyTargetGroupAttributes", handleNATSRequest(d.node, d.elbv2Service.ModifyTargetGroupAttributes), "spinifex-workers"}, + natsSub{"elbv2.DescribeTargetGroupAttributes", handleNATSRequest(d.node, d.elbv2Service.DescribeTargetGroupAttributes), "spinifex-workers"}, + natsSub{"elbv2.ModifyLoadBalancerAttributes", handleNATSRequest(d.node, d.elbv2Service.ModifyLoadBalancerAttributes), "spinifex-workers"}, + natsSub{"elbv2.DescribeLoadBalancerAttributes", handleNATSRequest(d.node, d.elbv2Service.DescribeLoadBalancerAttributes), "spinifex-workers"}, + natsSub{"elbv2.SetSecurityGroups", handleNATSRequest(d.node, d.elbv2Service.SetSecurityGroups), "spinifex-workers"}, + natsSub{"elbv2.SetIpAddressType", handleNATSRequest(d.node, d.elbv2Service.SetIpAddressType), "spinifex-workers"}, + natsSub{"elbv2.SetSubnets", handleNATSRequest(d.node, d.elbv2Service.SetSubnets), "spinifex-workers"}, + natsSub{"elbv2.AddListenerCertificates", handleNATSRequest(d.node, d.elbv2Service.AddListenerCertificates), "spinifex-workers"}, + natsSub{"elbv2.RemoveListenerCertificates", handleNATSRequest(d.node, d.elbv2Service.RemoveListenerCertificates), "spinifex-workers"}, + natsSub{"elbv2.DescribeListenerCertificates", handleNATSRequest(d.node, d.elbv2Service.DescribeListenerCertificates), "spinifex-workers"}, + natsSub{"elbv2.DescribeSSLPolicies", handleNATSRequest(d.node, d.elbv2Service.DescribeSSLPolicies), "spinifex-workers"}, ) } @@ -1063,84 +1063,84 @@ func (d *Daemon) subscribeAll() error { // stable while real bodies land. if d.eksService != nil { subs = append(subs, - natsSub{"eks.CreateCluster", handleNATSRequestWithPrincipal(d.eksService.CreateCluster), "spinifex-workers"}, - natsSub{"eks.DescribeCluster", handleNATSRequest(d.eksService.DescribeCluster), "spinifex-workers"}, - natsSub{"eks.ListClusters", handleNATSRequest(d.eksService.ListClusters), "spinifex-workers"}, - natsSub{"eks.UpdateClusterConfig", handleNATSRequest(d.eksService.UpdateClusterConfig), "spinifex-workers"}, - natsSub{"eks.UpdateClusterVersion", handleNATSRequest(d.eksService.UpdateClusterVersion), "spinifex-workers"}, - natsSub{"eks.DeleteCluster", handleNATSRequest(d.eksService.DeleteCluster), "spinifex-workers"}, - natsSub{"eks.CreateNodegroup", handleNATSRequest(d.eksService.CreateNodegroup), "spinifex-workers"}, - natsSub{"eks.DescribeNodegroup", handleNATSRequest(d.eksService.DescribeNodegroup), "spinifex-workers"}, - natsSub{"eks.ListNodegroups", handleNATSRequest(d.eksService.ListNodegroups), "spinifex-workers"}, - natsSub{"eks.UpdateNodegroupConfig", handleNATSRequest(d.eksService.UpdateNodegroupConfig), "spinifex-workers"}, - natsSub{"eks.UpdateNodegroupVersion", handleNATSRequest(d.eksService.UpdateNodegroupVersion), "spinifex-workers"}, - natsSub{"eks.DeleteNodegroup", handleNATSRequest(d.eksService.DeleteNodegroup), "spinifex-workers"}, - natsSub{"eks.CreateAccessEntry", handleNATSRequest(d.eksService.CreateAccessEntry), "spinifex-workers"}, - natsSub{"eks.DescribeAccessEntry", handleNATSRequest(d.eksService.DescribeAccessEntry), "spinifex-workers"}, - natsSub{"eks.ListAccessEntries", handleNATSRequest(d.eksService.ListAccessEntries), "spinifex-workers"}, - natsSub{"eks.UpdateAccessEntry", handleNATSRequest(d.eksService.UpdateAccessEntry), "spinifex-workers"}, - natsSub{"eks.DeleteAccessEntry", handleNATSRequest(d.eksService.DeleteAccessEntry), "spinifex-workers"}, - natsSub{"eks.AssociateAccessPolicy", handleNATSRequest(d.eksService.AssociateAccessPolicy), "spinifex-workers"}, - natsSub{"eks.DisassociateAccessPolicy", handleNATSRequest(d.eksService.DisassociateAccessPolicy), "spinifex-workers"}, - natsSub{"eks.ListAssociatedAccessPolicies", handleNATSRequest(d.eksService.ListAssociatedAccessPolicies), "spinifex-workers"}, - natsSub{"eks.ListAccessPolicies", handleNATSRequest(d.eksService.ListAccessPolicies), "spinifex-workers"}, - natsSub{"eks.ListAddons", handleNATSRequest(d.eksService.ListAddons), "spinifex-workers"}, - natsSub{"eks.DescribeAddonVersions", handleNATSRequest(d.eksService.DescribeAddonVersions), "spinifex-workers"}, - natsSub{"eks.CreateAddon", handleNATSRequest(d.eksService.CreateAddon), "spinifex-workers"}, - natsSub{"eks.DeleteAddon", handleNATSRequest(d.eksService.DeleteAddon), "spinifex-workers"}, - natsSub{"eks.DescribeAddon", handleNATSRequest(d.eksService.DescribeAddon), "spinifex-workers"}, - natsSub{"eks.UpdateAddon", handleNATSRequest(d.eksService.UpdateAddon), "spinifex-workers"}, - natsSub{"eks.ListStagedAddonManifests", handleNATSRequest(d.eksService.ListStagedAddonManifests), "spinifex-workers"}, - natsSub{"eks.GetRecoveryDirective", handleNATSRequest(d.eksService.GetRecoveryDirective), "spinifex-workers"}, - natsSub{"eks.SetRecoveryDirective", handleNATSRequest(d.eksService.SetRecoveryDirective), "spinifex-workers"}, - natsSub{"eks.RestoreSnapshot", handleNATSRequest(d.eksService.RestoreSnapshot), "spinifex-workers"}, - natsSub{"eks.AssociateIdentityProviderConfig", handleNATSRequest(d.eksService.AssociateIdentityProviderConfig), "spinifex-workers"}, - natsSub{"eks.DescribeIdentityProviderConfig", handleNATSRequest(d.eksService.DescribeIdentityProviderConfig), "spinifex-workers"}, - natsSub{"eks.ListIdentityProviderConfigs", handleNATSRequest(d.eksService.ListIdentityProviderConfigs), "spinifex-workers"}, - natsSub{"eks.DisassociateIdentityProviderConfig", handleNATSRequest(d.eksService.DisassociateIdentityProviderConfig), "spinifex-workers"}, - natsSub{"eks.TagResource", handleNATSRequest(d.eksService.TagResource), "spinifex-workers"}, - natsSub{"eks.UntagResource", handleNATSRequest(d.eksService.UntagResource), "spinifex-workers"}, - natsSub{"eks.ListTagsForResource", handleNATSRequest(d.eksService.ListTagsForResource), "spinifex-workers"}, + natsSub{"eks.CreateCluster", handleNATSRequestWithPrincipal(d.node, d.eksService.CreateCluster), "spinifex-workers"}, + natsSub{"eks.DescribeCluster", handleNATSRequest(d.node, d.eksService.DescribeCluster), "spinifex-workers"}, + natsSub{"eks.ListClusters", handleNATSRequest(d.node, d.eksService.ListClusters), "spinifex-workers"}, + natsSub{"eks.UpdateClusterConfig", handleNATSRequest(d.node, d.eksService.UpdateClusterConfig), "spinifex-workers"}, + natsSub{"eks.UpdateClusterVersion", handleNATSRequest(d.node, d.eksService.UpdateClusterVersion), "spinifex-workers"}, + natsSub{"eks.DeleteCluster", handleNATSRequest(d.node, d.eksService.DeleteCluster), "spinifex-workers"}, + natsSub{"eks.CreateNodegroup", handleNATSRequest(d.node, d.eksService.CreateNodegroup), "spinifex-workers"}, + natsSub{"eks.DescribeNodegroup", handleNATSRequest(d.node, d.eksService.DescribeNodegroup), "spinifex-workers"}, + natsSub{"eks.ListNodegroups", handleNATSRequest(d.node, d.eksService.ListNodegroups), "spinifex-workers"}, + natsSub{"eks.UpdateNodegroupConfig", handleNATSRequest(d.node, d.eksService.UpdateNodegroupConfig), "spinifex-workers"}, + natsSub{"eks.UpdateNodegroupVersion", handleNATSRequest(d.node, d.eksService.UpdateNodegroupVersion), "spinifex-workers"}, + natsSub{"eks.DeleteNodegroup", handleNATSRequest(d.node, d.eksService.DeleteNodegroup), "spinifex-workers"}, + natsSub{"eks.CreateAccessEntry", handleNATSRequest(d.node, d.eksService.CreateAccessEntry), "spinifex-workers"}, + natsSub{"eks.DescribeAccessEntry", handleNATSRequest(d.node, d.eksService.DescribeAccessEntry), "spinifex-workers"}, + natsSub{"eks.ListAccessEntries", handleNATSRequest(d.node, d.eksService.ListAccessEntries), "spinifex-workers"}, + natsSub{"eks.UpdateAccessEntry", handleNATSRequest(d.node, d.eksService.UpdateAccessEntry), "spinifex-workers"}, + natsSub{"eks.DeleteAccessEntry", handleNATSRequest(d.node, d.eksService.DeleteAccessEntry), "spinifex-workers"}, + natsSub{"eks.AssociateAccessPolicy", handleNATSRequest(d.node, d.eksService.AssociateAccessPolicy), "spinifex-workers"}, + natsSub{"eks.DisassociateAccessPolicy", handleNATSRequest(d.node, d.eksService.DisassociateAccessPolicy), "spinifex-workers"}, + natsSub{"eks.ListAssociatedAccessPolicies", handleNATSRequest(d.node, d.eksService.ListAssociatedAccessPolicies), "spinifex-workers"}, + natsSub{"eks.ListAccessPolicies", handleNATSRequest(d.node, d.eksService.ListAccessPolicies), "spinifex-workers"}, + natsSub{"eks.ListAddons", handleNATSRequest(d.node, d.eksService.ListAddons), "spinifex-workers"}, + natsSub{"eks.DescribeAddonVersions", handleNATSRequest(d.node, d.eksService.DescribeAddonVersions), "spinifex-workers"}, + natsSub{"eks.CreateAddon", handleNATSRequest(d.node, d.eksService.CreateAddon), "spinifex-workers"}, + natsSub{"eks.DeleteAddon", handleNATSRequest(d.node, d.eksService.DeleteAddon), "spinifex-workers"}, + natsSub{"eks.DescribeAddon", handleNATSRequest(d.node, d.eksService.DescribeAddon), "spinifex-workers"}, + natsSub{"eks.UpdateAddon", handleNATSRequest(d.node, d.eksService.UpdateAddon), "spinifex-workers"}, + natsSub{"eks.ListStagedAddonManifests", handleNATSRequest(d.node, d.eksService.ListStagedAddonManifests), "spinifex-workers"}, + natsSub{"eks.GetRecoveryDirective", handleNATSRequest(d.node, d.eksService.GetRecoveryDirective), "spinifex-workers"}, + natsSub{"eks.SetRecoveryDirective", handleNATSRequest(d.node, d.eksService.SetRecoveryDirective), "spinifex-workers"}, + natsSub{"eks.RestoreSnapshot", handleNATSRequest(d.node, d.eksService.RestoreSnapshot), "spinifex-workers"}, + natsSub{"eks.AssociateIdentityProviderConfig", handleNATSRequest(d.node, d.eksService.AssociateIdentityProviderConfig), "spinifex-workers"}, + natsSub{"eks.DescribeIdentityProviderConfig", handleNATSRequest(d.node, d.eksService.DescribeIdentityProviderConfig), "spinifex-workers"}, + natsSub{"eks.ListIdentityProviderConfigs", handleNATSRequest(d.node, d.eksService.ListIdentityProviderConfigs), "spinifex-workers"}, + natsSub{"eks.DisassociateIdentityProviderConfig", handleNATSRequest(d.node, d.eksService.DisassociateIdentityProviderConfig), "spinifex-workers"}, + natsSub{"eks.TagResource", handleNATSRequest(d.node, d.eksService.TagResource), "spinifex-workers"}, + natsSub{"eks.UntagResource", handleNATSRequest(d.node, d.eksService.UntagResource), "spinifex-workers"}, + natsSub{"eks.ListTagsForResource", handleNATSRequest(d.node, d.eksService.ListTagsForResource), "spinifex-workers"}, ) } // ECS gateway → daemon subscriptions (control plane; per-account KV). if d.ecsService != nil { subs = append(subs, - natsSub{"ecs.CreateCluster", handleNATSRequest(d.ecsService.CreateCluster), "spinifex-workers"}, - natsSub{"ecs.DeleteCluster", handleNATSRequest(d.ecsService.DeleteCluster), "spinifex-workers"}, - natsSub{"ecs.DescribeClusters", handleNATSRequest(d.ecsService.DescribeClusters), "spinifex-workers"}, - natsSub{"ecs.ListClusters", handleNATSRequest(d.ecsService.ListClusters), "spinifex-workers"}, - natsSub{"ecs.RegisterTaskDefinition", handleNATSRequest(d.ecsService.RegisterTaskDefinition), "spinifex-workers"}, - natsSub{"ecs.DeregisterTaskDefinition", handleNATSRequest(d.ecsService.DeregisterTaskDefinition), "spinifex-workers"}, - natsSub{"ecs.DescribeTaskDefinition", handleNATSRequest(d.ecsService.DescribeTaskDefinition), "spinifex-workers"}, - natsSub{"ecs.ListTaskDefinitions", handleNATSRequest(d.ecsService.ListTaskDefinitions), "spinifex-workers"}, - natsSub{"ecs.RegisterContainerInstance", handleNATSRequest(d.ecsService.RegisterContainerInstance), "spinifex-workers"}, - natsSub{"ecs.DeregisterContainerInstance", handleNATSRequest(d.ecsService.DeregisterContainerInstance), "spinifex-workers"}, - natsSub{"ecs.UpdateContainerInstancesState", handleNATSRequest(d.ecsService.UpdateContainerInstancesState), "spinifex-workers"}, - natsSub{"ecs.DescribeContainerInstances", handleNATSRequest(d.ecsService.DescribeContainerInstances), "spinifex-workers"}, - natsSub{"ecs.ListContainerInstances", handleNATSRequest(d.ecsService.ListContainerInstances), "spinifex-workers"}, - natsSub{"ecs.RunTask", handleNATSRequest(d.ecsService.RunTask), "spinifex-workers"}, - natsSub{"ecs.StartTask", handleNATSRequest(d.ecsService.StartTask), "spinifex-workers"}, - natsSub{"ecs.StopTask", handleNATSRequest(d.ecsService.StopTask), "spinifex-workers"}, - natsSub{"ecs.DescribeTasks", handleNATSRequest(d.ecsService.DescribeTasks), "spinifex-workers"}, - natsSub{"ecs.ListTasks", handleNATSRequest(d.ecsService.ListTasks), "spinifex-workers"}, - natsSub{"ecs.CreateService", handleNATSRequest(d.ecsService.CreateService), "spinifex-workers"}, - natsSub{"ecs.UpdateService", handleNATSRequest(d.ecsService.UpdateService), "spinifex-workers"}, - natsSub{"ecs.DeleteService", handleNATSRequest(d.ecsService.DeleteService), "spinifex-workers"}, - natsSub{"ecs.DescribeServices", handleNATSRequest(d.ecsService.DescribeServices), "spinifex-workers"}, - natsSub{"ecs.ListServices", handleNATSRequest(d.ecsService.ListServices), "spinifex-workers"}, - natsSub{"ecs.SubmitTaskStateChange", handleNATSRequest(d.ecsService.SubmitTaskStateChange), "spinifex-workers"}, - natsSub{"ecs.PollAssignments", handleNATSRequest(d.ecsService.PollAssignments), "spinifex-workers"}, - natsSub{"ecs.ReportTaskGPU", handleNATSRequest(d.ecsService.ReportTaskGPU), "spinifex-workers"}, - natsSub{"ecs.ProvisionCapacity", handleNATSRequest(d.ecsService.ProvisionCapacity), "spinifex-workers"}, - natsSub{"ecs.TagResource", handleNATSRequest(d.ecsService.TagResource), "spinifex-workers"}, - natsSub{"ecs.UntagResource", handleNATSRequest(d.ecsService.UntagResource), "spinifex-workers"}, - natsSub{"ecs.ListTagsForResource", handleNATSRequest(d.ecsService.ListTagsForResource), "spinifex-workers"}, - natsSub{"ecs.PutClusterCapacityProviders", handleNATSRequest(d.ecsService.PutClusterCapacityProviders), "spinifex-workers"}, - natsSub{"ecs.CreateCapacityProvider", handleNATSRequest(d.ecsService.CreateCapacityProvider), "spinifex-workers"}, - natsSub{"ecs.DescribeCapacityProviders", handleNATSRequest(d.ecsService.DescribeCapacityProviders), "spinifex-workers"}, - natsSub{"ecs.DeleteCapacityProvider", handleNATSRequest(d.ecsService.DeleteCapacityProvider), "spinifex-workers"}, + natsSub{"ecs.CreateCluster", handleNATSRequest(d.node, d.ecsService.CreateCluster), "spinifex-workers"}, + natsSub{"ecs.DeleteCluster", handleNATSRequest(d.node, d.ecsService.DeleteCluster), "spinifex-workers"}, + natsSub{"ecs.DescribeClusters", handleNATSRequest(d.node, d.ecsService.DescribeClusters), "spinifex-workers"}, + natsSub{"ecs.ListClusters", handleNATSRequest(d.node, d.ecsService.ListClusters), "spinifex-workers"}, + natsSub{"ecs.RegisterTaskDefinition", handleNATSRequest(d.node, d.ecsService.RegisterTaskDefinition), "spinifex-workers"}, + natsSub{"ecs.DeregisterTaskDefinition", handleNATSRequest(d.node, d.ecsService.DeregisterTaskDefinition), "spinifex-workers"}, + natsSub{"ecs.DescribeTaskDefinition", handleNATSRequest(d.node, d.ecsService.DescribeTaskDefinition), "spinifex-workers"}, + natsSub{"ecs.ListTaskDefinitions", handleNATSRequest(d.node, d.ecsService.ListTaskDefinitions), "spinifex-workers"}, + natsSub{"ecs.RegisterContainerInstance", handleNATSRequest(d.node, d.ecsService.RegisterContainerInstance), "spinifex-workers"}, + natsSub{"ecs.DeregisterContainerInstance", handleNATSRequest(d.node, d.ecsService.DeregisterContainerInstance), "spinifex-workers"}, + natsSub{"ecs.UpdateContainerInstancesState", handleNATSRequest(d.node, d.ecsService.UpdateContainerInstancesState), "spinifex-workers"}, + natsSub{"ecs.DescribeContainerInstances", handleNATSRequest(d.node, d.ecsService.DescribeContainerInstances), "spinifex-workers"}, + natsSub{"ecs.ListContainerInstances", handleNATSRequest(d.node, d.ecsService.ListContainerInstances), "spinifex-workers"}, + natsSub{"ecs.RunTask", handleNATSRequest(d.node, d.ecsService.RunTask), "spinifex-workers"}, + natsSub{"ecs.StartTask", handleNATSRequest(d.node, d.ecsService.StartTask), "spinifex-workers"}, + natsSub{"ecs.StopTask", handleNATSRequest(d.node, d.ecsService.StopTask), "spinifex-workers"}, + natsSub{"ecs.DescribeTasks", handleNATSRequest(d.node, d.ecsService.DescribeTasks), "spinifex-workers"}, + natsSub{"ecs.ListTasks", handleNATSRequest(d.node, d.ecsService.ListTasks), "spinifex-workers"}, + natsSub{"ecs.CreateService", handleNATSRequest(d.node, d.ecsService.CreateService), "spinifex-workers"}, + natsSub{"ecs.UpdateService", handleNATSRequest(d.node, d.ecsService.UpdateService), "spinifex-workers"}, + natsSub{"ecs.DeleteService", handleNATSRequest(d.node, d.ecsService.DeleteService), "spinifex-workers"}, + natsSub{"ecs.DescribeServices", handleNATSRequest(d.node, d.ecsService.DescribeServices), "spinifex-workers"}, + natsSub{"ecs.ListServices", handleNATSRequest(d.node, d.ecsService.ListServices), "spinifex-workers"}, + natsSub{"ecs.SubmitTaskStateChange", handleNATSRequest(d.node, d.ecsService.SubmitTaskStateChange), "spinifex-workers"}, + natsSub{"ecs.PollAssignments", handleNATSRequest(d.node, d.ecsService.PollAssignments), "spinifex-workers"}, + natsSub{"ecs.ReportTaskGPU", handleNATSRequest(d.node, d.ecsService.ReportTaskGPU), "spinifex-workers"}, + natsSub{"ecs.ProvisionCapacity", handleNATSRequest(d.node, d.ecsService.ProvisionCapacity), "spinifex-workers"}, + natsSub{"ecs.TagResource", handleNATSRequest(d.node, d.ecsService.TagResource), "spinifex-workers"}, + natsSub{"ecs.UntagResource", handleNATSRequest(d.node, d.ecsService.UntagResource), "spinifex-workers"}, + natsSub{"ecs.ListTagsForResource", handleNATSRequest(d.node, d.ecsService.ListTagsForResource), "spinifex-workers"}, + natsSub{"ecs.PutClusterCapacityProviders", handleNATSRequest(d.node, d.ecsService.PutClusterCapacityProviders), "spinifex-workers"}, + natsSub{"ecs.CreateCapacityProvider", handleNATSRequest(d.node, d.ecsService.CreateCapacityProvider), "spinifex-workers"}, + natsSub{"ecs.DescribeCapacityProviders", handleNATSRequest(d.node, d.ecsService.DescribeCapacityProviders), "spinifex-workers"}, + natsSub{"ecs.DeleteCapacityProvider", handleNATSRequest(d.node, d.ecsService.DeleteCapacityProvider), "spinifex-workers"}, ) } @@ -1148,34 +1148,34 @@ func (d *Daemon) subscribeAll() error { // the gateway relays onto, addressing only — the payload is authoritative. if d.rdsService != nil { subs = append(subs, - natsSub{handlers_rds.SubjectRegisterWildcard, handleNATSRequest(d.rdsService.RegisterDBInstance), "spinifex-workers"}, - natsSub{handlers_rds.SubjectHealthWildcard, handleNATSRequest(d.rdsService.SubmitDBStateChange), "spinifex-workers"}, - natsSub{handlers_rds.SubjectGetDBBootstrapConfig, handleNATSRequest(d.rdsService.GetDBBootstrapConfig), "spinifex-workers"}, - natsSub{handlers_rds.SubjectAcknowledgeDBBootstrap, handleNATSRequest(d.rdsService.AcknowledgeDBBootstrap), "spinifex-workers"}, - natsSub{handlers_rds.SubjectCreateDBInstance, handleNATSRequest(d.rdsService.CreateDBInstance), "spinifex-workers"}, - natsSub{handlers_rds.SubjectDescribeDBInstances, handleNATSRequest(d.rdsService.DescribeDBInstances), "spinifex-workers"}, - natsSub{handlers_rds.SubjectRebootDBInstance, handleNATSRequest(d.rdsService.RebootDBInstance), "spinifex-workers"}, - natsSub{handlers_rds.SubjectStartDBInstance, handleNATSRequest(d.rdsService.StartDBInstance), "spinifex-workers"}, - natsSub{handlers_rds.SubjectStopDBInstance, handleNATSRequest(d.rdsService.StopDBInstance), "spinifex-workers"}, - natsSub{handlers_rds.SubjectModifyDBInstance, handleNATSRequest(d.rdsService.ModifyDBInstance), "spinifex-workers"}, - natsSub{handlers_rds.SubjectDeleteDBInstance, handleNATSRequest(d.rdsService.DeleteDBInstance), "spinifex-workers"}, - natsSub{handlers_rds.SubjectCreateDBSnapshot, handleNATSRequest(d.rdsService.CreateDBSnapshot), "spinifex-workers"}, - natsSub{handlers_rds.SubjectDescribeDBSnapshots, handleNATSRequest(d.rdsService.DescribeDBSnapshots), "spinifex-workers"}, - natsSub{handlers_rds.SubjectDeleteDBSnapshot, handleNATSRequest(d.rdsService.DeleteDBSnapshot), "spinifex-workers"}, - natsSub{handlers_rds.SubjectRestoreDBInstanceFromDBSnapshot, handleNATSRequest(d.rdsService.RestoreDBInstanceFromDBSnapshot), "spinifex-workers"}, - natsSub{handlers_rds.SubjectDescribeDBInstanceAutomatedBackups, handleNATSRequest(d.rdsService.DescribeDBInstanceAutomatedBackups), "spinifex-workers"}, - natsSub{handlers_rds.SubjectDescribeEvents, handleNATSRequest(d.rdsService.DescribeEvents), "spinifex-workers"}, - natsSub{handlers_rds.SubjectAddTagsToResource, handleNATSRequest(d.rdsService.AddTagsToResource), "spinifex-workers"}, - natsSub{handlers_rds.SubjectRemoveTagsFromResource, handleNATSRequest(d.rdsService.RemoveTagsFromResource), "spinifex-workers"}, - natsSub{handlers_rds.SubjectListTagsForResource, handleNATSRequest(d.rdsService.ListTagsForResource), "spinifex-workers"}, - natsSub{handlers_rds.SubjectCreateDBSubnetGroup, handleNATSRequest(d.rdsService.CreateDBSubnetGroup), "spinifex-workers"}, - natsSub{handlers_rds.SubjectDescribeDBSubnetGroups, handleNATSRequest(d.rdsService.DescribeDBSubnetGroups), "spinifex-workers"}, - natsSub{handlers_rds.SubjectDeleteDBSubnetGroup, handleNATSRequest(d.rdsService.DeleteDBSubnetGroup), "spinifex-workers"}, - natsSub{handlers_rds.SubjectCreateDBParameterGroup, handleNATSRequest(d.rdsService.CreateDBParameterGroup), "spinifex-workers"}, - natsSub{handlers_rds.SubjectDescribeDBParameterGroups, handleNATSRequest(d.rdsService.DescribeDBParameterGroups), "spinifex-workers"}, - natsSub{handlers_rds.SubjectModifyDBParameterGroup, handleNATSRequest(d.rdsService.ModifyDBParameterGroup), "spinifex-workers"}, - natsSub{handlers_rds.SubjectDescribeDBParameters, handleNATSRequest(d.rdsService.DescribeDBParameters), "spinifex-workers"}, - natsSub{handlers_rds.SubjectDeleteDBParameterGroup, handleNATSRequest(d.rdsService.DeleteDBParameterGroup), "spinifex-workers"}, + natsSub{handlers_rds.SubjectRegisterWildcard, handleNATSRequest(d.node, d.rdsService.RegisterDBInstance), "spinifex-workers"}, + natsSub{handlers_rds.SubjectHealthWildcard, handleNATSRequest(d.node, d.rdsService.SubmitDBStateChange), "spinifex-workers"}, + natsSub{handlers_rds.SubjectGetDBBootstrapConfig, handleNATSRequest(d.node, d.rdsService.GetDBBootstrapConfig), "spinifex-workers"}, + natsSub{handlers_rds.SubjectAcknowledgeDBBootstrap, handleNATSRequest(d.node, d.rdsService.AcknowledgeDBBootstrap), "spinifex-workers"}, + natsSub{handlers_rds.SubjectCreateDBInstance, handleNATSRequest(d.node, d.rdsService.CreateDBInstance), "spinifex-workers"}, + natsSub{handlers_rds.SubjectDescribeDBInstances, handleNATSRequest(d.node, d.rdsService.DescribeDBInstances), "spinifex-workers"}, + natsSub{handlers_rds.SubjectRebootDBInstance, handleNATSRequest(d.node, d.rdsService.RebootDBInstance), "spinifex-workers"}, + natsSub{handlers_rds.SubjectStartDBInstance, handleNATSRequest(d.node, d.rdsService.StartDBInstance), "spinifex-workers"}, + natsSub{handlers_rds.SubjectStopDBInstance, handleNATSRequest(d.node, d.rdsService.StopDBInstance), "spinifex-workers"}, + natsSub{handlers_rds.SubjectModifyDBInstance, handleNATSRequest(d.node, d.rdsService.ModifyDBInstance), "spinifex-workers"}, + natsSub{handlers_rds.SubjectDeleteDBInstance, handleNATSRequest(d.node, d.rdsService.DeleteDBInstance), "spinifex-workers"}, + natsSub{handlers_rds.SubjectCreateDBSnapshot, handleNATSRequest(d.node, d.rdsService.CreateDBSnapshot), "spinifex-workers"}, + natsSub{handlers_rds.SubjectDescribeDBSnapshots, handleNATSRequest(d.node, d.rdsService.DescribeDBSnapshots), "spinifex-workers"}, + natsSub{handlers_rds.SubjectDeleteDBSnapshot, handleNATSRequest(d.node, d.rdsService.DeleteDBSnapshot), "spinifex-workers"}, + natsSub{handlers_rds.SubjectRestoreDBInstanceFromDBSnapshot, handleNATSRequest(d.node, d.rdsService.RestoreDBInstanceFromDBSnapshot), "spinifex-workers"}, + natsSub{handlers_rds.SubjectDescribeDBInstanceAutomatedBackups, handleNATSRequest(d.node, d.rdsService.DescribeDBInstanceAutomatedBackups), "spinifex-workers"}, + natsSub{handlers_rds.SubjectDescribeEvents, handleNATSRequest(d.node, d.rdsService.DescribeEvents), "spinifex-workers"}, + natsSub{handlers_rds.SubjectAddTagsToResource, handleNATSRequest(d.node, d.rdsService.AddTagsToResource), "spinifex-workers"}, + natsSub{handlers_rds.SubjectRemoveTagsFromResource, handleNATSRequest(d.node, d.rdsService.RemoveTagsFromResource), "spinifex-workers"}, + natsSub{handlers_rds.SubjectListTagsForResource, handleNATSRequest(d.node, d.rdsService.ListTagsForResource), "spinifex-workers"}, + natsSub{handlers_rds.SubjectCreateDBSubnetGroup, handleNATSRequest(d.node, d.rdsService.CreateDBSubnetGroup), "spinifex-workers"}, + natsSub{handlers_rds.SubjectDescribeDBSubnetGroups, handleNATSRequest(d.node, d.rdsService.DescribeDBSubnetGroups), "spinifex-workers"}, + natsSub{handlers_rds.SubjectDeleteDBSubnetGroup, handleNATSRequest(d.node, d.rdsService.DeleteDBSubnetGroup), "spinifex-workers"}, + natsSub{handlers_rds.SubjectCreateDBParameterGroup, handleNATSRequest(d.node, d.rdsService.CreateDBParameterGroup), "spinifex-workers"}, + natsSub{handlers_rds.SubjectDescribeDBParameterGroups, handleNATSRequest(d.node, d.rdsService.DescribeDBParameterGroups), "spinifex-workers"}, + natsSub{handlers_rds.SubjectModifyDBParameterGroup, handleNATSRequest(d.node, d.rdsService.ModifyDBParameterGroup), "spinifex-workers"}, + natsSub{handlers_rds.SubjectDescribeDBParameters, handleNATSRequest(d.node, d.rdsService.DescribeDBParameters), "spinifex-workers"}, + natsSub{handlers_rds.SubjectDeleteDBParameterGroup, handleNATSRequest(d.node, d.rdsService.DeleteDBParameterGroup), "spinifex-workers"}, ) } @@ -1184,31 +1184,31 @@ func (d *Daemon) subscribeAll() error { // touching JetStream directly). if d.bedrockService != nil { subs = append(subs, - natsSub{handlers_bedrock.SubjectEnsureEndpoint, handleNATSRequest(d.bedrockService.Ensure), "spinifex-workers"}, - natsSub{handlers_bedrock.SubjectDescribeEndpoint, handleNATSRequest(d.bedrockService.Describe), "spinifex-workers"}, - natsSub{handlers_bedrock.SubjectListEndpoints, handleNATSRequest(d.bedrockService.List), "spinifex-workers"}, - natsSub{handlers_bedrock.SubjectDeleteEndpoint, handleNATSRequest(d.bedrockService.Delete), "spinifex-workers"}, + natsSub{handlers_bedrock.SubjectEnsureEndpoint, handleNATSRequest(d.node, d.bedrockService.Ensure), "spinifex-workers"}, + natsSub{handlers_bedrock.SubjectDescribeEndpoint, handleNATSRequest(d.node, d.bedrockService.Describe), "spinifex-workers"}, + natsSub{handlers_bedrock.SubjectListEndpoints, handleNATSRequest(d.node, d.bedrockService.List), "spinifex-workers"}, + natsSub{handlers_bedrock.SubjectDeleteEndpoint, handleNATSRequest(d.node, d.bedrockService.Delete), "spinifex-workers"}, ) } // ACM gateway → daemon subscriptions (certificate import and managed issuance). if d.acmService != nil { subs = append(subs, - natsSub{"acm.ImportCertificate", handleNATSRequest(d.acmService.ImportCertificate), "spinifex-workers"}, - natsSub{"acm.RequestCertificate", handleNATSRequest(d.acmService.RequestCertificate), "spinifex-workers"}, - natsSub{"acm.DescribeCertificate", handleNATSRequest(d.acmService.DescribeCertificate), "spinifex-workers"}, - natsSub{"acm.GetCertificate", handleNATSRequest(d.acmService.GetCertificate), "spinifex-workers"}, - natsSub{"acm.ListCertificates", handleNATSRequest(d.acmService.ListCertificates), "spinifex-workers"}, - natsSub{"acm.DeleteCertificate", handleNATSRequest(d.acmService.DeleteCertificate), "spinifex-workers"}, - natsSub{"acm.ListTagsForCertificate", handleNATSRequest(d.acmService.ListTagsForCertificate), "spinifex-workers"}, - natsSub{"acm.AddTagsToCertificate", handleNATSRequest(d.acmService.AddTagsToCertificate), "spinifex-workers"}, - natsSub{"acm.RemoveTagsFromCertificate", handleNATSRequest(d.acmService.RemoveTagsFromCertificate), "spinifex-workers"}, + natsSub{"acm.ImportCertificate", handleNATSRequest(d.node, d.acmService.ImportCertificate), "spinifex-workers"}, + natsSub{"acm.RequestCertificate", handleNATSRequest(d.node, d.acmService.RequestCertificate), "spinifex-workers"}, + natsSub{"acm.DescribeCertificate", handleNATSRequest(d.node, d.acmService.DescribeCertificate), "spinifex-workers"}, + natsSub{"acm.GetCertificate", handleNATSRequest(d.node, d.acmService.GetCertificate), "spinifex-workers"}, + natsSub{"acm.ListCertificates", handleNATSRequest(d.node, d.acmService.ListCertificates), "spinifex-workers"}, + natsSub{"acm.DeleteCertificate", handleNATSRequest(d.node, d.acmService.DeleteCertificate), "spinifex-workers"}, + natsSub{"acm.ListTagsForCertificate", handleNATSRequest(d.node, d.acmService.ListTagsForCertificate), "spinifex-workers"}, + natsSub{"acm.AddTagsToCertificate", handleNATSRequest(d.node, d.acmService.AddTagsToCertificate), "spinifex-workers"}, + natsSub{"acm.RemoveTagsFromCertificate", handleNATSRequest(d.node, d.acmService.RemoveTagsFromCertificate), "spinifex-workers"}, // ForceRenewCertificate is not an AWS API — it is the operator- // triggered path `spx admin cert force-renew` uses to reissue a // PRIVATE_CA certificate immediately, bypassing the renewal worker's // window and failure backoff, primarily to verify the ELBv2 fan-out // without waiting out the proportional window. - natsSub{"acm.ForceRenewCertificate", handleNATSRequest(d.acmRenewalWorker.ForceRenewCertificate), "spinifex-workers"}, + natsSub{"acm.ForceRenewCertificate", handleNATSRequest(d.node, d.acmRenewalWorker.ForceRenewCertificate), "spinifex-workers"}, ) } @@ -1222,28 +1222,28 @@ func (d *Daemon) subscribeAll() error { // JetStream KV metadata; blob/manifest bytes never traverse these subjects. if d.ecrMetaService != nil { subs = append(subs, - natsSub{handlers_ecr.SubjectRepoCreate, handleNATSRequest(d.ecrMetaService.RepoCreate), "spinifex-workers"}, - natsSub{handlers_ecr.SubjectRepoDescribe, handleNATSRequest(d.ecrMetaService.RepoDescribe), "spinifex-workers"}, - natsSub{handlers_ecr.SubjectRepoList, handleNATSRequest(d.ecrMetaService.RepoList), "spinifex-workers"}, - natsSub{handlers_ecr.SubjectRepoDelete, handleNATSRequest(d.ecrMetaService.RepoDelete), "spinifex-workers"}, - natsSub{handlers_ecr.SubjectPolicyPut, handleNATSRequest(d.ecrMetaService.PolicyPut), "spinifex-workers"}, - natsSub{handlers_ecr.SubjectPolicyGet, handleNATSRequest(d.ecrMetaService.PolicyGet), "spinifex-workers"}, - natsSub{handlers_ecr.SubjectPolicyDelete, handleNATSRequest(d.ecrMetaService.PolicyDelete), "spinifex-workers"}, - natsSub{handlers_ecr.SubjectLifecyclePut, handleNATSRequest(d.ecrMetaService.LifecyclePut), "spinifex-workers"}, - natsSub{handlers_ecr.SubjectLifecycleGet, handleNATSRequest(d.ecrMetaService.LifecycleGet), "spinifex-workers"}, - natsSub{handlers_ecr.SubjectLifecycleDelete, handleNATSRequest(d.ecrMetaService.LifecycleDelete), "spinifex-workers"}, - natsSub{handlers_ecr.SubjectTagPut, handleNATSRequest(d.ecrMetaService.TagPut), "spinifex-workers"}, - natsSub{handlers_ecr.SubjectTagGet, handleNATSRequest(d.ecrMetaService.TagGet), "spinifex-workers"}, - natsSub{handlers_ecr.SubjectTagList, handleNATSRequest(d.ecrMetaService.TagList), "spinifex-workers"}, - natsSub{handlers_ecr.SubjectTagDelete, handleNATSRequest(d.ecrMetaService.TagDelete), "spinifex-workers"}, - natsSub{handlers_ecr.SubjectManifestPut, handleNATSRequest(d.ecrMetaService.ManifestPut), "spinifex-workers"}, - natsSub{handlers_ecr.SubjectManifestDescribe, handleNATSRequest(d.ecrMetaService.ManifestDescribe), "spinifex-workers"}, - natsSub{handlers_ecr.SubjectManifestList, handleNATSRequest(d.ecrMetaService.ManifestList), "spinifex-workers"}, - natsSub{handlers_ecr.SubjectManifestDelete, handleNATSRequest(d.ecrMetaService.ManifestDelete), "spinifex-workers"}, - natsSub{handlers_ecr.SubjectUploadCreate, handleNATSRequest(d.ecrMetaService.UploadCreate), "spinifex-workers"}, - natsSub{handlers_ecr.SubjectUploadGet, handleNATSRequest(d.ecrMetaService.UploadGet), "spinifex-workers"}, - natsSub{handlers_ecr.SubjectUploadUpdate, handleNATSRequest(d.ecrMetaService.UploadUpdate), "spinifex-workers"}, - natsSub{handlers_ecr.SubjectUploadDelete, handleNATSRequest(d.ecrMetaService.UploadDelete), "spinifex-workers"}, + natsSub{handlers_ecr.SubjectRepoCreate, handleNATSRequest(d.node, d.ecrMetaService.RepoCreate), "spinifex-workers"}, + natsSub{handlers_ecr.SubjectRepoDescribe, handleNATSRequest(d.node, d.ecrMetaService.RepoDescribe), "spinifex-workers"}, + natsSub{handlers_ecr.SubjectRepoList, handleNATSRequest(d.node, d.ecrMetaService.RepoList), "spinifex-workers"}, + natsSub{handlers_ecr.SubjectRepoDelete, handleNATSRequest(d.node, d.ecrMetaService.RepoDelete), "spinifex-workers"}, + natsSub{handlers_ecr.SubjectPolicyPut, handleNATSRequest(d.node, d.ecrMetaService.PolicyPut), "spinifex-workers"}, + natsSub{handlers_ecr.SubjectPolicyGet, handleNATSRequest(d.node, d.ecrMetaService.PolicyGet), "spinifex-workers"}, + natsSub{handlers_ecr.SubjectPolicyDelete, handleNATSRequest(d.node, d.ecrMetaService.PolicyDelete), "spinifex-workers"}, + natsSub{handlers_ecr.SubjectLifecyclePut, handleNATSRequest(d.node, d.ecrMetaService.LifecyclePut), "spinifex-workers"}, + natsSub{handlers_ecr.SubjectLifecycleGet, handleNATSRequest(d.node, d.ecrMetaService.LifecycleGet), "spinifex-workers"}, + natsSub{handlers_ecr.SubjectLifecycleDelete, handleNATSRequest(d.node, d.ecrMetaService.LifecycleDelete), "spinifex-workers"}, + natsSub{handlers_ecr.SubjectTagPut, handleNATSRequest(d.node, d.ecrMetaService.TagPut), "spinifex-workers"}, + natsSub{handlers_ecr.SubjectTagGet, handleNATSRequest(d.node, d.ecrMetaService.TagGet), "spinifex-workers"}, + natsSub{handlers_ecr.SubjectTagList, handleNATSRequest(d.node, d.ecrMetaService.TagList), "spinifex-workers"}, + natsSub{handlers_ecr.SubjectTagDelete, handleNATSRequest(d.node, d.ecrMetaService.TagDelete), "spinifex-workers"}, + natsSub{handlers_ecr.SubjectManifestPut, handleNATSRequest(d.node, d.ecrMetaService.ManifestPut), "spinifex-workers"}, + natsSub{handlers_ecr.SubjectManifestDescribe, handleNATSRequest(d.node, d.ecrMetaService.ManifestDescribe), "spinifex-workers"}, + natsSub{handlers_ecr.SubjectManifestList, handleNATSRequest(d.node, d.ecrMetaService.ManifestList), "spinifex-workers"}, + natsSub{handlers_ecr.SubjectManifestDelete, handleNATSRequest(d.node, d.ecrMetaService.ManifestDelete), "spinifex-workers"}, + natsSub{handlers_ecr.SubjectUploadCreate, handleNATSRequest(d.node, d.ecrMetaService.UploadCreate), "spinifex-workers"}, + natsSub{handlers_ecr.SubjectUploadGet, handleNATSRequest(d.node, d.ecrMetaService.UploadGet), "spinifex-workers"}, + natsSub{handlers_ecr.SubjectUploadUpdate, handleNATSRequest(d.node, d.ecrMetaService.UploadUpdate), "spinifex-workers"}, + natsSub{handlers_ecr.SubjectUploadDelete, handleNATSRequest(d.node, d.ecrMetaService.UploadDelete), "spinifex-workers"}, ) } @@ -1251,12 +1251,12 @@ func (d *Daemon) subscribeAll() error { // the disabled stub, so reads return empty lists and mutations a clean // UnsupportedOperation instead of a NATS timeout. subs = append(subs, - natsSub{"ec2.AllocateAddress", handleNATSRequest(d.eipService.AllocateAddress), "spinifex-workers"}, - natsSub{"ec2.ReleaseAddress", handleNATSRequest(d.eipService.ReleaseAddress), "spinifex-workers"}, - natsSub{"ec2.AssociateAddress", handleNATSRequest(d.eipService.AssociateAddress), "spinifex-workers"}, - natsSub{"ec2.DisassociateAddress", handleNATSRequest(d.eipService.DisassociateAddress), "spinifex-workers"}, - natsSub{"ec2.DescribeAddresses", handleNATSRequest(d.eipService.DescribeAddresses), "spinifex-workers"}, - natsSub{"ec2.DescribeAddressesAttribute", handleNATSRequest(d.eipService.DescribeAddressesAttribute), "spinifex-workers"}, + natsSub{"ec2.AllocateAddress", handleNATSRequest(d.node, d.eipService.AllocateAddress), "spinifex-workers"}, + natsSub{"ec2.ReleaseAddress", handleNATSRequest(d.node, d.eipService.ReleaseAddress), "spinifex-workers"}, + natsSub{"ec2.AssociateAddress", handleNATSRequest(d.node, d.eipService.AssociateAddress), "spinifex-workers"}, + natsSub{"ec2.DisassociateAddress", handleNATSRequest(d.node, d.eipService.DisassociateAddress), "spinifex-workers"}, + natsSub{"ec2.DescribeAddresses", handleNATSRequest(d.node, d.eipService.DescribeAddresses), "spinifex-workers"}, + natsSub{"ec2.DescribeAddressesAttribute", handleNATSRequest(d.node, d.eipService.DescribeAddressesAttribute), "spinifex-workers"}, // vpcd holds the leases, but the records naming those addresses live // here, so the reconcile request flows daemon-ward. natsSub{dhcp.TopicLeaseChanged, d.handleDHCPLeaseChanged, "spinifex-workers"}, @@ -2617,7 +2617,7 @@ func (d *Daemon) persistState(nodeID string, vms map[string]*vm.VM) error { // cannot change underneath either write. if d.jsManager != nil { d.jsManager.WriteNodeMarkerBestEffort(nodeID, kvSyncTimeout) - d.jsManager.WriteRunningSet(nodeID, vms) + d.jsManager.WriteRunningSet(nodeID, d.config.AZ, vms) } if localErr != nil { @@ -2788,7 +2788,7 @@ func (d *Daemon) respondWithVolumeAttachment(msg *nats.Msg, volumeID, instanceID jsonResp, err := json.Marshal(attachment) if err != nil { slog.Error("Failed to marshal VolumeAttachment response", "err", err) - respondWithError(msg, awserrors.ErrorServerInternal) + respondWithError(d.node, msg, awserrors.ErrorServerInternal) return } diff --git a/spinifex/daemon/daemon_handlers.go b/spinifex/daemon/daemon_handlers.go index 148edfccb..1536c20ba 100644 --- a/spinifex/daemon/daemon_handlers.go +++ b/spinifex/daemon/daemon_handlers.go @@ -46,59 +46,69 @@ func endOpSpan(span trace.Span, err error) { span.End() } -// respondWithError sends an error payload for the given error code on the NATS message. -func respondWithError(msg *nats.Msg, errCode string) { - if err := msg.Respond(utils.GenerateErrorPayload(errCode)); err != nil { +// respondNATSMsg sends data as the reply to msg via RespondMsg rather than +// Respond, so the reply can carry the X-Node-ID header identifying which +// daemon answered. nodeID "" (an unconfigured caller) omits the header rather +// than sending an empty one, which downstream fan-outs already treat as absent. +func respondNATSMsg(nodeID string, msg *nats.Msg, data []byte) { + reply := nats.NewMsg("") + reply.Data = data + if nodeID != "" { + reply.Header.Set(utils.NodeIDHeader, nodeID) + } + if err := msg.RespondMsg(reply); err != nil { slog.Error("Failed to respond to NATS request", "err", err) } } +// respondWithError sends an error payload for the given error code on the NATS message. +func respondWithError(nodeID string, msg *nats.Msg, errCode string) { + respondNATSMsg(nodeID, msg, utils.GenerateErrorPayload(errCode)) +} + // respondWithServiceError sends the sanitized error code AND the handler's // message, so a caller sees the actionable reason rather than a bare code. Use // it for any error originating in a service call: the code alone collapses a // specific refusal ("only PRIVATE_CA certificates can be force-renewed") into // an opaque ServerInternal, leaving the reason visible only in the daemon log. // Mirrors utils.ServeNATSRequestCtx, which has always preserved the message. -func respondWithServiceError(msg *nats.Msg, err error) { +func respondWithServiceError(nodeID string, msg *nats.Msg, err error) { payload := utils.GenerateErrorPayloadWithMessage(awserrors.ValidErrorCodeFromError(err), err.Error()) - if respErr := msg.Respond(payload); respErr != nil { - slog.Error("Failed to respond to NATS request", "err", respErr) - } + respondNATSMsg(nodeID, msg, payload) } // respondErrorOutcome answers with errCode and reports the outcome that code // implies, so a handler classifies its failure by the code it already chose // rather than by a second judgement that could disagree with it. -func respondErrorOutcome(msg *nats.Msg, errCode string) string { - respondWithError(msg, errCode) +func respondErrorOutcome(nodeID string, msg *nats.Msg, errCode string) string { + respondWithError(nodeID, msg, errCode) return outcomeForCode(errCode) } // respondServiceErrorOutcome is respondErrorOutcome for an error value. -func respondServiceErrorOutcome(msg *nats.Msg, err error) string { - respondWithServiceError(msg, err) +func respondServiceErrorOutcome(nodeID string, msg *nats.Msg, err error) string { + respondWithServiceError(nodeID, msg, err) return outcomeForError(err) } // respondWithJSON marshals data to JSON and sends it as a NATS response. // On marshal failure it responds with an internal server error. -func respondWithJSON(msg *nats.Msg, data any) { +func respondWithJSON(nodeID string, msg *nats.Msg, data any) { jsonResponse, err := json.Marshal(data) if err != nil { slog.Error("Failed to marshal response", "type", fmt.Sprintf("%T", data), "err", err) - respondWithError(msg, awserrors.ErrorServerInternal) + respondWithError(nodeID, msg, awserrors.ErrorServerInternal) return } - if err := msg.Respond(jsonResponse); err != nil { - slog.Error("Failed to respond to NATS request", "err", err) - } + respondNATSMsg(nodeID, msg, jsonResponse) } // handleNATSRequest returns the nats.MsgHandler for the common unmarshal → service → // marshal → respond pattern. Per message the handler opens a consumer span joining the // producer's trace, extracts the account ID from the NATS message header, and passes -// both to the service function. -func handleNATSRequest[I any, O any](serviceFn func(context.Context, *I, string) (*O, error)) natsHandler { +// both to the service function. nodeID is stamped on every reply path — including the +// unmarshal-failure branch — so a fan-out can attribute this daemon's frame by identity. +func handleNATSRequest[I any, O any](nodeID string, serviceFn func(context.Context, *I, string) (*O, error)) natsHandler { return func(msg *nats.Msg) string { ctx, span := utils.StartConsumerSpan(msg) defer span.End() @@ -110,9 +120,7 @@ func handleNATSRequest[I any, O any](serviceFn func(context.Context, *I, string) input := new(I) if errResp := utils.UnmarshalJsonPayload(input, msg.Data); errResp != nil { utils.MarkSpanError(span, errors.New(awserrors.ErrorInvalidParameterValue)) - if err := msg.Respond(errResp); err != nil { - slog.ErrorContext(ctx, "Failed to respond to NATS request", "err", err) - } + respondNATSMsg(nodeID, msg, errResp) // A payload the daemon cannot parse is the caller's mistake, not a // fault of its own. return outcomeClientError @@ -124,10 +132,10 @@ func handleNATSRequest[I any, O any](serviceFn func(context.Context, *I, string) // that says whether the daemon or its caller was at fault. logHandlerError(ctx, "handleNATSRequest: service call failed", msg.Subject, err) utils.MarkSpanError(span, err) - respondWithServiceError(msg, err) + respondWithServiceError(nodeID, msg, err) return outcomeForError(err) } - respondWithJSON(msg, output) + respondWithJSON(nodeID, msg, output) return outcomeSuccess } } @@ -136,7 +144,7 @@ func handleNATSRequest[I any, O any](serviceFn func(context.Context, *I, string) // also need the caller's IAM principal ARN (X-Principal-ARN header) — e.g. EKS // CreateCluster, which mints the bootstrap-creator-admin AccessEntry for the // caller. -func handleNATSRequestWithPrincipal[I any, O any](serviceFn func(context.Context, *I, string, string) (*O, error)) natsHandler { +func handleNATSRequestWithPrincipal[I any, O any](nodeID string, serviceFn func(context.Context, *I, string, string) (*O, error)) natsHandler { return func(msg *nats.Msg) string { ctx, span := utils.StartConsumerSpan(msg) defer span.End() @@ -146,19 +154,17 @@ func handleNATSRequestWithPrincipal[I any, O any](serviceFn func(context.Context input := new(I) if errResp := utils.UnmarshalJsonPayload(input, msg.Data); errResp != nil { utils.MarkSpanError(span, errors.New(awserrors.ErrorInvalidParameterValue)) - if err := msg.Respond(errResp); err != nil { - slog.ErrorContext(ctx, "Failed to respond to NATS request", "err", err) - } + respondNATSMsg(nodeID, msg, errResp) return outcomeClientError } output, err := serviceFn(ctx, input, accountID, principalARN) if err != nil { logHandlerError(ctx, "handleNATSRequestWithPrincipal: service call failed", msg.Subject, err) utils.MarkSpanError(span, err) - respondWithServiceError(msg, err) + respondWithServiceError(nodeID, msg, err) return outcomeForError(err) } - respondWithJSON(msg, output) + respondWithJSON(nodeID, msg, output) return outcomeSuccess } } @@ -188,7 +194,7 @@ func (d *Daemon) dispatchEC2Command(msg *nats.Msg) (string, string) { if err := json.Unmarshal(msg.Data, &command); err != nil { slog.ErrorContext(ctx, "Error unmarshaling EC2 instance command", "err", err) utils.MarkSpanError(span, err) - respondWithError(msg, awserrors.ErrorServerInternal) + respondWithError(d.node, msg, awserrors.ErrorServerInternal) return "unknown", outcomeError } @@ -199,11 +205,11 @@ func (d *Daemon) dispatchEC2Command(msg *nats.Msg) (string, string) { instance, ok := d.vmMgr.Get(command.ID) if !ok { slog.WarnContext(ctx, "Instance is not running on this node", "id", command.ID) - return name, respondErrorOutcome(msg, awserrors.ErrorInvalidInstanceIDNotFound) + return name, respondErrorOutcome(d.node, msg, awserrors.ErrorInvalidInstanceIDNotFound) } // Verify the caller owns this instance - if !checkInstanceOwnership(msg, command.ID, instance.AccountID) { + if !checkInstanceOwnership(d.node, msg, command.ID, instance.AccountID) { return name, outcomeClientError } @@ -245,7 +251,7 @@ func (d *Daemon) dispatchEC2Command(msg *nats.Msg) (string, string) { endOpSpan(opSpan, err) if err != nil { utils.MarkSpanError(span, err) - return name, respondServiceErrorOutcome(msg, err) + return name, respondServiceErrorOutcome(d.node, msg, err) } if err := msg.Respond(fmt.Appendf(nil, `{"status":"running","instanceId":"%s"}`, instance.ID)); err != nil { slog.ErrorContext(ctx, "Failed to respond to NATS request", "err", err) @@ -257,7 +263,7 @@ func (d *Daemon) dispatchEC2Command(msg *nats.Msg) (string, string) { endOpSpan(opSpan, err) if err != nil { utils.MarkSpanError(span, err) - return name, respondServiceErrorOutcome(msg, err) + return name, respondServiceErrorOutcome(d.node, msg, err) } if err := msg.Respond([]byte(`{}`)); err != nil { slog.ErrorContext(ctx, "Failed to respond to NATS request", "err", err) @@ -273,7 +279,7 @@ func (d *Daemon) dispatchEC2Command(msg *nats.Msg) (string, string) { endOpSpan(opSpan, err) if err != nil { utils.MarkSpanError(span, err) - return name, respondServiceErrorOutcome(msg, err) + return name, respondServiceErrorOutcome(d.node, msg, err) } if err := msg.Respond([]byte(`{}`)); err != nil { slog.ErrorContext(ctx, "Failed to respond to NATS request", "err", err) @@ -281,7 +287,7 @@ func (d *Daemon) dispatchEC2Command(msg *nats.Msg) (string, string) { return name, outcomeSuccess default: slog.WarnContext(ctx, "Unhandled EC2 instance command", "id", command.ID, "attributes", command.Attributes) - return name, respondErrorOutcome(msg, awserrors.ErrorServerInternal) + return name, respondErrorOutcome(d.node, msg, awserrors.ErrorServerInternal) } } @@ -345,7 +351,7 @@ func (d *Daemon) handleHealthCheck(msg *nats.Msg) string { Uptime: int64(time.Since(d.startTime).Seconds()), } - respondWithJSON(msg, response) + respondWithJSON(d.node, msg, response) slog.Debug("Health check responded", "node", d.node, "epoch", d.clusterConfig.Epoch) return outcomeSuccess } @@ -357,7 +363,7 @@ func (d *Daemon) handleNodeDiscover(msg *nats.Msg) string { Node: d.node, } - respondWithJSON(msg, response) + respondWithJSON(d.node, msg, response) slog.Debug("Node discovery responded", "node", d.node) return outcomeSuccess } @@ -439,7 +445,7 @@ func (d *Daemon) handleNodeStatus(msg *nats.Msg) string { } wg.Wait() - respondWithJSON(msg, resp) + respondWithJSON(d.node, msg, resp) return outcomeSuccess } @@ -627,7 +633,7 @@ func (d *Daemon) handleNodeVMs(msg *nats.Msg) string { VMs: vms, } - respondWithJSON(msg, resp) + respondWithJSON(d.node, msg, resp) return outcomeSuccess } diff --git a/spinifex/daemon/daemon_handlers_capacityreservation.go b/spinifex/daemon/daemon_handlers_capacityreservation.go index c78ce0f7e..965bf0050 100644 --- a/spinifex/daemon/daemon_handlers_capacityreservation.go +++ b/spinifex/daemon/daemon_handlers_capacityreservation.go @@ -30,7 +30,7 @@ func (d *Daemon) handleEC2CreateCapacityReservation(msg *nats.Msg) string { instanceType := aws.StringValue(input.InstanceType) it := d.resourceMgr.instanceTypes[instanceType] if it == nil || instancetypes.IsGPUType(it) { - respondWithError(msg, awserrors.ErrorInvalidInstanceType) + respondWithError(d.node, msg, awserrors.ErrorInvalidInstanceType) return outcomeError } @@ -58,7 +58,7 @@ func (d *Daemon) handleEC2CreateCapacityReservation(msg *nats.Msg) string { } if err := d.resourceMgr.CreateReservation(rec); err != nil { - respondWithServiceError(msg, err) + respondWithServiceError(d.node, msg, err) return outcomeError } @@ -71,11 +71,11 @@ func (d *Daemon) handleEC2CreateCapacityReservation(msg *nats.Msg) string { slog.Error("Failed to subscribe reservation launch subject, rolling back reservation", "crId", rec.ID, "err", err) d.resourceMgr.CancelReservation(rec.ID, accountID) - respondWithError(msg, awserrors.ErrorServerInternal) + respondWithError(d.node, msg, awserrors.ErrorServerInternal) return outcomeError } - respondWithJSON(msg, rec.toAWSCapacityReservation()) + respondWithJSON(d.node, msg, rec.toAWSCapacityReservation()) return outcomeSuccess } @@ -89,7 +89,7 @@ func (d *Daemon) handleEC2DescribeCapacityReservations(msg *nats.Msg) string { for _, rec := range d.resourceMgr.ListReservations(accountID) { out.CapacityReservations = append(out.CapacityReservations, rec.toAWSCapacityReservation()) } - respondWithJSON(msg, out) + respondWithJSON(d.node, msg, out) return outcomeSuccess } @@ -112,7 +112,7 @@ func (d *Daemon) handleEC2CancelCapacityReservation(msg *nats.Msg) string { // hits no responder → InvalidCapacityReservationId.NotFound at the gateway. d.unsubscribeReservationLaunch(aws.StringValue(input.CapacityReservationId)) } - respondWithJSON(msg, &ec2.CancelCapacityReservationOutput{Return: aws.Bool(found)}) + respondWithJSON(d.node, msg, &ec2.CancelCapacityReservationOutput{Return: aws.Bool(found)}) return outcomeSuccess } diff --git a/spinifex/daemon/daemon_handlers_console.go b/spinifex/daemon/daemon_handlers_console.go index fb38185dd..453e15efc 100644 --- a/spinifex/daemon/daemon_handlers_console.go +++ b/spinifex/daemon/daemon_handlers_console.go @@ -26,7 +26,7 @@ func (d *Daemon) handleEC2GetConsoleOutput(msg *nats.Msg) { } if input.InstanceId == nil { - respondWithError(msg, awserrors.ErrorMissingParameter) + respondWithError(d.node, msg, awserrors.ErrorMissingParameter) return } @@ -35,12 +35,12 @@ func (d *Daemon) handleEC2GetConsoleOutput(msg *nats.Msg) { // Find the instance on this node instance, exists := d.vmMgr.Get(instanceID) if !exists { - respondWithError(msg, awserrors.ErrorInvalidInstanceIDNotFound) + respondWithError(d.node, msg, awserrors.ErrorInvalidInstanceIDNotFound) return } // Verify the caller owns this instance - if !checkInstanceOwnership(msg, instanceID, instance.AccountID) { + if !checkInstanceOwnership(d.node, msg, instanceID, instance.AccountID) { return } @@ -84,6 +84,6 @@ func (d *Daemon) handleEC2GetConsoleOutput(msg *nats.Msg) { Timestamp: &modTime, } - respondWithJSON(msg, output) + respondWithJSON(d.node, msg, output) slog.Info("handleEC2GetConsoleOutput completed", "instance_id", instanceID, "output_bytes", len(outputData)) } diff --git a/spinifex/daemon/daemon_handlers_eks_test.go b/spinifex/daemon/daemon_handlers_eks_test.go index 6da376f1c..d914e1813 100644 --- a/spinifex/daemon/daemon_handlers_eks_test.go +++ b/spinifex/daemon/daemon_handlers_eks_test.go @@ -71,39 +71,39 @@ func TestDaemonHandleEKS_AllHandlersDispatchToService(t *testing.T) { handler nats.MsgHandler wantCode string }{ - {"eks.UpdateClusterConfig", asMsgHandler(handleNATSRequest(d.eksService.UpdateClusterConfig)), notImpl}, - {"eks.UpdateClusterVersion", asMsgHandler(handleNATSRequest(d.eksService.UpdateClusterVersion)), notImpl}, - {"eks.CreateNodegroup", asMsgHandler(handleNATSRequest(d.eksService.CreateNodegroup)), unavailable}, - {"eks.DescribeNodegroup", asMsgHandler(handleNATSRequest(d.eksService.DescribeNodegroup)), invalid}, - {"eks.ListNodegroups", asMsgHandler(handleNATSRequest(d.eksService.ListNodegroups)), invalid}, - {"eks.UpdateNodegroupConfig", asMsgHandler(handleNATSRequest(d.eksService.UpdateNodegroupConfig)), unavailable}, - {"eks.UpdateNodegroupVersion", asMsgHandler(handleNATSRequest(d.eksService.UpdateNodegroupVersion)), notImpl}, - {"eks.DeleteNodegroup", asMsgHandler(handleNATSRequest(d.eksService.DeleteNodegroup)), unavailable}, - {"eks.CreateAccessEntry", asMsgHandler(handleNATSRequest(d.eksService.CreateAccessEntry)), invalid}, - {"eks.DescribeAccessEntry", asMsgHandler(handleNATSRequest(d.eksService.DescribeAccessEntry)), invalid}, - {"eks.ListAccessEntries", asMsgHandler(handleNATSRequest(d.eksService.ListAccessEntries)), invalid}, - {"eks.UpdateAccessEntry", asMsgHandler(handleNATSRequest(d.eksService.UpdateAccessEntry)), invalid}, - {"eks.DeleteAccessEntry", asMsgHandler(handleNATSRequest(d.eksService.DeleteAccessEntry)), invalid}, - {"eks.AssociateAccessPolicy", asMsgHandler(handleNATSRequest(d.eksService.AssociateAccessPolicy)), invalid}, - {"eks.DisassociateAccessPolicy", asMsgHandler(handleNATSRequest(d.eksService.DisassociateAccessPolicy)), invalid}, - {"eks.ListAssociatedAccessPolicies", asMsgHandler(handleNATSRequest(d.eksService.ListAssociatedAccessPolicies)), invalid}, - {"eks.ListAccessPolicies", asMsgHandler(handleNATSRequest(d.eksService.ListAccessPolicies)), ""}, + {"eks.UpdateClusterConfig", asMsgHandler(handleNATSRequest(d.node, d.eksService.UpdateClusterConfig)), notImpl}, + {"eks.UpdateClusterVersion", asMsgHandler(handleNATSRequest(d.node, d.eksService.UpdateClusterVersion)), notImpl}, + {"eks.CreateNodegroup", asMsgHandler(handleNATSRequest(d.node, d.eksService.CreateNodegroup)), unavailable}, + {"eks.DescribeNodegroup", asMsgHandler(handleNATSRequest(d.node, d.eksService.DescribeNodegroup)), invalid}, + {"eks.ListNodegroups", asMsgHandler(handleNATSRequest(d.node, d.eksService.ListNodegroups)), invalid}, + {"eks.UpdateNodegroupConfig", asMsgHandler(handleNATSRequest(d.node, d.eksService.UpdateNodegroupConfig)), unavailable}, + {"eks.UpdateNodegroupVersion", asMsgHandler(handleNATSRequest(d.node, d.eksService.UpdateNodegroupVersion)), notImpl}, + {"eks.DeleteNodegroup", asMsgHandler(handleNATSRequest(d.node, d.eksService.DeleteNodegroup)), unavailable}, + {"eks.CreateAccessEntry", asMsgHandler(handleNATSRequest(d.node, d.eksService.CreateAccessEntry)), invalid}, + {"eks.DescribeAccessEntry", asMsgHandler(handleNATSRequest(d.node, d.eksService.DescribeAccessEntry)), invalid}, + {"eks.ListAccessEntries", asMsgHandler(handleNATSRequest(d.node, d.eksService.ListAccessEntries)), invalid}, + {"eks.UpdateAccessEntry", asMsgHandler(handleNATSRequest(d.node, d.eksService.UpdateAccessEntry)), invalid}, + {"eks.DeleteAccessEntry", asMsgHandler(handleNATSRequest(d.node, d.eksService.DeleteAccessEntry)), invalid}, + {"eks.AssociateAccessPolicy", asMsgHandler(handleNATSRequest(d.node, d.eksService.AssociateAccessPolicy)), invalid}, + {"eks.DisassociateAccessPolicy", asMsgHandler(handleNATSRequest(d.node, d.eksService.DisassociateAccessPolicy)), invalid}, + {"eks.ListAssociatedAccessPolicies", asMsgHandler(handleNATSRequest(d.node, d.eksService.ListAssociatedAccessPolicies)), invalid}, + {"eks.ListAccessPolicies", asMsgHandler(handleNATSRequest(d.node, d.eksService.ListAccessPolicies)), ""}, // Addon handlers are implemented (Sprint 6 P1): an empty body fails // cluster validation with InvalidParameterValue, except // DescribeAddonVersions which returns the static catalogue (success). - {"eks.ListAddons", asMsgHandler(handleNATSRequest(d.eksService.ListAddons)), invalid}, - {"eks.DescribeAddonVersions", asMsgHandler(handleNATSRequest(d.eksService.DescribeAddonVersions)), ""}, - {"eks.CreateAddon", asMsgHandler(handleNATSRequest(d.eksService.CreateAddon)), invalid}, - {"eks.DeleteAddon", asMsgHandler(handleNATSRequest(d.eksService.DeleteAddon)), invalid}, - {"eks.DescribeAddon", asMsgHandler(handleNATSRequest(d.eksService.DescribeAddon)), invalid}, - {"eks.UpdateAddon", asMsgHandler(handleNATSRequest(d.eksService.UpdateAddon)), invalid}, - {"eks.AssociateIdentityProviderConfig", asMsgHandler(handleNATSRequest(d.eksService.AssociateIdentityProviderConfig)), notImpl}, - {"eks.DescribeIdentityProviderConfig", asMsgHandler(handleNATSRequest(d.eksService.DescribeIdentityProviderConfig)), notImpl}, - {"eks.ListIdentityProviderConfigs", asMsgHandler(handleNATSRequest(d.eksService.ListIdentityProviderConfigs)), notImpl}, - {"eks.DisassociateIdentityProviderConfig", asMsgHandler(handleNATSRequest(d.eksService.DisassociateIdentityProviderConfig)), notImpl}, - {"eks.TagResource", asMsgHandler(handleNATSRequest(d.eksService.TagResource)), notImpl}, - {"eks.UntagResource", asMsgHandler(handleNATSRequest(d.eksService.UntagResource)), notImpl}, - {"eks.ListTagsForResource", asMsgHandler(handleNATSRequest(d.eksService.ListTagsForResource)), notImpl}, + {"eks.ListAddons", asMsgHandler(handleNATSRequest(d.node, d.eksService.ListAddons)), invalid}, + {"eks.DescribeAddonVersions", asMsgHandler(handleNATSRequest(d.node, d.eksService.DescribeAddonVersions)), ""}, + {"eks.CreateAddon", asMsgHandler(handleNATSRequest(d.node, d.eksService.CreateAddon)), invalid}, + {"eks.DeleteAddon", asMsgHandler(handleNATSRequest(d.node, d.eksService.DeleteAddon)), invalid}, + {"eks.DescribeAddon", asMsgHandler(handleNATSRequest(d.node, d.eksService.DescribeAddon)), invalid}, + {"eks.UpdateAddon", asMsgHandler(handleNATSRequest(d.node, d.eksService.UpdateAddon)), invalid}, + {"eks.AssociateIdentityProviderConfig", asMsgHandler(handleNATSRequest(d.node, d.eksService.AssociateIdentityProviderConfig)), notImpl}, + {"eks.DescribeIdentityProviderConfig", asMsgHandler(handleNATSRequest(d.node, d.eksService.DescribeIdentityProviderConfig)), notImpl}, + {"eks.ListIdentityProviderConfigs", asMsgHandler(handleNATSRequest(d.node, d.eksService.ListIdentityProviderConfigs)), notImpl}, + {"eks.DisassociateIdentityProviderConfig", asMsgHandler(handleNATSRequest(d.node, d.eksService.DisassociateIdentityProviderConfig)), notImpl}, + {"eks.TagResource", asMsgHandler(handleNATSRequest(d.node, d.eksService.TagResource)), notImpl}, + {"eks.UntagResource", asMsgHandler(handleNATSRequest(d.node, d.eksService.UntagResource)), notImpl}, + {"eks.ListTagsForResource", asMsgHandler(handleNATSRequest(d.node, d.eksService.ListTagsForResource)), notImpl}, } require.Len(t, cases, 30, "expected exactly one handler per non-lifecycle AWS EKS action") diff --git a/spinifex/daemon/daemon_handlers_eni.go b/spinifex/daemon/daemon_handlers_eni.go index d95f682b4..e328b7636 100644 --- a/spinifex/daemon/daemon_handlers_eni.go +++ b/spinifex/daemon/daemon_handlers_eni.go @@ -24,7 +24,7 @@ func (d *Daemon) handleAttachNetworkInterface(ctx context.Context, msg *nats.Msg slog.InfoContext(ctx, "Attaching ENI to instance", "instanceId", command.ID) if command.AttachENIData == nil || command.AttachENIData.NetworkInterfaceID == "" { - return respondErrorOutcome(msg, awserrors.ErrorInvalidParameterValue) + return respondErrorOutcome(d.node, msg, awserrors.ErrorInvalidParameterValue) } eniID := command.AttachENIData.NetworkInterfaceID @@ -32,20 +32,20 @@ func (d *Daemon) handleAttachNetworkInterface(ctx context.Context, msg *nats.Msg accountID := utils.AccountIDFromMsg(msg) if status := d.vmMgr.Status(instance); status != vm.StateRunning { - return respondErrorOutcome(msg, awserrors.ErrorIncorrectInstanceState) + return respondErrorOutcome(d.node, msg, awserrors.ErrorIncorrectInstanceState) } record, err := d.vpcService.GetENIRecord(accountID, eniID) if err != nil { - return respondServiceErrorOutcome(msg, err) + return respondServiceErrorOutcome(d.node, msg, err) } if record.Status == "in-use" && record.InstanceId != instance.ID { - return respondErrorOutcome(msg, awserrors.ErrorInvalidNetworkInterfaceInUse) + return respondErrorOutcome(d.node, msg, awserrors.ErrorInvalidNetworkInterfaceInUse) } attachmentID, err := d.vpcService.AttachENI(accountID, eniID, instance.ID, deviceIndex) if err != nil { - return respondServiceErrorOutcome(msg, err) + return respondServiceErrorOutcome(d.node, msg, err) } if err := d.vpcService.UpdateENI(accountID, eniID, func(r *handlers_ec2_vpc.ENIRecord) { r.AttachmentStatus = "attaching" @@ -69,7 +69,7 @@ func (d *Daemon) handleAttachNetworkInterface(ctx context.Context, msg *nats.Msg r.HotPlugSlot = 0 r.LastAttachError = hotPlugErr.Error() }) - return respondErrorOutcome(msg, eniHotplugErrorCode(hotPlugErr)) + return respondErrorOutcome(d.node, msg, eniHotplugErrorCode(hotPlugErr)) } if err := d.vpcService.UpdateENI(accountID, eniID, func(r *handlers_ec2_vpc.ENIRecord) { @@ -89,7 +89,7 @@ func (d *Daemon) handleAttachNetworkInterface(ctx context.Context, msg *nats.Msg "deviceIndex": deviceIndex, }) - respondWithJSON(msg, ec2.AttachNetworkInterfaceOutput{ + respondWithJSON(d.node, msg, ec2.AttachNetworkInterfaceOutput{ AttachmentId: aws.String(attachmentID), }) return outcomeSuccess @@ -102,7 +102,7 @@ func (d *Daemon) handleDetachNetworkInterface(ctx context.Context, msg *nats.Msg slog.InfoContext(ctx, "Detaching ENI from instance", "instanceId", command.ID) if command.DetachENIData == nil || command.DetachENIData.AttachmentID == "" { - return respondErrorOutcome(msg, awserrors.ErrorInvalidParameterValue) + return respondErrorOutcome(d.node, msg, awserrors.ErrorInvalidParameterValue) } attachmentID := command.DetachENIData.AttachmentID @@ -111,14 +111,14 @@ func (d *Daemon) handleDetachNetworkInterface(ctx context.Context, msg *nats.Msg record, err := d.vpcService.FindENIByAttachment(accountID, attachmentID) if err != nil { - return respondServiceErrorOutcome(msg, err) + return respondServiceErrorOutcome(d.node, msg, err) } if record.InstanceId != instance.ID { - return respondErrorOutcome(msg, awserrors.ErrorInvalidAttachmentIDNotFound) + return respondErrorOutcome(d.node, msg, awserrors.ErrorInvalidAttachmentIDNotFound) } if status := d.vmMgr.Status(instance); status != vm.StateRunning { - return respondErrorOutcome(msg, awserrors.ErrorIncorrectInstanceState) + return respondErrorOutcome(d.node, msg, awserrors.ErrorIncorrectInstanceState) } if err := d.vpcService.UpdateENI(accountID, record.NetworkInterfaceId, func(r *handlers_ec2_vpc.ENIRecord) { @@ -137,7 +137,7 @@ func (d *Daemon) handleDetachNetworkInterface(ctx context.Context, msg *nats.Msg _ = d.vpcService.UpdateENI(accountID, record.NetworkInterfaceId, func(r *handlers_ec2_vpc.ENIRecord) { r.DetachInFlight = false }) - return respondErrorOutcome(msg, eniHotplugErrorCode(err)) + return respondErrorOutcome(d.node, msg, eniHotplugErrorCode(err)) } if err := d.vpcService.DetachENI(ctx, accountID, record.NetworkInterfaceId); err != nil { @@ -158,7 +158,7 @@ func (d *Daemon) handleDetachNetworkInterface(ctx context.Context, msg *nats.Msg "attachmentId": attachmentID, }) - respondWithJSON(msg, ec2.DetachNetworkInterfaceOutput{}) + respondWithJSON(d.node, msg, ec2.DetachNetworkInterfaceOutput{}) return outcomeSuccess } diff --git a/spinifex/daemon/daemon_handlers_error_test.go b/spinifex/daemon/daemon_handlers_error_test.go index c6987251d..e933a1a2d 100644 --- a/spinifex/daemon/daemon_handlers_error_test.go +++ b/spinifex/daemon/daemon_handlers_error_test.go @@ -30,7 +30,7 @@ func requestErrorEnvelope(t *testing.T, subject string, fn func(context.Context, require.NoError(t, err) defer nc.Close() - sub, err := nc.Subscribe(subject, asMsgHandler(handleNATSRequest(fn))) + sub, err := nc.Subscribe(subject, asMsgHandler(handleNATSRequest("test-node", fn))) require.NoError(t, err) defer func() { _ = sub.Unsubscribe() }() diff --git a/spinifex/daemon/daemon_handlers_iam_profile.go b/spinifex/daemon/daemon_handlers_iam_profile.go index 1f4e34143..93b2d45da 100644 --- a/spinifex/daemon/daemon_handlers_iam_profile.go +++ b/spinifex/daemon/daemon_handlers_iam_profile.go @@ -14,8 +14,8 @@ import ( func (d *Daemon) handleAssociateIamInstanceProfile(ctx context.Context, msg *nats.Msg, command types.EC2InstanceCommand, instance *vm.VM) string { result, err := d.instanceService.AssociateIamInstanceProfile(ctx, instance, command) if err != nil { - return respondServiceErrorOutcome(msg, err) + return respondServiceErrorOutcome(d.node, msg, err) } - respondWithJSON(msg, result) + respondWithJSON(d.node, msg, result) return outcomeSuccess } diff --git a/spinifex/daemon/daemon_handlers_image.go b/spinifex/daemon/daemon_handlers_image.go index d25bacbc8..2dcc372e8 100644 --- a/spinifex/daemon/daemon_handlers_image.go +++ b/spinifex/daemon/daemon_handlers_image.go @@ -24,7 +24,7 @@ func (d *Daemon) handleSpinifexPromoteImage(msg *nats.Msg) string { ) return admin.PromoteSystemImage(store, d.config.Predastore.Bucket, *input) } - return handleNATSRequest(promoteImage)(msg) + return handleNATSRequest(d.node, promoteImage)(msg) } // stoppedInstanceOwnerElector is implemented by state stores that can @@ -50,7 +50,7 @@ func (d *Daemon) handleEC2CreateImage(msg *nats.Msg) string { accountID := utils.AccountIDFromMsg(msg) if input.InstanceId == nil || *input.InstanceId == "" { - respondWithError(msg, awserrors.ErrorMissingParameter) + respondWithError(d.node, msg, awserrors.ErrorMissingParameter) return outcomeError } @@ -92,7 +92,7 @@ func (d *Daemon) handleEC2CreateImage(msg *nats.Msg) string { } if stopped == nil { slog.Warn("CreateImage: instance not found", "instanceId", instanceID) - respondWithError(msg, awserrors.ErrorInvalidInstanceIDNotFound) + respondWithError(d.node, msg, awserrors.ErrorInvalidInstanceIDNotFound) return outcomeError } @@ -107,7 +107,7 @@ func (d *Daemon) handleEC2CreateImage(msg *nats.Msg) string { elector, canElect := d.stateStore.(stoppedInstanceOwnerElector) if !canElect { slog.Error("CreateImage: state store cannot elect owner for legacy stopped instance", "instanceId", instanceID) - respondWithError(msg, awserrors.ErrorInvalidInstanceIDNotFound) + respondWithError(d.node, msg, awserrors.ErrorInvalidInstanceIDNotFound) return outcomeError } elected, uerr := elector.UpdateStoppedInstance(instanceID, func(v *vm.VM) { @@ -117,7 +117,7 @@ func (d *Daemon) handleEC2CreateImage(msg *nats.Msg) string { }) if uerr != nil || elected == nil || elected.LastNode == "" { slog.Warn("CreateImage: failed to elect owner for legacy stopped instance", "instanceId", instanceID, "err", uerr) - respondWithError(msg, awserrors.ErrorInvalidInstanceIDNotFound) + respondWithError(d.node, msg, awserrors.ErrorInvalidInstanceIDNotFound) return outcomeError } stopped.LastNode = elected.LastNode @@ -125,7 +125,7 @@ func (d *Daemon) handleEC2CreateImage(msg *nats.Msg) string { if stopped.LastNode != d.node { slog.Info("CreateImage: declining, instance owned by another node", "instanceId", instanceID, "lastNode", stopped.LastNode) - respondWithError(msg, awserrors.ErrorInvalidInstanceIDNotFound) + respondWithError(d.node, msg, awserrors.ErrorInvalidInstanceIDNotFound) return outcomeError } @@ -145,19 +145,19 @@ func (d *Daemon) handleEC2CreateImage(msg *nats.Msg) string { } // Verify the caller owns this instance - if !checkInstanceOwnership(msg, instanceID, instance.AccountID) { + if !checkInstanceOwnership(d.node, msg, instanceID, instance.AccountID) { return outcomeError } if status != vm.StateRunning && status != vm.StateStopped { slog.Warn("CreateImage: instance not in valid state", "instanceId", instanceID, "status", status) - respondWithError(msg, awserrors.ErrorIncorrectInstanceState) + respondWithError(d.node, msg, awserrors.ErrorIncorrectInstanceState) return outcomeError } if rootVolumeID == "" { slog.Error("CreateImage: no root volume found", "instanceId", instanceID) - respondWithError(msg, awserrors.ErrorServerInternal) + respondWithError(d.node, msg, awserrors.ErrorServerInternal) return outcomeError } @@ -171,11 +171,11 @@ func (d *Daemon) handleEC2CreateImage(msg *nats.Msg) string { output, err := d.imageService.CreateImageFromInstance(params, accountID) if err != nil { slog.Error("CreateImage: service failed", "instanceId", instanceID, "err", err) - respondWithServiceError(msg, err) + respondWithServiceError(d.node, msg, err) return outcomeError } - respondWithJSON(msg, output) + respondWithJSON(d.node, msg, output) slog.Info("CreateImage completed", "instanceId", instanceID, "imageId", *output.ImageId) return outcomeSuccess } diff --git a/spinifex/daemon/daemon_handlers_instance.go b/spinifex/daemon/daemon_handlers_instance.go index a4bd97bed..362049619 100644 --- a/spinifex/daemon/daemon_handlers_instance.go +++ b/spinifex/daemon/daemon_handlers_instance.go @@ -35,7 +35,7 @@ func (d *Daemon) handleSetInstanceTags(ctx context.Context, msg *nats.Msg, comma remove := command.Attributes.RemoveInstanceTags data := command.InstanceTagsData if data == nil || (!remove && len(data.Tags) == 0) { - return respondErrorOutcome(msg, awserrors.ErrorMissingParameter) + return respondErrorOutcome(d.node, msg, awserrors.ErrorMissingParameter) } var newTags []*ec2.Tag @@ -48,14 +48,14 @@ func (d *Daemon) handleSetInstanceTags(ctx context.Context, msg *nats.Msg, comma newTags = handlers_ec2_instance.ApplyInstanceTagMutation(v.Instance.Tags, data, remove) }) if missingRecord { - return respondErrorOutcome(msg, awserrors.ErrorServerInternal) + return respondErrorOutcome(d.node, msg, awserrors.ErrorServerInternal) } accountID := utils.AccountIDFromMsg(msg) if err := d.tagsService.PutResourceTags(ctx, accountID, instance.ID, handlers_ec2_instance.TagsToMap(newTags)); err != nil { slog.ErrorContext(ctx, "SetInstanceTags: central tag store write failed", "instanceId", instance.ID, "err", err) - return respondErrorOutcome(msg, awserrors.ErrorServerInternal) + return respondErrorOutcome(d.node, msg, awserrors.ErrorServerInternal) } found, err := d.vmMgr.UpdateAndPersist(instance.ID, func(v *vm.VM) bool { @@ -66,10 +66,10 @@ func (d *Daemon) handleSetInstanceTags(ctx context.Context, msg *nats.Msg, comma return true }) if err != nil { - return respondServiceErrorOutcome(msg, err) + return respondServiceErrorOutcome(d.node, msg, err) } if !found { - return respondErrorOutcome(msg, awserrors.ErrorInvalidInstanceIDNotFound) + return respondErrorOutcome(d.node, msg, awserrors.ErrorInvalidInstanceIDNotFound) } if err := msg.Respond([]byte(`{}`)); err != nil { @@ -85,7 +85,7 @@ func (d *Daemon) handleSetInstanceTags(ctx context.Context, msg *nats.Msg, comma func (d *Daemon) handleSetInstanceMonitoring(ctx context.Context, msg *nats.Msg, command types.EC2InstanceCommand, instance *vm.VM) string { data := command.InstanceMonitoringData if data == nil { - return respondErrorOutcome(msg, awserrors.ErrorMissingParameter) + return respondErrorOutcome(d.node, msg, awserrors.ErrorMissingParameter) } found, err := d.vmMgr.UpdateAndPersist(instance.ID, func(v *vm.VM) bool { @@ -99,10 +99,10 @@ func (d *Daemon) handleSetInstanceMonitoring(ctx context.Context, msg *nats.Msg, return true }) if err != nil { - return respondServiceErrorOutcome(msg, err) + return respondServiceErrorOutcome(d.node, msg, err) } if !found { - return respondErrorOutcome(msg, awserrors.ErrorInvalidInstanceIDNotFound) + return respondErrorOutcome(d.node, msg, awserrors.ErrorInvalidInstanceIDNotFound) } // Off the manager lock: the refresh takes the VM's own ENI lock. @@ -129,7 +129,7 @@ func (d *Daemon) handleEC2RunInstances(msg *nats.Msg) string { accountID := utils.AccountIDFromMsg(msg) if accountID == "" { slog.Error("handleEC2RunInstances: missing account ID in NATS header") - respondWithError(msg, awserrors.ErrorServerInternal) + respondWithError(d.node, msg, awserrors.ErrorServerInternal) return outcomeError } @@ -150,7 +150,7 @@ func (d *Daemon) handleEC2RunInstances(msg *nats.Msg) string { if reservationID != "" { if it, ok := d.resourceMgr.instanceTypes[aws.StringValue(input.InstanceType)]; ok { if vErr := d.resourceMgr.ValidateReservationTarget(reservationID, accountID, it); vErr != nil { - respondWithServiceError(msg, vErr) + respondWithServiceError(d.node, msg, vErr) return outcomeError } } @@ -160,7 +160,7 @@ func (d *Daemon) handleEC2RunInstances(msg *nats.Msg) string { reservation, instances, instanceType, err := d.instanceService.PrepareRunInstances(ctx, input, accountID, reservationID) endOpSpan(prepSpan, err) if err != nil { - respondWithServiceError(msg, err) + respondWithServiceError(d.node, msg, err) return outcomeError } @@ -174,7 +174,7 @@ func (d *Daemon) handleEC2RunInstances(msg *nats.Msg) string { jsonResponse, err := json.Marshal(reservation) if err != nil { slog.Error("handleEC2RunInstances failed to marshal reservation", "err", err) - respondWithError(msg, awserrors.ErrorServerInternal) + respondWithError(d.node, msg, awserrors.ErrorServerInternal) for range instances { if reservationID == "" { d.resourceMgr.deallocate(instanceType) @@ -256,7 +256,7 @@ func (d *Daemon) handleEC2StartStoppedInstance(msg *nats.Msg) string { if err := json.Unmarshal(msg.Data, &peek); err != nil || peek.InstanceID == "" { // Can't determine target node — fall through to local start which will // return the appropriate error (missing parameter / unmarshal failure). - return handleNATSRequest(d.instanceService.StartStoppedInstance)(msg) + return handleNATSRequest(d.node, d.instanceService.StartStoppedInstance)(msg) } lastNode := d.instanceService.StoppedInstanceNode(peek.InstanceID) @@ -304,5 +304,5 @@ func (d *Daemon) handleEC2StartStoppedInstance(msg *nats.Msg) string { } } - return handleNATSRequest(d.instanceService.StartStoppedInstance)(msg) + return handleNATSRequest(d.node, d.instanceService.StartStoppedInstance)(msg) } diff --git a/spinifex/daemon/daemon_handlers_instance_test.go b/spinifex/daemon/daemon_handlers_instance_test.go index 39e12f3dc..977995a2e 100644 --- a/spinifex/daemon/daemon_handlers_instance_test.go +++ b/spinifex/daemon/daemon_handlers_instance_test.go @@ -214,7 +214,7 @@ func TestHandleEC2TerminateStoppedInstance_LoadError(t *testing.T) { d := daemonWithFakeStateStore(t, store) body, _ := json.Marshal(handlers_ec2_instance.TerminateStoppedInstanceInput{InstanceID: "i-load-fail"}) - reply := requestHandler(t, d.natsConn, "ec2.terminate.test1", asMsgHandler(handleNATSRequest(d.instanceService.TerminateStoppedInstance)), testAccountID, body) + reply := requestHandler(t, d.natsConn, "ec2.terminate.test1", asMsgHandler(handleNATSRequest(d.node, d.instanceService.TerminateStoppedInstance)), testAccountID, body) assert.Equal(t, awserrors.ErrorServerInternal, decodeError(t, reply.Data)["Code"]) } @@ -222,7 +222,7 @@ func TestHandleEC2TerminateStoppedInstance_StateStoreNil(t *testing.T) { d := createTestDaemon(t, sharedNATSURL) body, _ := json.Marshal(handlers_ec2_instance.TerminateStoppedInstanceInput{InstanceID: "i-no-store"}) - reply := requestHandler(t, d.natsConn, "ec2.terminate.test2", asMsgHandler(handleNATSRequest(d.instanceService.TerminateStoppedInstance)), testAccountID, body) + reply := requestHandler(t, d.natsConn, "ec2.terminate.test2", asMsgHandler(handleNATSRequest(d.node, d.instanceService.TerminateStoppedInstance)), testAccountID, body) assert.Equal(t, awserrors.ErrorServerInternal, decodeError(t, reply.Data)["Code"]) } @@ -236,7 +236,7 @@ func TestHandleEC2TerminateStoppedInstance_WriteTerminatedFailureAborts(t *testi d := daemonWithFakeStateStore(t, store) body, _ := json.Marshal(handlers_ec2_instance.TerminateStoppedInstanceInput{InstanceID: v.ID}) - reply := requestHandler(t, d.natsConn, "ec2.terminate.test3", asMsgHandler(handleNATSRequest(d.instanceService.TerminateStoppedInstance)), testAccountID, body) + reply := requestHandler(t, d.natsConn, "ec2.terminate.test3", asMsgHandler(handleNATSRequest(d.node, d.instanceService.TerminateStoppedInstance)), testAccountID, body) assert.Equal(t, awserrors.ErrorServerInternal, decodeError(t, reply.Data)["Code"]) _, stillStopped := store.Stopped[v.ID] @@ -257,7 +257,7 @@ func TestHandleEC2TerminateStoppedInstance_DeleteRetrySucceeds(t *testing.T) { d := daemonWithFakeStateStore(t, store) body, _ := json.Marshal(handlers_ec2_instance.TerminateStoppedInstanceInput{InstanceID: v.ID}) - reply := requestHandler(t, d.natsConn, "ec2.terminate.test4", asMsgHandler(handleNATSRequest(d.instanceService.TerminateStoppedInstance)), testAccountID, body) + reply := requestHandler(t, d.natsConn, "ec2.terminate.test4", asMsgHandler(handleNATSRequest(d.node, d.instanceService.TerminateStoppedInstance)), testAccountID, body) var resp map[string]string require.NoError(t, json.Unmarshal(reply.Data, &resp)) @@ -282,7 +282,7 @@ func TestHandleEC2TerminateStoppedInstance_DeleteAlwaysFailsKeepsTerminated(t *t d := daemonWithFakeStateStore(t, store) body, _ := json.Marshal(handlers_ec2_instance.TerminateStoppedInstanceInput{InstanceID: v.ID}) - reply := requestHandler(t, d.natsConn, "ec2.terminate.test5", asMsgHandler(handleNATSRequest(d.instanceService.TerminateStoppedInstance)), testAccountID, body) + reply := requestHandler(t, d.natsConn, "ec2.terminate.test5", asMsgHandler(handleNATSRequest(d.node, d.instanceService.TerminateStoppedInstance)), testAccountID, body) var resp map[string]string require.NoError(t, json.Unmarshal(reply.Data, &resp)) @@ -298,7 +298,7 @@ func TestHandleEC2TerminateStoppedInstance_CrossTenantRejected(t *testing.T) { d := daemonWithFakeStateStore(t, store) body, _ := json.Marshal(handlers_ec2_instance.TerminateStoppedInstanceInput{InstanceID: "i-foreign-term"}) - reply := requestHandler(t, d.natsConn, "ec2.terminate.test6", asMsgHandler(handleNATSRequest(d.instanceService.TerminateStoppedInstance)), testAccountID, body) + reply := requestHandler(t, d.natsConn, "ec2.terminate.test6", asMsgHandler(handleNATSRequest(d.node, d.instanceService.TerminateStoppedInstance)), testAccountID, body) assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, decodeError(t, reply.Data)["Code"]) _, inTerminated := store.Terminated["i-foreign-term"] @@ -321,7 +321,7 @@ func TestHandleEC2ModifyInstanceAttribute_WriteFailureReturnsServerInternal(t *t InstanceType: &ec2.AttributeValue{Value: aws.String("t3.large")}, } body, _ := json.Marshal(input) - reply := requestHandler(t, d.natsConn, "ec2.ModifyInstanceAttribute.test1", asMsgHandler(handleNATSRequest(d.instanceService.ModifyInstanceAttribute)), testAccountID, body) + reply := requestHandler(t, d.natsConn, "ec2.ModifyInstanceAttribute.test1", asMsgHandler(handleNATSRequest(d.node, d.instanceService.ModifyInstanceAttribute)), testAccountID, body) assert.Equal(t, awserrors.ErrorServerInternal, decodeError(t, reply.Data)["Code"]) } @@ -335,7 +335,7 @@ func TestHandleEC2ModifyInstanceAttribute_LoadFailureReturnsServerInternal(t *te InstanceType: &ec2.AttributeValue{Value: aws.String("t3.large")}, } body, _ := json.Marshal(input) - reply := requestHandler(t, d.natsConn, "ec2.ModifyInstanceAttribute.test2", asMsgHandler(handleNATSRequest(d.instanceService.ModifyInstanceAttribute)), testAccountID, body) + reply := requestHandler(t, d.natsConn, "ec2.ModifyInstanceAttribute.test2", asMsgHandler(handleNATSRequest(d.node, d.instanceService.ModifyInstanceAttribute)), testAccountID, body) assert.Equal(t, awserrors.ErrorServerInternal, decodeError(t, reply.Data)["Code"]) } @@ -359,7 +359,7 @@ func TestHandleEC2ModifyInstanceAttribute_NilInstanceFieldGuard(t *testing.T) { InstanceType: &ec2.AttributeValue{Value: aws.String("t3.large")}, } body, _ := json.Marshal(input) - reply := requestHandler(t, d.natsConn, "ec2.ModifyInstanceAttribute.test3", asMsgHandler(handleNATSRequest(d.instanceService.ModifyInstanceAttribute)), testAccountID, body) + reply := requestHandler(t, d.natsConn, "ec2.ModifyInstanceAttribute.test3", asMsgHandler(handleNATSRequest(d.node, d.instanceService.ModifyInstanceAttribute)), testAccountID, body) assert.Equal(t, awserrors.ErrorServerInternal, decodeError(t, reply.Data)["Code"]) } @@ -374,7 +374,7 @@ func TestHandleEC2ModifyInstanceAttribute_EmptyInstanceTypeRejected(t *testing.T InstanceType: &ec2.AttributeValue{Value: aws.String("")}, } body, _ := json.Marshal(input) - reply := requestHandler(t, d.natsConn, "ec2.ModifyInstanceAttribute.test4", asMsgHandler(handleNATSRequest(d.instanceService.ModifyInstanceAttribute)), testAccountID, body) + reply := requestHandler(t, d.natsConn, "ec2.ModifyInstanceAttribute.test4", asMsgHandler(handleNATSRequest(d.node, d.instanceService.ModifyInstanceAttribute)), testAccountID, body) assert.Equal(t, awserrors.ErrorInvalidInstanceAttributeValue, decodeError(t, reply.Data)["Code"]) } @@ -388,7 +388,7 @@ func TestHandleEC2ModifyInstanceAttribute_CrossTenantRejected(t *testing.T) { InstanceType: &ec2.AttributeValue{Value: aws.String("t3.large")}, } body, _ := json.Marshal(input) - reply := requestHandler(t, d.natsConn, "ec2.ModifyInstanceAttribute.test5", asMsgHandler(handleNATSRequest(d.instanceService.ModifyInstanceAttribute)), testAccountID, body) + reply := requestHandler(t, d.natsConn, "ec2.ModifyInstanceAttribute.test5", asMsgHandler(handleNATSRequest(d.node, d.instanceService.ModifyInstanceAttribute)), testAccountID, body) assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, decodeError(t, reply.Data)["Code"]) } @@ -406,7 +406,7 @@ func TestHandleEC2DescribeInstanceAttribute_StoppedFallback_LoadError(t *testing Attribute: aws.String(ec2.InstanceAttributeNameInstanceType), } body, _ := json.Marshal(input) - reply := requestHandler(t, d.natsConn, "ec2.DescribeInstanceAttribute.test1", asMsgHandler(handleNATSRequest(d.instanceService.DescribeInstanceAttribute)), testAccountID, body) + reply := requestHandler(t, d.natsConn, "ec2.DescribeInstanceAttribute.test1", asMsgHandler(handleNATSRequest(d.node, d.instanceService.DescribeInstanceAttribute)), testAccountID, body) assert.Equal(t, awserrors.ErrorServerInternal, decodeError(t, reply.Data)["Code"]) } @@ -423,7 +423,7 @@ func TestHandleEC2DescribeInstanceAttribute_StoppedFallback_HitsKV(t *testing.T) Attribute: aws.String(ec2.InstanceAttributeNameInstanceType), } body, _ := json.Marshal(input) - reply := requestHandler(t, d.natsConn, "ec2.DescribeInstanceAttribute.test2", asMsgHandler(handleNATSRequest(d.instanceService.DescribeInstanceAttribute)), testAccountID, body) + reply := requestHandler(t, d.natsConn, "ec2.DescribeInstanceAttribute.test2", asMsgHandler(handleNATSRequest(d.node, d.instanceService.DescribeInstanceAttribute)), testAccountID, body) var output ec2.DescribeInstanceAttributeOutput require.NoError(t, json.Unmarshal(reply.Data, &output)) @@ -442,7 +442,7 @@ func TestHandleEC2DescribeInstanceAttribute_StateStoreNil(t *testing.T) { Attribute: aws.String(ec2.InstanceAttributeNameInstanceType), } body, _ := json.Marshal(input) - reply := requestHandler(t, d.natsConn, "ec2.DescribeInstanceAttribute.test3", asMsgHandler(handleNATSRequest(d.instanceService.DescribeInstanceAttribute)), testAccountID, body) + reply := requestHandler(t, d.natsConn, "ec2.DescribeInstanceAttribute.test3", asMsgHandler(handleNATSRequest(d.node, d.instanceService.DescribeInstanceAttribute)), testAccountID, body) assert.Equal(t, awserrors.ErrorServerInternal, decodeError(t, reply.Data)["Code"]) } @@ -453,7 +453,7 @@ func TestHandleEC2DescribeStoppedInstances_ListError(t *testing.T) { store.ListStoppedErr = errors.New("list failed") d := daemonWithFakeStateStore(t, store) - reply := requestHandler(t, d.natsConn, "ec2.DescribeStoppedInstances.test1", asMsgHandler(handleNATSRequest(d.instanceService.DescribeStoppedInstances)), testAccountID, []byte("{}")) + reply := requestHandler(t, d.natsConn, "ec2.DescribeStoppedInstances.test1", asMsgHandler(handleNATSRequest(d.node, d.instanceService.DescribeStoppedInstances)), testAccountID, []byte("{}")) assert.Equal(t, awserrors.ErrorServerInternal, decodeError(t, reply.Data)["Code"]) } @@ -462,7 +462,7 @@ func TestHandleEC2DescribeTerminatedInstances_ListError(t *testing.T) { store.ListTerminatedErr = errors.New("list failed") d := daemonWithFakeStateStore(t, store) - reply := requestHandler(t, d.natsConn, "ec2.DescribeTerminatedInstances.test1", asMsgHandler(handleNATSRequest(d.instanceService.DescribeTerminatedInstances)), testAccountID, []byte("{}")) + reply := requestHandler(t, d.natsConn, "ec2.DescribeTerminatedInstances.test1", asMsgHandler(handleNATSRequest(d.node, d.instanceService.DescribeTerminatedInstances)), testAccountID, []byte("{}")) assert.Equal(t, awserrors.ErrorServerInternal, decodeError(t, reply.Data)["Code"]) } @@ -472,7 +472,7 @@ func TestHandleEC2DescribeStoppedInstances_CrossAccountIsolation(t *testing.T) { store.Stopped["i-yours"] = stoppedVMFixture("i-yours", "999988887777") d := daemonWithFakeStateStore(t, store) - reply := requestHandler(t, d.natsConn, "ec2.DescribeStoppedInstances.test3", asMsgHandler(handleNATSRequest(d.instanceService.DescribeStoppedInstances)), testAccountID, []byte("{}")) + reply := requestHandler(t, d.natsConn, "ec2.DescribeStoppedInstances.test3", asMsgHandler(handleNATSRequest(d.node, d.instanceService.DescribeStoppedInstances)), testAccountID, []byte("{}")) var output ec2.DescribeInstancesOutput require.NoError(t, json.Unmarshal(reply.Data, &output)) diff --git a/spinifex/daemon/daemon_handlers_password.go b/spinifex/daemon/daemon_handlers_password.go index 8f17f79ab..569b42ec5 100644 --- a/spinifex/daemon/daemon_handlers_password.go +++ b/spinifex/daemon/daemon_handlers_password.go @@ -78,7 +78,7 @@ func (d *Daemon) handleEC2GetPasswordData(msg *nats.Msg) { } if input.InstanceId == nil { - respondWithError(msg, awserrors.ErrorMissingParameter) + respondWithError(d.node, msg, awserrors.ErrorMissingParameter) return } @@ -87,12 +87,12 @@ func (d *Daemon) handleEC2GetPasswordData(msg *nats.Msg) { // Find the instance on this node instance, exists := d.vmMgr.Get(instanceID) if !exists { - respondWithError(msg, awserrors.ErrorInvalidInstanceIDNotFound) + respondWithError(d.node, msg, awserrors.ErrorInvalidInstanceIDNotFound) return } // Verify the caller owns this instance - if !checkInstanceOwnership(msg, instanceID, instance.AccountID) { + if !checkInstanceOwnership(d.node, msg, instanceID, instance.AccountID) { return } @@ -121,7 +121,7 @@ func (d *Daemon) handleEC2GetPasswordData(msg *nats.Msg) { // worth retrying. A non-RSA launch key never will, so name that case instead // of returning the same empty string for a condition that cannot resolve. if passwordData == "" && d.launchKeyCannotEncrypt(instance) { - respondWithError(msg, awserrors.ErrorInvalidKeyPairType) + respondWithError(d.node, msg, awserrors.ErrorInvalidKeyPairType) return } @@ -136,7 +136,7 @@ func (d *Daemon) handleEC2GetPasswordData(msg *nats.Msg) { Timestamp: &modTime, } - respondWithJSON(msg, output) + respondWithJSON(d.node, msg, output) // Log the ciphertext length only; the payload itself is never logged. slog.Info("handleEC2GetPasswordData completed", "instance_id", instanceID, "password_data_bytes", len(passwordData)) } diff --git a/spinifex/daemon/daemon_handlers_spotinstance.go b/spinifex/daemon/daemon_handlers_spotinstance.go index b98c8ea8b..736a5d13e 100644 --- a/spinifex/daemon/daemon_handlers_spotinstance.go +++ b/spinifex/daemon/daemon_handlers_spotinstance.go @@ -17,7 +17,7 @@ import ( // the lifecycle is always spot for this command. func (d *Daemon) handleSetSpotLineage(ctx context.Context, msg *nats.Msg, command types.EC2InstanceCommand) string { if command.SpotLineageData == nil { - return respondErrorOutcome(msg, awserrors.ErrorMissingParameter) + return respondErrorOutcome(d.node, msg, awserrors.ErrorMissingParameter) } found, err := d.vmMgr.UpdateAndPersist(command.ID, func(v *vm.VM) bool { @@ -26,10 +26,10 @@ func (d *Daemon) handleSetSpotLineage(ctx context.Context, msg *nats.Msg, comman return true }) if err != nil { - return respondServiceErrorOutcome(msg, err) + return respondServiceErrorOutcome(d.node, msg, err) } if !found { - return respondErrorOutcome(msg, awserrors.ErrorInvalidInstanceIDNotFound) + return respondErrorOutcome(d.node, msg, awserrors.ErrorInvalidInstanceIDNotFound) } if err := msg.Respond([]byte(`{}`)); err != nil { diff --git a/spinifex/daemon/daemon_handlers_storage.go b/spinifex/daemon/daemon_handlers_storage.go index bd55caaba..49a6ee3af 100644 --- a/spinifex/daemon/daemon_handlers_storage.go +++ b/spinifex/daemon/daemon_handlers_storage.go @@ -59,14 +59,14 @@ func (d *Daemon) handleStorageConfig(msg *nats.Msg) string { data, err := os.ReadFile(predastorePath) if err != nil { slog.Debug("handleStorageConfig: failed to read predastore config", "path", predastorePath, "err", err) - respondWithError(msg, "InternalError") + respondWithError(d.node, msg, "InternalError") return outcomeError } var cfg predastoreTOML if err := toml.Unmarshal(data, &cfg); err != nil { slog.Error("handleStorageConfig: failed to parse predastore config", "err", err) - respondWithError(msg, "InternalError") + respondWithError(d.node, msg, "InternalError") return outcomeError } @@ -111,6 +111,6 @@ func (d *Daemon) handleStorageConfig(msg *nats.Msg) string { resp.Buckets = []types.StorageBucket{} } - respondWithJSON(msg, resp) + respondWithJSON(d.node, msg, resp) return outcomeSuccess } diff --git a/spinifex/daemon/daemon_handlers_test.go b/spinifex/daemon/daemon_handlers_test.go index ed2e65593..bd3917a13 100644 --- a/spinifex/daemon/daemon_handlers_test.go +++ b/spinifex/daemon/daemon_handlers_test.go @@ -38,6 +38,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/testutil" "github.com/mulgadc/spinifex/spinifex/testutil/ebsfake" "github.com/mulgadc/spinifex/spinifex/types" + "github.com/mulgadc/spinifex/spinifex/utils" "github.com/mulgadc/spinifex/spinifex/vm" vmmock "github.com/mulgadc/spinifex/spinifex/vm/mock" "github.com/nats-io/nats.go" @@ -164,7 +165,7 @@ func TestHandleNATSRequest_ValidRequest(t *testing.T) { return &testOutput{Greeting: "hello " + in.Name}, nil } - sub, err := nc.Subscribe("test.greet", asMsgHandler(handleNATSRequest(serviceFn))) + sub, err := nc.Subscribe("test.greet", asMsgHandler(handleNATSRequest("test-node", serviceFn))) require.NoError(t, err) defer sub.Unsubscribe() @@ -189,7 +190,7 @@ func TestHandleNATSRequest_MalformedJSON(t *testing.T) { return &testOutput{Greeting: "hello"}, nil } - sub, err := nc.Subscribe("test.malformed", asMsgHandler(handleNATSRequest(serviceFn))) + sub, err := nc.Subscribe("test.malformed", asMsgHandler(handleNATSRequest("test-node", serviceFn))) require.NoError(t, err) defer sub.Unsubscribe() @@ -213,7 +214,7 @@ func TestHandleNATSRequest_ServiceError(t *testing.T) { return nil, fmt.Errorf("something went wrong") } - sub, err := nc.Subscribe("test.err", asMsgHandler(handleNATSRequest(serviceFn))) + sub, err := nc.Subscribe("test.err", asMsgHandler(handleNATSRequest("test-node", serviceFn))) require.NoError(t, err) defer sub.Unsubscribe() @@ -239,7 +240,7 @@ func TestHandleNATSRequest_WrappedAWSServiceError(t *testing.T) { return nil, fmt.Errorf("allocate public address: %w", cause) } - sub, err := nc.Subscribe("test.err.wrapped", asMsgHandler(handleNATSRequest(serviceFn))) + sub, err := nc.Subscribe("test.err.wrapped", asMsgHandler(handleNATSRequest("test-node", serviceFn))) require.NoError(t, err) defer sub.Unsubscribe() @@ -260,7 +261,7 @@ func TestHandleEC2CreateKeyPair_RoundTrip(t *testing.T) { daemon := createFullTestDaemon(t, natsURL) - sub, err := daemon.natsConn.QueueSubscribe("ec2.CreateKeyPair", "spinifex-workers", asMsgHandler(handleNATSRequest(daemon.keyService.CreateKeyPair))) + sub, err := daemon.natsConn.QueueSubscribe("ec2.CreateKeyPair", "spinifex-workers", asMsgHandler(handleNATSRequest(daemon.node, daemon.keyService.CreateKeyPair))) require.NoError(t, err) defer sub.Unsubscribe() @@ -1254,7 +1255,7 @@ func TestHandleEC2DescribeStoppedInstances_ReturnsStoppedInstances(t *testing.T) err := daemon.jsManager.WriteStoppedInstance(stoppedVM.ID, stoppedVM) require.NoError(t, err) - sub, err := daemon.natsConn.QueueSubscribe("ec2.DescribeStoppedInstances", "spinifex-workers", asMsgHandler(handleNATSRequest(daemon.instanceService.DescribeStoppedInstances))) + sub, err := daemon.natsConn.QueueSubscribe("ec2.DescribeStoppedInstances", "spinifex-workers", asMsgHandler(handleNATSRequest(daemon.node, daemon.instanceService.DescribeStoppedInstances))) require.NoError(t, err) defer sub.Unsubscribe() @@ -1308,7 +1309,7 @@ func TestHandleEC2DescribeStoppedInstances_WithFilter(t *testing.T) { require.NoError(t, err) } - sub, err := daemon.natsConn.QueueSubscribe("ec2.DescribeStoppedInstances", "spinifex-workers", asMsgHandler(handleNATSRequest(daemon.instanceService.DescribeStoppedInstances))) + sub, err := daemon.natsConn.QueueSubscribe("ec2.DescribeStoppedInstances", "spinifex-workers", asMsgHandler(handleNATSRequest(daemon.node, daemon.instanceService.DescribeStoppedInstances))) require.NoError(t, err) defer sub.Unsubscribe() @@ -1526,7 +1527,7 @@ func TestHandleEC2ModifyInstanceAttribute_WrapperRoundTrip(t *testing.T) { daemon := createFullTestDaemonWithJetStream(t, natsURL) - sub, err := daemon.natsConn.QueueSubscribe("ec2.ModifyInstanceAttribute", "spinifex-workers", asMsgHandler(handleNATSRequest(daemon.instanceService.ModifyInstanceAttribute))) + sub, err := daemon.natsConn.QueueSubscribe("ec2.ModifyInstanceAttribute", "spinifex-workers", asMsgHandler(handleNATSRequest(daemon.node, daemon.instanceService.ModifyInstanceAttribute))) require.NoError(t, err) defer sub.Unsubscribe() @@ -1568,7 +1569,7 @@ func TestHandleEC2DescribeInstanceAttribute_RunningInstance_InstanceType(t *test daemon := createFullTestDaemonWithJetStream(t, natsURL) - sub, err := daemon.natsConn.Subscribe("ec2.DescribeInstanceAttribute", asMsgHandler(handleNATSRequest(daemon.instanceService.DescribeInstanceAttribute))) + sub, err := daemon.natsConn.Subscribe("ec2.DescribeInstanceAttribute", asMsgHandler(handleNATSRequest(daemon.node, daemon.instanceService.DescribeInstanceAttribute))) require.NoError(t, err) defer sub.Unsubscribe() @@ -1610,7 +1611,7 @@ func TestHandleEC2DescribeInstanceAttribute_StoppedInstance_InstanceType(t *test daemon := createFullTestDaemonWithJetStream(t, natsURL) - sub, err := daemon.natsConn.Subscribe("ec2.DescribeInstanceAttribute", asMsgHandler(handleNATSRequest(daemon.instanceService.DescribeInstanceAttribute))) + sub, err := daemon.natsConn.Subscribe("ec2.DescribeInstanceAttribute", asMsgHandler(handleNATSRequest(daemon.node, daemon.instanceService.DescribeInstanceAttribute))) require.NoError(t, err) defer sub.Unsubscribe() @@ -1650,7 +1651,7 @@ func TestHandleEC2DescribeInstanceAttribute_UserData(t *testing.T) { daemon := createFullTestDaemonWithJetStream(t, natsURL) - sub, err := daemon.natsConn.Subscribe("ec2.DescribeInstanceAttribute", asMsgHandler(handleNATSRequest(daemon.instanceService.DescribeInstanceAttribute))) + sub, err := daemon.natsConn.Subscribe("ec2.DescribeInstanceAttribute", asMsgHandler(handleNATSRequest(daemon.node, daemon.instanceService.DescribeInstanceAttribute))) require.NoError(t, err) defer sub.Unsubscribe() @@ -1692,7 +1693,7 @@ func TestHandleEC2DescribeInstanceAttribute_DefaultAttribute_DisableApiTerminati daemon := createFullTestDaemonWithJetStream(t, natsURL) - sub, err := daemon.natsConn.Subscribe("ec2.DescribeInstanceAttribute", asMsgHandler(handleNATSRequest(daemon.instanceService.DescribeInstanceAttribute))) + sub, err := daemon.natsConn.Subscribe("ec2.DescribeInstanceAttribute", asMsgHandler(handleNATSRequest(daemon.node, daemon.instanceService.DescribeInstanceAttribute))) require.NoError(t, err) defer sub.Unsubscribe() @@ -1736,7 +1737,7 @@ func TestHandleEC2DescribeInstanceAttribute_DefaultAttribute_ShutdownBehavior(t daemon := createFullTestDaemonWithJetStream(t, natsURL) - sub, err := daemon.natsConn.Subscribe("ec2.DescribeInstanceAttribute", asMsgHandler(handleNATSRequest(daemon.instanceService.DescribeInstanceAttribute))) + sub, err := daemon.natsConn.Subscribe("ec2.DescribeInstanceAttribute", asMsgHandler(handleNATSRequest(daemon.node, daemon.instanceService.DescribeInstanceAttribute))) require.NoError(t, err) defer sub.Unsubscribe() @@ -1774,7 +1775,7 @@ func TestHandleEC2DescribeInstanceAttribute_GroupSet_WithSecurityGroups(t *testi daemon := createFullTestDaemonWithJetStream(t, natsURL) - sub, err := daemon.natsConn.Subscribe("ec2.DescribeInstanceAttribute", asMsgHandler(handleNATSRequest(daemon.instanceService.DescribeInstanceAttribute))) + sub, err := daemon.natsConn.Subscribe("ec2.DescribeInstanceAttribute", asMsgHandler(handleNATSRequest(daemon.node, daemon.instanceService.DescribeInstanceAttribute))) require.NoError(t, err) defer sub.Unsubscribe() @@ -1820,7 +1821,7 @@ func TestHandleEC2DescribeInstanceAttribute_GroupSet_NilInstance(t *testing.T) { daemon := createFullTestDaemonWithJetStream(t, natsURL) - sub, err := daemon.natsConn.Subscribe("ec2.DescribeInstanceAttribute", asMsgHandler(handleNATSRequest(daemon.instanceService.DescribeInstanceAttribute))) + sub, err := daemon.natsConn.Subscribe("ec2.DescribeInstanceAttribute", asMsgHandler(handleNATSRequest(daemon.node, daemon.instanceService.DescribeInstanceAttribute))) require.NoError(t, err) defer sub.Unsubscribe() @@ -1857,7 +1858,7 @@ func TestHandleEC2DescribeInstanceAttribute_InstanceNotFound(t *testing.T) { daemon := createFullTestDaemonWithJetStream(t, natsURL) - sub, err := daemon.natsConn.Subscribe("ec2.DescribeInstanceAttribute", asMsgHandler(handleNATSRequest(daemon.instanceService.DescribeInstanceAttribute))) + sub, err := daemon.natsConn.Subscribe("ec2.DescribeInstanceAttribute", asMsgHandler(handleNATSRequest(daemon.node, daemon.instanceService.DescribeInstanceAttribute))) require.NoError(t, err) defer sub.Unsubscribe() @@ -1880,7 +1881,7 @@ func TestHandleEC2DescribeInstanceAttribute_UnsupportedAttribute(t *testing.T) { daemon := createFullTestDaemonWithJetStream(t, natsURL) - sub, err := daemon.natsConn.Subscribe("ec2.DescribeInstanceAttribute", asMsgHandler(handleNATSRequest(daemon.instanceService.DescribeInstanceAttribute))) + sub, err := daemon.natsConn.Subscribe("ec2.DescribeInstanceAttribute", asMsgHandler(handleNATSRequest(daemon.node, daemon.instanceService.DescribeInstanceAttribute))) require.NoError(t, err) defer sub.Unsubscribe() @@ -1916,7 +1917,7 @@ func TestHandleEC2DescribeInstanceAttribute_InvalidJSON(t *testing.T) { daemon := createFullTestDaemonWithJetStream(t, natsURL) - sub, err := daemon.natsConn.Subscribe("ec2.DescribeInstanceAttribute", asMsgHandler(handleNATSRequest(daemon.instanceService.DescribeInstanceAttribute))) + sub, err := daemon.natsConn.Subscribe("ec2.DescribeInstanceAttribute", asMsgHandler(handleNATSRequest(daemon.node, daemon.instanceService.DescribeInstanceAttribute))) require.NoError(t, err) defer sub.Unsubscribe() @@ -1984,68 +1985,68 @@ func TestDelegateHandlers_RoundTrip(t *testing.T) { { name: "DeleteKeyPair", topic: "ec2.test.DeleteKeyPair", - handler: asMsgHandler(handleNATSRequest(daemon.keyService.DeleteKeyPair)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.keyService.DeleteKeyPair)), input: &ec2.DeleteKeyPairInput{KeyName: aws.String("nonexistent-key")}, // Idempotent delete: missing key returns a success response. }, { name: "ImportKeyPair", topic: "ec2.test.ImportKeyPair", - handler: asMsgHandler(handleNATSRequest(daemon.keyService.ImportKeyPair)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.keyService.ImportKeyPair)), input: &ec2.ImportKeyPairInput{KeyName: aws.String("imported-key"), PublicKeyMaterial: []byte("ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAITest test@test")}, expectedCode: awserrors.ErrorInvalidKeyFormat, }, { name: "CreateVolume", topic: "ec2.test.CreateVolume", - handler: asMsgHandler(handleNATSRequest(daemon.volumeService.CreateVolume)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.volumeService.CreateVolume)), input: &ec2.CreateVolumeInput{AvailabilityZone: aws.String("us-east-1a"), Size: aws.Int64(10)}, expectedCode: awserrors.ErrorInvalidAvailabilityZone, }, { name: "DescribeVolumeStatus", topic: "ec2.test.DescribeVolumeStatus", - handler: asMsgHandler(handleNATSRequest(daemon.volumeService.DescribeVolumeStatus)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.volumeService.DescribeVolumeStatus)), input: &ec2.DescribeVolumeStatusInput{}, }, { name: "DeleteVolume", topic: "ec2.test.DeleteVolume", - handler: asMsgHandler(handleNATSRequest(daemon.volumeService.DeleteVolume)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.volumeService.DeleteVolume)), input: &ec2.DeleteVolumeInput{VolumeId: aws.String("vol-nonexistent")}, expectedCode: awserrors.ErrorInvalidVolumeNotFound, }, { name: "CreateSnapshot", topic: "ec2.test.CreateSnapshot", - handler: asMsgHandler(handleNATSRequest(daemon.snapshotService.CreateSnapshot)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.snapshotService.CreateSnapshot)), input: &ec2.CreateSnapshotInput{VolumeId: aws.String("vol-nonexistent")}, expectedCode: awserrors.ErrorInvalidVolumeNotFound, }, { name: "DescribeSnapshots", topic: "ec2.test.DescribeSnapshots", - handler: asMsgHandler(handleNATSRequest(daemon.snapshotService.DescribeSnapshots)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.snapshotService.DescribeSnapshots)), input: &ec2.DescribeSnapshotsInput{}, }, { name: "DeleteSnapshot", topic: "ec2.test.DeleteSnapshot", - handler: asMsgHandler(handleNATSRequest(daemon.snapshotService.DeleteSnapshot)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.snapshotService.DeleteSnapshot)), input: &ec2.DeleteSnapshotInput{SnapshotId: aws.String("snap-nonexistent")}, expectedCode: awserrors.ErrorInvalidSnapshotNotFound, }, { name: "CopySnapshot", topic: "ec2.test.CopySnapshot", - handler: asMsgHandler(handleNATSRequest(daemon.snapshotService.CopySnapshot)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.snapshotService.CopySnapshot)), input: &ec2.CopySnapshotInput{SourceRegion: aws.String("us-east-1"), SourceSnapshotId: aws.String("snap-nonexistent")}, expectedCode: awserrors.ErrorInvalidSnapshotNotFound, }, { name: "DeleteTags", topic: "ec2.test.DeleteTags", - handler: asMsgHandler(handleNATSRequest(daemon.deleteTags)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.deleteTags)), // Instance IDs route to the owning daemon; with no owner // subscribed the mutation is rejected rather than written blind. input: &ec2.DeleteTagsInput{Resources: []*string{aws.String("i-12345678")}}, @@ -2054,7 +2055,7 @@ func TestDelegateHandlers_RoundTrip(t *testing.T) { { name: "DescribeTags", topic: "ec2.test.DescribeTags", - handler: asMsgHandler(handleNATSRequest(daemon.tagsService.DescribeTags)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.tagsService.DescribeTags)), input: &ec2.DescribeTagsInput{}, }, } @@ -2358,26 +2359,26 @@ func TestDelegateHandlers_VPC(t *testing.T) { { name: "CreateVpc", topic: "ec2.test.CreateVpc", - handler: asMsgHandler(handleNATSRequest(daemon.vpcService.CreateVpc)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.vpcService.CreateVpc)), input: &ec2.CreateVpcInput{CidrBlock: aws.String("10.0.0.0/16")}, }, { name: "DeleteVpc", topic: "ec2.test.DeleteVpc", - handler: asMsgHandler(handleNATSRequest(daemon.vpcService.DeleteVpc)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.vpcService.DeleteVpc)), input: &ec2.DeleteVpcInput{VpcId: aws.String("vpc-nonexistent")}, expectedCode: awserrors.ErrorInvalidVpcIDNotFound, }, { name: "DescribeVpcs", topic: "ec2.test.DescribeVpcs", - handler: asMsgHandler(handleNATSRequest(daemon.vpcService.DescribeVpcs)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.vpcService.DescribeVpcs)), input: &ec2.DescribeVpcsInput{}, }, { name: "CreateSubnet", topic: "ec2.test.CreateSubnet", - handler: asMsgHandler(handleNATSRequest(daemon.vpcService.CreateSubnet)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.vpcService.CreateSubnet)), input: &ec2.CreateSubnetInput{ VpcId: aws.String("vpc-nonexistent"), CidrBlock: aws.String("10.0.1.0/24"), @@ -2387,34 +2388,34 @@ func TestDelegateHandlers_VPC(t *testing.T) { { name: "DeleteSubnet", topic: "ec2.test.DeleteSubnet", - handler: asMsgHandler(handleNATSRequest(daemon.vpcService.DeleteSubnet)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.vpcService.DeleteSubnet)), input: &ec2.DeleteSubnetInput{SubnetId: aws.String("subnet-nonexistent")}, expectedCode: awserrors.ErrorInvalidSubnetIDNotFound, }, { name: "DescribeSubnets", topic: "ec2.test.DescribeSubnets", - handler: asMsgHandler(handleNATSRequest(daemon.vpcService.DescribeSubnets)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.vpcService.DescribeSubnets)), input: &ec2.DescribeSubnetsInput{}, }, { name: "CreateNetworkInterface", topic: "ec2.test.CreateNetworkInterface", - handler: asMsgHandler(handleNATSRequest(daemon.vpcService.CreateNetworkInterface)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.vpcService.CreateNetworkInterface)), input: &ec2.CreateNetworkInterfaceInput{SubnetId: aws.String("subnet-nonexistent")}, expectedCode: awserrors.ErrorInvalidSubnetIDNotFound, }, { name: "DeleteNetworkInterface", topic: "ec2.test.DeleteNetworkInterface", - handler: asMsgHandler(handleNATSRequest(daemon.vpcService.DeleteNetworkInterface)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.vpcService.DeleteNetworkInterface)), input: &ec2.DeleteNetworkInterfaceInput{NetworkInterfaceId: aws.String("eni-nonexistent")}, expectedCode: awserrors.ErrorInvalidNetworkInterfaceIDNotFound, }, { name: "DescribeNetworkInterfaces", topic: "ec2.test.DescribeNetworkInterfaces", - handler: asMsgHandler(handleNATSRequest(daemon.vpcService.DescribeNetworkInterfaces)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.vpcService.DescribeNetworkInterfaces)), input: &ec2.DescribeNetworkInterfacesInput{}, }, } @@ -2429,26 +2430,26 @@ func TestDelegateHandlers_IGW(t *testing.T) { { name: "CreateInternetGateway", topic: "ec2.test.CreateInternetGateway", - handler: asMsgHandler(handleNATSRequest(daemon.igwService.CreateInternetGateway)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.igwService.CreateInternetGateway)), input: &ec2.CreateInternetGatewayInput{}, }, { name: "DeleteInternetGateway", topic: "ec2.test.DeleteInternetGateway", - handler: asMsgHandler(handleNATSRequest(daemon.igwService.DeleteInternetGateway)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.igwService.DeleteInternetGateway)), input: &ec2.DeleteInternetGatewayInput{InternetGatewayId: aws.String("igw-nonexistent")}, expectedCode: awserrors.ErrorInvalidInternetGatewayIDNotFound, }, { name: "DescribeInternetGateways", topic: "ec2.test.DescribeInternetGateways", - handler: asMsgHandler(handleNATSRequest(daemon.igwService.DescribeInternetGateways)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.igwService.DescribeInternetGateways)), input: &ec2.DescribeInternetGatewaysInput{}, }, { name: "AttachInternetGateway", topic: "ec2.test.AttachInternetGateway", - handler: asMsgHandler(handleNATSRequest(daemon.igwService.AttachInternetGateway)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.igwService.AttachInternetGateway)), input: &ec2.AttachInternetGatewayInput{ InternetGatewayId: aws.String("igw-nonexistent"), VpcId: aws.String("vpc-nonexistent"), @@ -2458,7 +2459,7 @@ func TestDelegateHandlers_IGW(t *testing.T) { { name: "DetachInternetGateway", topic: "ec2.test.DetachInternetGateway", - handler: asMsgHandler(handleNATSRequest(daemon.igwService.DetachInternetGateway)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.igwService.DetachInternetGateway)), input: &ec2.DetachInternetGatewayInput{ InternetGatewayId: aws.String("igw-nonexistent"), VpcId: aws.String("vpc-nonexistent"), @@ -2473,7 +2474,7 @@ func TestDelegateHandlers_IGW(t *testing.T) { func TestHandleEC2CreateVpc_SuccessPath(t *testing.T) { daemon := createVPCTestDaemon(t) - sub, err := daemon.natsConn.QueueSubscribe("ec2.CreateVpc", "spinifex-workers", asMsgHandler(handleNATSRequest(daemon.vpcService.CreateVpc))) + sub, err := daemon.natsConn.QueueSubscribe("ec2.CreateVpc", "spinifex-workers", asMsgHandler(handleNATSRequest(daemon.node, daemon.vpcService.CreateVpc))) require.NoError(t, err) defer sub.Unsubscribe() @@ -2494,11 +2495,11 @@ func TestHandleEC2CreateVpc_SuccessPath(t *testing.T) { func TestHandleEC2CreateAndDescribeVpc_RoundTrip(t *testing.T) { daemon := createVPCTestDaemon(t) - createSub, err := daemon.natsConn.QueueSubscribe("ec2.CreateVpc", "spinifex-workers", asMsgHandler(handleNATSRequest(daemon.vpcService.CreateVpc))) + createSub, err := daemon.natsConn.QueueSubscribe("ec2.CreateVpc", "spinifex-workers", asMsgHandler(handleNATSRequest(daemon.node, daemon.vpcService.CreateVpc))) require.NoError(t, err) defer createSub.Unsubscribe() - describeSub, err := daemon.natsConn.QueueSubscribe("ec2.DescribeVpcs", "spinifex-workers", asMsgHandler(handleNATSRequest(daemon.vpcService.DescribeVpcs))) + describeSub, err := daemon.natsConn.QueueSubscribe("ec2.DescribeVpcs", "spinifex-workers", asMsgHandler(handleNATSRequest(daemon.node, daemon.vpcService.DescribeVpcs))) require.NoError(t, err) defer describeSub.Unsubscribe() @@ -2534,7 +2535,7 @@ func TestHandleEC2CreateAndDescribeVpc_RoundTrip(t *testing.T) { func TestHandleEC2CreateInternetGateway_SuccessPath(t *testing.T) { daemon := createVPCTestDaemon(t) - sub, err := daemon.natsConn.QueueSubscribe("ec2.CreateInternetGateway", "spinifex-workers", asMsgHandler(handleNATSRequest(daemon.igwService.CreateInternetGateway))) + sub, err := daemon.natsConn.QueueSubscribe("ec2.CreateInternetGateway", "spinifex-workers", asMsgHandler(handleNATSRequest(daemon.node, daemon.igwService.CreateInternetGateway))) require.NoError(t, err) defer sub.Unsubscribe() @@ -2554,11 +2555,11 @@ func TestHandleEC2CreateInternetGateway_SuccessPath(t *testing.T) { func TestHandleEC2CreateSubnet_SuccessPath(t *testing.T) { daemon := createVPCTestDaemon(t) - createVpcSub, err := daemon.natsConn.QueueSubscribe("ec2.CreateVpc", "spinifex-workers", asMsgHandler(handleNATSRequest(daemon.vpcService.CreateVpc))) + createVpcSub, err := daemon.natsConn.QueueSubscribe("ec2.CreateVpc", "spinifex-workers", asMsgHandler(handleNATSRequest(daemon.node, daemon.vpcService.CreateVpc))) require.NoError(t, err) defer createVpcSub.Unsubscribe() - createSubnetSub, err := daemon.natsConn.QueueSubscribe("ec2.CreateSubnet", "spinifex-workers", asMsgHandler(handleNATSRequest(daemon.vpcService.CreateSubnet))) + createSubnetSub, err := daemon.natsConn.QueueSubscribe("ec2.CreateSubnet", "spinifex-workers", asMsgHandler(handleNATSRequest(daemon.node, daemon.vpcService.CreateSubnet))) require.NoError(t, err) defer createSubnetSub.Unsubscribe() @@ -2605,21 +2606,21 @@ func TestDelegateHandlers_EIGW(t *testing.T) { { name: "CreateEgressOnlyInternetGateway", topic: "ec2.test.CreateEgressOnlyIGW", - handler: asMsgHandler(handleNATSRequest(daemon.eigwService.CreateEgressOnlyInternetGateway)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.eigwService.CreateEgressOnlyInternetGateway)), input: &ec2.CreateEgressOnlyInternetGatewayInput{VpcId: aws.String("vpc-123")}, expectedCode: awserrors.ErrorInvalidVpcIDNotFound, }, { name: "DeleteEgressOnlyInternetGateway", topic: "ec2.test.DeleteEgressOnlyIGW", - handler: asMsgHandler(handleNATSRequest(daemon.eigwService.DeleteEgressOnlyInternetGateway)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.eigwService.DeleteEgressOnlyInternetGateway)), input: &ec2.DeleteEgressOnlyInternetGatewayInput{EgressOnlyInternetGatewayId: aws.String("eigw-nonexistent")}, expectedCode: awserrors.ErrorInvalidEgressOnlyInternetGatewayIdNotFound, }, { name: "DescribeEgressOnlyInternetGateways", topic: "ec2.test.DescribeEgressOnlyIGWs", - handler: asMsgHandler(handleNATSRequest(daemon.eigwService.DescribeEgressOnlyInternetGateways)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.eigwService.DescribeEgressOnlyInternetGateways)), input: &ec2.DescribeEgressOnlyInternetGatewaysInput{}, }, } @@ -2675,7 +2676,7 @@ func TestHandleEC2ModifyVolume_Success(t *testing.T) { func TestHandleEC2DescribeInstanceTypes_CapacityFilter(t *testing.T) { daemon := createFullTestDaemon(t, sharedNATSURL) - sub, err := daemon.natsConn.QueueSubscribe("ec2.DescribeInstanceTypes", "spinifex-workers", asMsgHandler(handleNATSRequest(daemon.instanceService.DescribeInstanceTypes))) + sub, err := daemon.natsConn.QueueSubscribe("ec2.DescribeInstanceTypes", "spinifex-workers", asMsgHandler(handleNATSRequest(daemon.node, daemon.instanceService.DescribeInstanceTypes))) require.NoError(t, err) defer sub.Unsubscribe() @@ -2702,7 +2703,7 @@ func TestHandleEC2DescribeInstanceTypes_CapacityFilter(t *testing.T) { func TestHandleEC2DescribeInstanceTypes_NoFilter(t *testing.T) { daemon := createFullTestDaemon(t, sharedNATSURL) - sub, err := daemon.natsConn.QueueSubscribe("ec2.DescribeInstanceTypes.nofilter", "spinifex-workers", asMsgHandler(handleNATSRequest(daemon.instanceService.DescribeInstanceTypes))) + sub, err := daemon.natsConn.QueueSubscribe("ec2.DescribeInstanceTypes.nofilter", "spinifex-workers", asMsgHandler(handleNATSRequest(daemon.node, daemon.instanceService.DescribeInstanceTypes))) require.NoError(t, err) defer sub.Unsubscribe() @@ -3268,7 +3269,7 @@ func TestHandleEC2RunInstances_MalformedInput(t *testing.T) { func TestHandleEC2DescribeInstances_MalformedInstanceID(t *testing.T) { daemon := createFullTestDaemon(t, sharedNATSURL) - sub, err := daemon.natsConn.Subscribe("ec2.DescribeInstances.malformed", asMsgHandler(handleNATSRequest(daemon.instanceService.DescribeInstances))) + sub, err := daemon.natsConn.Subscribe("ec2.DescribeInstances.malformed", asMsgHandler(handleNATSRequest(daemon.node, daemon.instanceService.DescribeInstances))) require.NoError(t, err) defer sub.Unsubscribe() @@ -3300,7 +3301,7 @@ func TestHandleEC2DescribeInstanceStatus_RoundTrip(t *testing.T) { daemon.vmMgr.Insert(runningVM) t.Cleanup(func() { daemon.vmMgr.Delete(runningVM.ID) }) - sub, err := daemon.natsConn.Subscribe("ec2.DescribeInstanceStatus.rt", asMsgHandler(handleNATSRequest(daemon.instanceService.DescribeInstanceStatus))) + sub, err := daemon.natsConn.Subscribe("ec2.DescribeInstanceStatus.rt", asMsgHandler(handleNATSRequest(daemon.node, daemon.instanceService.DescribeInstanceStatus))) require.NoError(t, err) defer sub.Unsubscribe() @@ -3326,7 +3327,7 @@ func TestHandleEC2DescribeInstanceStatus_RoundTrip(t *testing.T) { func TestHandleEC2DescribeInstanceStatus_MalformedInstanceID(t *testing.T) { daemon := createFullTestDaemon(t, sharedNATSURL) - sub, err := daemon.natsConn.Subscribe("ec2.DescribeInstanceStatus.malformed", asMsgHandler(handleNATSRequest(daemon.instanceService.DescribeInstanceStatus))) + sub, err := daemon.natsConn.Subscribe("ec2.DescribeInstanceStatus.malformed", asMsgHandler(handleNATSRequest(daemon.node, daemon.instanceService.DescribeInstanceStatus))) require.NoError(t, err) defer sub.Unsubscribe() @@ -3361,7 +3362,7 @@ func TestHandleEC2DescribeTerminatedInstances_ReturnsTerminatedInstances(t *test err := daemon.jsManager.WriteTerminatedInstance(terminatedVM.ID, terminatedVM) require.NoError(t, err) - sub, err := daemon.natsConn.QueueSubscribe("ec2.DescribeTerminatedInstances", "spinifex-workers", asMsgHandler(handleNATSRequest(daemon.instanceService.DescribeTerminatedInstances))) + sub, err := daemon.natsConn.QueueSubscribe("ec2.DescribeTerminatedInstances", "spinifex-workers", asMsgHandler(handleNATSRequest(daemon.node, daemon.instanceService.DescribeTerminatedInstances))) require.NoError(t, err) defer sub.Unsubscribe() @@ -3410,7 +3411,7 @@ func TestHandleEC2DescribeTerminatedInstances_WithFilter(t *testing.T) { require.NoError(t, daemon.jsManager.WriteTerminatedInstance(v.ID, v)) } - sub, err := daemon.natsConn.QueueSubscribe("ec2.DescribeTerminatedInstances", "spinifex-workers", asMsgHandler(handleNATSRequest(daemon.instanceService.DescribeTerminatedInstances))) + sub, err := daemon.natsConn.QueueSubscribe("ec2.DescribeTerminatedInstances", "spinifex-workers", asMsgHandler(handleNATSRequest(daemon.node, daemon.instanceService.DescribeTerminatedInstances))) require.NoError(t, err) defer sub.Unsubscribe() @@ -3446,7 +3447,7 @@ func TestHandleEC2TerminateStoppedInstance_WritesToTerminatedKV(t *testing.T) { } require.NoError(t, daemon.jsManager.WriteStoppedInstance(stoppedVM.ID, stoppedVM)) - sub, err := daemon.natsConn.QueueSubscribe("ec2.terminate", "spinifex-workers", asMsgHandler(handleNATSRequest(daemon.instanceService.TerminateStoppedInstance))) + sub, err := daemon.natsConn.QueueSubscribe("ec2.terminate", "spinifex-workers", asMsgHandler(handleNATSRequest(daemon.node, daemon.instanceService.TerminateStoppedInstance))) require.NoError(t, err) defer sub.Unsubscribe() @@ -3489,34 +3490,34 @@ func TestDelegateHandlers_EIP(t *testing.T) { { name: "AllocateAddress", topic: "ec2.test.AllocateAddress", - handler: asMsgHandler(handleNATSRequest(daemon.eipService.AllocateAddress)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.eipService.AllocateAddress)), input: &ec2.AllocateAddressInput{}, }, { name: "ReleaseAddress", topic: "ec2.test.ReleaseAddress", - handler: asMsgHandler(handleNATSRequest(daemon.eipService.ReleaseAddress)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.eipService.ReleaseAddress)), input: &ec2.ReleaseAddressInput{AllocationId: aws.String("eipalloc-nonexistent")}, expectedCode: awserrors.ErrorInvalidAllocationIDNotFound, }, { name: "AssociateAddress", topic: "ec2.test.AssociateAddress", - handler: asMsgHandler(handleNATSRequest(daemon.eipService.AssociateAddress)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.eipService.AssociateAddress)), input: &ec2.AssociateAddressInput{AllocationId: aws.String("eipalloc-nonexistent")}, expectedCode: awserrors.ErrorInvalidAllocationIDNotFound, }, { name: "DisassociateAddress", topic: "ec2.test.DisassociateAddress", - handler: asMsgHandler(handleNATSRequest(daemon.eipService.DisassociateAddress)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.eipService.DisassociateAddress)), input: &ec2.DisassociateAddressInput{AssociationId: aws.String("eipassoc-nonexistent")}, expectedCode: awserrors.ErrorInvalidAssociationIDNotFound, }, { name: "DescribeAddresses", topic: "ec2.test.DescribeAddresses", - handler: asMsgHandler(handleNATSRequest(daemon.eipService.DescribeAddresses)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.eipService.DescribeAddresses)), input: &ec2.DescribeAddressesInput{}, }, } @@ -3530,7 +3531,7 @@ func TestDelegateHandlers_SecurityGroup(t *testing.T) { daemon := createVPCTestDaemon(t) // Create a VPC first so SG operations have a valid target - createVpcSub, err := daemon.natsConn.QueueSubscribe("ec2.CreateVpc", "spinifex-workers", asMsgHandler(handleNATSRequest(daemon.vpcService.CreateVpc))) + createVpcSub, err := daemon.natsConn.QueueSubscribe("ec2.CreateVpc", "spinifex-workers", asMsgHandler(handleNATSRequest(daemon.node, daemon.vpcService.CreateVpc))) require.NoError(t, err) defer createVpcSub.Unsubscribe() @@ -3547,7 +3548,7 @@ func TestDelegateHandlers_SecurityGroup(t *testing.T) { { name: "CreateSecurityGroup", topic: "ec2.test.CreateSecurityGroup", - handler: asMsgHandler(handleNATSRequest(daemon.vpcService.CreateSecurityGroup)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.vpcService.CreateSecurityGroup)), input: &ec2.CreateSecurityGroupInput{ GroupName: aws.String("test-sg"), Description: aws.String("test security group"), @@ -3557,41 +3558,41 @@ func TestDelegateHandlers_SecurityGroup(t *testing.T) { { name: "DescribeSecurityGroups", topic: "ec2.test.DescribeSecurityGroups", - handler: asMsgHandler(handleNATSRequest(daemon.vpcService.DescribeSecurityGroups)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.vpcService.DescribeSecurityGroups)), input: &ec2.DescribeSecurityGroupsInput{}, }, { name: "AuthorizeSecurityGroupIngress", topic: "ec2.test.AuthorizeSecurityGroupIngress", - handler: asMsgHandler(handleNATSRequest(daemon.vpcService.AuthorizeSecurityGroupIngress)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.vpcService.AuthorizeSecurityGroupIngress)), input: &ec2.AuthorizeSecurityGroupIngressInput{GroupId: aws.String("sg-nonexistent")}, expectedCode: awserrors.ErrorInvalidGroupNotFound, }, { name: "AuthorizeSecurityGroupEgress", topic: "ec2.test.AuthorizeSecurityGroupEgress", - handler: asMsgHandler(handleNATSRequest(daemon.vpcService.AuthorizeSecurityGroupEgress)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.vpcService.AuthorizeSecurityGroupEgress)), input: &ec2.AuthorizeSecurityGroupEgressInput{GroupId: aws.String("sg-nonexistent")}, expectedCode: awserrors.ErrorInvalidGroupNotFound, }, { name: "RevokeSecurityGroupIngress", topic: "ec2.test.RevokeSecurityGroupIngress", - handler: asMsgHandler(handleNATSRequest(daemon.vpcService.RevokeSecurityGroupIngress)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.vpcService.RevokeSecurityGroupIngress)), input: &ec2.RevokeSecurityGroupIngressInput{GroupId: aws.String("sg-nonexistent")}, expectedCode: awserrors.ErrorInvalidGroupNotFound, }, { name: "RevokeSecurityGroupEgress", topic: "ec2.test.RevokeSecurityGroupEgress", - handler: asMsgHandler(handleNATSRequest(daemon.vpcService.RevokeSecurityGroupEgress)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.vpcService.RevokeSecurityGroupEgress)), input: &ec2.RevokeSecurityGroupEgressInput{GroupId: aws.String("sg-nonexistent")}, expectedCode: awserrors.ErrorInvalidGroupNotFound, }, { name: "DeleteSecurityGroup", topic: "ec2.test.DeleteSecurityGroup", - handler: asMsgHandler(handleNATSRequest(daemon.vpcService.DeleteSecurityGroup)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.vpcService.DeleteSecurityGroup)), input: &ec2.DeleteSecurityGroupInput{GroupId: aws.String("sg-nonexistent")}, expectedCode: awserrors.ErrorInvalidGroupNotFound, }, @@ -3616,27 +3617,27 @@ func TestDelegateHandlers_RouteTable(t *testing.T) { { name: "CreateRouteTable", topic: "ec2.test.CreateRouteTable", - handler: asMsgHandler(handleNATSRequest(daemon.routeTableService.CreateRouteTable)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.routeTableService.CreateRouteTable)), input: &ec2.CreateRouteTableInput{VpcId: aws.String("vpc-nonexistent")}, expectedCode: awserrors.ErrorInvalidVpcIDNotFound, }, { name: "DeleteRouteTable", topic: "ec2.test.DeleteRouteTable", - handler: asMsgHandler(handleNATSRequest(daemon.routeTableService.DeleteRouteTable)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.routeTableService.DeleteRouteTable)), input: &ec2.DeleteRouteTableInput{RouteTableId: aws.String("rtb-nonexistent")}, expectedCode: awserrors.ErrorInvalidRouteTableIDNotFound, }, { name: "DescribeRouteTables", topic: "ec2.test.DescribeRouteTables", - handler: asMsgHandler(handleNATSRequest(daemon.routeTableService.DescribeRouteTables)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.routeTableService.DescribeRouteTables)), input: &ec2.DescribeRouteTablesInput{}, }, { name: "CreateRoute", topic: "ec2.test.CreateRoute", - handler: asMsgHandler(handleNATSRequest(daemon.routeTableService.CreateRoute)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.routeTableService.CreateRoute)), input: &ec2.CreateRouteInput{ RouteTableId: aws.String("rtb-nonexistent"), DestinationCidrBlock: aws.String("0.0.0.0/0"), @@ -3647,7 +3648,7 @@ func TestDelegateHandlers_RouteTable(t *testing.T) { { name: "DeleteRoute", topic: "ec2.test.DeleteRoute", - handler: asMsgHandler(handleNATSRequest(daemon.routeTableService.DeleteRoute)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.routeTableService.DeleteRoute)), input: &ec2.DeleteRouteInput{ RouteTableId: aws.String("rtb-nonexistent"), DestinationCidrBlock: aws.String("0.0.0.0/0"), @@ -3657,7 +3658,7 @@ func TestDelegateHandlers_RouteTable(t *testing.T) { { name: "ReplaceRoute", topic: "ec2.test.ReplaceRoute", - handler: asMsgHandler(handleNATSRequest(daemon.routeTableService.ReplaceRoute)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.routeTableService.ReplaceRoute)), input: &ec2.ReplaceRouteInput{ RouteTableId: aws.String("rtb-nonexistent"), DestinationCidrBlock: aws.String("0.0.0.0/0"), @@ -3668,7 +3669,7 @@ func TestDelegateHandlers_RouteTable(t *testing.T) { { name: "AssociateRouteTable", topic: "ec2.test.AssociateRouteTable", - handler: asMsgHandler(handleNATSRequest(daemon.routeTableService.AssociateRouteTable)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.routeTableService.AssociateRouteTable)), input: &ec2.AssociateRouteTableInput{ RouteTableId: aws.String("rtb-nonexistent"), SubnetId: aws.String("subnet-nonexistent"), @@ -3678,14 +3679,14 @@ func TestDelegateHandlers_RouteTable(t *testing.T) { { name: "DisassociateRouteTable", topic: "ec2.test.DisassociateRouteTable", - handler: asMsgHandler(handleNATSRequest(daemon.routeTableService.DisassociateRouteTable)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.routeTableService.DisassociateRouteTable)), input: &ec2.DisassociateRouteTableInput{AssociationId: aws.String("rtbassoc-nonexistent")}, expectedCode: awserrors.ErrorInvalidAssociationIDNotFound, }, { name: "ReplaceRouteTableAssociation", topic: "ec2.test.ReplaceRouteTableAssociation", - handler: asMsgHandler(handleNATSRequest(daemon.routeTableService.ReplaceRouteTableAssociation)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.routeTableService.ReplaceRouteTableAssociation)), input: &ec2.ReplaceRouteTableAssociationInput{ AssociationId: aws.String("rtbassoc-nonexistent"), RouteTableId: aws.String("rtb-nonexistent"), @@ -3712,7 +3713,7 @@ func TestDelegateHandlers_PlacementGroup(t *testing.T) { { name: "CreatePlacementGroup", topic: "ec2.test.CreatePlacementGroup", - handler: asMsgHandler(handleNATSRequest(daemon.placementGroupService.CreatePlacementGroup)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.placementGroupService.CreatePlacementGroup)), input: &ec2.CreatePlacementGroupInput{ GroupName: aws.String("test-pg"), Strategy: aws.String("spread"), @@ -3721,20 +3722,20 @@ func TestDelegateHandlers_PlacementGroup(t *testing.T) { { name: "DescribePlacementGroups", topic: "ec2.test.DescribePlacementGroups", - handler: asMsgHandler(handleNATSRequest(daemon.placementGroupService.DescribePlacementGroups)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.placementGroupService.DescribePlacementGroups)), input: &ec2.DescribePlacementGroupsInput{}, }, { name: "DeletePlacementGroup", topic: "ec2.test.DeletePlacementGroup", - handler: asMsgHandler(handleNATSRequest(daemon.placementGroupService.DeletePlacementGroup)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.placementGroupService.DeletePlacementGroup)), input: &ec2.DeletePlacementGroupInput{GroupName: aws.String("pg-nonexistent")}, expectedCode: awserrors.ErrorInvalidPlacementGroupUnknown, }, { name: "ReserveSpreadNodes", topic: "ec2.test.ReserveSpreadNodes", - handler: asMsgHandler(handleNATSRequest(daemon.placementGroupService.ReserveSpreadNodes)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.placementGroupService.ReserveSpreadNodes)), input: &handlers_ec2_placementgroup.ReserveSpreadNodesInput{ GroupName: "pg-nonexistent", EligibleNodes: []string{"node-1"}, @@ -3746,7 +3747,7 @@ func TestDelegateHandlers_PlacementGroup(t *testing.T) { { name: "FinalizeSpreadInstances", topic: "ec2.test.FinalizeSpreadInstances", - handler: asMsgHandler(handleNATSRequest(daemon.placementGroupService.FinalizeSpreadInstances)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.placementGroupService.FinalizeSpreadInstances)), input: &handlers_ec2_placementgroup.FinalizeSpreadInstancesInput{ GroupName: "pg-nonexistent", NodeInstances: map[string][]string{"node-1": {"i-123"}}, @@ -3756,7 +3757,7 @@ func TestDelegateHandlers_PlacementGroup(t *testing.T) { { name: "ReleaseSpreadNodes", topic: "ec2.test.ReleaseSpreadNodes", - handler: asMsgHandler(handleNATSRequest(daemon.placementGroupService.ReleaseSpreadNodes)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.placementGroupService.ReleaseSpreadNodes)), input: &handlers_ec2_placementgroup.ReleaseSpreadNodesInput{ GroupName: "pg-nonexistent", Nodes: []string{"node-1"}, @@ -3766,7 +3767,7 @@ func TestDelegateHandlers_PlacementGroup(t *testing.T) { { name: "RemoveInstanceFromPlacementGroup", topic: "ec2.test.RemoveInstanceFromPlacementGroup", - handler: asMsgHandler(handleNATSRequest(daemon.placementGroupService.RemoveInstance)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.placementGroupService.RemoveInstance)), input: &handlers_ec2_placementgroup.RemoveInstanceInput{ GroupName: "pg-nonexistent", NodeName: "node-1", @@ -3779,7 +3780,7 @@ func TestDelegateHandlers_PlacementGroup(t *testing.T) { { name: "ReserveClusterNode", topic: "ec2.test.ReserveClusterNode", - handler: asMsgHandler(handleNATSRequest(daemon.placementGroupService.ReserveClusterNode)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.placementGroupService.ReserveClusterNode)), input: &handlers_ec2_placementgroup.ReserveClusterNodeInput{ GroupName: "pg-nonexistent", EligibleNodes: []string{"node-1"}, @@ -3789,7 +3790,7 @@ func TestDelegateHandlers_PlacementGroup(t *testing.T) { { name: "FinalizeClusterInstances", topic: "ec2.test.FinalizeClusterInstances", - handler: asMsgHandler(handleNATSRequest(daemon.placementGroupService.FinalizeClusterInstances)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.placementGroupService.FinalizeClusterInstances)), input: &handlers_ec2_placementgroup.FinalizeClusterInstancesInput{ GroupName: "pg-nonexistent", NodeInstances: map[string][]string{"node-1": {"i-123"}}, @@ -3807,7 +3808,7 @@ func TestDelegateHandlers_VPCAttributes(t *testing.T) { daemon := createVPCTestDaemon(t) // Create a VPC first - createVpcSub, err := daemon.natsConn.QueueSubscribe("ec2.CreateVpc", "spinifex-workers", asMsgHandler(handleNATSRequest(daemon.vpcService.CreateVpc))) + createVpcSub, err := daemon.natsConn.QueueSubscribe("ec2.CreateVpc", "spinifex-workers", asMsgHandler(handleNATSRequest(daemon.node, daemon.vpcService.CreateVpc))) require.NoError(t, err) defer createVpcSub.Unsubscribe() @@ -3824,7 +3825,7 @@ func TestDelegateHandlers_VPCAttributes(t *testing.T) { { name: "ModifySubnetAttribute", topic: "ec2.test.ModifySubnetAttribute", - handler: asMsgHandler(handleNATSRequest(daemon.vpcService.ModifySubnetAttribute)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.vpcService.ModifySubnetAttribute)), input: &ec2.ModifySubnetAttributeInput{SubnetId: aws.String("subnet-nonexistent")}, expectedCode: awserrors.ErrorInvalidSubnetIDNotFound, }, @@ -3834,14 +3835,14 @@ func TestDelegateHandlers_VPCAttributes(t *testing.T) { // the validation contract. name: "ModifyVpcAttribute", topic: "ec2.test.ModifyVpcAttribute", - handler: asMsgHandler(handleNATSRequest(daemon.vpcService.ModifyVpcAttribute)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.vpcService.ModifyVpcAttribute)), input: &ec2.ModifyVpcAttributeInput{VpcId: aws.String(vpcID)}, expectedCode: awserrors.ErrorInvalidParameterValue, }, { name: "DescribeVpcAttribute", topic: "ec2.test.DescribeVpcAttribute", - handler: asMsgHandler(handleNATSRequest(daemon.vpcService.DescribeVpcAttribute)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.vpcService.DescribeVpcAttribute)), input: &ec2.DescribeVpcAttributeInput{ VpcId: aws.String(vpcID), Attribute: aws.String("enableDnsSupport"), @@ -3852,7 +3853,7 @@ func TestDelegateHandlers_VPCAttributes(t *testing.T) { // "exactly one attribute" guard before the ENI lookup runs. name: "ModifyNetworkInterfaceAttribute", topic: "ec2.test.ModifyNetworkInterfaceAttribute", - handler: asMsgHandler(handleNATSRequest(daemon.vpcService.ModifyNetworkInterfaceAttribute)), + handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.vpcService.ModifyNetworkInterfaceAttribute)), input: &ec2.ModifyNetworkInterfaceAttributeInput{NetworkInterfaceId: aws.String("eni-nonexistent")}, expectedCode: awserrors.ErrorInvalidParameterValue, }, @@ -3882,7 +3883,7 @@ func TestRespondWithJSON_MarshalSuccess(t *testing.T) { want := testOutput{Greeting: "hello world"} sub, err := nc.Subscribe("test.respond.ok", func(msg *nats.Msg) { - respondWithJSON(msg, want) + respondWithJSON("node-a", msg, want) }) require.NoError(t, err) defer sub.Unsubscribe() @@ -3893,6 +3894,7 @@ func TestRespondWithJSON_MarshalSuccess(t *testing.T) { var got testOutput require.NoError(t, json.Unmarshal(reply.Data, &got)) assert.Equal(t, want, got) + assert.Equal(t, "node-a", reply.Header.Get(utils.NodeIDHeader)) } func TestRespondWithJSON_MarshalFailureReturnsServerInternal(t *testing.T) { @@ -3905,7 +3907,7 @@ func TestRespondWithJSON_MarshalFailureReturnsServerInternal(t *testing.T) { sub, err := nc.Subscribe("test.respond.marshalfail", func(msg *nats.Msg) { // Channels are not marshalable by encoding/json, forcing the // marshal-failure branch. - respondWithJSON(msg, make(chan int)) + respondWithJSON("node-a", msg, make(chan int)) }) require.NoError(t, err) defer sub.Unsubscribe() @@ -3927,7 +3929,7 @@ func TestRespondWithJSON_RespondFailureLogs(t *testing.T) { msg := &nats.Msg{Subject: "test.respond.unbound"} assert.NotPanics(t, func() { - respondWithJSON(msg, testOutput{Greeting: "ignored"}) + respondWithJSON("node-a", msg, testOutput{Greeting: "ignored"}) }) logged := buf.String() diff --git a/spinifex/daemon/daemon_handlers_volume.go b/spinifex/daemon/daemon_handlers_volume.go index 060e03279..d403af10c 100644 --- a/spinifex/daemon/daemon_handlers_volume.go +++ b/spinifex/daemon/daemon_handlers_volume.go @@ -26,7 +26,7 @@ func (d *Daemon) handleAttachVolume(ctx context.Context, msg *nats.Msg, command if command.AttachVolumeData == nil || command.AttachVolumeData.VolumeID == "" { slog.ErrorContext(ctx, "AttachVolume: missing attach volume data") - return respondErrorOutcome(msg, awserrors.ErrorInvalidParameterValue) + return respondErrorOutcome(d.node, msg, awserrors.ErrorInvalidParameterValue) } volumeID := command.AttachVolumeData.VolumeID @@ -37,13 +37,13 @@ func (d *Daemon) handleAttachVolume(ctx context.Context, msg *nats.Msg, command if status != vm.StateRunning { slog.ErrorContext(ctx, "AttachVolume: instance not running", "instanceId", command.ID, "status", status) - return respondErrorOutcome(msg, awserrors.ErrorIncorrectInstanceState) + return respondErrorOutcome(d.node, msg, awserrors.ErrorIncorrectInstanceState) } volMeta, err := d.volumeService.GetVolumeMetadata(volumeID) if err != nil { slog.ErrorContext(ctx, "AttachVolume: failed to get volume metadata", "volumeId", volumeID, "err", err) - return respondErrorOutcome(msg, awserrors.ErrorInvalidVolumeNotFound) + return respondErrorOutcome(d.node, msg, awserrors.ErrorInvalidVolumeNotFound) } callerAccountID := utils.AccountIDFromMsg(msg) @@ -52,7 +52,7 @@ func (d *Daemon) handleAttachVolume(ctx context.Context, msg *nats.Msg, command "volumeId", volumeID, "callerAccount", callerAccountID, "ownerAccount", volMeta.TenantID) - return respondErrorOutcome(msg, awserrors.ErrorInvalidVolumeNotFound) + return respondErrorOutcome(d.node, msg, awserrors.ErrorInvalidVolumeNotFound) } if volMeta.State != "available" { @@ -65,7 +65,7 @@ func (d *Daemon) handleAttachVolume(ctx context.Context, msg *nats.Msg, command "attachedDevice", volMeta.DeviceName) // AWS returns VolumeInUse (not InvalidParameterValue) when a // re-attach targets a device other than the one already in use. - return respondErrorOutcome(msg, awserrors.ErrorVolumeInUse) + return respondErrorOutcome(d.node, msg, awserrors.ErrorVolumeInUse) } // Volume is already attached to this instance (e.g. a CSI @@ -79,7 +79,7 @@ func (d *Daemon) handleAttachVolume(ctx context.Context, msg *nats.Msg, command slog.ErrorContext(ctx, "AttachVolume: volume not available", "volumeId", volumeID, "state", volMeta.State) - return respondErrorOutcome(msg, awserrors.ErrorVolumeInUse) + return respondErrorOutcome(d.node, msg, awserrors.ErrorVolumeInUse) } if volMeta.AvailabilityZone != "" && d.config.AZ != "" && @@ -88,12 +88,12 @@ func (d *Daemon) handleAttachVolume(ctx context.Context, msg *nats.Msg, command "volumeId", volumeID, "volumeAZ", volMeta.AvailabilityZone, "instanceAZ", d.config.AZ) - return respondErrorOutcome(msg, awserrors.ErrorInvalidVolumeZoneMismatch) + return respondErrorOutcome(d.node, msg, awserrors.ErrorInvalidVolumeZoneMismatch) } device, err := d.vmMgr.AttachVolume(ctx, instance.ID, volumeID, command.AttachVolumeData.Device) if err != nil { - return respondErrorOutcome(msg, attachDetachErrorCode(err)) + return respondErrorOutcome(d.node, msg, attachDetachErrorCode(err)) } // AttachVolume returns the API-form device name (/dev/sd[f-p]), not @@ -114,7 +114,7 @@ func (d *Daemon) handleDetachVolume(ctx context.Context, msg *nats.Msg, command if command.DetachVolumeData == nil || command.DetachVolumeData.VolumeID == "" { slog.ErrorContext(ctx, "DetachVolume: missing detach volume data") - return respondErrorOutcome(msg, awserrors.ErrorInvalidParameterValue) + return respondErrorOutcome(d.node, msg, awserrors.ErrorInvalidParameterValue) } deviceName, err := d.vmMgr.DetachVolume( @@ -125,7 +125,7 @@ func (d *Daemon) handleDetachVolume(ctx context.Context, msg *nats.Msg, command command.DetachVolumeData.Force, ) if err != nil { - return respondErrorOutcome(msg, attachDetachErrorCode(err)) + return respondErrorOutcome(d.node, msg, attachDetachErrorCode(err)) } d.respondWithVolumeAttachment(msg, command.DetachVolumeData.VolumeID, command.ID, deviceName, "detaching") @@ -146,7 +146,7 @@ func (d *Daemon) handleDrainVolume(ctx context.Context, msg *nats.Msg, command t if command.DrainVolumeData == nil || command.DrainVolumeData.VolumeID == "" { slog.ErrorContext(ctx, "DrainVolume: missing drain volume data", "instanceId", command.ID) - return respondErrorOutcome(msg, awserrors.ErrorInvalidParameterValue) + return respondErrorOutcome(d.node, msg, awserrors.ErrorInvalidParameterValue) } volumeID := command.DrainVolumeData.VolumeID @@ -158,7 +158,7 @@ func (d *Daemon) handleDrainVolume(ctx context.Context, msg *nats.Msg, command t if status == vm.StateStopped || status == vm.StateTerminated { slog.InfoContext(ctx, "DrainVolume: instance teardown is complete, nothing to drain", "volumeId", volumeID, "instanceId", command.ID, "status", status) - respondWithJSON(msg, types.DrainVolumeResponse{VolumeID: volumeID, Status: types.DrainVolumeStatusNotRunning}) + respondWithJSON(d.node, msg, types.DrainVolumeResponse{VolumeID: volumeID, Status: types.DrainVolumeStatusNotRunning}) return outcomeSuccess } @@ -169,10 +169,10 @@ func (d *Daemon) handleDrainVolume(ctx context.Context, msg *nats.Msg, command t // read a stale checkpoint. if err = handlers_ec2_snapshot.DrainVolumeSocket(d.config.DataDir, volumeID); err != nil { slog.ErrorContext(ctx, "DrainVolume: drain failed", "volumeId", volumeID, "instanceId", command.ID, "err", err) - return respondErrorOutcome(msg, awserrors.ErrorServerInternal) + return respondErrorOutcome(d.node, msg, awserrors.ErrorServerInternal) } - respondWithJSON(msg, types.DrainVolumeResponse{VolumeID: volumeID, Status: types.DrainVolumeStatusDrained}) + respondWithJSON(d.node, msg, types.DrainVolumeResponse{VolumeID: volumeID, Status: types.DrainVolumeStatusDrained}) return outcomeSuccess } @@ -226,11 +226,11 @@ func (d *Daemon) handleEC2ModifyVolume(msg *nats.Msg) string { if err != nil { slog.ErrorContext(ctx, "handleEC2ModifyVolume service.ModifyVolume failed", "err", err) utils.MarkSpanError(span, err) - respondWithServiceError(msg, err) + respondWithServiceError(d.node, msg, err) return outcomeError } - respondWithJSON(msg, output) + respondWithJSON(d.node, msg, output) // Notify viperblockd to reload state after volume modification (e.g. resize) if modifyVolumeInput.VolumeId != nil { diff --git a/spinifex/daemon/daemon_test.go b/spinifex/daemon/daemon_test.go index 611ff0174..65fe88a22 100644 --- a/spinifex/daemon/daemon_test.go +++ b/spinifex/daemon/daemon_test.go @@ -384,7 +384,7 @@ func TestHandleEC2DescribeInstanceTypes(t *testing.T) { daemon := createTestDaemon(t, natsURL) // Subscribe to DescribeInstanceTypes (no queue group for fan-out) - sub, err := daemon.natsConn.Subscribe("ec2.DescribeInstanceTypes", asMsgHandler(handleNATSRequest(daemon.instanceService.DescribeInstanceTypes))) + sub, err := daemon.natsConn.Subscribe("ec2.DescribeInstanceTypes", asMsgHandler(handleNATSRequest(daemon.node, daemon.instanceService.DescribeInstanceTypes))) require.NoError(t, err, "Failed to subscribe to ec2.DescribeInstanceTypes") defer sub.Unsubscribe() @@ -4576,3 +4576,38 @@ func TestConfigureEBSProvider_ViperblockdInjectsSameInstanceEverywhere(t *testin assert.Same(t, d.ebsProvider, d.snapshotService.EBSProvider()) assert.Same(t, d.ebsProvider, d.volumeService.EBSProvider()) } + +// TestNodeIDNamespace_Agrees pins the one node ID namespace the identity-mode +// fan-out depends on: the X-Node-ID reply header, config.ClusterConfig.Nodes' +// keys, daemon.Heartbeat.Node (via NodeDiscoverResponse, which carries the +// same value), and vm.VM.LastNode's source (vm.Deps.NodeID) must all be the +// same identifier. If any of these diverged, responder-set coverage would +// silently never match a configured node, and every fan-out would read as +// permanently incomplete. +func TestNodeIDNamespace_Agrees(t *testing.T) { + daemon := createTestDaemon(t, sharedNATSURL) + + // config.ClusterConfig.Nodes' key for this node must be the node's own ID. + _, present := daemon.clusterConfig.Nodes[daemon.node] + require.True(t, present, "clusterConfig.Nodes must be keyed by this node's own ID") + + // vm.Deps.NodeID (the source of vm.VM.LastNode) must be the same ID. + deps := daemon.buildVMManagerDeps() + assert.Equal(t, daemon.node, deps.NodeID) + + // The X-Node-ID reply header, and NodeDiscoverResponse.Node (the source + // fed into daemon.Heartbeat.Node elsewhere), must both be the same ID too. + sub, err := daemon.natsConn.Subscribe("test.nodediscover", asMsgHandler(daemon.handleNodeDiscover)) + require.NoError(t, err) + defer func() { _ = sub.Unsubscribe() }() + + reply, err := daemon.natsConn.Request("test.nodediscover", nil, 5*time.Second) + require.NoError(t, err) + + assert.Equal(t, daemon.node, reply.Header.Get(utils.NodeIDHeader), + "the reply header must carry the same node ID as everything else") + + var resp types.NodeDiscoverResponse + require.NoError(t, json.Unmarshal(reply.Data, &resp)) + assert.Equal(t, daemon.node, resp.Node) +} diff --git a/spinifex/daemon/instance_running_set.go b/spinifex/daemon/instance_running_set.go index 67668f17b..810e69804 100644 --- a/spinifex/daemon/instance_running_set.go +++ b/spinifex/daemon/instance_running_set.go @@ -34,7 +34,7 @@ type runningSetState struct { // source of truth for the node's own instances, so a failure here leaves the // cluster's view stale rather than losing anything, and is logged rather than // returned. The next state change repeats whatever did not land. -func (m *JetStreamManager) WriteRunningSet(nodeID string, vms map[string]*vm.VM) { +func (m *JetStreamManager) WriteRunningSet(nodeID, az string, vms map[string]*vm.VM) { if m.records == nil { return } @@ -60,6 +60,7 @@ func (m *JetStreamManager) WriteRunningSet(nodeID string, vms map[string]*vm.VM) // so nothing lists it and nothing retires it. record := instance.Record() record.Status.LastNode = nodeID + record.Status.AZ = az sum, err := recordDigest(record) if err != nil { diff --git a/spinifex/daemon/instance_running_set_test.go b/spinifex/daemon/instance_running_set_test.go index 6f278fa77..41d1df95f 100644 --- a/spinifex/daemon/instance_running_set_test.go +++ b/spinifex/daemon/instance_running_set_test.go @@ -16,6 +16,7 @@ import ( ) const localNode = "node-1" +const localAZ = "us-east-1a" func runningVM(id, instanceType string) *vm.VM { return &vm.VM{ID: id, InstanceType: instanceType, Status: vm.StateRunning, LastNode: localNode} @@ -35,7 +36,7 @@ func runningSet(vms ...*vm.VM) map[string]*vm.VM { func writeRunningSet(t *testing.T, m *daemon.JetStreamManager, vms ...*vm.VM) { t.Helper() require.NoError(t, m.WriteNodeMarker(localNode)) - m.WriteRunningSet(localNode, runningSet(vms...)) + m.WriteRunningSet(localNode, localAZ, runningSet(vms...)) } // recordRevision reads the revision of a record straight from the bucket, so a @@ -162,12 +163,13 @@ func TestWriteRunningSet_StampsOwnership(t *testing.T) { unowned := &vm.VM{ID: "i-1", InstanceType: "t3.nano", Status: vm.StateRunning} require.NoError(t, m.WriteNodeMarker(localNode)) - m.WriteRunningSet(localNode, runningSet(unowned)) + m.WriteRunningSet(localNode, localAZ, runningSet(unowned)) record, err := m.LoadInstanceRecord("i-1") require.NoError(t, err) require.NotNil(t, record) assert.Equal(t, localNode, record.Status.LastNode) + assert.Equal(t, localAZ, record.Status.AZ, "the node's own AZ must reach the record") } func TestWriteRunningSet_RetiresAMemberThatLeft(t *testing.T) { diff --git a/spinifex/daemon/jetstream_test.go b/spinifex/daemon/jetstream_test.go index ef8a4007b..93bdd2ac2 100644 --- a/spinifex/daemon/jetstream_test.go +++ b/spinifex/daemon/jetstream_test.go @@ -26,7 +26,7 @@ func writeNodeState(t *testing.T, m *JetStreamManager, nodeID string, vms map[st if err := m.WriteNodeMarker(nodeID); err != nil { return err } - m.WriteRunningSet(nodeID, vms) + m.WriteRunningSet(nodeID, "", vms) return nil } diff --git a/spinifex/daemon/nats_outcome_test.go b/spinifex/daemon/nats_outcome_test.go index ceac2ac5b..bd629099d 100644 --- a/spinifex/daemon/nats_outcome_test.go +++ b/spinifex/daemon/nats_outcome_test.go @@ -130,7 +130,7 @@ func TestHandleNATSRequestReportsOutcome(t *testing.T) { defer nc.Close() handler := natsMetricsHandler(tc.subject, - handleNATSRequest(func(context.Context, *testInput, string) (*testOutput, error) { + handleNATSRequest("test-node", func(context.Context, *testInput, string) (*testOutput, error) { if tc.svcErr != nil { return nil, tc.svcErr } @@ -162,7 +162,7 @@ func TestHandleNATSRequestReportsOutcomeForBadPayload(t *testing.T) { defer nc.Close() handler := natsMetricsHandler(subject, - handleNATSRequest(func(context.Context, *testInput, string) (*testOutput, error) { + handleNATSRequest("test-node", func(context.Context, *testInput, string) (*testOutput, error) { t.Fatal("service must not be reached for a malformed payload") return nil, nil })) diff --git a/spinifex/daemon/ochre_deps.go b/spinifex/daemon/ochre_deps.go index bc91d729a..7b7ad3548 100644 --- a/spinifex/daemon/ochre_deps.go +++ b/spinifex/daemon/ochre_deps.go @@ -101,7 +101,7 @@ func (d *Daemon) startOchreVector() { d.ochreAppliance = appliance d.mu.Unlock() if err := d.registerNatsSubs([]natsSub{ - {handlers_ochrevector.SubjectTeardownAppliance, handleNATSRequest(d.handleOchreApplianceTeardown), "spinifex-workers"}, + {handlers_ochrevector.SubjectTeardownAppliance, handleNATSRequest(d.node, d.handleOchreApplianceTeardown), "spinifex-workers"}, }); err != nil { slog.Error("Ochre vector store: failed to register appliance teardown subject", "err", err) } @@ -154,14 +154,14 @@ func (d *Daemon) startOchreVector() { // table-driven mechanism subscribeAll itself uses, so a queue-group // registration here is indistinguishable from one made at boot. subs := []natsSub{ - {handlers_ochrevector.SubjectCreateIndex, handleNATSRequest(vectorService.CreateIndex), "spinifex-workers"}, - {handlers_ochrevector.SubjectDeleteIndex, handleNATSRequest(vectorService.DeleteIndex), "spinifex-workers"}, - {handlers_ochrevector.SubjectListIndexes, handleNATSRequest(vectorService.ListIndexes), "spinifex-workers"}, - {handlers_ochrevector.SubjectIngest, handleNATSRequest(vectorService.Ingest), "spinifex-workers"}, - {handlers_ochrevector.SubjectDescribeJob, handleNATSRequest(vectorService.DescribeJob), "spinifex-workers"}, - {handlers_ochrevector.SubjectQuery, handleNATSRequest(vectorService.Query), "spinifex-workers"}, - {handlers_ochrevector.SubjectListJobs, handleNATSRequest(vectorService.ListJobs), "spinifex-workers"}, - {handlers_ochrevector.SubjectStopJob, handleNATSRequest(vectorService.StopJob), "spinifex-workers"}, + {handlers_ochrevector.SubjectCreateIndex, handleNATSRequest(d.node, vectorService.CreateIndex), "spinifex-workers"}, + {handlers_ochrevector.SubjectDeleteIndex, handleNATSRequest(d.node, vectorService.DeleteIndex), "spinifex-workers"}, + {handlers_ochrevector.SubjectListIndexes, handleNATSRequest(d.node, vectorService.ListIndexes), "spinifex-workers"}, + {handlers_ochrevector.SubjectIngest, handleNATSRequest(d.node, vectorService.Ingest), "spinifex-workers"}, + {handlers_ochrevector.SubjectDescribeJob, handleNATSRequest(d.node, vectorService.DescribeJob), "spinifex-workers"}, + {handlers_ochrevector.SubjectQuery, handleNATSRequest(d.node, vectorService.Query), "spinifex-workers"}, + {handlers_ochrevector.SubjectListJobs, handleNATSRequest(d.node, vectorService.ListJobs), "spinifex-workers"}, + {handlers_ochrevector.SubjectStopJob, handleNATSRequest(d.node, vectorService.StopJob), "spinifex-workers"}, } // backup/restore need RegrantAccount alongside EnsureAccount, which is @@ -174,8 +174,8 @@ func (d *Daemon) startOchreVector() { d.ochreBackupService = backupSvc d.mu.Unlock() subs = append(subs, - natsSub{handlers_ochrevector.SubjectBackupAccount, handleNATSRequest(backupSvc.Backup), "spinifex-workers"}, - natsSub{handlers_ochrevector.SubjectRestoreAccount, handleNATSRequest(backupSvc.Restore), "spinifex-workers"}, + natsSub{handlers_ochrevector.SubjectBackupAccount, handleNATSRequest(d.node, backupSvc.Backup), "spinifex-workers"}, + natsSub{handlers_ochrevector.SubjectRestoreAccount, handleNATSRequest(d.node, backupSvc.Restore), "spinifex-workers"}, ) } else { slog.Warn("Ochre vector store: backend does not support account backup/restore; subjects not registered") diff --git a/spinifex/daemon/ownership.go b/spinifex/daemon/ownership.go index 44f5fb283..9714e105c 100644 --- a/spinifex/daemon/ownership.go +++ b/spinifex/daemon/ownership.go @@ -10,14 +10,14 @@ import ( // checkInstanceOwnership verifies the caller owns the instance. Returns true if // allowed; false after sending an error. Empty ownerAccountID is root-only. -func checkInstanceOwnership(msg *nats.Msg, instanceID, ownerAccountID string) bool { +func checkInstanceOwnership(nodeID string, msg *nats.Msg, instanceID, ownerAccountID string) bool { callerAccountID := utils.AccountIDFromMsg(msg) if ownerAccountID == "" { if callerAccountID != utils.GlobalAccountID { slog.Warn("Untenanted instance access denied (not root)", "instanceId", instanceID, "callerAccount", callerAccountID) - respondWithError(msg, awserrors.ErrorInvalidInstanceIDNotFound) + respondWithError(nodeID, msg, awserrors.ErrorInvalidInstanceIDNotFound) return false } return true @@ -26,7 +26,7 @@ func checkInstanceOwnership(msg *nats.Msg, instanceID, ownerAccountID string) bo if callerAccountID != ownerAccountID { slog.Warn("Account does not own instance", "instanceId", instanceID, "callerAccount", callerAccountID, "ownerAccount", ownerAccountID) - respondWithError(msg, awserrors.ErrorInvalidInstanceIDNotFound) + respondWithError(nodeID, msg, awserrors.ErrorInvalidInstanceIDNotFound) return false } return true diff --git a/spinifex/gateway/ec2.go b/spinifex/gateway/ec2.go index 101289cc3..46f38b36f 100644 --- a/spinifex/gateway/ec2.go +++ b/spinifex/gateway/ec2.go @@ -145,7 +145,7 @@ func ec2HandlerWithReq[In any](handler func(ctx context.Context, input *In, gw * var ec2Actions = map[string]ec2Action{ "DescribeInstances": ec2Handler(func(ctx context.Context, input *ec2.DescribeInstancesInput, gw *GatewayConfig, accountID string) (any, error) { - out, err := gateway_ec2_instance.DescribeInstancesChecked(ctx, input, gw.NATSConn, gw.DiscoverActiveNodes(ctx), accountID) + out, err := gateway_ec2_instance.DescribeInstancesChecked(ctx, input, gw.NATSConn, 0, gw.NodeIDs, accountID) if err != nil { return out, err } diff --git a/spinifex/gateway/ec2/capacityreservation/CancelCapacityReservation.go b/spinifex/gateway/ec2/capacityreservation/CancelCapacityReservation.go index 644c2372a..2b6070344 100644 --- a/spinifex/gateway/ec2/capacityreservation/CancelCapacityReservation.go +++ b/spinifex/gateway/ec2/capacityreservation/CancelCapacityReservation.go @@ -48,7 +48,7 @@ func CancelCapacityReservation(ctx context.Context, input *ec2.CancelCapacityRes for _, frame := range frames { var ack ec2.CancelCapacityReservationOutput - if json.Unmarshal(frame, &ack) == nil && aws.BoolValue(ack.Return) { + if json.Unmarshal(frame.Data, &ack) == nil && aws.BoolValue(ack.Return) { output.Return = aws.Bool(true) return output, nil } diff --git a/spinifex/gateway/ec2/capacityreservation/DescribeCapacityReservations.go b/spinifex/gateway/ec2/capacityreservation/DescribeCapacityReservations.go index ef6ec38e7..50e1d6941 100644 --- a/spinifex/gateway/ec2/capacityreservation/DescribeCapacityReservations.go +++ b/spinifex/gateway/ec2/capacityreservation/DescribeCapacityReservations.go @@ -56,7 +56,7 @@ func DescribeCapacityReservations(ctx context.Context, input *ec2.DescribeCapaci var all []*ec2.CapacityReservation for _, frame := range frames { var o ec2.DescribeCapacityReservationsOutput - if json.Unmarshal(frame, &o) == nil { + if json.Unmarshal(frame.Data, &o) == nil { all = append(all, o.CapacityReservations...) } } diff --git a/spinifex/gateway/ec2/capacityreservation/fanout.go b/spinifex/gateway/ec2/capacityreservation/fanout.go index a41b15a72..0eada6d00 100644 --- a/spinifex/gateway/ec2/capacityreservation/fanout.go +++ b/spinifex/gateway/ec2/capacityreservation/fanout.go @@ -42,7 +42,7 @@ func collectCensus(ctx context.Context, natsConn *nats.Conn, expectedNodes int, var census []nodeCensus for _, frame := range frames { var status types.NodeStatusResponse - if err := json.Unmarshal(frame, &status); err != nil { + if err := json.Unmarshal(frame.Data, &status); err != nil { slog.DebugContext(ctx, "collectCensus: failed to unmarshal response", "err", err) continue } diff --git a/spinifex/gateway/ec2/instance/DescribeInstanceAttribute.go b/spinifex/gateway/ec2/instance/DescribeInstanceAttribute.go index ab7999af6..372b4598b 100644 --- a/spinifex/gateway/ec2/instance/DescribeInstanceAttribute.go +++ b/spinifex/gateway/ec2/instance/DescribeInstanceAttribute.go @@ -61,7 +61,7 @@ func DescribeInstanceAttribute(ctx context.Context, input *ec2.DescribeInstanceA if len(frames) > 0 { var out ec2.DescribeInstanceAttributeOutput - if json.Unmarshal(frames[0], &out) == nil { + if json.Unmarshal(frames[0].Data, &out) == nil { slog.InfoContext(ctx, "DescribeInstanceAttribute: Completed successfully", "instance_id", *input.InstanceId, "responses", sum.Received) return &out, nil diff --git a/spinifex/gateway/ec2/instance/DescribeInstanceStatus.go b/spinifex/gateway/ec2/instance/DescribeInstanceStatus.go index 5702d6141..382e4eb0e 100644 --- a/spinifex/gateway/ec2/instance/DescribeInstanceStatus.go +++ b/spinifex/gateway/ec2/instance/DescribeInstanceStatus.go @@ -44,7 +44,7 @@ func DescribeInstanceStatus(ctx context.Context, input *ec2.DescribeInstanceStat var allStatuses []*ec2.InstanceStatus for _, frame := range frames { var nodeOutput ec2.DescribeInstanceStatusOutput - if json.Unmarshal(frame, &nodeOutput) == nil { + if json.Unmarshal(frame.Data, &nodeOutput) == nil { allStatuses = append(allStatuses, nodeOutput.InstanceStatuses...) } } diff --git a/spinifex/gateway/ec2/instance/DescribeInstanceTypes.go b/spinifex/gateway/ec2/instance/DescribeInstanceTypes.go index cebb0e7dc..6216bd733 100644 --- a/spinifex/gateway/ec2/instance/DescribeInstanceTypes.go +++ b/spinifex/gateway/ec2/instance/DescribeInstanceTypes.go @@ -29,7 +29,7 @@ func DescribeInstanceTypes(ctx context.Context, input *ec2.DescribeInstanceTypes var allInstanceTypes []*ec2.InstanceTypeInfo for _, frame := range frames { var nodeOutput ec2.DescribeInstanceTypesOutput - if json.Unmarshal(frame, &nodeOutput) == nil { + if json.Unmarshal(frame.Data, &nodeOutput) == nil { allInstanceTypes = append(allInstanceTypes, nodeOutput.InstanceTypes...) } } diff --git a/spinifex/gateway/ec2/instance/DescribeInstances.go b/spinifex/gateway/ec2/instance/DescribeInstances.go index eb33b953b..8106fa6c1 100644 --- a/spinifex/gateway/ec2/instance/DescribeInstances.go +++ b/spinifex/gateway/ec2/instance/DescribeInstances.go @@ -17,10 +17,40 @@ import ( "github.com/nats-io/nats.go" ) +// maxConcurrentAbsenceProofs bounds how many describes may hold a full fan-out +// deadline at once. Sized so the gateway can absorb a burst of genuine misses +// while the ceiling on pinned state stays flat. +const maxConcurrentAbsenceProofs = 64 + +// absenceProofSlots is the budget for describes that cannot settle early. A +// healthy cluster answers from every node and releases its slot in +// milliseconds; slots only pile up when a node has gone quiet, which is +// exactly when a caller could otherwise pin the gateway with free requests. +var absenceProofSlots = make(chan struct{}, maxConcurrentAbsenceProofs) + +// acquireAbsenceProofSlot takes a slot without blocking, returning a release +// func and whether it succeeded. It never queues: waiting for a slot would let +// the backlog grow, which is the thing being defended against. The decision is +// made before any reply is read, so it cannot depend on whether the instance +// exists — a nonexistent id and another account's id throttle identically. +func acquireAbsenceProofSlot() (release func(), ok bool) { + select { + case absenceProofSlots <- struct{}{}: + return func() { <-absenceProofSlots }, true + default: + return nil, false + } +} + // defaultDescribeFanoutTimeout is the hard deadline the describe fan-out waits // for node replies before treating the sweep as incomplete. const defaultDescribeFanoutTimeout = 3 * time.Second +// describeResponderGrace is how long the fan-out keeps listening after every +// configured node has answered. Sized well above the sub-millisecond spread +// between node replies, so a loaded node still lands inside it. +const describeResponderGrace = 150 * time.Millisecond + // describeConfig holds the tunable knobs of the describe fan-out. type describeConfig struct { fanoutTimeout time.Duration @@ -47,7 +77,7 @@ func WithFanoutTimeout(d time.Duration) DescribeOption { // Callers that must not act on a partial view at all (the quota reconcile) // use DescribeInstancesForReconcile. func DescribeInstances(ctx context.Context, input *ec2.DescribeInstancesInput, natsConn *nats.Conn, expectedNodes int, accountID string, opts ...DescribeOption) (*ec2.DescribeInstancesOutput, error) { - reservations, _, firstClient4xx, err := gatherInstances(ctx, input, natsConn, expectedNodes, accountID, opts...) + reservations, _, firstClient4xx, err := gatherInstances(ctx, input, natsConn, expectedNodes, nil, accountID, opts...) if err != nil { return nil, err } @@ -61,14 +91,23 @@ func DescribeInstances(ctx context.Context, input *ec2.DescribeInstancesInput, n // DescribeInstancesChecked is the customer-facing variant served by the // gateway's DescribeInstances action. On top of DescribeInstances' aggregation, // it asserts InvalidInstanceID.NotFound for any explicitly named instance ID -// absent from the result — but only when the sweep is provably complete -// (every expected node answered, both KV buckets succeeded). A partial sweep -// stays silent, the same as DescribeInstances: a node timing out during the -// fan-out must never turn into a false NotFound for an instance that actually -// exists on that node. A --filters-only query (no explicit IDs) is never -// affected and keeps returning an empty list with no error. -func DescribeInstancesChecked(ctx context.Context, input *ec2.DescribeInstancesInput, natsConn *nats.Conn, expectedNodes int, accountID string, opts ...DescribeOption) (*ec2.DescribeInstancesOutput, error) { - reservations, complete, firstClient4xx, err := gatherInstances(ctx, input, natsConn, expectedNodes, accountID, opts...) +// absent from the result — but only when the sweep is provably complete. A +// partial sweep stays silent, the same as DescribeInstances: a node timing +// out during the fan-out must never turn into a false NotFound for an +// instance that actually exists on that node. A --filters-only query (no +// explicit IDs) is never affected and keeps returning an empty list with no +// error. +// +// nodeIDs is the configured cluster node set. When non-empty, completeness +// for the InstanceIds assertion is judged by responder identity — the +// configured nodes whose frames actually decoded — rather than by +// expectedNodes, and the fan-out collects to its full deadline rather than +// exiting once data arrives, since absence cannot be proven from a prefix of +// the replies. An empty or unset nodeIDs falls back to expectedNodes, which a +// caller that does not supply one leaves at 0 — never complete, so the +// assertion is never made. +func DescribeInstancesChecked(ctx context.Context, input *ec2.DescribeInstancesInput, natsConn *nats.Conn, expectedNodes int, nodeIDs []string, accountID string, opts ...DescribeOption) (*ec2.DescribeInstancesOutput, error) { + reservations, complete, firstClient4xx, err := gatherInstances(ctx, input, natsConn, expectedNodes, nodeIDs, accountID, opts...) if err != nil { return nil, err } @@ -103,17 +142,22 @@ func DescribeInstancesChecked(ctx context.Context, input *ec2.DescribeInstancesI // sweep reads the instance record space instead of this; the retype gate and the // volume delete guard are what still call it. func DescribeInstancesForReconcile(ctx context.Context, input *ec2.DescribeInstancesInput, natsConn *nats.Conn, expectedNodes int, accountID string) (reservations []*ec2.Reservation, complete bool, err error) { - reservations, complete, _, err = gatherInstances(ctx, input, natsConn, expectedNodes, accountID) + reservations, complete, _, err = gatherInstances(ctx, input, natsConn, expectedNodes, nil, accountID) return reservations, complete, err } // gatherInstances runs the running-instance fan-out plus the stopped/terminated -// KV bucket queries and aggregates every reservation. complete reports whether -// the sweep saw a success frame from all expectedNodes without timing out and -// both bucket queries succeeded — the precondition reconcile needs before it may -// lower a counter. firstClient4xx carries the first deterministic 4xx for the -// lenient caller to surface when nothing was collected. -func gatherInstances(ctx context.Context, input *ec2.DescribeInstancesInput, natsConn *nats.Conn, expectedNodes int, accountID string, opts ...DescribeOption) (reservations []*ec2.Reservation, complete bool, firstClient4xx string, err error) { +// KV bucket queries and aggregates every reservation. firstClient4xx carries +// the first deterministic 4xx for the lenient caller to surface when nothing +// was collected. +// +// With nodeIDs empty, complete is judged the historical way: a success frame +// from all expectedNodes, without timing out. With nodeIDs set, complete is +// judged by responder identity instead — see DescribeInstancesChecked — and an +// explicit-ID request collects to its deadline under CollectUntilDeadline; a +// request with no explicit IDs stays on CollectServeData regardless, since +// nothing there is being proved. +func gatherInstances(ctx context.Context, input *ec2.DescribeInstancesInput, natsConn *nats.Conn, expectedNodes int, nodeIDs []string, accountID string, opts ...DescribeOption) (reservations []*ec2.Reservation, complete bool, firstClient4xx string, err error) { cfg := describeConfig{fanoutTimeout: defaultDescribeFanoutTimeout} for _, opt := range opts { opt(&cfg) @@ -125,48 +169,201 @@ func gatherInstances(ctx context.Context, input *ec2.DescribeInstancesInput, nat return nil, false, "", fmt.Errorf("failed to marshal input: %w", err) } - frames, sum, err := utils.Gather(ctx, natsConn, "ec2.DescribeInstances", jsonData, - utils.GatherOpts{Timeout: cfg.fanoutTimeout, ExpectedNodes: expectedNodes, AccountID: accountID}) - if err != nil { - return nil, false, "", err - } - - var allReservations []*ec2.Reservation - for _, frame := range frames { - var nodeOutput ec2.DescribeInstancesOutput - if json.Unmarshal(frame, &nodeOutput) == nil && nodeOutput.Reservations != nil { - allReservations = append(allReservations, nodeOutput.Reservations...) - } - } - - // The fan-out is complete only when every expected node answered with a - // success frame and the deadline was not hit; an error frame or a missing - // node leaves the view partial. - fanoutComplete := expectedNodes > 0 && !sum.TimedOut && sum.Successes >= expectedNodes - - // Both topics use queue groups (single responder each); query in parallel. + // The buckets run alongside the fan-out rather than after it, so a stopped + // or terminated instance can settle the collection too. Queried on queue + // groups, so one responder each. var kvMu sync.Mutex var kvWg sync.WaitGroup bucketsOK := true + var bucketReservations []*ec2.Reservation + settler := newInstanceSettler(input.InstanceIds) for _, topic := range []string{"ec2.DescribeStoppedInstances", "ec2.DescribeTerminatedInstances"} { kvWg.Add(1) go func(topic string) { defer kvWg.Done() reservations, ok := queryInstanceBucket(ctx, natsConn, topic, jsonData, accountID) + settler.observe(reservations) kvMu.Lock() defer kvMu.Unlock() if !ok { bucketsOK = false } - allReservations = append(allReservations, reservations...) + bucketReservations = append(bucketReservations, reservations...) }(topic) } + + identity := len(nodeIDs) > 0 + gatherOpts := utils.GatherOpts{Timeout: cfg.fanoutTimeout, AccountID: accountID} + if identity { + gatherOpts.ExpectedResponders = len(nodeIDs) + if len(input.InstanceIds) > 0 { + release, ok := acquireAbsenceProofSlot() + if !ok { + kvWg.Wait() + slog.WarnContext(ctx, "DescribeInstances: absence proof budget exhausted, throttling", + "limit", maxConcurrentAbsenceProofs) + return nil, false, "", errors.New(awserrors.ErrorRequestLimitExceeded) + } + defer release() + + // Only the path that could assert absence collects past the point + // where every node has answered; a filters-only listing may still + // exit on the first frames. + gatherOpts.Mode = utils.CollectUntilDeadline + gatherOpts.ResponderGrace = describeResponderGrace + gatherOpts.Settled = settler.settled + } + } else { + gatherOpts.ExpectedNodes = expectedNodes + } + + frames, sum, err := utils.Gather(ctx, natsConn, "ec2.DescribeInstances", jsonData, gatherOpts) + if err != nil { + kvWg.Wait() + return nil, false, "", err + } + + var allReservations []*ec2.Reservation + var fanoutComplete bool + if identity { + allReservations, fanoutComplete = judgeIdentityCompleteness(ctx, frames, sum, nodeIDs) + } else { + for _, frame := range frames { + var nodeOutput ec2.DescribeInstancesOutput + if json.Unmarshal(frame.Data, &nodeOutput) == nil && nodeOutput.Reservations != nil { + allReservations = append(allReservations, nodeOutput.Reservations...) + } + } + // The fan-out is complete only when every expected node answered with a + // success frame and the deadline was not hit; an error frame or a missing + // node leaves the view partial. + fanoutComplete = expectedNodes > 0 && !sum.TimedOut && sum.Successes >= expectedNodes + } + kvWg.Wait() + allReservations = append(allReservations, bucketReservations...) slog.InfoContext(ctx, "DescribeInstances: Aggregated response", "total_reservations", len(allReservations)) return allReservations, fanoutComplete && bucketsOK, sum.FirstClient4xx, nil } +// instanceSettler tracks which of the requested instance ids have turned up, +// across both the running-instance fan-out and the stopped and terminated +// bucket queries that run alongside it. Once none are outstanding there is +// nothing left to prove and collection can stop without waiting out the +// responder grace window. +// +// A bucket answer is only consulted when the next fan-out frame arrives, so it +// shortens the wait only when it lands first. Absence collects past this point, +// and an instance the caller cannot see is filtered out of every source, so it +// takes the absence path too — the two NotFound answers stay indistinguishable +// in time. +type instanceSettler struct { + mu sync.Mutex + wanted map[string]bool + consumed int +} + +func newInstanceSettler(instanceIDs []*string) *instanceSettler { + wanted := map[string]bool{} + for _, id := range instanceIDs { + if id != nil { + wanted[*id] = true + } + } + return &instanceSettler{wanted: wanted} +} + +// observe records ids found outside the fan-out. Safe to call concurrently +// with settled, which is what the bucket goroutines do. +func (s *instanceSettler) observe(reservations []*ec2.Reservation) { + s.mu.Lock() + defer s.mu.Unlock() + s.forget(reservations) +} + +// settled is the Gather predicate. It decodes only the frames it has not seen +// before, so a large fan-out does not re-decode its whole backlog per frame. +func (s *instanceSettler) settled(frames []utils.Frame, _ utils.Summary) bool { + s.mu.Lock() + defer s.mu.Unlock() + for ; s.consumed < len(frames); s.consumed++ { + var out ec2.DescribeInstancesOutput + if json.Unmarshal(frames[s.consumed].Data, &out) != nil { + continue + } + s.forget(out.Reservations) + } + return len(s.wanted) == 0 +} + +// forget drops every id present in reservations. Caller holds the lock. +func (s *instanceSettler) forget(reservations []*ec2.Reservation) { + for _, res := range reservations { + if res == nil { + continue + } + for _, inst := range res.Instances { + if inst != nil && inst.InstanceId != nil { + delete(s.wanted, *inst.InstanceId) + } + } + } +} + +// judgeIdentityCompleteness decodes every frame and builds ValidResponders — +// the configured nodes whose payload actually decoded as DescribeInstancesOutput. +// A node with nil Reservations still decoded, so it is a valid responder, not +// an invalid one; only a failed decode withholds a node from the set. +// Completeness requires ValidResponders to cover nodeIDs and fails closed on +// any ambiguity: a frame with no node ID, a node that answered as both a +// success and an error, or a node whose repeated frames disagreed. +func judgeIdentityCompleteness(ctx context.Context, frames []utils.Frame, sum utils.Summary, nodeIDs []string) (reservations []*ec2.Reservation, complete bool) { + validResponders := map[string]bool{} + for _, frame := range frames { + var nodeOutput ec2.DescribeInstancesOutput + if json.Unmarshal(frame.Data, &nodeOutput) != nil { + if frame.NodeID != "" { + slog.WarnContext(ctx, "DescribeInstances: frame did not decode as DescribeInstancesOutput", "node", frame.NodeID) + } + continue + } + if frame.NodeID != "" { + validResponders[frame.NodeID] = true + } + reservations = append(reservations, nodeOutput.Reservations...) + } + + ambiguous := sum.Unidentified > 0 || len(sum.ConflictNodes) > 0 || sum.CapHit + var bothSets []string + for node := range sum.SuccessResponders { + if sum.ErrorResponders[node] { + ambiguous = true + bothSets = append(bothSets, node) + } + } + + var missing, erroring []string + for _, id := range nodeIDs { + if !validResponders[id] { + missing = append(missing, id) + } + if sum.ErrorResponders[id] { + erroring = append(erroring, id) + } + } + + complete = len(nodeIDs) > 0 && len(missing) == 0 && !ambiguous + // A settled early exit leaves nodes unheard by design, so the shortfall is + // the caller's own doing and says nothing about cluster health. + if !complete && !sum.SettledEarly { + slog.InfoContext(ctx, "DescribeInstances: fan-out incomplete", + "missing_nodes", missing, "erroring_nodes", erroring, "conflict_nodes", len(sum.ConflictNodes), + "unidentified_frames", sum.Unidentified, "nodes_answering_as_both", bothSets, "cap_hit", sum.CapHit) + } + return reservations, complete +} + // EnrichInstanceProfileIDs resolves IamInstanceProfile.Id for every instance // that carries only an ARN. Results are cached per ARN to avoid repeated RPCs. // Misses are warn-logged and leave Id empty (graceful degradation for deleted profiles). diff --git a/spinifex/gateway/ec2/instance/DescribeInstances_test.go b/spinifex/gateway/ec2/instance/DescribeInstances_test.go index 94dfc9b18..3a1f1dec7 100644 --- a/spinifex/gateway/ec2/instance/DescribeInstances_test.go +++ b/spinifex/gateway/ec2/instance/DescribeInstances_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "sync" "testing" "time" @@ -297,7 +298,7 @@ func TestDescribeInstancesChecked_ExplicitIDNotFound_CompleteSweep(t *testing.T) nc.Flush() input := &ec2.DescribeInstancesInput{InstanceIds: []*string{aws.String("i-doesnotexist0000000")}} - _, err = DescribeInstancesChecked(context.Background(), input, nc, 1, "123456789012") + _, err = DescribeInstancesChecked(context.Background(), input, nc, 1, nil, "123456789012") require.Error(t, err) assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, err.Error()) @@ -353,7 +354,7 @@ func TestDescribeInstancesChecked_ExplicitIDFound_CompleteSweep(t *testing.T) { nc.Flush() input := &ec2.DescribeInstancesInput{InstanceIds: []*string{aws.String("i-real")}} - output, err := DescribeInstancesChecked(context.Background(), input, nc, 1, "123456789012") + output, err := DescribeInstancesChecked(context.Background(), input, nc, 1, nil, "123456789012") require.NoError(t, err) require.Len(t, output.Reservations, 1) @@ -378,7 +379,7 @@ func TestDescribeInstancesChecked_FilterOnlyNoMatch_ReturnsEmptyNotNotFound(t *t input := &ec2.DescribeInstancesInput{ Filters: []*ec2.Filter{{Name: aws.String("tag:nope"), Values: []*string{aws.String("nothing")}}}, } - output, err := DescribeInstancesChecked(context.Background(), input, nc, 1, "123456789012") + output, err := DescribeInstancesChecked(context.Background(), input, nc, 1, nil, "123456789012") require.NoError(t, err) require.NotNil(t, output) @@ -405,7 +406,7 @@ func TestDescribeInstancesChecked_ExplicitIDNotFound_NodeTimeout(t *testing.T) { nc.Flush() input := &ec2.DescribeInstancesInput{InstanceIds: []*string{aws.String("i-on-the-slow-node")}} - output, err := DescribeInstancesChecked(context.Background(), input, nc, 2, "123456789012", + output, err := DescribeInstancesChecked(context.Background(), input, nc, 2, nil, "123456789012", WithFanoutTimeout(300*time.Millisecond)) require.NoError(t, err, "an incomplete sweep must never assert a false NotFound") @@ -413,6 +414,156 @@ func TestDescribeInstancesChecked_ExplicitIDNotFound_NodeTimeout(t *testing.T) { assert.Empty(t, output.Reservations) } +// --- Identity-mode completeness (nodeIDs set) --- + +// A missing configured node suppresses NotFound even though the requested ID +// never showed up in what did arrive: the sweep is provably incomplete, so +// silence is the only safe answer. +func TestDescribeInstancesChecked_Identity_MissingNodeSuppressesNotFound(t *testing.T) { + t.Parallel() + _, nc := startTestNATSServer(t) + subscribeEmptyInstanceBuckets(t, nc) + + data, _ := json.Marshal(&ec2.DescribeInstancesOutput{ + Reservations: []*ec2.Reservation{{ + ReservationId: aws.String("r-1"), + Instances: []*ec2.Instance{{InstanceId: aws.String("i-real")}}, + }}, + }) + subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", data) + // node-2 is configured but never answers. + + input := &ec2.DescribeInstancesInput{InstanceIds: []*string{aws.String("i-doesnotexist0000000")}} + output, err := DescribeInstancesChecked(context.Background(), input, nc, 0, []string{"node-1", "node-2"}, "123456789012", + WithFanoutTimeout(300*time.Millisecond)) + + // The sweep stays lenient on a partial view — node-1's reservations are + // still returned — it is only the NotFound assertion that a missing node + // must suppress. + require.NoError(t, err, "a node missing from the configured set must never assert a false NotFound") + require.NotNil(t, output) +} + +// The positive counterpart: every configured node answers, so the sweep is +// complete and the assertion fires for a genuinely absent instance ID. +func TestDescribeInstancesChecked_Identity_AllNodesAnswer_NotFoundAsserted(t *testing.T) { + t.Parallel() + _, nc := startTestNATSServer(t) + subscribeEmptyInstanceBuckets(t, nc) + + data, _ := json.Marshal(&ec2.DescribeInstancesOutput{ + Reservations: []*ec2.Reservation{{ + ReservationId: aws.String("r-1"), + Instances: []*ec2.Instance{{InstanceId: aws.String("i-real")}}, + }}, + }) + subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", data) + subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-2", + mustMarshalDescribeOutput(t, &ec2.DescribeInstancesOutput{Reservations: []*ec2.Reservation{}})) + + input := &ec2.DescribeInstancesInput{InstanceIds: []*string{aws.String("i-doesnotexist0000000")}} + _, err := DescribeInstancesChecked(context.Background(), input, nc, 0, []string{"node-1", "node-2"}, "123456789012", + WithFanoutTimeout(2*time.Second)) + + require.Error(t, err) + assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, err.Error()) +} + +// A node that answers with a decodable error envelope (a 5xx from that node) +// never becomes a valid responder, so it counts the same as a missing node — +// the sweep stays incomplete and NotFound is suppressed. +func TestDescribeInstancesChecked_Identity_ErroringNodeSuppressesNotFound(t *testing.T) { + t.Parallel() + _, nc := startTestNATSServer(t) + subscribeEmptyInstanceBuckets(t, nc) + + subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", + mustMarshalDescribeOutput(t, &ec2.DescribeInstancesOutput{Reservations: []*ec2.Reservation{}})) + subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-2", + utils.GenerateErrorPayload(awserrors.ErrorBandwidthLimitExceeded)) + + input := &ec2.DescribeInstancesInput{InstanceIds: []*string{aws.String("i-doesnotexist0000000")}} + output, err := DescribeInstancesChecked(context.Background(), input, nc, 0, []string{"node-1", "node-2"}, "123456789012", + WithFanoutTimeout(300*time.Millisecond)) + + require.NoError(t, err, "an erroring node must suppress NotFound, not be treated as an empty-handed valid responder") + require.NotNil(t, output) + assert.Empty(t, output.Reservations) +} + +// A payload that is neither a valid error envelope nor a decodable +// DescribeInstancesOutput (a malformed reply) leaves its node out of +// ValidResponders, so the sweep is incomplete and NotFound is suppressed — +// the same as a node that never answered at all. +func TestDescribeInstancesChecked_Identity_UndecodablePayloadSuppressesNotFound(t *testing.T) { + t.Parallel() + _, nc := startTestNATSServer(t) + subscribeEmptyInstanceBuckets(t, nc) + + subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", + mustMarshalDescribeOutput(t, &ec2.DescribeInstancesOutput{Reservations: []*ec2.Reservation{}})) + subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-2", []byte("not valid json at all")) + + input := &ec2.DescribeInstancesInput{InstanceIds: []*string{aws.String("i-doesnotexist0000000")}} + output, err := DescribeInstancesChecked(context.Background(), input, nc, 0, []string{"node-1", "node-2"}, "123456789012", + WithFanoutTimeout(300*time.Millisecond)) + + require.NoError(t, err) + require.NotNil(t, output) + assert.Empty(t, output.Reservations) +} + +// A node with nil Reservations still decoded successfully, so it is a valid +// responder — this is the trap the plan calls out explicitly: nil +// Reservations is not the same thing as a failed decode. +func TestDescribeInstancesChecked_Identity_NilReservationsIsValidResponder(t *testing.T) { + t.Parallel() + _, nc := startTestNATSServer(t) + subscribeEmptyInstanceBuckets(t, nc) + + // {} decodes to DescribeInstancesOutput{Reservations: nil}. + subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`)) + subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-2", []byte(`{}`)) + + input := &ec2.DescribeInstancesInput{InstanceIds: []*string{aws.String("i-doesnotexist0000000")}} + _, err := DescribeInstancesChecked(context.Background(), input, nc, 0, []string{"node-1", "node-2"}, "123456789012", + WithFanoutTimeout(2*time.Second)) + + require.Error(t, err, "a nil-Reservations node is a complete, valid responder, so the sweep is complete and NotFound must fire") + assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, err.Error()) +} + +// An empty or unset nodeIDs falls back to the legacy expectedNodes path. +// DescribeInstancesChecked's only production caller always passes +// expectedNodes=0 alongside its node set, so an empty node set (e.g. an +// unconfigured cluster) can never satisfy expectedNodes>0 and NotFound stays +// suppressed — the fallback fails closed rather than trusting a stale count. +func TestDescribeInstancesChecked_Identity_EmptyNodeIDsFallsBackAndStaysSuppressed(t *testing.T) { + t.Parallel() + _, nc := startTestNATSServer(t) + subscribeEmptyInstanceBuckets(t, nc) + + subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", + mustMarshalDescribeOutput(t, &ec2.DescribeInstancesOutput{Reservations: []*ec2.Reservation{}})) + + input := &ec2.DescribeInstancesInput{InstanceIds: []*string{aws.String("i-doesnotexist0000000")}} + output, err := DescribeInstancesChecked(context.Background(), input, nc, 0, nil, "123456789012", + WithFanoutTimeout(300*time.Millisecond)) + + require.NoError(t, err) + require.NotNil(t, output) + assert.Empty(t, output.Reservations) +} + +// mustMarshalDescribeOutput is a t.Helper wrapper so the identity-mode tests +// above stay one line each instead of repeating the error check. +func mustMarshalDescribeOutput(t *testing.T, out *ec2.DescribeInstancesOutput) []byte { + t.Helper() + data, err := json.Marshal(out) + require.NoError(t, err) + return data +} + func TestDescribeInstances_ClosedConnection(t *testing.T) { t.Parallel() _, nc := startTestNATSServer(t) @@ -502,3 +653,155 @@ func TestEnrichInstanceProfileIDs_NoOpInputs(t *testing.T) { assert.Nil(t, noProfile.Reservations[0].Instances[0].IamInstanceProfile, "instance with no profile is untouched") } + +// frameWithInstances builds one daemon reply carrying the given instance IDs. +func frameWithInstances(t *testing.T, nodeID string, instanceIDs ...string) utils.Frame { + t.Helper() + instances := make([]*ec2.Instance, 0, len(instanceIDs)) + for _, id := range instanceIDs { + instances = append(instances, &ec2.Instance{InstanceId: aws.String(id)}) + } + data, err := json.Marshal(ec2.DescribeInstancesOutput{ + Reservations: []*ec2.Reservation{{Instances: instances}}, + }) + require.NoError(t, err) + return utils.Frame{NodeID: nodeID, Data: data} +} + +// reservationsWith builds the bucket-query shape the settler observes. +func reservationsWith(instanceIDs ...string) []*ec2.Reservation { + instances := make([]*ec2.Instance, 0, len(instanceIDs)) + for _, id := range instanceIDs { + instances = append(instances, &ec2.Instance{InstanceId: aws.String(id)}) + } + return []*ec2.Reservation{{Instances: instances}} +} + +func TestInstanceSettler_SettlesOnlyWhenEveryIDIsSeen(t *testing.T) { + t.Parallel() + settler := newInstanceSettler([]*string{aws.String("i-a"), aws.String("i-b")}) + + frames := []utils.Frame{frameWithInstances(t, "node-1", "i-a")} + assert.False(t, settler.settled(frames, utils.Summary{}), "one of two ids found is not settled") + + frames = append(frames, frameWithInstances(t, "node-2", "i-b")) + assert.True(t, settler.settled(frames, utils.Summary{}), "both ids found, nothing left to prove") +} + +// An id that no node holds must never settle, so absence keeps paying the full +// deadline — that is the whole reason CollectUntilDeadline exists. +func TestInstanceSettler_AbsentIDNeverSettles(t *testing.T) { + t.Parallel() + settler := newInstanceSettler([]*string{aws.String("i-missing")}) + + frames := []utils.Frame{ + frameWithInstances(t, "node-1", "i-other"), + frameWithInstances(t, "node-2", "i-another"), + } + assert.False(t, settler.settled(frames, utils.Summary{})) +} + +func TestInstanceSettler_UndecodableFrameDoesNotSettle(t *testing.T) { + t.Parallel() + settler := newInstanceSettler([]*string{aws.String("i-a")}) + + frames := []utils.Frame{{NodeID: "node-1", Data: []byte("not json")}} + assert.False(t, settler.settled(frames, utils.Summary{})) + + frames = append(frames, frameWithInstances(t, "node-2", "i-a")) + assert.True(t, settler.settled(frames, utils.Summary{}), "a later decodable frame still settles") +} + +// A stopped or terminated instance never appears in a fan-out frame, so the +// bucket queries have to be able to settle the collection on their own. +func TestInstanceSettler_BucketHitAloneSettles(t *testing.T) { + t.Parallel() + settler := newInstanceSettler([]*string{aws.String("i-stopped")}) + + frames := []utils.Frame{frameWithInstances(t, "node-1", "i-running")} + assert.False(t, settler.settled(frames, utils.Summary{})) + + settler.observe(reservationsWith("i-stopped")) + assert.True(t, settler.settled(frames, utils.Summary{}), "the bucket answered, so nothing is outstanding") +} + +func TestInstanceSettler_NoIDsSettlesImmediately(t *testing.T) { + t.Parallel() + assert.True(t, newInstanceSettler(nil).settled(nil, utils.Summary{})) + assert.True(t, newInstanceSettler([]*string{nil}).settled(nil, utils.Summary{})) +} + +// observe races settled in production: the bucket goroutines write while Gather +// reads. Run under -race to make the lock earn its place. +func TestInstanceSettler_ConcurrentObserveAndSettled(t *testing.T) { + t.Parallel() + settler := newInstanceSettler([]*string{aws.String("i-a"), aws.String("i-b")}) + frames := []utils.Frame{frameWithInstances(t, "node-1", "i-a")} + + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + settler.observe(reservationsWith("i-b")) + }() + go func() { + defer wg.Done() + settler.settled(frames, utils.Summary{}) + }() + wg.Wait() + + assert.True(t, settler.settled(frames, utils.Summary{})) +} + +func TestAcquireAbsenceProofSlot_ThrottlesPastTheLimitAndRecovers(t *testing.T) { + releases := make([]func(), 0, maxConcurrentAbsenceProofs) + t.Cleanup(func() { + for _, release := range releases { + release() + } + }) + + for i := range maxConcurrentAbsenceProofs { + release, ok := acquireAbsenceProofSlot() + require.True(t, ok, "slot %d of the budget must be available", i) + releases = append(releases, release) + } + + _, ok := acquireAbsenceProofSlot() + assert.False(t, ok, "the budget must refuse rather than queue once exhausted") + + releases[0]() + releases = releases[1:] + regained, ok := acquireAbsenceProofSlot() + require.True(t, ok, "releasing a slot must return it to the budget") + releases = append(releases, regained) +} + +// The throttle decision is made before any reply is read, so it cannot vary +// with whether the instance exists — otherwise it would reintroduce the timing +// oracle the full deadline exists to close. +func TestAcquireAbsenceProofSlot_DoesNotBlockWhenExhausted(t *testing.T) { + releases := make([]func(), 0, maxConcurrentAbsenceProofs) + t.Cleanup(func() { + for _, release := range releases { + release() + } + }) + for range maxConcurrentAbsenceProofs { + release, ok := acquireAbsenceProofSlot() + require.True(t, ok) + releases = append(releases, release) + } + + done := make(chan struct{}) + go func() { + defer close(done) + _, _ = acquireAbsenceProofSlot() + }() + + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("acquireAbsenceProofSlot blocked; it must refuse immediately so the backlog cannot grow") + } +} diff --git a/spinifex/gateway/ec2/instance/IamInstanceProfileAssociation.go b/spinifex/gateway/ec2/instance/IamInstanceProfileAssociation.go index d0e93ed5d..da71d0000 100644 --- a/spinifex/gateway/ec2/instance/IamInstanceProfileAssociation.go +++ b/spinifex/gateway/ec2/instance/IamInstanceProfileAssociation.go @@ -198,7 +198,7 @@ func broadcastForAssociation(ctx context.Context, natsConn *nats.Conn, subject s // Non-owners reply JSON null (a success frame that unmarshals to nil). for _, f := range frames { var assoc *ec2.IamInstanceProfileAssociation - if json.Unmarshal(f, &assoc) == nil && assoc != nil { + if json.Unmarshal(f.Data, &assoc) == nil && assoc != nil { return assoc, nil } } @@ -229,7 +229,7 @@ func broadcastDescribeAssociations(ctx context.Context, natsConn *nats.Conn, inp var associations []*ec2.IamInstanceProfileAssociation for _, f := range frames { var resp ec2.DescribeIamInstanceProfileAssociationsOutput - if json.Unmarshal(f, &resp) == nil { + if json.Unmarshal(f.Data, &resp) == nil { associations = append(associations, resp.IamInstanceProfileAssociations...) } } diff --git a/spinifex/gateway/ec2/instance/TerminateInstances_test.go b/spinifex/gateway/ec2/instance/TerminateInstances_test.go index 2c562afa0..9b1aaf23c 100644 --- a/spinifex/gateway/ec2/instance/TerminateInstances_test.go +++ b/spinifex/gateway/ec2/instance/TerminateInstances_test.go @@ -113,7 +113,6 @@ func TestTerminateInstances_NilInstanceIdSkipped(t *testing.T) { func TestTerminateInstances_NATSRequestFails(t *testing.T) { _, nc := startTestNATSServer(t) - noopTerminateRetrySleep(t) instanceID := "i-nosubscriber" @@ -130,7 +129,6 @@ func TestTerminateInstances_NATSRequestFails(t *testing.T) { func TestTerminateInstances_MixedSuccessAndFailure(t *testing.T) { _, nc := startTestNATSServer(t) - noopTerminateRetrySleep(t) goodID := "i-good" badID := "i-bad" @@ -178,7 +176,6 @@ func TestTerminateInstances_VerifiesQMPAttributes(t *testing.T) { func TestTerminateInstances_StoppedInstanceFallback(t *testing.T) { _, nc := startTestNATSServer(t) - noopTerminateRetrySleep(t) instanceID := "i-stopped-term" @@ -210,7 +207,6 @@ func TestTerminateInstances_StoppedInstanceFallback(t *testing.T) { func TestTerminateInstances_MixedRunningAndStopped(t *testing.T) { _, nc := startTestNATSServer(t) - noopTerminateRetrySleep(t) runningID := "i-running-mix" stoppedID := "i-stopped-mix" diff --git a/spinifex/gateway/ec2/instance/helpers_test.go b/spinifex/gateway/ec2/instance/helpers_test.go index 43d91a4e4..550a6298b 100644 --- a/spinifex/gateway/ec2/instance/helpers_test.go +++ b/spinifex/gateway/ec2/instance/helpers_test.go @@ -1,12 +1,15 @@ package gateway_ec2_instance import ( + "os" "testing" "time" "github.com/mulgadc/spinifex/spinifex/testutil" + "github.com/mulgadc/spinifex/spinifex/utils" "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" + "github.com/stretchr/testify/require" ) // startTestNATSServer starts an embedded NATS server for testing. @@ -15,11 +18,27 @@ func startTestNATSServer(t *testing.T) (*server.Server, *nats.Conn) { return testutil.StartTestNATS(t) } -// noopTerminateRetrySleep replaces the terminate NoResponders backoff with a -// no-op for the duration of the test, so retry paths do not burn real seconds. -func noopTerminateRetrySleep(t *testing.T) { - t.Helper() - prev := terminateRetrySleep +// TestMain neuters the terminate NoResponders backoff for the whole binary, so +// retry paths do not burn real seconds. Installed once rather than swapped per +// test: the seam is a package global and the tests that need it run in +// parallel, so their restores raced each other. +func TestMain(m *testing.M) { terminateRetrySleep = func(time.Duration) {} - t.Cleanup(func() { terminateRetrySleep = prev }) + os.Exit(m.Run()) +} + +// subscribeAsNode replies on subject as nodeID, carrying the X-Node-ID header +// a real daemon reply would set, so identity-mode completeness can attribute +// the frame. +func subscribeAsNode(t *testing.T, nc *nats.Conn, subject, nodeID string, data []byte) { + t.Helper() + _, err := nc.Subscribe(subject, func(msg *nats.Msg) { + reply := nats.NewMsg(msg.Reply) + reply.Data = data + if nodeID != "" { + reply.Header.Set(utils.NodeIDHeader, nodeID) + } + _ = msg.RespondMsg(reply) + }) + require.NoError(t, err) } diff --git a/spinifex/gateway/ec2/instance/placement.go b/spinifex/gateway/ec2/instance/placement.go index 7f4bffb83..152bff221 100644 --- a/spinifex/gateway/ec2/instance/placement.go +++ b/spinifex/gateway/ec2/instance/placement.go @@ -73,7 +73,7 @@ func queryNodeCapacity(ctx context.Context, natsConn *nats.Conn, instanceType st var nodes []nodeAllocation for _, frame := range frames { var status types.NodeStatusResponse - if err := json.Unmarshal(frame, &status); err != nil { + if err := json.Unmarshal(frame.Data, &status); err != nil { slog.DebugContext(ctx, "queryNodeCapacity: failed to unmarshal response", "err", err) continue } diff --git a/spinifex/gateway/ec2/instance/placement_spread_test.go b/spinifex/gateway/ec2/instance/placement_spread_test.go index 121d89fa0..faefcbd30 100644 --- a/spinifex/gateway/ec2/instance/placement_spread_test.go +++ b/spinifex/gateway/ec2/instance/placement_spread_test.go @@ -260,7 +260,6 @@ func TestDistributeInstancesSpread_PartialSuccessMeetsMinCount(t *testing.T) { func TestDistributeInstancesSpread_BelowMinCountRollsBackAndReleases(t *testing.T) { t.Parallel() _, nc := startTestNATSServer(t) - noopTerminateRetrySleep(t) h := mockSpreadCluster(t, nc, map[string]int{"node-1": 2, "node-2": 2}, @@ -310,7 +309,6 @@ func TestDistributeInstancesSpread_PropagatesClientError(t *testing.T) { func TestDistributeInstancesSpread_FinalizeFailureRollsBack(t *testing.T) { t.Parallel() _, nc := startTestNATSServer(t) - noopTerminateRetrySleep(t) h := mockSpreadCluster(t, nc, map[string]int{"node-1": 2, "node-2": 2}, diff --git a/spinifex/gateway/ec2/instance/placement_test.go b/spinifex/gateway/ec2/instance/placement_test.go index 08737da3f..cfa945248 100644 --- a/spinifex/gateway/ec2/instance/placement_test.go +++ b/spinifex/gateway/ec2/instance/placement_test.go @@ -267,7 +267,6 @@ func TestAggregateResults_PartialSuccessMeetsMinCount(t *testing.T) { func TestAggregateResults_PartialFailureBelowMinCount(t *testing.T) { _, nc := startTestNATSServer(t) - noopTerminateRetrySleep(t) results := []nodeLaunchResult{ { @@ -340,7 +339,6 @@ func TestAggregateResults_TimeoutSurfacedNotMasked(t *testing.T) { // without erroring) keeps the generic capacity code. func TestAggregateResults_ShortWithoutNodeErrors(t *testing.T) { _, nc := startTestNATSServer(t) - noopTerminateRetrySleep(t) results := []nodeLaunchResult{ { NodeID: "node-1", diff --git a/spinifex/gateway/gateway.go b/spinifex/gateway/gateway.go index e013f0ed4..04632b65f 100644 --- a/spinifex/gateway/gateway.go +++ b/spinifex/gateway/gateway.go @@ -104,6 +104,11 @@ type GatewayConfig struct { RootCAs *x509.CertPool ExpectedNodes int // Number of expected spinifex nodes for multi-node operations Region string // Region this gateway is running in + // NodeIDs is the configured cluster node set — the keys of + // config.ClusterConfig.Nodes — used to judge a fan-out's completeness by + // responder identity rather than by count. Distinct from ExpectedNodes, + // which stays a count for the callers that only want one. + NodeIDs []string // The last discovered node count and when it was discovered, so the // discovery fan-out runs once per activeNodesTTL rather than once per // request that needs to know how many nodes to wait for. @@ -992,7 +997,7 @@ func (gw *GatewayConfig) DiscoverActiveNodes(ctx context.Context) int { nodesSeen := make(map[string]bool) for _, frame := range frames { var response types.NodeDiscoverResponse - if err := json.Unmarshal(frame, &response); err != nil { + if err := json.Unmarshal(frame.Data, &response); err != nil { slog.DebugContext(ctx, "DiscoverActiveNodes: Failed to unmarshal response", "err", err) continue } diff --git a/spinifex/gateway/spx/spx.go b/spinifex/gateway/spx/spx.go index 0b83526e6..02b9fe006 100644 --- a/spinifex/gateway/spx/spx.go +++ b/spinifex/gateway/spx/spx.go @@ -55,7 +55,7 @@ func GetNodes(ctx context.Context, nc *nats.Conn, expectedNodes int) (*GetNodesO nodes := make([]types.NodeStatusResponse, 0, len(frames)) for _, frame := range frames { var node types.NodeStatusResponse - if json.Unmarshal(frame, &node) == nil { + if json.Unmarshal(frame.Data, &node) == nil { nodes = append(nodes, node) } } @@ -94,7 +94,7 @@ func GetVMs(ctx context.Context, nc *nats.Conn, expectedNodes int) (*GetVMsOutpu allVMs := make([]VMInfoWithNode, 0) for _, frame := range frames { var nodeResp types.NodeVMsResponse - if json.Unmarshal(frame, &nodeResp) != nil { + if json.Unmarshal(frame.Data, &nodeResp) != nil { continue } for _, vm := range nodeResp.VMs { diff --git a/spinifex/handlers/ec2/image/service_nats.go b/spinifex/handlers/ec2/image/service_nats.go index 2ebbe383d..dc0ab0a5d 100644 --- a/spinifex/handlers/ec2/image/service_nats.go +++ b/spinifex/handlers/ec2/image/service_nats.go @@ -46,7 +46,7 @@ func (s *NATSImageService) CreateImage(ctx context.Context, input *ec2.CreateIma if len(frames) > 0 { var out ec2.CreateImageOutput - if err := json.Unmarshal(frames[0], &out); err != nil { + if err := json.Unmarshal(frames[0].Data, &out); err != nil { return nil, fmt.Errorf("failed to unmarshal CreateImage response: %w", err) } return &out, nil diff --git a/spinifex/handlers/ec2/instance/service_impl.go b/spinifex/handlers/ec2/instance/service_impl.go index e7286cb09..4b00be443 100644 --- a/spinifex/handlers/ec2/instance/service_impl.go +++ b/spinifex/handlers/ec2/instance/service_impl.go @@ -1684,6 +1684,16 @@ func IsInstanceVisible(callerAccountID, ownerAccountID string) bool { return callerAccountID == ownerAccountID } +// IsInstanceVisibleToCaller combines account ownership with the +// platform-managed hide rule: LB/EKS control-plane VMs are system-account-owned +// and stay hidden from customer listings, visible only to the Global account. +func IsInstanceVisibleToCaller(accountID string, v *vm.VM) bool { + if !IsInstanceVisible(accountID, v.AccountID) { + return false + } + return v.ManagedBy == "" || accountID == utils.GlobalAccountID +} + // instanceMatchesFilters checks whether a VM + its built ec2.Instance copy satisfy all parsed filters. func instanceMatchesFilters(inst *vm.VM, ic *ec2.Instance, filters map[string][]string) bool { for name, values := range filters { @@ -1776,14 +1786,7 @@ func (s *InstanceServiceImpl) DescribeInstances(ctx context.Context, input *ec2. // Iterate under the manager lock to avoid races on Status, Instance, PublicIP, etc. s.vmMgr.View(func(vms map[string]*vm.VM) { for _, instance := range vms { - if !IsInstanceVisible(accountID, instance.AccountID) { - continue - } - // Platform-managed VMs (LB, EKS control plane) are system-account-owned - // and hidden from customer listings; only the system/Global account - // sees them. The EKS reconciler resolves its control-plane VM's state - // by running describe as the system account. - if instance.ManagedBy != "" && accountID != utils.GlobalAccountID { + if !IsInstanceVisibleToCaller(accountID, instance) { continue } if len(instanceIDFilter) > 0 && !instanceIDFilter[instance.ID] { diff --git a/spinifex/handlers/imds/instance_lookup.go b/spinifex/handlers/imds/instance_lookup.go index 6f2672ec4..76e300581 100644 --- a/spinifex/handlers/imds/instance_lookup.go +++ b/spinifex/handlers/imds/instance_lookup.go @@ -9,12 +9,12 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" - gateway_ec2_instance "github.com/mulgadc/spinifex/spinifex/gateway/ec2/instance" - "github.com/nats-io/nats.go" + handlers_ec2_instance "github.com/mulgadc/spinifex/spinifex/handlers/ec2/instance" + "github.com/mulgadc/spinifex/spinifex/vm" ) -// imdsLookupRetries bounds transient-gather retries for IMDS instance lookups so -// a momentary NATS gather timeout under load does not starve the guest bootstrap. +// imdsLookupRetries bounds transient-gather retries for the record-space +// fallback so a momentary JetStream hiccup does not starve guest bootstrap. const imdsLookupRetries = 3 // retryBackoff sleeps a short increasing delay between IMDS lookup attempts, @@ -46,86 +46,116 @@ func retryGather[T any](ctx context.Context, label, instanceID string, fn func() return out, err } -// natsInstanceLookup resolves instance-only metadata fields via DescribeInstances fan-out. -type natsInstanceLookup struct { - nc *nats.Conn - expectedNodes int +// localStateReader reads this node's on-disk VM state. Narrowed to what IMDS +// consumes: it does not know the state lives in a file, so caching and +// invalidation are the implementation's business, not this package's. +type localStateReader interface { + LocalVM(instanceID string) (*vm.VM, error) } -var _ instanceLookup = (*natsInstanceLookup)(nil) +// recordLoader is the shared instance record space accessor the fallback +// path needs. Satisfied by *daemon.JetStreamManager; narrowed to an interface +// so the fallback can be exercised without a live JetStream connection. +type recordLoader interface { + LoadInstanceRecord(instanceID string) (*vm.InstanceRecord, error) +} -func (l *natsInstanceLookup) describe(ctx context.Context, accountID, instanceID string) (*instanceFacts, error) { - out, err := retryGather(ctx, "describe instance", instanceID, func() (*ec2.DescribeInstancesOutput, error) { - return gateway_ec2_instance.DescribeInstances(ctx, &ec2.DescribeInstancesInput{ - InstanceIds: []*string{aws.String(instanceID)}, - }, l.nc, l.expectedNodes, accountID) +// localInstanceLookup resolves instance-only metadata fields local-first: a +// guest is only ever served by the node it runs on, so the node's own on-disk +// VM state is the authoritative, instant answer. It falls back to a direct +// read of the shared instance record space and never fans out over NATS. +type localInstanceLookup struct { + local localStateReader // nil-safe: local hit is skipped without it + records recordLoader // nil-safe: fallback is skipped without it +} + +var _ instanceLookup = (*localInstanceLookup)(nil) + +func (l *localInstanceLookup) describe(ctx context.Context, accountID, instanceID string) (*instanceFacts, error) { + v := l.localVM(ctx, instanceID) + if v == nil { + var err error + v, err = l.recordVM(ctx, instanceID) + if err != nil { + return nil, err + } + } + if v == nil || !handlers_ec2_instance.IsInstanceVisibleToCaller(accountID, v) { + return nil, nil // not present, or not visible to the caller + } + return instanceFactsFromVM(v), nil +} + +// localVM reads the instance straight off this node's on-disk VM state. +// Returns nil if there is no reader, the instance is not on this node, or the +// read fails — a read failure logs and falls through to the record-space +// fallback rather than erroring the whole lookup. +func (l *localInstanceLookup) localVM(ctx context.Context, instanceID string) *vm.VM { + if l.local == nil { + return nil + } + v, err := l.local.LocalVM(instanceID) + if err != nil { + slog.WarnContext(ctx, "IMDS: local instance state unavailable, falling back to record space", "err", err) + return nil + } + return v +} + +// recordVM falls back to the shared instance record space for an instance not +// (yet, or no longer) on this node. Returns (nil, nil) on a genuine miss. +func (l *localInstanceLookup) recordVM(ctx context.Context, instanceID string) (*vm.VM, error) { + if l.records == nil { + return nil, nil + } + record, err := retryGather(ctx, "load instance record", instanceID, func() (*vm.InstanceRecord, error) { + return l.records.LoadInstanceRecord(instanceID) }) if err != nil { - return nil, fmt.Errorf("describe instance %s: %w", instanceID, err) + return nil, fmt.Errorf("load instance record %s: %w", instanceID, err) } + if record == nil { + return nil, nil + } + return vm.VMFromRecord(record), nil +} - res, inst := firstReservationInstance(out) - if inst == nil { - return nil, nil // terminating or not yet visible; treat as miss +// instanceFactsFromVM projects the write-once spec fields IMDS serves. Returns +// nil if the instance has no observed instance/reservation yet (mid-launch). +func instanceFactsFromVM(v *vm.VM) *instanceFacts { + if v.Instance == nil || v.Reservation == nil { + return nil } + inst := v.Instance facts := &instanceFacts{ - instanceType: aws.StringValue(inst.InstanceType), - imageID: aws.StringValue(inst.ImageId), - keyName: aws.StringValue(inst.KeyName), - architecture: aws.StringValue(inst.Architecture), - amiLaunchIndex: aws.Int64Value(inst.AmiLaunchIndex), - reservationID: aws.StringValue(res.ReservationId), - lifecycleType: aws.StringValue(inst.InstanceLifecycle), - pendingTime: aws.TimeValue(inst.LaunchTime), - } - if inst.IamInstanceProfile != nil { - facts.iamInstanceProfileArn = aws.StringValue(inst.IamInstanceProfile.Arn) + instanceType: v.InstanceType, + imageID: aws.StringValue(inst.ImageId), + keyName: aws.StringValue(inst.KeyName), + architecture: aws.StringValue(inst.Architecture), + amiLaunchIndex: aws.Int64Value(inst.AmiLaunchIndex), + reservationID: aws.StringValue(v.Reservation.ReservationId), + iamInstanceProfileArn: v.IamInstanceProfileArn, + lifecycleType: v.InstanceLifecycle, + pendingTime: aws.TimeValue(inst.LaunchTime), + userData: decodeUserData(v.RunInstancesInput), } if inst.MetadataOptions != nil { facts.httpTokens = aws.StringValue(inst.MetadataOptions.HttpTokens) } - - facts.userData = l.userData(ctx, accountID, instanceID) - return facts, nil + return facts } -// userData fetches and decodes the instance's base64 user-data, returning nil on miss or error. -func (l *natsInstanceLookup) userData(ctx context.Context, accountID, instanceID string) []byte { - attr, err := retryGather(ctx, "describe instance attribute", instanceID, func() (*ec2.DescribeInstanceAttributeOutput, error) { - return gateway_ec2_instance.DescribeInstanceAttribute(ctx, &ec2.DescribeInstanceAttributeInput{ - InstanceId: aws.String(instanceID), - Attribute: aws.String("userData"), - }, l.nc, l.expectedNodes, accountID) - }) - if err != nil || attr == nil || attr.UserData == nil || attr.UserData.Value == nil { +// decodeUserData extracts and decodes the launch-time base64 user-data, +// returning nil on absence or a decode error. +func decodeUserData(input *ec2.RunInstancesInput) []byte { + if input == nil || input.UserData == nil { return nil } - decoded, err := base64.StdEncoding.DecodeString(aws.StringValue(attr.UserData.Value)) + decoded, err := base64.StdEncoding.DecodeString(aws.StringValue(input.UserData)) if err != nil { - slog.WarnContext(ctx, "IMDS: failed to decode instance user-data", "instance_id", instanceID, "err", err) + slog.Warn("IMDS: failed to decode instance user-data", "err", err) return nil } return decoded } - -// firstReservationInstance returns the first instance in a DescribeInstances -// response along with its owning reservation, or (nil, nil) when none. The -// reservation is always non-nil when the instance is, since an instance is only -// returned from a non-nil reservation that contains it. -func firstReservationInstance(out *ec2.DescribeInstancesOutput) (*ec2.Reservation, *ec2.Instance) { - if out == nil { - return nil, nil - } - for _, res := range out.Reservations { - if res == nil { - continue - } - for _, inst := range res.Instances { - if inst != nil { - return res, inst - } - } - } - return nil, nil -} diff --git a/spinifex/handlers/imds/instance_lookup_test.go b/spinifex/handlers/imds/instance_lookup_test.go index fa1a645bf..a0a316bd4 100644 --- a/spinifex/handlers/imds/instance_lookup_test.go +++ b/spinifex/handlers/imds/instance_lookup_test.go @@ -1,42 +1,169 @@ package handlers_imds import ( + "context" + "errors" "testing" + "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mulgadc/spinifex/spinifex/vm" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestFirstReservationInstance_NilAndEmpty(t *testing.T) { - for _, out := range []*ec2.DescribeInstancesOutput{ - nil, - {}, - {Reservations: []*ec2.Reservation{nil, {Instances: []*ec2.Instance{nil}}}}, - } { - res, inst := firstReservationInstance(out) - assert.Nil(t, res) - assert.Nil(t, inst) +// fakeLocalStateReader stands in for the vpcd-side on-disk state adapter. +type fakeLocalStateReader struct { + vms map[string]*vm.VM + err error +} + +func (f fakeLocalStateReader) LocalVM(instanceID string) (*vm.VM, error) { + if f.err != nil { + return nil, f.err + } + return f.vms[instanceID], nil +} + +// noCallRecordLoader fails the test if the fallback is ever invoked, proving a +// local hit resolves without reaching for the record space (and so never +// fans out over NATS: the fallback is the only path off this node). +type noCallRecordLoader struct{ t *testing.T } + +func (n noCallRecordLoader) LoadInstanceRecord(instanceID string) (*vm.InstanceRecord, error) { + n.t.Fatalf("record-space fallback called for %s despite a local hit", instanceID) + return nil, nil +} + +// fakeRecordLoader stands in for the JetStream-backed accessor in fallback tests. +type fakeRecordLoader struct { + record *vm.InstanceRecord + err error +} + +func (f fakeRecordLoader) LoadInstanceRecord(string) (*vm.InstanceRecord, error) { + return f.record, f.err +} + +// testVM builds a fully resolvable instance for lookup tests. +func testVM(id string) *vm.VM { + launch := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + return &vm.VM{ + ID: id, + AccountID: "111111111111", + InstanceType: "t3.micro", + IamInstanceProfileArn: "arn:aws:iam::111111111111:instance-profile/demo", + InstanceLifecycle: "spot", + RunInstancesInput: &ec2.RunInstancesInput{ + UserData: aws.String("aGVsbG8="), // "hello" + }, + Reservation: &ec2.Reservation{ + ReservationId: aws.String("r-abc123"), + }, + Instance: &ec2.Instance{ + ImageId: aws.String("ami-123"), + KeyName: aws.String("demo-key"), + Architecture: aws.String("x86_64"), + AmiLaunchIndex: aws.Int64(0), + LaunchTime: aws.Time(launch), + MetadataOptions: &ec2.InstanceMetadataOptionsResponse{HttpTokens: aws.String("optional")}, + }, } } -// firstReservationInstance skips nil reservations and nil instance slots and -// returns the first concrete instance with its owning reservation. -func TestFirstReservationInstance_SkipsNilsAndReturnsFirst(t *testing.T) { - out := &ec2.DescribeInstancesOutput{ - Reservations: []*ec2.Reservation{ - nil, - {Instances: []*ec2.Instance{nil}}, - { - ReservationId: aws.String("r-123"), - Instances: []*ec2.Instance{{InstanceId: aws.String("i-first")}, {InstanceId: aws.String("i-second")}}, - }, - }, +func TestDescribe_LocalHit_ResolvesWithoutRecordSpace(t *testing.T) { + l := &localInstanceLookup{ + local: fakeLocalStateReader{vms: map[string]*vm.VM{"i-local": testVM("i-local")}}, + records: noCallRecordLoader{t: t}, } - res, inst := firstReservationInstance(out) - require.NotNil(t, inst) - assert.Equal(t, "i-first", aws.StringValue(inst.InstanceId)) - require.NotNil(t, res) - assert.Equal(t, "r-123", aws.StringValue(res.ReservationId)) + facts, err := l.describe(context.Background(), "111111111111", "i-local") + require.NoError(t, err) + require.NotNil(t, facts) + + assert.Equal(t, "t3.micro", facts.instanceType) + assert.Equal(t, "ami-123", facts.imageID) + assert.Equal(t, "demo-key", facts.keyName) + assert.Equal(t, "x86_64", facts.architecture) + assert.Equal(t, "arn:aws:iam::111111111111:instance-profile/demo", facts.iamInstanceProfileArn) + assert.Equal(t, "spot", facts.lifecycleType) + assert.Equal(t, "optional", facts.httpTokens) + assert.Equal(t, int64(0), facts.amiLaunchIndex) + assert.Equal(t, "r-abc123", facts.reservationID) + assert.Equal(t, []byte("hello"), facts.userData) +} + +func TestDescribe_LocalMiss_FallsBackToRecordSpace(t *testing.T) { + fallbackVM := testVM("i-remote") + + l := &localInstanceLookup{ + local: fakeLocalStateReader{vms: map[string]*vm.VM{}}, // instance not on this node + records: fakeRecordLoader{record: fallbackVM.Record()}, + } + facts, err := l.describe(context.Background(), "111111111111", "i-remote") + require.NoError(t, err) + require.NotNil(t, facts) + assert.Equal(t, "t3.micro", facts.instanceType) + assert.Equal(t, "ami-123", facts.imageID) +} + +func TestDescribe_RecordSpaceMiss_ReturnsMiss(t *testing.T) { + l := &localInstanceLookup{ + local: fakeLocalStateReader{vms: map[string]*vm.VM{}}, + records: fakeRecordLoader{record: nil}, + } + facts, err := l.describe(context.Background(), "111111111111", "i-nowhere") + require.NoError(t, err) + assert.Nil(t, facts) +} + +func TestDescribe_NoLocalReaderOrRecordLoader_IsAMiss(t *testing.T) { + l := &localInstanceLookup{} // both left nil, as when neither adapter is available + facts, err := l.describe(context.Background(), "111111111111", "i-nowhere") + require.NoError(t, err) + assert.Nil(t, facts) +} + +// TestDescribe_LocalReadError_FallsBackToRecordSpace proves a local-state +// read failure does not error the whole lookup or leave IMDS stuck: it falls +// through to the record space like a genuine local miss. +func TestDescribe_LocalReadError_FallsBackToRecordSpace(t *testing.T) { + l := &localInstanceLookup{ + local: fakeLocalStateReader{err: errors.New("state file corrupt")}, + records: fakeRecordLoader{record: testVM("i-a").Record()}, + } + facts, err := l.describe(context.Background(), "111111111111", "i-a") + require.NoError(t, err) + require.NotNil(t, facts, "a local read error must fall back to the record space, not error out") +} + +func TestDescribe_ManagedByHidesFromNonGlobalAccount(t *testing.T) { + managed := testVM("i-managed") + managed.ManagedBy = "eks" + + l := &localInstanceLookup{ + local: fakeLocalStateReader{vms: map[string]*vm.VM{"i-managed": managed}}, + records: noCallRecordLoader{t: t}, + } + facts, err := l.describe(context.Background(), "111111111111", "i-managed") + require.NoError(t, err) + assert.Nil(t, facts, "platform-managed instance must stay hidden from a non-global account") +} + +func TestDescribe_WrongAccount_IsAMiss(t *testing.T) { + l := &localInstanceLookup{ + local: fakeLocalStateReader{vms: map[string]*vm.VM{"i-local": testVM("i-local")}}, + records: noCallRecordLoader{t: t}, + } + facts, err := l.describe(context.Background(), "222222222222", "i-local") + require.NoError(t, err) + assert.Nil(t, facts) +} + +// TestLocalInstanceLookupHasNoNATSSurface pins the "never fan out" contract at +// the type level: localInstanceLookup carries only the narrow local-state and +// record-space interfaces, no NATS connection to fan a request out over. +func TestLocalInstanceLookupHasNoNATSSurface(t *testing.T) { + l := localInstanceLookup{} + var _ instanceLookup = &l } diff --git a/spinifex/handlers/imds/service.go b/spinifex/handlers/imds/service.go index 02667e4f0..583b35cad 100644 --- a/spinifex/handlers/imds/service.go +++ b/spinifex/handlers/imds/service.go @@ -83,8 +83,13 @@ type IMDSServiceImpl struct { // non-empty, each per-tap responder also serves the VPC DNS shim on // 169.254.169.253:53, relaying to northstar's unprivileged wildcard listener. // caCertPath is the deployment CA served at /spinifex/ca.pem; empty 404s it. +// localState and records back the local-first instance lookup: localState +// reads this node's own VM state, records falls back to the shared instance +// record space for an instance that has moved off this node. Either may be +// nil, degrading that source to a miss rather than failing IMDS to start — +// the caller (vpcd) owns their construction and availability. // ctx bounds the bucket opens only; each served request carries its own. -func NewIMDSServiceImpl(ctx context.Context, natsConn *nats.Conn, sts stsAssumer, iamSvc profileLookup, pubKeys publicKeyLookup, expectedNodes int, listTaps listTapsFunc, baseDomain, internalDomain, caCertPath string, resolverIPs []string) (*IMDSServiceImpl, error) { +func NewIMDSServiceImpl(ctx context.Context, natsConn *nats.Conn, sts stsAssumer, iamSvc profileLookup, pubKeys publicKeyLookup, localState localStateReader, records recordLoader, listTaps listTapsFunc, baseDomain, internalDomain, caCertPath string, resolverIPs []string) (*IMDSServiceImpl, error) { if natsConn == nil { return nil, errors.New("nil NATS connection") } @@ -137,7 +142,7 @@ func NewIMDSServiceImpl(ctx context.Context, natsConn *nats.Conn, sts stsAssumer sgKV: sgKV, subnetKV: subnetKV, vpcKV: vpcKV, - lookup: &natsInstanceLookup{nc: natsConn, expectedNodes: expectedNodes}, + lookup: &localInstanceLookup{local: localState, records: records}, }, tokens: newTokenStore(), v1Allow: newV1AllowCache(), diff --git a/spinifex/services/awsgw/awsgw.go b/spinifex/services/awsgw/awsgw.go index 97c7c8f65..7ac328f42 100644 --- a/spinifex/services/awsgw/awsgw.go +++ b/spinifex/services/awsgw/awsgw.go @@ -476,6 +476,14 @@ func launchService(config *config.ClusterConfig) error { } go gateway_bedrock.NewUsageConsumer(bedrockUsage, bedrockPrices).Run(janitorCtx, usageConsumer) + // nodeIDs is the configured cluster node set, in the same namespace as the + // daemon reply header, so a fan-out's completeness can be judged by + // responder identity rather than by count. + nodeIDs := make([]string, 0, len(config.Nodes)) + for nodeID := range config.Nodes { + nodeIDs = append(nodeIDs, nodeID) + } + gw := gateway.GatewayConfig{ Debug: nodeConfig.AWSGW.Debug, DisableLogging: false, @@ -483,6 +491,7 @@ func launchService(config *config.ClusterConfig) error { RootCAs: rootCAs, Config: nodeConfig.AWSGW.Config, ExpectedNodes: len(config.Nodes), + NodeIDs: nodeIDs, Region: nodeConfig.Region, InternalSuffix: config.AWS.InternalSuffix, RegistryPort: registryPort, diff --git a/spinifex/utils/nats.go b/spinifex/utils/nats.go index 788032c79..a76188895 100644 --- a/spinifex/utils/nats.go +++ b/spinifex/utils/nats.go @@ -2,6 +2,7 @@ package utils import ( "context" + "crypto/sha256" "crypto/tls" "crypto/x509" "encoding/json" @@ -369,8 +370,52 @@ const ( maxScatterGatherResponseSize = 10 * 1024 * 1024 // maxScatterGatherUnboundedResponses caps responses when expectedNodes is 0. maxScatterGatherUnboundedResponses = 256 + // maxIdentityGatherFrames caps total frames an identity-mode fan-out + // (CollectUntilDeadline, or CollectServeData with ExpectedResponders set) + // will process, independent of cluster size, so a stray publisher or a + // hot restart loop cannot hold collection open for the full deadline. + maxIdentityGatherFrames = 4096 + // maxIdentityGatherBytes caps total retained payload bytes for an + // identity-mode fan-out. Only the first payload per node is retained, so + // this bounds memory to roughly the cluster's healthy-case response size. + maxIdentityGatherBytes = 64 * 1024 * 1024 ) +// NodeIDHeader is the NATS reply header key carrying the responding daemon +// node's ID, set on every daemon reply so a fan-out can attribute a frame. +const NodeIDHeader = "X-Node-ID" + +// NodeIDFromMsg extracts the responding node's ID from a NATS reply header, or "" if absent. +func NodeIDFromMsg(msg *nats.Msg) string { + if msg == nil || msg.Header == nil { + return "" + } + return msg.Header.Get(NodeIDHeader) +} + +// CollectionMode governs when a Gather fan-out stops waiting for replies. +type CollectionMode int + +const ( + // CollectServeData exits once its stop condition is met — ExpectedNodes + // frames, ExpectedResponders unique nodes, or StopOnFirst. The zero value, + // so every caller that does not opt in keeps today's exact behavior. + CollectServeData CollectionMode = iota + // CollectUntilDeadline collects for the full Timeout. A fan-out proving a + // negative (nobody has it) cannot be established from a prefix of the + // replies — the frame that would refute it is the one most likely to be + // late. GatherOpts.Settled is the one way out, for the case where the + // replies so far mean nothing is left to prove. + CollectUntilDeadline +) + +// Frame is one daemon reply, tagged with the responding node's ID ("" when +// the reply carried no node ID header, e.g. an older daemon). +type Frame struct { + NodeID string + Data []byte +} + // Summary is a local tally of a fan-out; it is never sent over the wire. type Summary struct { Received int // frames seen (success + error) @@ -378,28 +423,69 @@ type Summary struct { ErrorCodes map[string]int // AWS error code -> count across error frames FirstClient4xx string // first deterministic 4xx code seen, "" if none TimedOut bool // deadline hit before the stop condition was met + + // The rest are populated only in identity mode (ExpectedResponders set, + // or Mode == CollectUntilDeadline). They stay nil/zero for a caller that + // does not opt in, so that caller's behavior is unchanged by construction. + Responders map[string]bool // node ID -> answered at all (success or error); terminates collection under CollectServeData + SuccessResponders map[string]bool // node ID -> answered with a decodable non-error frame + ErrorResponders map[string]bool // node ID -> answered with an error envelope + ConflictNodes map[string]bool // node ID -> replied more than once with payloads that disagree + Unidentified int // frames received with no node ID header + DuplicateFrames int // frames whose node had already answered; bytes dropped, identity kept + CapHit bool // the identity frame or byte cap ended collection early + SettledEarly bool // collection ended before the deadline with the answer settled, by Settled or by full responder coverage } // GatherOpts configures a Gather fan-out. type GatherOpts struct { - Timeout time.Duration // hard deadline for the whole fan-out - ExpectedNodes int // early-exit once this many frames arrive (0 = wait full Timeout) - StopOnFirst bool // return after the first non-error frame (first-wins) - AccountID string // sets X-Account-ID header when non-empty + Timeout time.Duration // hard deadline for the whole fan-out + ExpectedNodes int // early-exit once this many frames arrive (0 = wait full Timeout); mutually exclusive with ExpectedResponders + ExpectedResponders int // identity mode: early-exit once this many distinct nodes have answered, under CollectServeData; mutually exclusive with ExpectedNodes + Mode CollectionMode // CollectServeData (default) or CollectUntilDeadline + StopOnFirst bool // return after the first non-error frame (first-wins) + AccountID string // sets X-Account-ID header when non-empty + + // Settled ends a CollectUntilDeadline fan-out early; it is ignored in every + // other mode. Consulted after each retained frame, it returns true once the + // replies already answer the question, so only a caller still trying to + // prove a negative pays the rest of the deadline. + Settled func(frames []Frame, sum Summary) bool + + // ResponderGrace caps how long a CollectUntilDeadline fan-out keeps waiting + // after every ExpectedResponders node has answered. Zero keeps the full + // deadline. It exists because the only thing left to wait for at that point + // is a duplicate from a node already heard, and core NATS gives a request + // only to the subscribers present when it was published: an overlapping + // subscription replies alongside the first, and one that arrives later + // never receives the request at all, so waiting longer cannot find it. + ResponderGrace time.Duration } // Gather publishes payload to subject over a fresh inbox and collects reply -// frames until ExpectedNodes answer, StopOnFirst yields a success, or Timeout -// elapses. Error envelopes and oversized frames are dropped from frames but -// counted in sum; returned frames are raw daemon replies for the caller to -// decode and merge. It carries ctx's trace context onto the wire: it opens a -// producer span for the fan-out and injects traceparent so every consumer joins -// the same trace. -func Gather(ctx context.Context, conn *nats.Conn, subject string, payload []byte, opts GatherOpts) (frames [][]byte, sum Summary, err error) { +// frames until the stop condition for opts.Mode is met, or Timeout elapses. +// Error envelopes and oversized frames are dropped from frames but counted in +// sum; returned frames are raw daemon replies for the caller to decode and +// merge, tagged with the responding node's ID when the reply carried one. +// +// With ExpectedResponders unset and Mode left at its zero value +// (CollectServeData), Gather's behavior is unchanged from before identity +// mode existed: the stop condition is ExpectedNodes frames (or the unbounded +// cap), and Summary's identity fields stay nil/zero. Identity mode — set +// either by ExpectedResponders or by CollectUntilDeadline — additionally +// tracks per-node responder sets, retains only the first payload seen from +// each node, and is bounded by its own frame and byte caps independent of +// ExpectedNodes. It carries ctx's trace context onto the wire: it opens a +// producer span for the fan-out and injects traceparent so every consumer +// joins the same trace. +func Gather(ctx context.Context, conn *nats.Conn, subject string, payload []byte, opts GatherOpts) (frames []Frame, sum Summary, err error) { sum.ErrorCodes = map[string]int{} if conn == nil || !conn.IsConnected() { return nil, sum, ErrClusterUnavailable } + if opts.ExpectedNodes > 0 && opts.ExpectedResponders > 0 { + return nil, sum, fmt.Errorf("gather: ExpectedNodes and ExpectedResponders are mutually exclusive") + } ctx, span := startProducerSpan(ctx, subject, opts.AccountID) defer func() { endSpanWithError(span, err) }() @@ -422,23 +508,63 @@ func Gather(ctx context.Context, conn *nats.Conn, subject string, payload []byte return nil, sum, fmt.Errorf("failed to publish request: %w", err) } + identityMode := opts.ExpectedResponders > 0 || opts.Mode == CollectUntilDeadline + earlyExitOnResponders := opts.Mode == CollectServeData && opts.ExpectedResponders > 0 + + var seenHash map[string][32]byte + if identityMode { + sum.Responders = map[string]bool{} + sum.SuccessResponders = map[string]bool{} + sum.ErrorResponders = map[string]bool{} + sum.ConflictNodes = map[string]bool{} + seenHash = map[string][32]byte{} + } + maxResponses := maxScatterGatherUnboundedResponses if opts.ExpectedNodes > 0 { maxResponses = opts.ExpectedNodes } + coverageGrace := opts.Mode == CollectUntilDeadline && opts.ResponderGrace > 0 && opts.ExpectedResponders > 0 + + var retainedBytes int deadline := time.Now().Add(opts.Timeout) - for sum.Received < maxResponses { - remaining := time.Until(deadline) + var graceDeadline time.Time + for { + if earlyExitOnResponders { + if len(sum.Responders) >= opts.ExpectedResponders { + break + } + } else if !identityMode && sum.Received >= maxResponses { + break + } + + // Once every expected node has answered, only a duplicate from a node + // already heard can still change the verdict, and core NATS delivers + // that on the same timescale as the first reply. So wait a grace window + // rather than the whole deadline. Armed once and never extended, so a + // node replying repeatedly cannot hold the fan-out open. + if coverageGrace && graceDeadline.IsZero() && len(sum.Responders) >= opts.ExpectedResponders { + graceDeadline = time.Now().Add(opts.ResponderGrace) + } + + // Running out of the grace window is a settled exit, not a timeout: the + // nodes that were asked all answered. + effective, graceExpired := deadline, false + if !graceDeadline.IsZero() && graceDeadline.Before(deadline) { + effective, graceExpired = graceDeadline, true + } + + remaining := time.Until(effective) if remaining <= 0 { - sum.TimedOut = true + sum.TimedOut, sum.SettledEarly = !graceExpired, graceExpired break } msg, nerr := sub.NextMsg(remaining) if nerr != nil { if errors.Is(nerr, nats.ErrTimeout) || errors.Is(nerr, nats.ErrNoResponders) { - sum.TimedOut = true + sum.TimedOut, sum.SettledEarly = !graceExpired, graceExpired break } return frames, sum, fmt.Errorf("gather receive error on %s: %w", subject, nerr) @@ -451,6 +577,34 @@ func Gather(ctx context.Context, conn *nats.Conn, subject string, payload []byte continue } + nodeID := NodeIDFromMsg(msg) + + if identityMode { + if sum.Received > maxIdentityGatherFrames || retainedBytes > maxIdentityGatherBytes { + sum.CapHit = true + break + } + + if nodeID == "" { + sum.Unidentified++ + } else if prevHash, seen := seenHash[nodeID]; seen { + // A later frame from a node already seen: keep the identity and + // flag disagreement, but never let its bytes overwrite the first + // payload — there is no merge rule for two contradictory answers + // from one node, and inventing one would hide the fault. + sum.DuplicateFrames++ + if sha256.Sum256(msg.Data) != prevHash { + sum.ConflictNodes[nodeID] = true + } + if _, verr := ValidateErrorPayload(msg.Data); verr != nil { + sum.ErrorResponders[nodeID] = true + } else { + sum.SuccessResponders[nodeID] = true + } + continue + } + } + responseError, verr := ValidateErrorPayload(msg.Data) if verr != nil { code := "" @@ -465,14 +619,29 @@ func Gather(ctx context.Context, conn *nats.Conn, subject string, payload []byte } } slog.Debug("Gather: skipping error response", "code", code, "subject", subject) + if identityMode && nodeID != "" { + sum.Responders[nodeID] = true + sum.ErrorResponders[nodeID] = true + seenHash[nodeID] = sha256.Sum256(msg.Data) + } continue } sum.Successes++ - frames = append(frames, msg.Data) + if identityMode && nodeID != "" { + sum.Responders[nodeID] = true + sum.SuccessResponders[nodeID] = true + seenHash[nodeID] = sha256.Sum256(msg.Data) + retainedBytes += len(msg.Data) + } + frames = append(frames, Frame{NodeID: nodeID, Data: msg.Data}) if opts.StopOnFirst { return frames, sum, nil } + if opts.Mode == CollectUntilDeadline && opts.Settled != nil && opts.Settled(frames, sum) { + sum.SettledEarly = true + break + } } return frames, sum, nil diff --git a/spinifex/utils/nats_test.go b/spinifex/utils/nats_test.go index 51f426ab1..631796eed 100644 --- a/spinifex/utils/nats_test.go +++ b/spinifex/utils/nats_test.go @@ -860,7 +860,7 @@ func TestGather_StopOnFirstSkipsErrors(t *testing.T) { var out struct { Value string `json:"value"` } - require.NoError(t, json.Unmarshal(frames[0], &out)) + require.NoError(t, json.Unmarshal(frames[0].Data, &out)) assert.Equal(t, "found", out.Value) assert.Equal(t, 1, sum.Successes) assert.False(t, sum.TimedOut) @@ -925,7 +925,7 @@ func TestGather_AccountIDHeaderSet(t *testing.T) { require.Len(t, frames, 1) var echo gatherAcctEcho - require.NoError(t, json.Unmarshal(frames[0], &echo)) + require.NoError(t, json.Unmarshal(frames[0].Data, &echo)) assert.True(t, echo.Present) assert.Equal(t, "111122223333", echo.ID) } @@ -949,7 +949,7 @@ func TestGather_AccountIDHeaderAbsentWhenEmpty(t *testing.T) { require.Len(t, frames, 1) var echo gatherAcctEcho - require.NoError(t, json.Unmarshal(frames[0], &echo)) + require.NoError(t, json.Unmarshal(frames[0].Data, &echo)) assert.False(t, echo.Present) assert.Empty(t, echo.ID) } @@ -972,3 +972,340 @@ func TestGather_ClosedConn_ReturnsClusterUnavailable(t *testing.T) { GatherOpts{Timeout: 50 * time.Millisecond, ExpectedNodes: 1}) require.ErrorIs(t, err, ErrClusterUnavailable) } + +// --- Gather identity-mode tests --- + +// subscribeAsNode replies on subject as nodeID, carrying the X-Node-ID header +// a real daemon reply would set. delay, if non-zero, is applied before +// responding, so tests can control arrival order. +func subscribeAsNode(t *testing.T, nc *nats.Conn, subject, nodeID string, data []byte, delay time.Duration) { + t.Helper() + _, err := nc.Subscribe(subject, func(msg *nats.Msg) { + if delay > 0 { + time.Sleep(delay) + } + reply := nats.NewMsg(msg.Reply) + reply.Data = data + if nodeID != "" { + reply.Header.Set(NodeIDHeader, nodeID) + } + _ = msg.RespondMsg(reply) + }) + require.NoError(t, err) +} + +// A caller that sets neither ExpectedResponders nor CollectUntilDeadline gets +// exactly the pre-identity-mode Gather: the new Summary fields stay nil, not +// merely empty, so a caller checking len(sum.Responders) == 0 cannot +// accidentally read "identity mode ran and saw nobody" from "identity mode +// never ran". +func TestGather_LegacyMode_IdentityFieldsStayNil(t *testing.T) { + _, nc := testutil.StartTestNATS(t) + subscribeAsNode(t, nc, "test.gather.legacy", "node-a", []byte(`{"ok":true}`), 0) + + frames, sum, err := Gather(context.Background(), nc, "test.gather.legacy", []byte("{}"), + GatherOpts{Timeout: time.Second, ExpectedNodes: 1}) + + require.NoError(t, err) + require.Len(t, frames, 1) + assert.Equal(t, "node-a", frames[0].NodeID, "the frame still carries whatever node ID the reply set") + assert.Nil(t, sum.Responders) + assert.Nil(t, sum.SuccessResponders) + assert.Nil(t, sum.ErrorResponders) + assert.Nil(t, sum.ConflictNodes) +} + +// ExpectedResponders under the default CollectServeData mode exits as soon as +// the distinct-node count is met, same early-exit shape as ExpectedNodes, and +// tags each frame with the node that sent it. +func TestGather_IdentityMode_NodeIDCarriedOnFramesAndEarlyExit(t *testing.T) { + _, nc := testutil.StartTestNATS(t) + subscribeAsNode(t, nc, "test.gather.identity.early", "node-a", []byte(`{"ok":true}`), 0) + subscribeAsNode(t, nc, "test.gather.identity.early", "node-b", []byte(`{"ok":true}`), 0) + + start := time.Now() + frames, sum, err := Gather(context.Background(), nc, "test.gather.identity.early", []byte("{}"), + GatherOpts{Timeout: 5 * time.Second, ExpectedResponders: 2}) + elapsed := time.Since(start) + + require.NoError(t, err) + require.Len(t, frames, 2) + assert.Less(t, elapsed, 2*time.Second) + assert.False(t, sum.TimedOut) + gotIDs := map[string]bool{frames[0].NodeID: true, frames[1].NodeID: true} + assert.Equal(t, map[string]bool{"node-a": true, "node-b": true}, gotIDs) + assert.Len(t, sum.Responders, 2) +} + +// Responders is every node that answered at all; SuccessResponders and +// ErrorResponders partition it by how. A node cannot land in both from a +// single reply. +func TestGather_IdentityMode_RespondersPartitionSuccessAndError(t *testing.T) { + _, nc := testutil.StartTestNATS(t) + subscribeAsNode(t, nc, "test.gather.identity.partition", "node-ok", []byte(`{"ok":true}`), 0) + subscribeAsNode(t, nc, "test.gather.identity.partition", "node-err", + GenerateErrorPayload(awserrors.ErrorInvalidInstanceIDNotFound), 0) + + _, sum, err := Gather(context.Background(), nc, "test.gather.identity.partition", []byte("{}"), + GatherOpts{Timeout: 2 * time.Second, ExpectedResponders: 2}) + + require.NoError(t, err) + assert.Equal(t, map[string]bool{"node-ok": true, "node-err": true}, sum.Responders) + assert.Equal(t, map[string]bool{"node-ok": true}, sum.SuccessResponders) + assert.Equal(t, map[string]bool{"node-err": true}, sum.ErrorResponders) +} + +// A reply with no X-Node-ID header (an older daemon, or a stray publisher on +// the subject) cannot be attributed to any node, so it is counted separately +// rather than silently folded into the responder sets — a describe-level +// completeness judgement must be able to see it and refuse to trust the sweep. +func TestGather_IdentityMode_UnidentifiedFrameCounted(t *testing.T) { + _, nc := testutil.StartTestNATS(t) + subscribeAsNode(t, nc, "test.gather.identity.unident", "node-a", []byte(`{"ok":true}`), 0) + // No node ID set on this reply. + _, err := nc.Subscribe("test.gather.identity.unident", func(msg *nats.Msg) { + _ = msg.Respond([]byte(`{"ok":true}`)) + }) + require.NoError(t, err) + + _, sum, err := Gather(context.Background(), nc, "test.gather.identity.unident", []byte("{}"), + GatherOpts{Timeout: 500 * time.Millisecond, ExpectedResponders: 2}) + + require.NoError(t, err) + assert.Equal(t, 1, sum.Unidentified) + assert.Len(t, sum.Responders, 1, "only the identified reply counts toward the responder set") +} + +// A node that replies twice with disagreeing payloads (the restart-overlap +// case: an old subscription still draining alongside the new one) keeps only +// its first payload — never the second, win or lose — and is flagged in +// ConflictNodes so a completeness judgement downstream can refuse to trust it. +func TestGather_IdentityMode_DuplicateDisagreeingFrame_FirstPayloadWinsAndFlagsConflict(t *testing.T) { + _, nc := testutil.StartTestNATS(t) + // Two subscribers answering as the same node ID: the first reply must be + // the one retained regardless of arrival order tie-breaks, so give the + // disagreeing second reply a small delay to make the ordering deterministic. + subscribeAsNode(t, nc, "test.gather.identity.dup", "node-a", []byte(`{"value":"first"}`), 0) + subscribeAsNode(t, nc, "test.gather.identity.dup", "node-a", []byte(`{"value":"second"}`), 50*time.Millisecond) + + frames, sum, err := Gather(context.Background(), nc, "test.gather.identity.dup", []byte("{}"), + GatherOpts{Timeout: time.Second, Mode: CollectUntilDeadline, ExpectedResponders: 1}) + + require.NoError(t, err) + require.Len(t, frames, 1, "the disagreeing duplicate's bytes must never be retained") + assert.JSONEq(t, `{"value":"first"}`, string(frames[0].Data)) + assert.Equal(t, map[string]bool{"node-a": true}, sum.ConflictNodes) + assert.Equal(t, 1, sum.DuplicateFrames) +} + +// A node replying twice with identical payloads (a harmless at-least-once +// redelivery) is still counted as a duplicate frame, but is not a conflict: +// there is nothing to disagree about. +func TestGather_IdentityMode_DuplicateIdenticalFrame_NotFlaggedAsConflict(t *testing.T) { + _, nc := testutil.StartTestNATS(t) + subscribeAsNode(t, nc, "test.gather.identity.dupsame", "node-a", []byte(`{"value":"same"}`), 0) + subscribeAsNode(t, nc, "test.gather.identity.dupsame", "node-a", []byte(`{"value":"same"}`), 50*time.Millisecond) + + _, sum, err := Gather(context.Background(), nc, "test.gather.identity.dupsame", []byte("{}"), + GatherOpts{Timeout: 300 * time.Millisecond, Mode: CollectUntilDeadline, ExpectedResponders: 1}) + + require.NoError(t, err) + assert.Empty(t, sum.ConflictNodes) + assert.Equal(t, 1, sum.DuplicateFrames) +} + +// CollectUntilDeadline exists specifically because absence cannot be proven +// from a prefix of the replies: it must not exit early just because every +// currently-expected responder has already answered. +func TestGather_CollectUntilDeadline_DoesNotExitEarly(t *testing.T) { + _, nc := testutil.StartTestNATS(t) + subscribeAsNode(t, nc, "test.gather.identity.deadline", "node-a", []byte(`{"ok":true}`), 0) + + const budget = 400 * time.Millisecond + start := time.Now() + _, sum, err := Gather(context.Background(), nc, "test.gather.identity.deadline", []byte("{}"), + GatherOpts{Timeout: budget, Mode: CollectUntilDeadline, ExpectedResponders: 1}) + elapsed := time.Since(start) + + require.NoError(t, err) + assert.GreaterOrEqual(t, elapsed, budget-20*time.Millisecond, + "CollectUntilDeadline must run to the deadline even though the only expected responder already answered") + assert.True(t, sum.TimedOut) + assert.Equal(t, map[string]bool{"node-a": true}, sum.Responders) +} + +// Adapted restart-overlap acceptance scenario: node-1 has an old subscription +// still draining alongside its new one and answers twice with disagreeing +// payloads (simulating a daemon mid-restart); node-2 never answers within the +// deadline. The mechanism this pins is what a describe-level completeness +// judgement relies on: ConflictNodes and the missing node-2 are both visible +// in Summary, so the caller can refuse the sweep rather than trust a stale +// answer. It does not assert the instance is present in the result — under +// the documented first-payload-wins rule the disagreeing reply's data is +// dropped regardless, only the conflict flag survives it. +func TestGather_CollectUntilDeadline_RestartOverlap_SurfacesConflictAndMissingNode(t *testing.T) { + _, nc := testutil.StartTestNATS(t) + subscribeAsNode(t, nc, "test.gather.identity.restart", "node-1", []byte(`{"value":"stale"}`), 0) + subscribeAsNode(t, nc, "test.gather.identity.restart", "node-1", []byte(`{"value":"fresh"}`), 50*time.Millisecond) + // node-2 is expected but never answers. + + _, sum, err := Gather(context.Background(), nc, "test.gather.identity.restart", []byte("{}"), + GatherOpts{Timeout: 300 * time.Millisecond, Mode: CollectUntilDeadline, ExpectedResponders: 2}) + + require.NoError(t, err) + assert.True(t, sum.ConflictNodes["node-1"], "the restart overlap must be visible as a conflict, not silently resolved") + assert.False(t, sum.Responders["node-2"], "node-2 never answered and must not be reported as having done so") +} + +func TestGather_Settled_EndsCollectionBeforeTheDeadline(t *testing.T) { + _, nc := testutil.StartTestNATS(t) + subscribeAsNode(t, nc, "test.gather.settled.hit", "node-a", []byte(`{"ok":true}`), 0) + + const budget = 800 * time.Millisecond + start := time.Now() + _, sum, err := Gather(context.Background(), nc, "test.gather.settled.hit", []byte("{}"), + GatherOpts{ + Timeout: budget, Mode: CollectUntilDeadline, ExpectedResponders: 2, + Settled: func(frames []Frame, _ Summary) bool { return len(frames) >= 1 }, + }) + elapsed := time.Since(start) + + require.NoError(t, err) + assert.Less(t, elapsed, budget/2, "a settled fan-out must not wait out the deadline") + assert.True(t, sum.SettledEarly) + assert.False(t, sum.TimedOut, "settling is not a timeout") +} + +// A predicate that never settles must leave CollectUntilDeadline's guarantee +// intact, since that is the case where absence is still being proven. +func TestGather_Settled_NeverTrue_StillRunsToTheDeadline(t *testing.T) { + _, nc := testutil.StartTestNATS(t) + subscribeAsNode(t, nc, "test.gather.settled.miss", "node-a", []byte(`{"ok":true}`), 0) + + const budget = 400 * time.Millisecond + start := time.Now() + _, sum, err := Gather(context.Background(), nc, "test.gather.settled.miss", []byte("{}"), + GatherOpts{ + Timeout: budget, Mode: CollectUntilDeadline, ExpectedResponders: 1, + Settled: func([]Frame, Summary) bool { return false }, + }) + elapsed := time.Since(start) + + require.NoError(t, err) + assert.GreaterOrEqual(t, elapsed, budget-20*time.Millisecond) + assert.False(t, sum.SettledEarly) + assert.True(t, sum.TimedOut) +} + +func TestGather_Settled_IgnoredUnderCollectServeData(t *testing.T) { + _, nc := testutil.StartTestNATS(t) + subscribeAsNode(t, nc, "test.gather.settled.mode", "node-a", []byte(`{"ok":true}`), 0) + + called := false + _, sum, err := Gather(context.Background(), nc, "test.gather.settled.mode", []byte("{}"), + GatherOpts{ + Timeout: 300 * time.Millisecond, ExpectedResponders: 1, + Settled: func([]Frame, Summary) bool { called = true; return true }, + }) + + require.NoError(t, err) + assert.False(t, called, "Settled belongs to CollectUntilDeadline and must not run in serve-data mode") + assert.False(t, sum.SettledEarly) +} + +// Once every expected node has answered, the only reply still outstanding is a +// duplicate from a node already heard, which core NATS delivers alongside the +// first. Waiting out the deadline for it buys nothing. +func TestGather_ResponderGrace_EndsCollectionAfterFullCoverage(t *testing.T) { + _, nc := testutil.StartTestNATS(t) + subscribeAsNode(t, nc, "test.gather.grace.covered", "node-a", []byte(`{"ok":true}`), 0) + subscribeAsNode(t, nc, "test.gather.grace.covered", "node-b", []byte(`{"ok":true}`), 0) + + const budget = 2 * time.Second + start := time.Now() + _, sum, err := Gather(context.Background(), nc, "test.gather.grace.covered", []byte("{}"), + GatherOpts{ + Timeout: budget, Mode: CollectUntilDeadline, ExpectedResponders: 2, + ResponderGrace: 150 * time.Millisecond, + Settled: func([]Frame, Summary) bool { return false }, + }) + elapsed := time.Since(start) + + require.NoError(t, err) + assert.Len(t, sum.Responders, 2) + assert.Less(t, elapsed, budget/2, "full coverage plus grace must not wait out the deadline") + assert.True(t, sum.SettledEarly) + assert.False(t, sum.TimedOut, "hearing from everyone is not a timeout") +} + +// The grace window only opens on full coverage. A missing node is exactly the +// case absence proofs exist for, so it must still cost the whole deadline. +func TestGather_ResponderGrace_PartialCoverageStillRunsToTheDeadline(t *testing.T) { + _, nc := testutil.StartTestNATS(t) + subscribeAsNode(t, nc, "test.gather.grace.partial", "node-a", []byte(`{"ok":true}`), 0) + + const budget = 400 * time.Millisecond + start := time.Now() + _, sum, err := Gather(context.Background(), nc, "test.gather.grace.partial", []byte("{}"), + GatherOpts{ + Timeout: budget, Mode: CollectUntilDeadline, ExpectedResponders: 2, + ResponderGrace: 50 * time.Millisecond, + Settled: func([]Frame, Summary) bool { return false }, + }) + elapsed := time.Since(start) + + require.NoError(t, err) + assert.Len(t, sum.Responders, 1) + assert.GreaterOrEqual(t, elapsed, budget-20*time.Millisecond) + assert.True(t, sum.TimedOut) + assert.False(t, sum.SettledEarly) +} + +// The window is armed once and never extended, so a node replying over and +// over cannot hold the fan-out open past it. +func TestGather_ResponderGrace_ArmedOnceNotExtendedByRepeatReplies(t *testing.T) { + _, nc := testutil.StartTestNATS(t) + subscribeAsNode(t, nc, "test.gather.grace.chatty", "node-a", []byte(`{"ok":true}`), 0) + // Extra subscriptions on the same node id keep frames trickling in past + // the grace window without ever adding a new responder. + for i := 1; i <= 6; i++ { + subscribeAsNode(t, nc, "test.gather.grace.chatty", "node-a", []byte(`{"ok":true}`), + time.Duration(i)*100*time.Millisecond) + } + + const grace = 200 * time.Millisecond + start := time.Now() + _, sum, err := Gather(context.Background(), nc, "test.gather.grace.chatty", []byte("{}"), + GatherOpts{ + Timeout: 3 * time.Second, Mode: CollectUntilDeadline, ExpectedResponders: 1, + ResponderGrace: grace, + Settled: func([]Frame, Summary) bool { return false }, + }) + elapsed := time.Since(start) + + require.NoError(t, err) + assert.Len(t, sum.Responders, 1) + assert.Less(t, elapsed, grace*2, "repeat frames must not push the grace window out") + assert.True(t, sum.SettledEarly) +} + +// Zero ResponderGrace is the pre-existing behaviour: collect to the deadline +// even when every node has answered. +func TestGather_ResponderGrace_ZeroKeepsTheFullDeadline(t *testing.T) { + _, nc := testutil.StartTestNATS(t) + subscribeAsNode(t, nc, "test.gather.grace.zero", "node-a", []byte(`{"ok":true}`), 0) + + const budget = 400 * time.Millisecond + start := time.Now() + _, sum, err := Gather(context.Background(), nc, "test.gather.grace.zero", []byte("{}"), + GatherOpts{ + Timeout: budget, Mode: CollectUntilDeadline, ExpectedResponders: 1, + Settled: func([]Frame, Summary) bool { return false }, + }) + elapsed := time.Since(start) + + require.NoError(t, err) + assert.GreaterOrEqual(t, elapsed, budget-20*time.Millisecond) + assert.True(t, sum.TimedOut) + assert.False(t, sum.SettledEarly) +} diff --git a/spinifex/vm/record.go b/spinifex/vm/record.go index 9745f9b2b..989c931b0 100644 --- a/spinifex/vm/record.go +++ b/spinifex/vm/record.go @@ -59,6 +59,11 @@ type InstanceStatus struct { // CASes to take ownership. Ownership is carried here rather than in the key. LastNode string `json:"last_node,omitempty"` + // AZ is LastNode's own availability zone, stamped alongside it. Placement + // is observed by the node running the instance, so it lives in status + // rather than spec. + AZ string `json:"az,omitempty"` + MetadataServerAddress string `json:"metadata_server_address,omitempty"` ENIId string `json:"eni_id,omitempty"` @@ -124,6 +129,7 @@ func (v *VM) Record() *InstanceRecord { Reservation: v.Reservation, Health: v.Health, LastNode: v.LastNode, + AZ: v.AZ, MetadataServerAddress: v.MetadataServerAddress, ENIId: v.ENIId, ENIMac: v.ENIMac, @@ -178,6 +184,7 @@ func VMFromRecord(r *InstanceRecord) *VM { Reservation: r.Status.Reservation, Health: r.Status.Health, LastNode: r.Status.LastNode, + AZ: r.Status.AZ, MetadataServerAddress: r.Status.MetadataServerAddress, ENIId: r.Status.ENIId, ENIMac: r.Status.ENIMac, diff --git a/spinifex/vm/record_test.go b/spinifex/vm/record_test.go index 1544256a0..f4c725696 100644 --- a/spinifex/vm/record_test.go +++ b/spinifex/vm/record_test.go @@ -213,6 +213,7 @@ func populatedVM() *vm.VM { Reservation: &ec2.Reservation{ReservationId: aws.String("r-1")}, Health: vm.InstanceHealthState{CrashCount: 2, LastCrashReason: "oom"}, LastNode: "node-1", + AZ: "us-east-1a", MetadataServerAddress: "127.0.0.1:12345", ENIId: "eni-1", ENIMac: "aa:bb:cc:dd:ee:01", diff --git a/spinifex/vm/vm.go b/spinifex/vm/vm.go index d75e23229..c2e84126d 100644 --- a/spinifex/vm/vm.go +++ b/spinifex/vm/vm.go @@ -107,6 +107,11 @@ type VM struct { // Set when ownership is released on stop for shared KV storage. LastNode string `json:"last_node,omitempty"` + // AZ is the availability zone of the node that last ran this instance, + // stamped by that node from its own config. Placement observed by the + // owner, not something an operator requests. + AZ string `json:"az,omitempty"` + // Metadata server address (e.g., "127.0.0.1:12345") for EC2 metadata service MetadataServerAddress string `json:"metadata_server_address,omitempty"` diff --git a/spinifex/vpcd/export_test.go b/spinifex/vpcd/export_test.go new file mode 100644 index 000000000..5aca7b43b --- /dev/null +++ b/spinifex/vpcd/export_test.go @@ -0,0 +1,24 @@ +package vpcd + +import ( + "github.com/mulgadc/spinifex/spinifex/daemon" + "github.com/mulgadc/spinifex/spinifex/vm" +) + +// LocalStateReaderForTest is the local instance state reader's consumer-facing +// shape, exported so an external test can exercise it without reaching into +// package internals. +type LocalStateReaderForTest interface { + LocalVM(instanceID string) (*vm.VM, error) +} + +// NewLocalVMStateReaderForTest builds a reader over dataDir. A non-nil +// readState replaces the parse step, so a test can count parses and prove the +// mtime/size cache is doing its job. +func NewLocalVMStateReaderForTest(dataDir string, readState func(path string) (*daemon.LocalState, error)) LocalStateReaderForTest { + r := newLocalVMStateReader(dataDir) + if readState != nil { + r.readState = readState + } + return r +} diff --git a/spinifex/vpcd/imds_instance_state.go b/spinifex/vpcd/imds_instance_state.go new file mode 100644 index 000000000..0337aed57 --- /dev/null +++ b/spinifex/vpcd/imds_instance_state.go @@ -0,0 +1,96 @@ +package vpcd + +import ( + "log/slog" + "os" + "sync" + "time" + + "github.com/mulgadc/spinifex/spinifex/daemon" + "github.com/mulgadc/spinifex/spinifex/vm" + "github.com/nats-io/nats.go" +) + +// localVMStateReader adapts this node's on-disk instance state to IMDS's +// local-first lookup. vpcd is the composition root for both the network and +// compute planes IMDS straddles, so this adapter — not handlers/imds — is +// what knows the state lives in a daemon-owned file. +type localVMStateReader struct { + dataDir string + + // readState parses the local state file; overridable in tests to count + // calls. Defaults to daemon.ReadLocalState. + readState func(path string) (*daemon.LocalState, error) + + mu sync.Mutex + mtime time.Time + size int64 + cached *daemon.LocalState +} + +func newLocalVMStateReader(dataDir string) *localVMStateReader { + return &localVMStateReader{dataDir: dataDir} +} + +// LocalVM reads the instance straight off this node's on-disk VM state, +// reusing the last parse while the file's mtime and size are unchanged. IMDS +// serves concurrent requests across per-tap responders, so this is +// mutex-guarded; a stat or parse failure clears the cache and returns the +// error rather than serving a stale hit. +func (r *localVMStateReader) LocalVM(instanceID string) (*vm.VM, error) { + r.mu.Lock() + defer r.mu.Unlock() + + path := daemon.LocalStatePath(r.dataDir) + info, err := os.Stat(path) + if err != nil { + r.cached = nil + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + + if r.cached == nil || !info.ModTime().Equal(r.mtime) || info.Size() != r.size { + read := r.readState + if read == nil { + read = daemon.ReadLocalState + } + state, readErr := read(path) + if readErr != nil { + r.cached = nil + return nil, readErr + } + r.cached = state + r.mtime = info.ModTime() + r.size = info.Size() + } + + if r.cached == nil { + return nil, nil + } + return r.cached.VMS[instanceID], nil +} + +// recordLoader mirrors handlers/imds's own narrow instance-record accessor +// interface, so a construction failure below returns a true nil interface +// rather than a *daemon.JetStreamManager typed nil boxed into one. +type recordLoader interface { + LoadInstanceRecord(instanceID string) (*vm.InstanceRecord, error) +} + +// newInstanceRecordLoader opens the shared instance record space best-effort: +// IMDS still serves local hits without it, and it only backs the fallback for +// an instance that has moved off this node. Returns nil on any failure. +func newInstanceRecordLoader(nc *nats.Conn) recordLoader { + jsm, err := daemon.NewJetStreamManager(nc, 1) + if err != nil { + slog.Warn("vpcd: JetStream manager unavailable, IMDS instance lookup will not fall back off this node", "err", err) + return nil + } + if err := jsm.InitKVBucket(); err != nil { + slog.Warn("vpcd: instance-state bucket unavailable, IMDS instance lookup will not fall back off this node", "err", err) + return nil + } + return jsm +} diff --git a/spinifex/vpcd/imds_instance_state_test.go b/spinifex/vpcd/imds_instance_state_test.go new file mode 100644 index 000000000..c28e1d1cc --- /dev/null +++ b/spinifex/vpcd/imds_instance_state_test.go @@ -0,0 +1,80 @@ +package vpcd_test + +import ( + "os" + "testing" + + "github.com/mulgadc/spinifex/spinifex/daemon" + "github.com/mulgadc/spinifex/spinifex/vm" + "github.com/mulgadc/spinifex/spinifex/vpcd" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func writeInstanceState(t *testing.T, dataDir string, vms map[string]*vm.VM) { + t.Helper() + data, err := daemon.MarshalLocalState(vms) + require.NoError(t, err) + require.NoError(t, daemon.WriteLocalStateBytes(daemon.LocalStatePath(dataDir), data)) +} + +func TestLocalVMStateReader_MissingFileIsAMiss(t *testing.T) { + r := vpcd.NewLocalVMStateReaderForTest(t.TempDir(), nil) + v, err := r.LocalVM("i-absent") + require.NoError(t, err) + assert.Nil(t, v) +} + +func TestLocalVMStateReader_ResolvesAKnownInstance(t *testing.T) { + dataDir := t.TempDir() + writeInstanceState(t, dataDir, map[string]*vm.VM{"i-a": {ID: "i-a", InstanceType: "t3.micro"}}) + + r := vpcd.NewLocalVMStateReaderForTest(dataDir, nil) + v, err := r.LocalVM("i-a") + require.NoError(t, err) + require.NotNil(t, v) + assert.Equal(t, "t3.micro", v.InstanceType) +} + +func TestLocalVMStateReader_CachesParseUntilFileChanges(t *testing.T) { + dataDir := t.TempDir() + writeInstanceState(t, dataDir, map[string]*vm.VM{"i-a": {ID: "i-a"}}) + + parses := 0 + r := vpcd.NewLocalVMStateReaderForTest(dataDir, func(path string) (*daemon.LocalState, error) { + parses++ + return daemon.ReadLocalState(path) + }) + + _, err := r.LocalVM("i-a") + require.NoError(t, err) + _, err = r.LocalVM("i-a") + require.NoError(t, err) + assert.Equal(t, 1, parses, "a second lookup on an unchanged file must reuse the cached parse") + + writeInstanceState(t, dataDir, map[string]*vm.VM{"i-a": {ID: "i-a"}, "i-b": {ID: "i-b"}}) + _, err = r.LocalVM("i-b") + require.NoError(t, err) + assert.Equal(t, 2, parses, "a changed file must be re-parsed") +} + +// TestLocalVMStateReader_CorruptFileDoesNotServeStaleCache proves a stat/parse +// failure clears the cache and surfaces the error rather than replaying the +// last good parse — the caller (localInstanceLookup) treats this as a +// fall-through to the record space, not a silently stale answer. +func TestLocalVMStateReader_CorruptFileDoesNotServeStaleCache(t *testing.T) { + dataDir := t.TempDir() + writeInstanceState(t, dataDir, map[string]*vm.VM{"i-a": {ID: "i-a"}}) + + r := vpcd.NewLocalVMStateReaderForTest(dataDir, nil) + v, err := r.LocalVM("i-a") + require.NoError(t, err) + require.NotNil(t, v, "warm the cache with a good parse") + + // Different size than the good state, so the cache cannot mistake this + // for an unchanged file and must attempt (and fail) a re-parse. + require.NoError(t, os.WriteFile(daemon.LocalStatePath(dataDir), []byte("not json"), 0o600)) + + _, err = r.LocalVM("i-a") + assert.Error(t, err, "a corrupt state file must surface the read failure, not the last good parse") +} diff --git a/spinifex/vpcd/vpcd.go b/spinifex/vpcd/vpcd.go index 2ce0c347b..0b4a80430 100644 --- a/spinifex/vpcd/vpcd.go +++ b/spinifex/vpcd/vpcd.go @@ -117,6 +117,9 @@ type Config struct { OVNSBAddr string // BaseDir is the base directory for PID files and state. BaseDir string + // DataDir is this node's data directory, holding its on-disk VM state. + // IMDS reads it directly for a local-first instance metadata lookup. + DataDir string // Debug enables debug logging. Debug bool // ExternalMode is "pool", "nat" (routed, outbound-only), or "" (disabled). @@ -533,13 +536,17 @@ func launchService(cfg *Config) error { } return live, nil } + // vpcd is the composition root for both planes IMDS straddles: it wires the + // on-disk VM state reader and the shared record space, so handlers/imds + // itself never needs to import the compute plane beyond vm.VM. imdsSvc, err := handlers_imds.NewIMDSServiceImpl( ctx, nc, handlers_imds.NewNATSSTSAssumer(nc), handlers_imds.NewNATSProfileLookup(nc), handlers_imds.NewNATSPublicKeyLookup(nc), - max(len(chassisNames), 1), + newLocalVMStateReader(cfg.DataDir), + newInstanceRecordLoader(nc), listTaps, cfg.NorthstarBaseDomain, cfg.NorthstarInternalDomain,